'''
import the necessary packages
'''
from utils.dataio import get_dataset
from model.AcquisitionLayer import Muestreo
from model.InitializationLayer import BackPropagationLayer, FSI_Initial, FSI_cell
import matplotlib.pyplot as plt
import tensorflow as tf
import numpy as np
import json
import sys
import os

gpus = tf.config.list_physical_devices('GPU')

if len(gpus)> 0:
    print("RUNNING ON GPU")
    for gpu in gpus:
        print("Name:", gpu.name, "  Type:", gpu.device_type)

    try:
        tf.config.set_visible_devices(gpus[0], 'GPU')
    except RuntimeError as e:
    # Visible devices must be set before GPUs have been initialized
        print(e)
else:
    print("running on cpu")

def plot_image(image, path2save):
    plt.imshow(image)
    plt.colorbar()
    plt.savefig(path2save)
    plt.close()

'''
parameters
'''

config_file = sys.argv[-1]
if os.path.exists(config_file):
    print("opening config file")
    print(config_file)
    with open(config_file) as json_file:
        params = json.load(json_file)
        data_params, forward_params, training_params, other_params = params["data"], params["forward_model"], params["training"], params["other"]
else:
    raise Exception("Config file not found: "+ config_file)

# data
SHAPE = data_params["image_shape"]
DATASET_NAME = data_params["dataset_name"]

# Forward model parameters
SNAPSHOTS =  forward_params["number_snapshots"]
TIPO_MUESTREO =  forward_params["tipo_muestreo"]
WAVE_LENGTH =  forward_params["wavelength"]
DX =  forward_params["pixel_size"]
DISTANCE_SENSOR =  forward_params["distance_sensor"]
NORMALIZE_MEASUREMENTS = forward_params["normalize_measurements"]

# Training
BATCH_SIZE = training_params["batch_size"]
RESULTS_FOLDER =  training_params["results_folder"]

if os.path.exists(RESULTS_FOLDER):
    for files in os.listdir(RESULTS_FOLDER):
        os.remove(os.path.join(RESULTS_FOLDER, files))
else:
    os.makedirs(RESULTS_FOLDER)

# other
FLOAT_DTYPE =  "float32"
COMPLEX_DTYPE =  "complex64"
P =  other_params["p"]
K_SIZE = other_params["k_size"]
N_ITER = other_params["n_iterations"]

'''
Data preprocessing
'''

_,  dataset = get_dataset(DATASET_NAME, shape = SHAPE, batch_size = BATCH_SIZE, num_classes = 10, complex_dtype=COMPLEX_DTYPE)
x_image, _ = next(iter(dataset))

'''
Acquisition Layer definition
'''
muestreo_layer = Muestreo(snapshots = SNAPSHOTS, wave_length = WAVE_LENGTH,
                dx = DX,distance = DISTANCE_SENSOR, tipo_muestreo = TIPO_MUESTREO,
                normalize_measurements = NORMALIZE_MEASUREMENTS, codificate = True,
                float_dtype = FLOAT_DTYPE, complex_dtype=COMPLEX_DTYPE, name="Muestreo")

muestras_entrada, mascara_usada, var_de_interes = muestreo_layer(x_image)

'''
image and matrices
'''
x_complex = tf.complex(x_image[0,...,0], x_image[0,...,1])

plot_image(x_image[0,...,0], os.path.join(RESULTS_FOLDER, "imagen_real.png"))
plot_image(x_image[0,...,1], os.path.join(RESULTS_FOLDER, "imagen_imag.png"))
plot_image(tf.math.abs(x_complex), os.path.join(RESULTS_FOLDER, "imagen_abs.png"))
plot_image(tf.math.angle(x_complex), os.path.join(RESULTS_FOLDER, "imagen_angle.png"))

if (TIPO_MUESTREO=="ASM" or TIPO_MUESTREO=="FRESNEL"):
    plot_image(tf.math.abs(var_de_interes[0,0,...]), os.path.join(RESULTS_FOLDER, "T_muestreo_.png"))
    plot_image(tf.math.angle(var_de_interes[0,0,...]), os.path.join(RESULTS_FOLDER, "T_muestreo_.png"))
plot_image(tf.math.abs(mascara_usada[0,0,]), os.path.join(RESULTS_FOLDER, "mascara_abs.png"))
plot_image(tf.math.angle(mascara_usada[0,0,]), os.path.join(RESULTS_FOLDER, "mascara_angle.png"))
#print("mascara rango", tf.reduce_min(tf.math.angle(mascara_usada[0,0,])), tf.reduce_max(tf.math.angle(mascara_usada[0,0,])))
plot_image(muestras_entrada[0,...,0], os.path.join(RESULTS_FOLDER, "muestras_imagen.png"))

'''
run back proopagation
'''

backpropagation_layer  = BackPropagationLayer(tipo_muestreo = TIPO_MUESTREO, complex_dtype = COMPLEX_DTYPE, name = "BackPropagationLayer")
back = backpropagation_layer([muestras_entrada,mascara_usada,var_de_interes])
back_real, back_imag = tf.unstack(back, num=2, axis=-1)
back_complex = tf.complex(back_real[0,], back_imag[0,])

plot_image(back_real[0,], os.path.join(RESULTS_FOLDER, "back_real.png"))
plot_image(back_imag[0,], os.path.join(RESULTS_FOLDER, "back_imag.png"))
plot_image(tf.math.abs(back_complex), os.path.join(RESULTS_FOLDER, "back_abs.png"))
plot_image(tf.math.angle(back_complex), os.path.join(RESULTS_FOLDER, "back_angle.png"))

'''
run fsi initialization
'''
init_initialzation = FSI_Initial(p = P, float_dtype=FLOAT_DTYPE,complex_dtype=COMPLEX_DTYPE, name = "init_initialzation")
Initialation = FSI_cell(p = P, k_size = K_SIZE, tipo_muestreo = TIPO_MUESTREO, float_dtype=FLOAT_DTYPE, complex_dtype=COMPLEX_DTYPE, train_initialization = True, name = "initialization_cell")
Ytr, Z = init_initialzation(muestras_entrada)

for i in range(N_ITER):
    Z = Initialation([Ytr, Z, mascara_usada, var_de_interes])

normalization = tf.reduce_max(tf.math.abs(Z), axis=(1,2,3), keepdims=True)

Z = tf.divide(Z, tf.cast(normalization, tf.complex64))
Z_abs = tf.squeeze(tf.math.abs(Z))
Z_angle = tf.squeeze(tf.math.angle(Z))

kernel_real, kernel_imag = Initialation.get_kernel_initialization()

plot_image(tf.math.real(Z)[0,0], os.path.join(RESULTS_FOLDER, "init_real.png"))
plot_image(tf.math.imag(Z)[0,0], os.path.join(RESULTS_FOLDER, "init_imag.png"))
plot_image(tf.math.abs(Z)[0,0], os.path.join(RESULTS_FOLDER, "init_abs.png"))
plot_image(tf.math.angle(Z)[0,0], os.path.join(RESULTS_FOLDER, "init_angle.png"))
plot_image(kernel_real[...,0,0], os.path.join(RESULTS_FOLDER, "kernel_real.png"))
plot_image(kernel_imag[...,0,0], os.path.join(RESULTS_FOLDER, "kernel_imag.png"))
print("The code ran successfully")


import numpy as np
import models

def rectangle(plane, x, y, w, h):

    assert isinstance(plane, models.Plane), "Expected plane to by of type Plane, got %s instead" % type(plane)

    distance = -plane.coefficients[3]

    vertices = np.array([[distance, w / 2, h / 2],
                         [distance, -(w / 2), h / 2],
                         [distance, -(w / 2), -(h / 2)],
                         [distance, w / 2, -(h / 2)]])
    vertices = vertices + np.array([0, x, y])
    edges = [(0, 1), (1, 2), (2, 3), (3, 0)]

    v1 = plane.coefficients[:3]
    if not np.isclose(v1[2], 1):
        v2 = np.array([-v1[1], v1[0], 0])
    else:
        v2 = np.array([0, v1[2], -v1[1]])
    v3 = np.cross(v1, v2)

    v2 = v2 / np.linalg.norm(v2)
    v3 = v3 / np.linalg.norm(v3)

    R = np.array([v1, v2, v3]).T
    vertices = np.dot(R, vertices.T).T

    return models.Polygon3D(vertices, edges, plane)

def ellipse(lambda1, lambda2, resolution, z_ref=0):

    A = list(reversed([[x, 2*lambda2] for x in np.linspace(-lambda1, lambda1, resolution + 2)[1:-1]]))
    B = [[lambda1, y] for y in np.linspace(0, 2*lambda2, resolution + 2)[1:-1]]

    v1 = np.array([lambda1, 0, z_ref])
    v2 = np.array([-lambda1, 0, z_ref])
    p1 = np.array([0, lambda2, z_ref])
    p2 = np.array([0, -lambda2, z_ref])

    # quadrant 1
    vertices_q1 = [v1]
    for i in range(resolution):

        px_top11 = np.linalg.det(np.array([v1[:2], A[i]]))
        px_top12 = np.linalg.det(np.array([[v1[0], 1], [A[i][0], 1]]))
        px_top21 = np.linalg.det(np.array([v2[:2], B[i]]))
        px_top22 = np.linalg.det(np.array([[v2[0], 1], [B[i][0], 1]]))

        py_top11 = np.linalg.det(np.array([v1[:2], A[i]]))
        py_top12 = np.linalg.det(np.array([[v1[1], 1], [A[i][1], 1]]))
        py_top21 = np.linalg.det(np.array([v2[:2], B[i]]))
        py_top22 = np.linalg.det(np.array([[v2[1], 1], [B[i][1], 1]]))

        bot11 = np.linalg.det(np.array([[v1[0], 1], [A[i][0], 1]]))
        bot12 = np.linalg.det(np.array([[v1[1], 1], [A[i][1], 1]]))
        bot21 = np.linalg.det(np.array([[v2[0], 1], [B[i][0], 1]]))
        bot22 = np.linalg.det(np.array([[v2[1], 1], [B[i][1], 1]]))
        denominator = np.linalg.det(np.array([[bot11, bot12], [bot21, bot22]]))

        px = np.linalg.det(np.array([[px_top11, px_top12], [px_top21, px_top22]])) / denominator
        py = np.linalg.det(np.array([[py_top11, py_top12], [py_top21, py_top22]])) / denominator

        vertices_q1.append(np.array([px, py, z_ref]))

    # quadrant 2
    vertices_q2 = [p1]
    for v1 in reversed(vertices_q1[1:]):
        vertices_q2.append(np.array([-v1[0], v1[1], z_ref]))

    # quadrant 3
    vertices_q3 = [v2]
    for v1 in vertices_q1[1:]:
        vertices_q3.append(np.array([-v1[0], -v1[1], z_ref]))

    # quadrant 4
    vertices_q4 = [p2]
    for v1 in reversed(vertices_q1[1:]):
        vertices_q4.append(np.array([v1[0], -v1[1], z_ref]))

    # vertices and edges
    vertices = np.array(vertices_q1 + vertices_q2 + vertices_q3 + vertices_q4)
    edges = list(zip(range(len(vertices)), range(1, len(vertices))))
    edges.append((len(vertices) - 1, 0))
    edges = set(edges)

    return models.ConvexPolygon2D(vertices, edges)

def random_evidence(n, z_ref, max_radius, min_range, max_range, resolution=4):

    min_range = np.array(min_range) + max_radius
    max_range = np.array(max_range) - max_radius

    evidence = []
    for i in range(n):
        lambda1 = max_radius * min(np.random.rand() + 0.1, 1)
        lambda2 = max_radius * min(np.random.rand() + 0.1, 1)

        lambda1, lambda2 = (lambda2, lambda1) if lambda2 > lambda1 else (lambda1, lambda2)

        e = ellipse(lambda1, lambda2, resolution, z_ref)

        v1 = (np.random.rand(2) - 0.5)
        v1 = v1 / np.linalg.norm(v1)
        v2 = np.array([-v1[1], v1[0]])

        R = np.array([v1, v2]).T
        t = ((max_range - min_range) * np.random.rand(2)) + min_range
        e.rigid(R, t)

        evidence.append(e)

    return evidence

def random_planes(num_planes, max_distance, a=None, b=None, c=None):

    coefs = np.random.rand(num_planes, 4) - 0.5

    if a is not None:
        coefs[:, 0] = a
    if b is not None:
        coefs[:, 1] = b
    if c is not None:
        coefs[:, 2] = c

    coefs[:, :3] = coefs[:, :3] / np.linalg.norm(coefs[:, :3], axis=1)[:, None]
    coefs[:, 3] = 2 * max_distance * coefs[:, 3]

    return [models.Plane(c) for c in coefs]


import numpy as np
import scipy.signal as signal
import matplotlib.pyplot as plt

def kaiserExp(length, alpha):
    K = length // 2 + length % 2
    p = (np.arange(1, length + 1) - K) / (K)
    return np.exp(alpha * np.sqrt(1 - p * p)) / np.exp(alpha)

def kaiserCosh(length, alpha):
    K = length // 2 + length % 2
    p = (np.arange(1, length + 1) - K) / (K)
    return np.cosh(alpha * np.sqrt(1 - p * p)) / np.cosh(alpha)

def testKaiser():
    length = 15
    alpha = 10
    win_scipy = signal.get_window(("kaiser", np.pi * alpha), length + length % 2)

    # alpha = 1.3
    win_exp = kaiserExp(length, np.pi * alpha)
    win_cosh = kaiserCosh(length, np.pi * alpha)

    if length % 2 == 1:
        win_scipy = win_scipy[1:]
    else:
        win_scipy = win_scipy[1:]
        win_exp = win_exp[:-1]
        win_cosh = win_cosh[:-1]

    print(len(win_scipy), len(win_cosh))
    print(win_scipy[0], win_scipy[-1])
    print(win_cosh[0], win_cosh[-1])

    plt.plot(win_scipy, label="scipy")
    plt.plot(win_exp, label="exp")
    plt.plot(win_cosh, label="cosh")
    plt.grid()
    plt.legend()
    plt.show()

def testHann():
    length = 16
    win_scipy = signal.get_window("hann", length + length % 2)[::-1]

    K = length + length % 2
    n = np.arange(1, length + 1)
    sn = np.sin(np.pi * n / K)
    win_hann = sn * sn

    print(win_scipy[0], win_scipy[-1])

    plt.plot(win_scipy, label="scipy")
    plt.plot(win_hann, label="naive")
    plt.grid()
    plt.legend()
    plt.show()

def testLowpass():
    length = 15
    cutoff = 0.1
    lp = np.zeros(length)
    K = length // 2 + length % 2
    for idx in range(length):
        x = np.pi * (idx + 1 - K)
        if x != 0:
            lp[idx] = np.sin(cutoff * x * 2) / x
        else:
            lp[idx] = cutoff
    plt.plot(lp)
    plt.show()


import argparse
import changelog_parser
import changelog_formatter

argparser = argparse.ArgumentParser()
argparser.add_argument("file")
args = argparser.parse_args()

file = open(args.file, "r")
changelog = changelog_parser.parse(file)
if changelog:
    formatted_changelog = changelog_formatter.format(changelog)

    # Format message to print newline character and not actual newlines
    message = "{}".format(formatted_changelog.replace("\n", r"\n"))
    print(message)


import json

fichier = "settings.json"

try:
    with open(fichier, "r") as f:
        settings = json.load(f)
    #print(settings)
except FileNotFoundError:
    print(f"Le fichier '{fichier}' n'a pas été trouvé.")
except json.JSONDecodeError:
    print(f"Erreur de décodage JSON dans le fichier '{fichier}'. Assurez-vous que le fichier est au format JSON valide.")
except Exception as e:
    print(f"Une erreur s'est produite : {e}")

settings["fontSize"]=15

with open(fichier, "w") as f:
    json.dump(settings, f, indent=4)

print()


import math
from heapq import heappop, heappush

def find_path (source_point, destination_point, mesh):

	"""
	Searches for a path from source_point to destination_point through the mesh

	Args:
		source_point: starting point of the pathfinder
		destination_point: the ultimate goal the pathfinder must reach
		mesh: pathway constraints the path adheres to

	Returns:

		A path (list of points) from source_point to destination_point if exists []
		A list of boxes explored by the algorithm {}
	"""
	path = []
	boxes = {}

	source_box = ()
	dest_box = ()
	for box in mesh['boxes']:                           #get the box for source_point and destination_point
		if point_within_box(source_point, box):
			source_box = box
		if point_within_box(destination_point, box):
			dest_box = box

	queue = [(0, source_box)]
	detail_points = {}
	detail_points[source_box] = source_point

	distances = {}
	distances[source_box] = 0

	backpointers = {}
	backpointers[source_point] = None

	#boxes[source_box] = (None, source_point)

	while queue:                                        #djikstras
		current_dist, current_box = heappop(queue)
		if (current_box == dest_box):
			break
		else:
			for adj_box in mesh['adj'][current_box]:
				current_point = detail_points[current_box]
				adj_point = findDetailPoint(current_box, adj_box, current_box)
				pathcost = distances[current_box] + euclidean(current_point, adj_point)
				if adj_box not in distances or pathcost < distances[adj_box]:
					detail_points[adj_box] = adj_point
					distances[adj_box] = pathcost
					backpointers[adj_point] = current_box
					queue_dist = euclidean(adj_point, destination_point)
					heappush(queue, (queue_dist, adj_box))
					boxes[adj_box] = current_box

	path.append(destination_point)
	while current_box:
		path.append(detail_points[current_box])
		current_box = boxes[current_box]
		path.append(detail_points[current_box])
		if (current_box == source_box):
			path.append(source_point)
			break
	return (path,distances.keys())

def point_within_box(point, box):                       #check if a point is within a given box
	if box[0] <= point[0] and box[1] >= point[0] and box[2] <= point[1] and box[3] >= point[1]:
		return True
	else:
		return False

def box_of_point(boxlist, point):
	for box in boxlist:                           #get the box for a given point
		if point_within_box(point, box):
			return box
	return False

def euclidean(tupleA, tupleB):
	base1 = (tupleA[0] - tupleB[0])
	base1 *= base1
	#print(base1)
	base2 = (tupleA[1] - tupleB[1])
	base2 *= base2
	#print(base2)
	return math.sqrt(base1+base2)

def findDetailPoint(boxA, boxB, originPoint):
	bAx1 = boxA[0]
	bAx2 = boxA[1]
	bAy1 = boxA[2]
	bAy2 = boxA[3]

	bBx1 = boxB[0]
	bBx2 = boxB[1]
	bBy1 = boxB[2]
	bBy2 = boxB[3]

	maxX, minX = [max(bAx1, bBx1), min(bAx2, bBx2)]
	maxY, minY = [max(bAy1, bBy1), min(bAy2, bBy2)]

	distanceA = euclidean(originPoint, (minX, minY))
	distanceB = euclidean(originPoint, (maxX, maxY))

	if minX == maxX: #vertical
		if distanceA < distanceB:
			return (minX, minY)
		elif distanceB < distanceA:
			return (maxX, maxY)
		else:
			newX = minX + ((maxX - minX)/2)
			return (newX, originPoint[1])
	if minY == maxY: #horizontal
		if distanceA < distanceB:
			return (minX, minY)
		elif distanceB < distanceA:
			return (maxX, maxY)
		else:
			newY = minY + ((maxY - minY)/2)
			return (originPoint[0], newY)

def box_center(box):
	return ((box[0] + box[2])/2, (box[1] + box[3])/2)


from flask_admin.contrib.sqla import ModelView
from flask_login import current_user

from app.db import CheckIn
from .formatters import DEFAULT_FORMATTERS

class CheckInsView(ModelView):
    column_list = [
        'student',
        'event',
        'timestamp',
        'number_of_scans',
        'is_cse',
        'degree'
    ]

    column_labels = {
        'is_cse': 'Is CSE?',
        'number_of_scans': 'Number of scans'
    }

    column_filters = [
        'event.name',
        'timestamp',
        'student.zid',
        'student.given_names',
        'student.surname',
        'degree',
        'enrolments.course',
        'is_cse',
        'number_of_scans'
    ]

    column_type_formatters = DEFAULT_FORMATTERS

    column_default_sort = ('timestamp', True)

    def is_accessible(self):
        return current_user.is_authenticated()

    def __init__(self, session, **kwargs):
        super(CheckInsView, self).__init__(CheckIn, session, **kwargs)


import os, random
import pickle
from typing import Optional
import torch
from torch.utils.data import Dataset, DataLoader
import pytorch_lightning as pl

class SeqAttnData(Dataset):
    def __init__(self, seq: dict):
        self.seq = seq
        self.idx2caseid = {
            i: case
            for i, case in enumerate(self.seq)
        }

    def __len__(self):
        return len(self.idx2caseid)

    def __getitem__(self, idx):
        case = self.idx2caseid[idx]
        data = self.seq[case]
        return case, data['L']['gesture'], data['R']['gesture'], data['label']

    @staticmethod
    def collate_fn(batch):
        # max seq len
        max_seq_len = 882
        max_len_L = max(
            len(L) for _, L, _, _ in batch
        )
        max_len_L = min(max_len_L, max_seq_len)
        input_ids_L = torch.zeros((len(batch), max_len_L))
        attn_mask_L = torch.zeros((len(batch), max_len_L))
        max_len_R = max(
            len(R) for _, _, R, _ in batch
        )
        max_len_R = min(max_len_R, max_seq_len)
        input_ids_R = torch.zeros((len(batch), max_len_R))
        attn_mask_R = torch.zeros((len(batch), max_len_R))
        labels = torch.zeros((len(batch),))
        cases = []
        for i, (case, L, R, label) in enumerate(batch):
            # shift right and add 3 (0,1,2 preserved)
            # 0: CLS
            # 1: SEP
            # 2: PAD
            L = (L + 3)[:(max_len_L - 2)].tolist()
            L_ids = (
                    [0] +
                    L +
                    [1] +
                    [2] * (max_len_L - 2 - len(L))
            )
            L_mask = [1] * (len(L) + 2) + [0] * (max_len_L - 2 - len(L))
            input_ids_L[i] = torch.tensor(L_ids)
            attn_mask_L[i] = torch.tensor(L_mask)

            R = (R + 3)[:(max_len_R - 2)].tolist()
            R_ids = (
                    [0] +
                    R +
                    [1] +
                    [2] * (max_len_R - 2 - len(R))
            )
            R_mask = [1] * (len(R) + 2) + [0] * (max_len_R - 2 - len(R))
            input_ids_R[i] = torch.tensor(R_ids)
            attn_mask_R[i] = torch.tensor(R_mask)

            labels[i] = label
            cases.append(case)

        return {
            "input_ids_L": input_ids_L.long(),
            "attn_mask_L": attn_mask_L.bool(),
            "input_ids_R": input_ids_R.long(),
            "attn_mask_R": attn_mask_R.bool(),
            "label": labels,
            "case": cases
        }

class MCDataModule(pl.LightningDataModule):
    def __init__(self,
        data_dir: str,
        batch_size: int, num_workers: int,
        k_fold: int,
        **kwargs
    ):
        super().__init__()
        self.batch_size = batch_size
        self.num_workers = num_workers
        self.data_dir = data_dir
        self.dataset = {}
        self.k_fold = k_fold

    def setup(self, stage: Optional[str] = None):
        seq = {}
        for split in ['train', 'valid', 'test']:
            # file = os.path.join(self.data_dir, f"k0", f"{split}_processed.pkl")
            file = os.path.join(self.data_dir, f"k0", f"{split}_combined.pkl")
            assert os.path.exists(file)

            with open(file, "rb") as f:
                seq = {**seq, **pickle.load(f)}

        keys = list(seq.keys())
        random.seed(self.k_fold)
        random.shuffle(keys)
        if "con" in self.data_dir:
            # drop 3 cases
            if "Gronau" in self.data_dir:
                assert len(seq) == 77
                traink, testk = keys[:62], keys[62:]
            else:
                assert len(seq) == 48
                traink, testk = keys[:41], keys[41:]
        elif "Gronau" in self.data_dir:
            assert len(seq) == 80
            traink, testk = keys[:64], keys[64:]
        else:
            assert len(seq) == 50
            traink, testk = keys[:40], keys[40:]
        train, test = {}, {}
        for k, v in seq.items():
            if k in traink:
                train = {**train, k:v}
            else:
                test = {**test, k:v}
        self.dataset["train"] = SeqAttnData(train)
        self.dataset["valid"] = SeqAttnData(test)

    def get_loader(self, split) -> DataLoader:
        return DataLoader(
            self.dataset[split], batch_size=self.batch_size,
            collate_fn=SeqAttnData.collate_fn, drop_last=False,
            num_workers=self.num_workers, pin_memory=True,
            shuffle=(split=="train")
        )

    def train_dataloader(self) -> DataLoader:
        return self.get_loader("train")

    def val_dataloader(self) -> DataLoader:
        return self.get_loader("valid")


"""Tests for search tasks."""

from unittest import mock
import pytest

from django.urls import reverse
from django.utils import timezone

from readthedocs.search.models import SearchQuery
from readthedocs.search import tasks

@pytest.mark.django_db
@pytest.mark.search
@pytest.mark.usefixtures("all_projects")
class TestSearchTasks:
    @classmethod
    def setup_class(cls):
        # This reverse needs to be inside the ``setup_class`` method because from
        # the Corporate site we don't define this URL if ``-ext`` module is not
        # installed
        cls.url = reverse("search_api")

    def test_search_query_recorded_when_results_not_zero(self, api_client):
        """Test if search query is recorded in a database when a search is made."""

        assert (
            SearchQuery.objects.all().count() == 0
        ), "no SearchQuery should be present if there is no search made."

        # `sphinx` is present in `documentation.json`
        # file of project `kuma`
        search_params = {"q": "sphinx", "project": "kuma", "version": "latest"}
        resp = api_client.get(self.url, search_params)

        assert resp.data["count"] == 1
        assert (
            SearchQuery.objects.all().count() == 1
        ), "there should be 1 obj since a search is made which returns one result."

    def test_partial_queries_are_not_recorded(self, api_client):
        """Test if partial queries are not recorded."""

        assert (
            SearchQuery.objects.all().count() == 0
        ), "no SearchQuery should be present if there is no search made."

        time = timezone.now()
        search_params = {"q": "stack", "project": "docs", "version": "latest"}

        with mock.patch("django.utils.timezone.now") as test_time:
            test_time.return_value = time
            resp = api_client.get(self.url, search_params)
            assert resp.status_code, 200

        assert (
            SearchQuery.objects.all().count() == 1
        ), "one SearchQuery should be present"

        # update the time and the search query and make another search request
        time = time + timezone.timedelta(seconds=2)
        search_params["q"] = "stack over"
        with mock.patch("django.utils.timezone.now") as test_time:
            test_time.return_value = time
            resp = api_client.get(self.url, search_params)
            assert resp.status_code, 200

        # update the time and the search query and make another search request
        time = time + timezone.timedelta(seconds=2)
        search_params["q"] = "stack overflow"
        with mock.patch("django.utils.timezone.now") as test_time:
            test_time.return_value = time
            resp = api_client.get(self.url, search_params)
            assert resp.status_code, 200

        assert (
            SearchQuery.objects.all().count() == 1
        ), "one SearchQuery should be present"
        assert (
            SearchQuery.objects.all().first().query == "stack overflow"
        ), "one SearchQuery should be there because partial queries gets updated"

    def test_search_query_recorded_when_results_are_zero(self, api_client):
        """Test that search queries are recorded when they have zero results."""

        assert (
            SearchQuery.objects.all().count() == 0
        ), "no SearchQuery should be present if there is no search made."

        # `readthedo` is NOT present in project `kuma`.
        search_params = {"q": "readthedo", "project": "kuma", "version": "latest"}
        resp = api_client.get(self.url, search_params)

        assert resp.data["count"] == 0
        assert SearchQuery.objects.all().count() == 1

    def test_delete_old_search_queries_from_db(self, project):
        """Test that the old search queries are being deleted."""

        assert (
            SearchQuery.objects.all().count() == 0
        ), "no SearchQuery should be present if there is no search made."

        obj = SearchQuery.objects.create(
            project=project, version=project.versions.all().first(), query="first"
        )
        obj.created = timezone.make_aware(timezone.datetime(2019, 1, 1))
        obj.save()

        assert SearchQuery.objects.all().count() == 1
        tasks.delete_old_search_queries_from_db()
        assert SearchQuery.objects.all().count() == 0


# Core libraries
import os
import sys
sys.path.append("../../../")
import cv2
import numpy as np
from tkinter import *
from tkinter.ttk import *

# My libraries
from Utilities.ImageUtils import ImageUtils

"""
Class implements the preview image GUI
"""

class ImagePreviewGUI():
	# Class constructor
	def __init__(self, super_interface):
		# Maximum image size
		self.__max_img_w = 400
		self.__max_img_h = 400

		# Maintain a reference to the superior interface
		self.__super_interface = super_interface

		# Create the empty image (matches the interface background)
		self.__np_blank_image = np.zeros((self.__max_img_w, self.__max_img_h, 3), np.uint8)
		self.__np_blank_image.fill(217)
		self.__blank_img = ImageUtils.convertImage(self.__np_blank_image)

		# Get this interface ready
		self.__initInterface()

	"""
	Public methods
	"""

	# Called when the mouse enters an ID, display the image larger
	def updateImage(self, np_image):
		# Resize the image
		np_image = ImageUtils.proportionallyResizeImageToMax(np_image, self.__max_img_w, self.__max_img_h)

		# Insert it at the top of the black image (so the label size stays consistent)
		padded_np_image = self.__np_blank_image.copy()
		padded_np_image[0:np_image.shape[0], 0:np_image.shape[1], :] = np_image

		# Update the stored image
		self.__image = ImageUtils.convertImage(padded_np_image)

		# Update the label
		self.__image_label.config(image=self.__image)

	# Called when the mouse leaves a particular ID, just display a black empty image
	def loseImage(self):
		self.__image_label.config(image=self.__blank_img)

	"""
	(Effectively) private methods
	"""

	def __initInterface(self):
		# Create our main frame
		self.__frame = Frame(self.__super_interface, relief=RAISED, borderwidth=2)
		self.__frame.pack(side=TOP, padx=5, pady=5)

		# Add a label to the top
		self.__label = Label(self.__frame, text="Image Preview")
		self.__label.pack(side=TOP, padx=5, pady=5)

		# Create the image label
		self.__image_label = Label(self.__frame, image=self.__blank_img)
		self.__image_label.pack(side=TOP, padx=5, pady=5)


from django.db import models
from _lib import panda
from django.db.models import Max
from django.core.validators import MaxValueValidator, MinValueValidator

from _mc_subdept.models import Subdept
# Create your models here.
class Category(models.Model):
    deptcode = models.CharField(db_column='DeptCode', max_length=6)  # Field name made lowercase.
    mcode = models.ForeignKey(Subdept, models.DO_NOTHING, db_column='MCode',related_name='category_key')  # Field name made lowercase.
    code = models.CharField(db_column='Code', primary_key=True, max_length=6,editable=False)  # Field name made lowercase.
    description = models.CharField(db_column='Description', max_length=40)  # Field name made lowercase.
    margin_min = models.DecimalField(max_digits=5, decimal_places=2, validators=[MinValueValidator(0.00)])
    margin_max = models.DecimalField(max_digits=5, decimal_places=2, validators=[MinValueValidator(0.00)])
    tax_code_purchase = models.CharField(max_length=10)
    tax_code_supply = models.CharField(max_length=10)
    margin = models.DecimalField(max_digits=5, decimal_places=2, validators=[MinValueValidator(0.00)])
    created_at = models.DateTimeField(db_column='CREATED_AT', null=True,editable=False)  # Field name made lowercase.
    created_by = models.CharField(db_column='CREATED_BY', max_length=30,editable=False)  # Field name made lowercase.
    updated_at = models.DateTimeField(db_column='UPDATED_AT', blank=True, null=True,editable=False)  # Field name made lowercase.
    updated_by = models.CharField(db_column='UPDATED_BY', max_length=30)  # Field name made lowercase.

    class Meta:
        managed = False
        db_table = 'category'
        ordering = ('code',)

    def __str__(self):
        return self.code

    def get_absolute_url(self):
        return f'/{self.code}/'

    # def next_running_number:
    #     maxCode = Category.objects.all().aggregate(Max('code'))
    #     if maxCode=='':
    #         return 1
    #     else:
    #         return maxCode+1

    def save(self, *args, **kwargs):
        print('1234567')
        print('self_group:',self.code)
        if self.created_by =='':
            maxCode = Category.objects.all().aggregate(Max('code'))
            if maxCode=='':
                next_running_number = 1
            else:
                print('print maxcode', maxCode)
                print(type(maxCode))
                next_running_number = int(maxCode['code__max'])+1
            self.code='{:04}'.format(next_running_number)
            self.created_at=panda.panda_today()
            self.created_by=self.updated_by
            self.updated_at=panda.panda_today()
        else:
            self.updated_at=panda.panda_today()
        super(Category, self).save(*args, **kwargs)


import os
import platform
import glob
from Tkinter import *
import ttk

class ImageViewer(Toplevel):
    def __init__(self, imgFileList):
        Toplevel.__init__(self)
        self.imgFileList = imgFileList  # a list of file names, one for each image in the series
        self.imgIndex = 0  # a numeric index indicating the current image in the list (self.imgFileList)
        self.highlightBox = (0, 0, 0, 0)
        self.currentFile = ""
        self.frame = Frame(self,  bg="red")
        self.frame.grid(column=0, row=0, sticky=N+S+E+W)
        self.xscrollbar = Scrollbar(self.frame, orient=HORIZONTAL)
        self.xscrollbar.grid(row=1, column=0, sticky=E+W)
        self.yscrollbar = Scrollbar(self.frame)
        self.yscrollbar.grid(row=0, column=1, sticky=N+S)
        self.canvas = Canvas(self.frame, width=550, height=550,  bg="gray", xscrollcommand=self.xscrollbar.set, yscrollcommand=self.yscrollbar.set)
        self.canvas.grid(column=0, row=0, sticky=N+S+E+W)
        self.xscrollbar.config(command=self.canvas.xview)
        self.yscrollbar.config(command=self.canvas.yview)
        self.updateImg()
        self.columnconfigure(0, weight=1)
        self.rowconfigure(0, weight=1)
        self.frame.columnconfigure(0, weight=1)
        self.frame.rowconfigure(0, weight=1)
        self.navFrame = ttk.Frame(self)
        self.navFrame.grid(column=0, row=1)
        self.butPrev = ttk.Button(self.navFrame, text='Prev', command=self.prior)
        self.butPrev.pack(side="left")
        self.butNext = ttk.Button(self.navFrame, text='Next', command=self.advance)
        self.butNext.pack()
        # key bindings
        self.bind("<Right>", self.keyPress)
        self.bind("<Left>", self.keyPress)
        if "Darwin" in platform.system():
            accel = 'Command'
        else:
            accel = 'Alt'
        self.bind('<'+accel+'-w>', lambda e: self.destroy())
        self.focus_force()
        #print "init - frame",self.frame.winfo_manager(), self.frame.winfo_width(), self.frame.winfo_reqheight()

    def keyPress(self, e):
        #print e.keysym
        if e.keysym == "Right":
            self.advance()
        elif e.keysym == "Left":
            self.prior()

    def setImage(self, imageName, highlightBox):
        if imageName in self.imgFileList:
            self.imgIndex = self.imgFileList.index(imageName)
            self.highlightBox = highlightBox
            self.updateImg()

    def advance(self):
        if self.imgIndex < len(self.imgFileList)-1:
            self.imgIndex += 1
            self.updateImg()

    def prior(self):
        if self.imgIndex > 0:
            self.imgIndex -= 1
            self.updateImg()

    def updateImg(self):
        gifFile = self.imgFileList[self.imgIndex]
        self.title(gifFile.rpartition('/')[2])
        imgobj = PhotoImage(file=gifFile)
        self.canvas.delete("image")
        image = self.canvas.create_image(0, 0, image=imgobj, anchor="nw", tag="image")
        self.canvas.tag_lower(image)
        self.canvas.config(scrollregion=self.canvas.bbox(ALL))
        self.canvas.img = imgobj  # need assignment to prevent garbage collection of PhotoImage object
        # (re)draw highlight box
        linecolor="green"
        self.canvas.delete("highlight")
        x1 = self.highlightBox[0]
        y1 = self.highlightBox[1]
        x2 = self.highlightBox[2]
        y2 = self.highlightBox[3]
        # need to convert from image to canvas coordinates?
        rect = self.canvas.create_polygon(x1, y1, x1, y2, x2, y2, x2, y1, fill="", outline=linecolor, width="2", tag="highlight")
        # scroll to put cell of interest in center
        # currentScrollPosX = self.xscrollbar.get()
        self.update_idletasks()
        fw = self.frame.winfo_width()
        fh = self.frame.winfo_height()
        cw = self.canvas.bbox(ALL)[2] - self.canvas.bbox(ALL)[0]
        ch = self.canvas.bbox(ALL)[3] - self.canvas.bbox(ALL)[1]
        cx = (x1+x2)/2.0
        cy = (y1+y2)/2.0
        xoffset = (cx - (fw/2.0)) / cw
        yoffset = (cy - (fh/2.0)) / ch
        self.canvas.xview_moveto(xoffset)
        self.canvas.yview_moveto(yoffset)

# def main():
#     workDir = "/Users/bbraun/Dropbox/python/4-28_Test_LAP"
#     os.chdir(workDir)
#     imgFileList = glob.glob("*Outline*.gif")
#     root = Tk()
#     iv = ImageViewer(imgFileList)
#     root.mainloop()
#     pass
#
#
# if __name__ == '__main__':
#     main()


import preprocess
import recognize
import printed_icr
from pdf2image import convert_from_path

def intel_char_recog(filename, texttype, filetype):
    output = 'default - empty string'
    if texttype.lower() == 'print':
        if filetype.lower() == 'pdf':
            pages = convert_from_path(filename, 500)
            # for page in pages:
            page[0].save('image.jpg', 'JPEG')
            output = printed_icr.printed_intel_char_recog('image.jpg')
        elif filetype.lower() == 'image':
            output = printed_icr.printed_intel_char_recog(filename)
    elif texttype.lower() == 'handwritten':
        if filetype.lower() == 'image':
            folder = preprocess.save_words_and_lines(filename)
            output = recognize.recog(folder)
    return output

def main():
    print(intel_char_recog('sample_segmentation_data/png1.jpg', 'handwritten', 'image'))

if __name__ == '__main__':
    main()


import base64

def get_secret_from_secret_manager(client, secret_id, decode="UTF-8", project_id='742741961768', version_id=1):
    secret_name = f"projects/{project_id}/secrets/{secret_id}/versions/{version_id}"
    response = client.access_secret_version(request={"name": secret_name})
    if decode == "UTF-8":
        value = response.payload.data.decode("UTF-8")
    else:
        value = base64.b64decode(response.payload.data.decode("UTF-8"))
    return value


#INFO: Preliminary TOF support
#TODO: Add different camera models
#TODO: Write the full evd data (noise vector, angles)
#TODO: Add the grayscale image values (of the render image)
#TODO: Add the foldback effect of ToF Cameras.
#TODO: Add output of depthmap
#TODO: Simulate the sampling (i.e. PMD outputs only 1500 different depth values)
#      See SR4000 user manual for details
#TODO: Implement radial distortion
#INFO: SR4000
#      176x144 pixel
#      Standard FOV: 43.6°(h) 34.6°(v)    Focal Length: 10mm
#      Wide FOV: 69°(h) 56°(v)            Focal Length: 5.8mm
#         pixel_width (mm): 0.0453
#         pixel_height(mm): 0.0430
#      Model 1: unambiguity distance 5m
#      Absolute accuracy of central pixels +-10mm
#      Repeatability of central pixels (sigma) 4mm
#
#      Model 2: unambiguity distance 10m
#      Absolute accuracy of central pixels +-15mm
#      Repeatability of central pixels (sigma) 6mm

import math
import sys
import os
import struct
import ctypes
import time
import random
import bpy
import numpy
import blensor.globals
import blensor.scan_interface
from blensor import evd
from blensor import mesh_utils

from mathutils import Vector, Euler, Matrix

def deg2rad(deg):
    return deg*math.pi/180.0

def rad2deg(rad):
    return rad*180.0/math.pi

def tuples_to_list(tuples):
    l = []
    for t in tuples:
        l.extend(t)
    return l

parameters = {"max_dist":20,"noise_mu":0.0,"noise_sigma":0.004, "backfolding":False, "xres": 176, "yres": 144, "lens_angle_w": 43.6, "lens_angle_h": 34.6, "flength": 10.0}

def addProperties(cType):
    global parameters
    cType.tof_max_dist = bpy.props.FloatProperty( name = "Scan distance", default = parameters["max_dist"], min = 0, max = 1000, description = "How far the laser can see" )
    cType.tof_noise_mu = bpy.props.FloatProperty( name = "Noise mu", default = parameters["noise_mu"], description = "The center of the gaussian noise" )
    cType.tof_noise_sigma = bpy.props.FloatProperty( name = "Noise sigma", default = parameters["noise_sigma"], description = "The sigma of the gaussian noise" )
    cType.tof_backfolding = bpy.props.BoolProperty( name = "Backfolding", default = parameters["backfolding"], description = "Should backfolding be simulated" )
    cType.tof_xres = bpy.props.IntProperty( name = "X resolution", default = parameters["xres"], description = "Horizontal resolution" )
    cType.tof_yres = bpy.props.IntProperty( name = "Y resolution", default = parameters["yres"], description = "Vertical resolution" )
    cType.tof_lens_angle_w = bpy.props.FloatProperty( name = "Horizontal FOV", default = parameters["lens_angle_w"], description = "Horizontal opening angle" )
    cType.tof_lens_angle_h = bpy.props.FloatProperty( name = "Vertical FOV", default = parameters["lens_angle_h"], description = "Vertical opening angle" )
    cType.tof_focal_length = bpy.props.FloatProperty( name = "Focal length", default = parameters["flength"], description = "Focal length" )

def scan_advanced(scanner_object, max_distance = 10.0, evd_file=None, add_blender_mesh = False,
                  add_noisy_blender_mesh = False, tof_res_x = 176, tof_res_y = 144,
                  lens_angle_w=43.6, lens_angle_h=34.6, flength = 10.0,  evd_last_scan=True,
                  noise_mu=0.0, noise_sigma=0.004, timestamp = 0.0, backfolding=False,
                  world_transformation=Matrix()):

    inv_scan_x = scanner_object.inv_scan_x
    inv_scan_y = scanner_object.inv_scan_y
    inv_scan_z = scanner_object.inv_scan_z

    start_time = time.time()

    #10.0mm is currently the distance between the focal point and the sensor
    sensor_width = 2 * math.tan(deg2rad(lens_angle_w/2.0)) * 10.0
    sensor_height = 2 * math.tan(deg2rad(lens_angle_h/2.0)) * 10.0

    if tof_res_x == 0 or tof_res_y == 0:
        raise ValueError("Resolution must be > 0")

    pixel_width = sensor_width / float(tof_res_x)
    pixel_height = sensor_height / float(tof_res_y)

    bpy.context.scene.render.resolution_percentage

    width = bpy.context.scene.render.resolution_x
    height = bpy.context.scene.render.resolution_y
    cx = float(tof_res_x) /2.0
    cy = float(tof_res_y) /2.0

    evd_buffer = []
    rays = []
    ray_info = []

    ray = Vector([0.0,0.0,0.0])
    for x in range(tof_res_x):
        for y in range(tof_res_y):
            """Calculate a vector that originates at the principal point
               and points to the pixel in the sensor. This vector is then
               scaled to the maximum scanning distance
            """
            physical_x = float(x-cx) * pixel_width
            physical_y = float(y-cy) * pixel_height
            physical_z = -float(flength)
            ray.xyz = [physical_x, physical_y, physical_z]
            ray.normalize()
            final_ray = max_distance*ray
            rays.extend([final_ray[0],final_ray[1],final_ray[2]])

            """ pitch and yaw are added for completeness, normally they are
                not provided by a ToF Camera but can be derived
                from the pixel position and the camera parameters.
            """
            yaw = math.atan(physical_x/flength)
            pitch = math.atan(physical_y/flength)

            ray_info.append([yaw, pitch, timestamp])

    returns = blensor.scan_interface.scan_rays(rays, max_distance, inv_scan_x = inv_scan_x, inv_scan_y = inv_scan_y, inv_scan_z = inv_scan_z)

    evd_storage = evd.evd_file(evd_file, tof_res_x, tof_res_y, max_distance)

    reusable_vector = Vector([0.0,0.0,0.0,0.0])
    for i in range(len(returns)):
        idx = returns[i][-1]
        distance_noise =  random.gauss(noise_mu, noise_sigma)
        #If everything works substitute the previous line with this
        #distance_noise =  pixel_noise[returns[idx][-1]] + random.gauss(noise_mu, noise_sigma)

        reusable_vector.xyzw = [returns[i][1],returns[i][2],returns[i][3],1.0]
        vt = (world_transformation * reusable_vector).xyz
        v = [returns[i][1],returns[i][2],returns[i][3]]
        vector_length = math.sqrt(v[0]**2+v[1]**2+v[2]**2)
        norm_vector = [v[0]/vector_length, v[1]/vector_length, v[2]/vector_length]

        vector_length_noise = vector_length+distance_noise
        if backfolding:
           #Distances > max_distance/2..max_distance are mapped to 0..max_distance/2
           if vector_length_noise >= max_distance/2.0:
               vector_length_noise = vector_length_noise - max_distance/2.0

        reusable_vector.xyzw = [norm_vector[0]*vector_length_noise, norm_vector[1]*vector_length_noise, norm_vector[2]*vector_length_noise,1.0]
        v_noise = (world_transformation * reusable_vector).xyz

        evd_storage.addEntry(timestamp = ray_info[idx][2], yaw =(ray_info[idx][0]+math.pi)%(2*math.pi), pitch=ray_info[idx][1], distance=vector_length, distance_noise=vector_length_noise, x=vt[0], y=vt[1], z=vt[2], x_noise=v_noise[0], y_noise=v_noise[1], z_noise=v_noise[2], object_id=returns[i][4], color=returns[i][5], idx=returns[i][-1])

    if evd_file:
        evd_storage.appendEvdFile()

    if not evd_storage.isEmpty():
        scan_data = numpy.array(evd_storage.buffer)
        additional_data = None
        if scanner_object.store_data_in_mesh:
            additional_data = evd_storage.buffer

        if add_blender_mesh:
            mesh_utils.add_mesh_from_points_tf(scan_data[:,5:8], "Scan", world_transformation, buffer=additional_data)

        if add_noisy_blender_mesh:
            mesh_utils.add_mesh_from_points_tf(scan_data[:,8:11], "NoisyScan", world_transformation, buffer=additional_data)

        bpy.context.scene.update()

    end_time = time.time()
    scan_time = end_time-start_time
    print ("Elapsed time: %.3f"%(scan_time))

    return True, 0.0, scan_time

# This Function creates scans over a range of frames

def scan_range(scanne_object, frame_start, frame_end, filename="/tmp/tof.evd", frame_time = (1.0/24.0), fps = 24, add_blender_mesh=False, add_noisy_blender_mesh=False, max_distance = 20.0, last_frame = True, noise_mu = 0.0, noise_sigma = 0.0, backfolding=False, tof_res_x = 176, tof_res_y=144, lens_angle_w=43.6, lens_angle_h=34.6, flength = 10.0, world_transformation=Matrix()):

    start_time = time.time()

    time_per_frame = 1.0 / float(fps)

    try:
        for i in range(frame_start,frame_end):

            bpy.context.scene.frame_current = i

            ok,start_radians,scan_time = scan_advanced(scanner_object=scanner_object,
                    evd_file = filename ,
                    add_blender_mesh=add_blender_mesh,
                    add_noisy_blender_mesh=add_noisy_blender_mesh,
                    max_distance=max_distance,
                    timestamp = float(i) * frame_time,
                    noise_mu = noise_mu, noise_sigma=noise_sigma, backfolding=backfolding,
                    tof_res_x = tof_res_x, tof_res_y = tof_res_y,
                    lens_angle_w = lens_angle_w, lens_angle_h = lens_angle_h, flength = flength,
                    world_transformation=world_transformation)

            if not ok:
                break
    except:
        print ("Scan aborted")

    if last_frame:
        evd_file = open(filename,"a")
        evd_file.buffer.write(struct.pack("i",-1))
        evd_file.close()

    end_time = time.time()
    print ("Total scan time: %.2f"%(end_time-start_time))

######################################################


import torch
import torch.nn as nn
import torch.nn.functional as F

class InceptionNet(nn.Module):
    def __init__(self, aux_logits=True, num_classes=2):
        super(InceptionNet, self).__init__()
        self.aux_logits = aux_logits
        self.conv1 = ConvBlock(
            in_channels=3,
            out_chanels=64,
            kernel_size=(7, 7),
            stride=(2, 2),
            padding=(3, 3),
        )
        self.conv2 = ConvBlock(64, 192, kernel_size=3, stride=1, padding=1)
        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
        self.inception3a = InceptionBlock(192, 64, 96, 128, 16, 32, 32)
        self.inception3b = InceptionBlock(256, 128, 128, 192, 32, 96, 64)
        self.inception4a = InceptionBlock(480, 192, 96, 208, 16, 48, 64)
        self.inception4b = InceptionBlock(512, 160, 112, 224, 24, 64, 64)
        self.inception4c = InceptionBlock(512, 128, 128, 256, 24, 64, 64)
        self.inception4d = InceptionBlock(512, 112, 144, 288, 32, 64, 64)
        self.inception4e = InceptionBlock(528, 256, 160, 320, 32, 128, 128)
        self.inception5a = InceptionBlock(832, 256, 160, 320, 32, 128, 128)
        self.inception5b = InceptionBlock(832, 384, 192, 384, 48, 128, 128)
        self.avgpool = nn.AvgPool2d(kernel_size=7, stride=1)
        self.dropout = nn.Dropout(p=0.4)
        self.fc = nn.Linear(1024, num_classes)

        if self.aux_logits:
            self.aux1 = InceptionAux(512, num_classes)
            self.aux2 = InceptionAux(528, num_classes)
        else:
            self.aux1 = self.aux2 = None

    def forward(self, x):
        x = self.conv1(x)
        x = self.maxpool(x)
        x = self.conv2(x)
        x = self.maxpool(x)
        x = self.inception3a(x)
        x = self.inception3b(x)
        x = self.maxpool(x)
        x = self.inception4a(x)

        if self.aux_logits and self.training:
            aux1 = self.aux1(x)

        x = self.inception4b(x)
        x = self.inception4c(x)
        x = self.inception4d(x)

        if self.aux_logits and self.training:
            aux2 = self.aux2(x)

        x = self.inception4e(x)
        x = self.maxpool(x)
        x = self.inception5a(x)
        x = self.inception5b(x)
        x = self.avgpool(x)
        x = x.reshape(x.shape[0], -1)
        x = self.dropout(x)
        x = self.fc(x)

        if self.aux_logits and self.training:
            return aux1, aux2, x
        return x

class ConvBlock(nn.Module):
    def __init__(self, in_channels, out_chanels, **kwargs):
        super(ConvBlock, self).__init__()
        self.conv = nn.Conv2d(in_channels, out_chanels, **kwargs)
        self.bn = nn.BatchNorm2d(out_chanels)

    def forward(self, x):
        return F.relu(self.bn(self.conv(x)))

class InceptionBlock(nn.Module):
    def __init__(
        self,
        in_channels,
        out_1x1,
        red_3x3,
        out_3x3,
        red_5x5,
        out_5x5,
        out_pool,
    ):
        super(InceptionBlock, self).__init__()
        self.branch1 = ConvBlock(in_channels, out_1x1, kernel_size=1)
        self.branch2 = nn.Sequential(
            ConvBlock(in_channels, red_3x3, kernel_size=1, padding=0),
            ConvBlock(red_3x3, out_3x3, kernel_size=3, padding=1),
        )
        self.branch3 = nn.Sequential(
            ConvBlock(in_channels, red_5x5, kernel_size=1),
            ConvBlock(red_5x5, out_5x5, kernel_size=5, padding=2),
        )
        self.branch4 = nn.Sequential(
            nn.MaxPool2d(kernel_size=3, padding=1, stride=1),
            ConvBlock(in_channels, out_pool, kernel_size=1),
        )

    def forward(self, x):
        branches = (self.branch1, self.branch2, self.branch3, self.branch4)
        return torch.cat([branch(x) for branch in branches], 1)

class InceptionAux(nn.Module):
    def __init__(self, in_channels, num_classes):
        super(InceptionAux, self).__init__()
        self.dropout = nn.Dropout(p=0.7)
        self.pool = nn.AvgPool2d(kernel_size=5, stride=3)
        self.conv = ConvBlock(in_channels, 128, kernel_size=1)
        self.fc1 = nn.Linear(2048, 1024)
        self.fc2 = nn.Linear(1024, num_classes)

    def forward(self, x):
        x = self.pool(x)
        x = self.conv(x)
        x = x.reshape(x.shape[0], -1)
        x = F.relu(self.fc1(x))
        x = self.dropout(x)
        x = self.fc2(x)
        return x


class Solution:
    def longestCommonPrefix(self, strs: List[str]) -> str:
        min_len=len(min(strs, key=len))
        prefix=""
        for i in range(min_len):
            pre=strs[0][i]
            for j in range(1, len(strs)):
                if strs[j][i]!=pre:
                    return prefix
            prefix+=pre
        return prefix

LeetCode Questions/14. Longest Common Prefix/code.py


import addonHandler
addonHandler.initTranslation()

import globalPluginHandler
import tones
import config
import ui
import api
import globalVars
import gui
from scriptHandler import script, getLastScriptRepeatCount

configName = 'objLocSound'
scriptCategory = _(u'物件位置音效')
if configName not in config.conf:
	config.conf[configName] = {'switch': True}

conf = config.conf[configName]
if conf['switch'] == u'False':
	conf['switch'] = False

class GlobalPlugin(globalPluginHandler.GlobalPlugin):
	def __init__(self):
		super(GlobalPlugin, self).__init__()
		self.currentObject = None
		self.currentObjectLocation = None
		# 持續觀察物件位置，用於動態物件，例如跑馬燈，或是某些沒有事件的焦點移動，比如瀏覽模式。
		self.timer = gui.NonReEntrantTimer(self.playLocationSound)
		if self.switch:
			self.start()

	def terminate(self):
		if hasattr(self, 'timer'):
			self.stop()

	def _get_switch(self):
		return conf.get('switch')

	def _set_switch(self, value):
		conf['switch'] = value

	def start(self):
		self.timer.Start(10)

	def stop(self):
		self.timer.Stop()

	@script(_(u'提示開關'), scriptCategory, 'KB:NVDA+l')
	def script_toggleSwitch(self, gesture):
		switch = not self.switch
		self.switch = switch
		methods = (self.stop, self.start)
		methods[switch]()
		messages = (_(u'關閉物件位置提示音效'), _(u'開啟物件位置提示音效'))
		ui.message(messages[switch])

	def playLocationSound(self):
		# 播放當前瀏覽物件中心點所在位置音效
		try:
			obj = api.getNavigatorObject()
			location = obj.location
			if location is None or self.currentObject == obj and location == self.currentObjectLocation:
				return

			self.currentObject = obj
			self.currentObjectLocation = location
			x, y = self.getCenterCoordinate(obj)

			self.playCoordinateSound(x, y)
		except TypeError:
			pass

	def getCenterCoordinate(self, obj):
		l, t, w, h = obj.location
		x = l + (w // 2)
		y = t + (h // 2)
		return (x, y)

	def playCoordinateSound(self, x, y):
		screenWidth, screenHeight = api.getDesktopObject().location[2:]
		if 0 <= x <= screenWidth and 0 <= y <= screenHeight:
			minPitch = config.conf['mouse']['audioCoordinates_minPitch']
			maxPitch = config.conf['mouse']['audioCoordinates_maxPitch']
			curPitch = minPitch + ((maxPitch - minPitch) * ((screenHeight - y) / float(screenHeight)))
			brightness = config.conf['mouse']['audioCoordinates_maxVolume']
			leftVolume = int((85 * ((screenWidth - float(x)) / screenWidth)) * brightness)
			rightVolume = int((85 * (float(x) / screenWidth)) * brightness)
			tones.beep(curPitch, 40, left=leftVolume, right=rightVolume)

	@script(u'播放當前物件左上與右下角座標音效，連按兩次報讀詳細資訊', scriptCategory, 'KB:NVDA+a')
	def script_playAreaSound(self, gesture):
		obj = api.getNavigatorObject()
		l, t, w, h = obj.location
		if getLastScriptRepeatCount():
			message = _(u'左上角：') + u' X:' + str(l) + u', Y:' + str(t) + u'\n'
			message += _(u'寬：') + u' ' + str(w) + u', ' + _(u'高：') + u' ' + str(h)
			return ui.message(message)

		self.playCoordinateSound(l, t)
		gui.wx.CallLater(100, self.playCoordinateSound, l+w, t+h).Start()


"""
You have \r\n, \r and \n as most common lineseps nowadays:
http://en.wikipedia.org/wiki/Newline
If there was a script that (badly) fixed it to \n\r instead of \r\n,
there is still only one line in notepad.exe, thus not noticing problems.
An other script will fail on that and introduces a new line \n\r\n
by fixing it because the line ends on the old \r standard.
Who would keep such a script running, knowing it did all that?
The second script was probably an FTP client that transferred .nfo files
in ascii mode while racing.
"""

if __name__ == '__main__':
    testfile = "linesep_test.txt"
    with open(testfile, 'w') as ofile:
        for _ in range(10):
            ofile.writelines("A line with r n.\r\n")
        for _ in range(10):
            ofile.writelines("A line with n r.\n\r")
        for _ in range(10):
            ofile.writelines("A line with n r n.\n\r\n")
    print("Done!")


import cv2 as cv
import sys

img = cv.imread(cv.samples.findFile("aisle.jpg"))

if img is None:
    sys.exit("Image not found.")

cv.imshow("Display Window", img)
# waits a key from keyboard and stores in k
k = cv.waitKey(0)

# if users press the s , imwrite saves image as png
if k == ord("s"):
    cv.imwrite("aisle.png", img)

cv.destroyWindow()


import unittest

class Solution:
    def findMedianSortedArrays(self, nums1, nums2):
        """
        Given two sorted arrays nums1 and nums2 of size m and n respectively,
        return the median of the two sorted arrays.

        The overall run time complexity should be O(log (m+n)).
        """
        median_position = (len(nums1) + len(nums2)) // 2
        is_even = len(nums1) + len(nums2) % 2 == 0
        idx_nums1 = 0
        idx_nums2 = 0
        i = 0
        sorted_arr = []
        while i < median_position:
            i += 1

            print(median_position, i, idx_nums1, idx_nums2)
            if idx_nums1 >= len(nums1):
                sorted_arr.append(nums2[idx_nums2])
                idx_nums2 += 1
                continue
            if idx_nums2 >= len(nums2):
                sorted_arr.append(nums1[idx_nums1])
                idx_nums1 += 1
                continue
            if nums1[idx_nums1] < nums2[idx_nums2]:
                sorted_arr.append(nums1[idx_nums1])
                idx_nums1 += 1
            elif nums2[idx_nums2] < nums1[idx_nums1]:
                sorted_arr.append(nums2[idx_nums2])
                idx_nums2 += 1
        print("final array:", sorted_arr)
        if is_even:
            return (sorted_arr[-2] + sorted_arr[-1]) / 2
        else:
            return float(sorted_arr[-1])

class Test_Median(unittest.TestCase):
    def test_median_is_one_from_each(self):
        soln = Solution()
        self.assertEqual(soln.findMedianSortedArrays([1, 2], [3, 4]), 2.5)

unittest.main()


pos=-1
def LinearSearch(list, n):
    for i in range(len(list)):
        if list[i]==n:
            globals()['pos']=i
            return True
    else:
        return False

list=[5,8,4,6,9,2]
n=int(input("Enter a Number to search: "))
if LinearSearch(list,n):
    print(n,"Found at Position",pos+1)
else:
    print("oops! the number you have entered is not in the list")


from typing import Dict
from models.flow import Flow
import warnings
from subprocess import Popen, PIPE

import numpy as np
import pyshark.tshark.tshark
import pyshark
from cryptography import x509
from cryptography.hazmat.backends import default_backend

class PcapReader(object):
    """Reader object for extracting features from .pcap files
    """

    def __init__(self, verbose=False):
        """Reader object for extracting features from .pcap files

        Args:
        --------
            verbose (bool, optional): Boolean indicating whether to be verbose in reading
        """
        # Set verbosity level
        self.verbose = verbose

    def read(self, path: str) -> dict:
        """Read TCP and UDP packets from .pcap file given by path.
            Automatically choses fastest available backend to use.

        Args:
        --------
            path (str): Path to .pcap file to read.

        Raises:
        -------
            ModuleNotFoundError: If TShark is not installed
            OSError: If there is problem reading file (usually caused because not being able to allocate memory)
        Returns:
        -------
            dict: Dictionary of flow_key -> Flow()
            Where features consist of:

                0) Filename of capture
                1) Protocol TCP/UDP
                2) TCP/UDP stream identifier
                3) Timestamp of packet
                4) Length of packet
                5) IP packet source
                6) IP packet destination
                7) TCP/UDP packet source port
                8) TCP/UDP packet destination port
                9) SSL/TLS certificate if exists, else None
        """
        # print(time.time(), ":\tstart reading pcap")
        # If verbose, print which file is currently being read
        if self.verbose:
            print("Reading {}...".format(path))
        # print(time.time(), ":\tend reading pcap")
        # Check if we can use fast tshark read or slow pyshark read
        packets = self.read_pyshark(path)
        return self.combine(packets)

    def read_tshark(self, path: str) -> np.ndarray:
        """Read TCP and UDP packets from file given by path using tshark backend
            Automatically choses fastest available backend to use.

        Args:
        --------
            path (str): Path to .pcap file to read.

        Returns:
        -------
            np.ndarray: Array of shape=(n_packets, n_features)
            Where features consist of:

                0) Filename of capture
                1) Protocol TCP/UDP
                2) TCP/UDP stream identifier
                3) Timestamp of packet
                4) Length of packet
                5) IP packet source
                6) IP packet destination
                7) TCP/UDP packet source port
                8) TCP/UDP packet destination port
                9) SSL/TLS certificate if exists, else None
        """
        # Create Tshark command
        command = ["tshark", "-r", path, "-Tfields",
                   "-e", "frame.time_epoch",
                   "-e", "tcp.stream",
                   "-e", "udp.stream",
                   "-e", "ip.proto",
                   "-e", "ip.src",
                   "-e", "tcp.srcport",
                   "-e", "udp.srcport",
                   "-e", "ip.dst",
                   "-e", "tcp.dstport",
                   "-e", "udp.dstport",
                   "-e", "ip.len"]
        ##"-e", "ssl.handshake.certificate"
        # Initialise result
        result = list()

        # Call Tshark on packets
        process = Popen(command, stdout=PIPE, stderr=PIPE)
        # Get output
        out, err = process.communicate()

        # Give warning message if any
        if err:
            warnings.warn("Error reading file: '{}'".format(
                err.decode('utf-8')))

        # Read each packet
        for packet in filter(None, out.decode('utf-8').split('\n')):
            # Get all data from packets
            packet = packet.split()

            # Perform check on packets
            if len(packet) < 8: continue

            # Perform check on multiple ip addresses
            packet[3] = packet[3].split(',')[0]
            packet[5] = packet[5].split(',')[0]
            packet[7] = packet[7].replace(',', '')

            # Parse certificate
            if len(packet) > 8:
                # Get first certificate
                cert = packet[8].split(',')[0]
                # Transform to hex
                cert = bytes.fromhex(cert.replace(':', ''))
                # Read as certificate
                cert = x509.load_der_x509_certificate(cert, default_backend())
                # Set packet as serial number
                packet[8] = cert.serial_number
            else:
                packet.append(None)

            # Add packet to result
            result.append([path] + packet)

        # Get result as numpy array
        result = np.asarray(result)

        # Check if any items exist
        if not result.shape[0]:
            return np.zeros((0, 8), dtype=object)

        # Change protocol number to text
        protocols = {'17': 'udp', '6': 'tcp'}
        result[:, 3] = [protocols.get(x, 'unknown') for x in result[:, 3]]

        # Return in original order
        return result[:, [0, 3, 2, 1, 8, 4, 6, 5, 7, 9]]

    def read_pyshark(self, path: str) -> np.ndarray:
        """Read TCP and UDP packets from file given by path
            using pyshark backend

        Args:
        --------
            path (str): Path to .pcap file to read.

        Returns:
        -------
            np.ndarray: Array of shape=(n_packets, n_features)
            Where features consist of:

                0) Filename of capture
                1) Protocol TCP/UDP
                2) TCP/UDP stream identifier
                3) Timestamp of packet
                4) Length of packet
                5) IP packet source
                6) IP packet destination
                7) TCP/UDP packet source port
                8) TCP/UDP packet destination port
                9) SSL/TLS certificate if exists, else None
        """
        # If verbose, print which file is currently being read
        if self.verbose:
            counter_a = 0
            counter_b = 0

        # Read pcap file
        pcap = iter(pyshark.FileCapture(path))

        # Initialise result
        result = list()

        # Loop over packets
        while True:
            try:
                packet = next(pcap)
            except ModuleNotFoundError as ex:
                raise ex
            except OSError as ex:
                raise ex
            except Exception as ex:
                warnings.warn("Pyshark error: '{}'".format(ex))
                break

            if not ("TCP" in packet or "UDP" in packet):
                continue

            try:

                d = []
                # Get required packet data
                if packet.layers[1].stream:
                    d = [path,
                        packet.layers[1].layer_name,  # Get
                        packet.layers[1].stream,  # Get stream ID
                        packet.sniff_timestamp,  # Get packet timestamp
                        packet.length,  # Get packet length
                        packet.layers[0].src,  # Get source IP or IPv6 (fixed)
                        packet.layers[0].dst,  # Get destination IP or IPv6 (fixed)
                        packet.layers[1].srcport,  # Get source port
                        packet.layers[1].dstport,  # Get destination port
                        None]
                elif packet.layers[2].stream:
                    d = [path,
                        packet.layers[2].layer_name,  # Get
                        packet.layers[2].stream,  # Get stream ID
                        packet.sniff_timestamp,  # Get packet timestamp
                        packet.length,  # Get packet length
                        packet.layers[1].src,  # Get source IP or IPv6 (fixed)
                        packet.layers[1].dst,  # Get destination IP or IPv6 (fixed)
                        packet.layers[2].srcport,  # Get source port
                        packet.layers[2].dstport,  # Get destination port
                        None]
                else:
                    continue

                # Check whether SSL/TLS certificate is in packet
                if "SSL" in packet and \
                        packet.ssl.get("handshake_certificate") is not None:
                    # Get certificate
                    cert = packet.ssl.get('handshake_certificate')
                    # Parse cert to bytes
                    cert = bytes.fromhex(cert.replace(':', ''))
                    # Parse x509 certificate as DER
                    cert = x509.load_der_x509_certificate(cert,
                                                        default_backend())
                    # Get serial number - TODO extend with other features?
                    d[-1] = cert.serial_number

                # Append data item to result
                result.append(d)
            except Exception as ex:
                warnings.warn(f"Cannot read pcap: {path}")

        # Close capture
        pcap.close()

        # Return result as numpy array
        return np.array(result)

    def combine(self, packets: np.ndarray) -> dict:
        """Combine individual packets into a flow representation

            Parameters
            ----------
            packets : np.array of shape=(n_samples_packets, n_features_packets)
                Output from Reader.read

            Returns
            -------
            flows : dict
                Dictionary of flow_key -> Flow()
            """
        # Initialise result
        result = dict()

        # For each packet, add it to a flow
        for packet in packets:
            key = (packet[0], packet[1], packet[2])
            # Add packet to flow
            result[key] = result.get(key, Flow()).add(packet)

        # Return result
        return result


# This file is part of sner4 project governed by MIT license, see the LICENSE.txt file.
"""
sner agent testssl
"""

import json
import re
import subprocess
import sys
from collections import defaultdict
from pprint import pprint
from zipfile import ZipFile

from sner.lib import file_from_zip
from sner.server.parser import ParsedItemsDb, ParserBase

class ParserModule(ParserBase):  # pylint: disable=too-few-public-methods
    """testssl parser"""

    ARCHIVE_PATHS = r'output\-[0-9]+\.json'
    FINDINGS_IGNORE = ['OK', 'INFO']

    @classmethod
    def parse_path(cls, path):
        """parse data from path"""

        pidb = ParsedItemsDb()

        with ZipFile(path) as fzip:
            for fname in filter(lambda x: re.match(cls.ARCHIVE_PATHS, x), fzip.namelist()):
                pidb = cls._parse_data(file_from_zip(path, fname).decode('utf-8'), pidb)

        return pidb

    @classmethod
    def _parse_data(cls, data, pidb):
        """parse raw string data"""

        json_data = json.loads(data)
        if not isinstance(json_data['scanTime'], int):  # pragma: no cover
            return pidb

        result = json_data['scanResult'][0]
        host_address = result['targetHost']
        service_port = int(result['port'])
        service_proto = 'tcp'

        note_data = {
            'output': data,
            'data': {},
            'findings': defaultdict(list)
        }

        for section_name, section_data in result.items():
            if isinstance(section_data, list):
                # pop findings for section
                for finding in section_data:
                    if finding['id'] == 'cert':
                        # rewrap certificate data
                        tmp = finding['finding'].split(' ')
                        tmp = ' '.join(tmp[:2]) + '\n' + '\n'.join(tmp[2:-2]) + '\n' + ' '.join(tmp[-2:])
                        # parse as auror tool
                        try:
                            note_data['cert_txt'] = subprocess.run(
                                ['openssl', 'x509', '-text', '-noout'],
                                input=tmp, check=True, capture_output=True, text=True
                            ).stdout
                        except subprocess.CalledProcessError as exc:  # pragma: no cover
                            note_data['cert_txt'] = str(exc)

                    if finding['severity'] not in cls.FINDINGS_IGNORE:
                        note_data['findings'][section_name].append(finding)
            else:
                # pop scalar data
                note_data['data'][section_name] = section_data

        pidb.upsert_note(host_address, 'testssl', service_proto, service_port, data=json.dumps(note_data))

        return pidb

if __name__ == '__main__':  # pragma: no cover
    pprint(ParserModule.parse_path(sys.argv[1]))


from building import *
Import('rtconfig')

src = Glob('*.c') + Glob('*.cpp')
cwd = GetCurrentDir()
group = []

CPPPATH = [cwd]
CPPDEFINES = ['RT_USING_MINILIBC']

if rtconfig.PLATFORM == 'gcc' and rtconfig.ARCH != 'sim' and not GetDepend('RT_USING_LIBC'):
    group = DefineGroup('libc', src, depend = [''],
        CPPPATH = CPPPATH, CPPDEFINES = CPPDEFINES)

Return('group')


import os

import pytest

from test.fixtures import KafkaFixture, ZookeeperFixture

@pytest.fixture(scope="module")
def version():
    if 'KAFKA_VERSION' not in os.environ:
        return ()
    return tuple(map(int, os.environ['KAFKA_VERSION'].split('.')))

@pytest.fixture(scope="module")
def zookeeper(version, request):
    assert version
    zk = ZookeeperFixture.instance()
    def fin():
        zk.close()
    request.addfinalizer(fin)
    return zk

@pytest.fixture(scope="module")
def kafka_broker(version, zookeeper, request):
    assert version
    k = KafkaFixture.instance(0, zookeeper.host, zookeeper.port,
                              partitions=4)
    def fin():
        k.close()
    request.addfinalizer(fin)
    return k


from pkgutil import iter_modules
from importlib import import_module

def load_configuration(name):
    '''
    加载配置。
    '''

    result = {}
    module = import_module(name)
    for _, child, _ in iter_modules(module.__path__):
        cn = f'{name}.{child}'
        cm = import_module(cn)
        vs = {}
        for k in dir(cm):
            if k[0] != '_':
                vs[k] = getattr(cm, k)
        result[child] = vs
    return result


import os
import sys
import platform
import numpy as np
import matplotlib.pyplot as mp

def init_chart():
    mp.gcf().set_facecolor(np.ones(3) * 240 / 255)
    mp.title('Polynomial Function', fontsize=20)
    mp.xlabel('x', fontsize=14)
    mp.ylabel('y', fontsize=14)
    mp.tick_params(which='both', top=True, right=True,
                   labelright=True, labelsize=10)
    mp.grid(linestyle=':')

def draw_chart(x, y):
    mp.plot(x, y, c='orangered', label='$y=x^3+2x^2+3x+4$')
    mp.legend()

def show_chart():
    mng = mp.get_current_fig_manager()
    if 'Windows' in platform.system():
        mng.window.state('zoomed')
    else:
        mng.resize(*mng.window.maxsize())
    mp.show()

def main(argc, argv, envp):
    x = np.linspace(-10, 10, 201)
    f = np.poly1d([1., 2., 3., 4.])
    y = f(x)
    init_chart()
    draw_chart(x, y)
    show_chart()
    return 0

if __name__ == '__main__':
    exit(main(len(sys.argv), sys.argv, os.environ))


#!/usr/bin/env python2.7
# -*- coding: utf-8 -*-

'''---------------------------------------------------------------------------|
                                                              _____           |
      Autor: Notsgnik                                       /||   /           |
      Email: Labruillere gmail.com                         / ||  /            |
      website: notsgnik.github.io                         /  || /             |
      License: GPL v3                                    /___||/              |
      																		  |
---------------------------------------------------------------------------!'''
from collections import OrderedDict

_somedata = [
	("e_magic","5a4d"),
	("e_lfanew", "00ffeeddccbbaa99887766"),
	("_IMAGE_FILE_HEADER",[
		("test","ab"),
		("test2",[
			("_name",[
				("subnested2","1337")
			])
		])
	])
]
_firstdata = [
	("e_magic","5a4d"),
	("e_lfanew", "00000040")
]

_TEST_STRUCTSMALL = [
	("e_magic" , 2), # signature
	("e_cblp" , 2), # bytes on last page of file
	("e_cp" , 2),
	("_IMAGE_FILE_HEADER",[
		("machine",4),
		("test",2),
		(
			[
				("nested",2),
				("_name",[
					("subnested", 4),
					("subnested2", 2)
				])
			],
			[
				"test1",
				"test2",
				"test3"
			]
		)
	]),
	("e_lfanew", 4)
]

_TEST_STRUCT = [
	("e_magic" , 4), # signature
	("e_cblp" , 2), # bytes on last page of file
	("e_cp" , 2), #Pages in File
	("e_crlc" , 2),
	("_IMAGE_FILE_HEADER" , [ # FileHeader
		("Machine" , 2),
		("NumberOfSections" , 2),
		("TimeDateStamp" , 4),
		("PointerToSymbolTable" , 4),
		("NumberOfSymbols" , 4),
		("SizeOfOptionalHeader" , 2),
		("Characteristics" , 2),
		(
			[
				("VirtualAdress" , 4),
				("Size" , 4)
			],
			[
				"_IMAGE_DATA_DIRECTORY_ENTRY_EXPORT" ,
				"_IMAGE_DATA_DIRECTORY_ENTRY_IMPORT",
				"_IMAGE_DATA_DIRECTORY_ENTRY_RESOURCE"
			]
		),
		("Characteristics" , 2)
	]),
	("e_crlv" , 2)
]

_IMAGE_DOS_HEADER = [
	("e_magic" , 2), # signature
	("e_cblp" , 2), # bytes on last page of file
	("e_cp" , 2),#Pages in File
	("e_crlc" , 2), #Relocations
	("e_cparhdr", 2), #Size of header in paragraphs
	("e_minalloc", 2), # minimum extra paragraphs
	("e_maxalloc", 2), # maxalloc
	("e_ss", 2 ),# initial ( relative ) ss
	("e_sp", 2 ),# initial sp
	("e_csum", 2), # checksum
	("e_iip", 2), # initial ip
	("e_ics", 2), # overlay number
	("e_otrt", 2), # Reserved
	("e_ovnb", 2), # Reserved
	("e_res0", 2), # Reserved
	("e_res1", 2), # Reserved
	("e_res2", 2), # Reserved
	("e_res3", 2), # Reserved
	("e_oemid", 2), # OEM id
	("e_oeminfo", 2), # OEM info
	("e_res4", 2), # Reserved
	("e_res5", 2), # Reserved
	("e_res6", 2), # Reserved
	("e_res7", 2), # Reserved
	("e_res8", 2), # Reserved
	("e_res9", 2), # Reserved
	("e_resa", 2), # Reserved
	("e_resb", 2), # Reserved
	("e_resc", 2), # Reserved
	("e_resd", 2), # Reserved
	("e_lfanew", 4) # Offset to new EXE Header
]

_IMAGE_NT_HEADERS_32 = [
	("Signature" , 4), # pe magic
	("_IMAGE_FILE_HEADER" , [ # FileHeader
		("Machine" , 2),
		("NumberOfSections" , 2),
		("TimeDateStamp" , 4),
		("PointerToSymbolTable" , 4),
		("NumberOfSymbols" , 4),
		("SizeOfOptionalHeader" , 2),
		("Characteristics" , 2)
	]),
	("_Image_Optional_HEADER" , [ #OptionalHeade
		("Magic" , 2 ),
		("MajorLinkerVersion" , 1 ),
		("MinorLinkerVersion" , 1 ),
		("SizeOfCode" , 4 ),
		("SizeOfInitializedData" , 4 ) ,
		("SizeOfUninitializedData" , 4  ),
		("AdressOfEntryPoint" , 4 ),
		("BaseOfCode" , 4 ),
		("BaseOfData" , 4 ),
		("ImageBase" , 4 ),
		("SectionAlignment", 4 ),
		("FileAlignement", 4 ),
		("MajorOperatingSystemVarsion", 2 ),
		("MinorOperatingSystemVersion", 2 ),
		("MajorImageVersion", 2 ),
		("MinorImageVersion", 2 ),
		("MajorSubsystemVersion", 2 ),
		("MinorSubsystemVersion", 2 ),
		("Win32VersionValue", 4 ),
		("SizeOfImage", 4 ),
		("SizeOfHEaders", 4 ),
		("CheckSum", 4 ),
		("Subsystem", 2 ),
		("DllCharacteristics", 2 ),
		("SizeOfStackReserve", 4 ),
		("SizeOfStackCommit", 4 ),
		("SizeOfHeapReserve", 4 ),
		("SizeOfHeapCommit" , 4 ),
		("LoaderFlags", 4 ),
		("NumberOfRvaAnSizes", 4),
		([
			("VirtualAdress", 4),
			("Size", 4)
		],[
			"_IMAGE_DATA_DIRECTORY_ENTRY_EXPORT",
			"_IMAGE_DATA_DIRECTORY_ENTRY_IMPORT",
			"_IMAGE_DATA_DIRECTORY_ENTRY_RESOURCE",
			"_IMAGE_DATA_DIRECTORY_ENTRY_EXCEPTION",
			"_IMAGE_DATA_DIRECTORY_ENTRY_SECURITY",
			"_IMAGE_DATA_DIRECTORY_ENTRY_BASERELOC",
			"_IMAGE_DATA_DIRECTORY_ENTRY_DEBUG",
			"_IMAGE_DATA_DIRECTORY_ENTRY_COPYRIGHT",
			"_IMAGE_DATA_DIRECTORY_ENTRY_GLOBALPTR",
			"_IMAGE_DATA_DIRECTORY_ENTRY_TLS",
			"_IMAGE_DATA_DIRECTORY_ENTRY_LOAD_CONFIG",
			"_IMAGE_DATA_DIRECTORY_ENTRY_BOUND_IMPORT",
			"_IMAGE_DATA_DIRECTORY_ENTRY_IAT",
			"_IMAGE_DATA_DIRECTORY_ENTRY_DELAY_IMPORT",
			"_IMAGE_DATA_DIRECTORY_ENTRY_COM_DESCRIPTOR",
			"_IMAGE_DATA_DIRECTORY_ENTRY_RESERVED"
		])
	])
]

_IMAGE_NT_HEADERS_32p = [
	("Signature" , 4), # pe magic
	("_IMAGE_FILE_HEADER" , [ # FileHeader
		("Machine", 2),
		("NumberOfSections", 2),
		("TimeDateStamp", 4),
		("PointerToSymbolTable", 4),
		("NumberOfSymbols", 4),
		("SizeOfOptionalHeader", 2),
		("Characteristics", 2)
	]),
	("_Image_Optional_HEADER", [ #OptionalHeader
		( "Magic", 2),
		( "MajorLinkerVersion", 1),
		( "MinorLinkerVersion", 1),
		( "SizeOfCode", 4),
		( "SizeOfInitializedData", 4),
		( "SizeOfUninitializedData", 4 ),
		( "AdressOfEntryPoint", 4),
		( "BaseOfCode", 4),
		( "ImageBase", 8),
		( "SectionAlignment", 4),
		( "FileAlignement", 4),
		( "MajorOperatingSystemVarsion", 2 ),
		( "MinorOperatingSystemVersion", 2 ),
		( "MajorImageVersion", 2 ),
		( "MinorImageVersion", 2 ),
		( "MajorSubsystemVersion", 2 ),
		( "MinorSubsystemVersion", 2 ),
		( "Win32VersionValue", 4 ),
		( "SizeOfImage", 4 ),
		( "SizeOfHEaders", 4 ),
		( "CheckSum", 4 ),
		( "Subsystem", 2 ),
		( "DllCharacteristics", 2),
		( "SizeOfStackReserve", 8),
		( "SizeOfStackCommit", 8),
		( "SizeOfHeapReserve", 8),
		( "SizeOfHeapCommit", 8),
		( "LoaderFlags", 4),
		( "NumberOfRvaAnSizes", 4),
		([
			("VirtualAdress" , 4),
			("Size" , 4)
		],
		[
			"_IMAGE_DATA_DIRECTORY_ENTRY_EXPORT",
			"_IMAGE_DATA_DIRECTORY_ENTRY_IMPORT",
			"_IMAGE_DATA_DIRECTORY_ENTRY_RESOURCE",
			"_IMAGE_DATA_DIRECTORY_ENTRY_EXCEPTION",
			"_IMAGE_DATA_DIRECTORY_ENTRY_SECURITY",
			"_IMAGE_DATA_DIRECTORY_ENTRY_BASERELOC",
			"_IMAGE_DATA_DIRECTORY_ENTRY_DEBUG",
			"_IMAGE_DATA_DIRECTORY_ENTRY_COPYRIGHT",
			"_IMAGE_DATA_DIRECTORY_ENTRY_GLOBALPTR",
			"_IMAGE_DATA_DIRECTORY_ENTRY_TLS",
			"_IMAGE_DATA_DIRECTORY_ENTRY_LOAD_CONFIG",
			"_IMAGE_DATA_DIRECTORY_ENTRY_BOUND_IMPORT",
			"_IMAGE_DATA_DIRECTORY_ENTRY_IAT",
			"_IMAGE_DATA_DIRECTORY_ENTRY_DELAY_IMPORT",
			"_IMAGE_DATA_DIRECTORY_ENTRY_COM_DESCRIPTOR",
			"_IMAGE_DATA_DIRECTORY_ENTRY_RESERVED"
		])
	])
]

_IMAGE_EXPORT_DIRECTORY = {
	"Characteristics" : 4,
	"TimeDateStamp" : 4,
	"MajorVersion" : 2,
	"MinorVersion" : 2,
	"Name" : 4,
	"Base" : 4,
	"NumberOfFunctions" : 4,
	"NumberOfNames" : 4,
	"AdressOfFunctions" : 4,
	"AdressOfNames" : 4,
	"AdressOfNAmeOrdinals" : 4
}

_IMAGE_IMPORT_DESCRIPTOR = [
	("Characteristics" , 4 ), # Can point to originalFirstThunk
	("TimeDateStamp" , 4),
	("ForwarderChain" , 4),
	("Name" , 4), # if bound to IAT has actual adresses
	("FirstThunk" , 4)
]

_IMAGE_RESOURCE_DIRECTORY = {
	"Characteristics" : 4,
	"TimeDateStamp" : 4,
	"MajorVersion" : 2,
	"MinorVersion" : 2,
	"NumberOfNamedEntries" : 2,
	"NumberOfIdEntries" : 2
}

_IMAGE_RESOURCE_DIRECTORY_ENTRIES = {
	"DataRVA" : 4,
	"Size" : 4,
	"CodePage" : 4,
	"Reserved" : 4
}

_IMAGE_RESOURCE_DIRECTORY_STRING = {
	"Length" : 4,
	"UString" : ("peLine",4)
}

_IMAGE_RESOURCE_DATA_ENTRIES = {
	"NameRVA" : 4,
	"IntegerID" : 4,
	"DataEntryRVA" : 4,
	"SubdirectoryRVA" : 4
}

_IMAGE_TLS_DIRECTORY_32 = {
	"StartAdressOfRawData" : 4,
	"EndAdressOfRawData" : 4,
	"AdressOfIndex" : 4,
	"AdressOfCallBacks" : 4,
	"SizeOfZeroFill" : 4,
	"Characteristics" : 4
}

_IMAGE_TLS_DIRECTORY_32p = {
	"StartAdressOfRawData" : 8,
	"EndAdressOfRawData" : 8,
	"AdressOfIndex" : 8,
	"AdressOfCallBacks" : 8,
	"SizeOfZeroFill" : 4,
	"Characteristics" : 4
}

_IMAGE_DELAY_IMPORT_DESCRIPTOR = {
	"grAttrs" : 4,
	"szName" : 4,
	"phmod" : 4,
	"pIAT" : 4,
	"pINT" : 4,
	"pBoundIAT" : 4,
	"pUnloadIAT" : 4,
	"dwTimeStamp" : 4
}

_IMAGE_SECTION_HEADER = {
	"Name" : 8,
	"VirtualSize" : 4,
	"VirtualAdress" : 4,
	"SizeOfRawData" : 4,
	"PointerToRawData" : 4,
	"PointerToReloacations" : 4,
	"PointerToLineNumbers" : 4,
	"NumberOfReloacations" : 2,
	"NumberOfLinenumbers" : 2,
	"Characteristics" : 4
}

_IMAGE_LOAD_CONFIGURATION_LAYOUT_32 = {
	"Characteristics" : 4,
	"TimeDateStamp" : 4,
	"MajorVersion" : 2,
	"MinorVersion" : 2,
	"GlobalFlagsClear" : 4,
	"GlobalFlagSet" : 4,
	"CriticalSectionDefaultTimeout" : 4,
	"DeCommitFreeBlockThreshold" : 8,
	"DeCommitTotalFreeThreshold" : 8,
	"LockPrefixTable" : 8,
	"MaximumAllocationSize" : 8,
	"VirtualMemoryThreshold" : 8,
	"ProcessAffinityMask" : 8,
	"ProcessHeapFlags" : 4,
	"CSDVersion" : 2,
	"Reserved" : 2,
	"EditList" : 8,
	"SecurityCookie" : 4,
	"SEHandlerTable" : 4,
	"SEHandlerCount" : 4
}
_IMAGE_LOAD_CONFIGURATION_LAYOUT_32p = {
	"Characteristics" : 4,
	"TimeDateStamp" : 4,
	"MajorVersion" : 2,
	"MinorVersion" : 2,
	"GlobalFlagsClear" : 4,
	"GlobalFlagSet" : 4,
	"CriticalSectionDefaultTimeout" : 4,
	"DeCommitFreeBlockThreshold" : 8,
	"DeCommitTotalFreeThreshold" : 8,
	"LockPrefixTable" : 8,
	"MaximumAllocationSize" : 8,
	"VirtualMemoryThreshold" : 8,
	"ProcessAffinityMask" : 8,
	"ProcessHeapFlags" : 4,
	"CSDVersion" : 2,
	"Reserved" : 2,
	"EditList" : 8,
	"SecurityCookie" : 8,
	"SEHandlerTable" : 8,
	"SEHandlerCount" : 8
}


import unittest
from datasets import Impl_DatasetsWindow
from unittest.mock import MagicMock
from PyQt5.QtWidgets import QMessageBox

class TestHelpWindow(unittest.TestCase):

    def setUp(self):
        self.window = Impl_DatasetsWindow()

    def test_help_button_click(self):
        # simulate the click event on the Help button
        self.window.btn_Help.click()

        # ensure that the QMessageBox was created and shown with the correct information
        mbox = QMessageBox()
        mbox.setIcon(QMessageBox.Information)
        mbox.setWindowTitle("Help")
        mbox.setText("Here is all the information about the DataSet Window")
        mbox.setInformativeText("Click the Datasets button, press the Load Dataset on the top left corner and select the csv or xml file that contains the dataset.(It seems to work mostly with the XMLs with the CVS you have to manually pick the “preset” on the right).The tool will recognize the type of report and preload the recommended features for best prediction accuracy.\nYou can add or remove columns and the start labelling the dataset which also can be saved to used later.")
        mbox.setStandardButtons(QMessageBox.Ok | QMessageBox.Cancel)
        mbox.exec_ = MagicMock()
        mbox.exec_()
        mbox.exec_.assert_called_once()


import os.path
import sys
import collections
import getopt
import re

from sklearn.neighbors import KNeighborsClassifier

def featureVectorProcessing(scoreFilesPaths, isAE, datasetType, aux_models_tag, sim_cal_id, resultDir):
	scoreObjs=[]
	for sf in scoreFilesPaths:
		scores={}
		with open(os.path.join(sf),"rt") as targetFile:
			for line in targetFile:
				parts = line.split(":")
				scores[parts[0].strip()]=float(" ".join(parts[1:]))
		scoreObjs.append(scores)
	label=0
	if isAE == "Yes":
		label=1

	features_IDs={}
	for key in scoreObjs[0]:
		features_IDs[key]=[]

	fileIDs=[]
	features=[]
	labels=[]
	for key in features_IDs:
		fileIDs.append(key)
		labels.append(label)
		feature=[]
		for scores in scoreObjs:
			feature.append(scores[key])
		features.append(feature)

	resultfilepath="feature_vectors_"+sim_cal_id
	if isAE == "Yes":
		resultfilepath+="_AE"
	else:
		resultfilepath+="_Benign"

	if datasetType == "Test":
		resultfilepath+="_Test"
	elif datasetType == "Train":
		resultfilepath+="_Train"
	elif datasetType == "Unseen":
		resultfilepath+="_Unseen"
	resultfilepath+="_"+aux_models_tag+".txt"
	resultfilepath=os.path.join(resultDir, resultfilepath)

	with open(resultfilepath, "w") as fv:
		length=len(fileIDs)
		for x in range(0,length):
			fs=str(round(features[x][0], 4))
			dim=len(features[x])
			for i in range(1, dim):
				fs+=","+str(round(features[x][i], 4))
			fv.write(fileIDs[x]+":"+fs+":"+str(labels[x])+"\n")

def main(argv):
	if (len(argv) < 6):
		print("\n===============================================================================================")
		print("\nUsage: python <this_script> resultDir isAE datasetType aux_models_tag sim_cal_id list_of_filepaths" )
		print("\n===============================================================================================")
		return

	resultDir = argv[0].strip()
	isAE = argv[1].strip()
	datasetType = argv[2].strip()
	aux_models_tag = argv[3].strip()
	sim_cal_id=argv[4].strip()
	listoffilepaths=argv[5:]

	featureVectorProcessing(listoffilepaths, isAE, datasetType, aux_models_tag, sim_cal_id, resultDir)

main(sys.argv[1:])


#!/usr/bin/env python3

# Definition for singly-linked list.
# class ListNode:
#     def __init__(self, val=0, next=None):
#         self.val = val
#         self.next = next
class Solution:
	def removeElements(self, head: Optional[ListNode], val: int) -> Optional[ListNode]:
		new_head = ListNode(next = head)
		cur_node = new_head
		while cur_node.next != None:
			if cur_node.next.val == val:
				cur_node.next = cur_node.next.next
			else:
				cur_node = cur_node.next
		return new_head.next


import os
import re
import pprint
from xml.dom import minidom

DEBUG = False

class Config(object):
	def __init__(self, filename):
		with open(filename, 'r') as file_:
			lines = file_.readlines()

		lines = self._remove_line_feed(lines)
		lines = self._remove_comment(lines)
		lines = self._remove_blank_lines(lines)

		basic, beats, visions = self._split_parts(lines)

		self._read_basics(basic)
		self._read_beats(beats)
		self._read_visions(visions)
		self._validate()

	def _remove_line_feed(self, lines):
		return [re.sub(r'\n', '', line) for line in lines]

	def _remove_comment(self, lines):
		return [re.sub(r'#.*', '', line) for line in lines]

	def _remove_blank_lines(self, lines):
		return [line for line in lines if line]

	def _split_parts(self, lines):
		basic, beats, visions = [], [], []

		target = basic
		for line in lines:
			if line.startswith('='):
				if target is basic:
					target = beats
				elif target is beats:
					target = visions
				continue

			target.append(line)

		return basic, beats, visions

	def _read_basics(self, basic):
		for line in basic:
			key, value = self._parse_key_value(line)
			setattr(self, key, value)

			if DEBUG:
				print('{}: {}'.format(key, value))

	def _read_beats(self, beats):
		self.beats = []

		position = 'left'
		for line in beats:
			singer, lyric, beats = self._parse_lyrics_beats(line)
			beat = {}
			beat['singer'] = singer
			beat['lyric'] = lyric
			beat['beats'] = beats
			if lyric is None:
				position = 'left'
			else:
				beat['position'] = position
				if position == 'left':
					position = 'right'
				elif position == 'right':
					position = 'left'
				else:
					assert False

			self.beats.append(beat)

			if DEBUG:
				pprint.pprint(beat)

	def _read_visions(self, visions):
		self.visions = []
		self._read_visions_basic(visions)
		self._read_visions_components(visions)

		if DEBUG:
			pprint.pprint(self.visions)

	def _read_visions_basic(self, visions):
		curr_beat = 0
		for line in visions:
			if line == 'clear':
				for vision in self.visions:
					if vision['end'] == -1:
						vision['end'] = curr_beat
			else:
				name, beat = self._parse_visions_beats(line)
				if name is not None:
					self.visions.append({
						'name': name,
						'start': curr_beat,
						'end': -1,
						'components': [],
					})
				curr_beat += beat

	def _read_visions_components(self, visions):
		for vision in self.visions:
			basename = '{}.svg'.format(vision['name'])
			src_filename = os.path.join(self.visions_dir_name, basename)
			document = minidom.parse(src_filename)
			groups = document.getElementsByTagName('g')
			for group in groups:
				layer_name = group.getAttribute('inkscape:label')
				if not layer_name: # not a layer
					continue
				rects = group.getElementsByTagName('rect')
				if not rects:
					continue
				rect = rects[0]
				style = rect.getAttribute('style')
				color = re.search('fill:#([0-9a-f]+)', style).group(1)
				vision['components'].append({
					'id': layer_name,
					'color': color,
				})

	def _validate(self):
		for key, type_ in self._get_settings_types().items():
			value = getattr(self, key)
			if type(value) is not type_:
				raise Exception('Invalid config type: {} - {}'.format(key, value))

	def _parse_key_value(self, line):
		match = re.match(r'^\s*(\S+)\s*:\s*(\S+)\s*$', line)
		if not match:
			raise Exception('Invalid config basic setting format: {}'.format(line))

		key = match.group(1)
		key = re.sub('-', '_', key)

		value = match.group(2)
		value = self._get_settings_types()[key](value)

		return key, value

	def _parse_lyrics_beats(self, line):
		match = re.match(r'^\s*((\S)\s+)*(\S+)\s*(.*?)\s*$', line)
		if not match:
			raise Exception('Invalid config beats format: {}'.format(line))

		singer = match.group(2)
		if singer:
			singer = self._get_singer_name(singer)

		lyric = match.group(3)
		if self._is_meta_label(lyric):
			lyric = None

		beats = match.group(4)
		beats = beats.split(',')
		beats = [beat.strip() for beat in beats]
		beats = [float(beat) for beat in beats]

		return singer, lyric, beats

	def _parse_visions_beats(self, line):
		match = re.match(r'^\s*(\S+)\s*(\S*)\s*$', line)
		if not match:
			raise Exception('Invalid config beats format: {}'.format(line))

		name = match.group(1)
		if self._is_meta_label(name):
			name = None

		beat = match.group(2)
		beat = float(beat)

		return name, beat

	def _is_meta_label(self, label):
		return label.startswith('[') and label.endswith(']')

	def _get_singer_name(self, singer_symbol):
		mapping = {
			'+': 'boy',
			'-': 'girl',
			'*': 'both',
		}

		if singer_symbol not in mapping:
			raise Exception('No such singer symbol: {}'.format(singer_symbol))

		singer_name = mapping[singer_symbol]
		return singer_name

	def _get_settings_types(self):
		return {
			'song_name': str,
			'lyric_writer': str,
			'melody_writer': str,
			'audio_filename': str,
			'output_dir_name': str,
			'visions_dir_name': str,
			'beat_per_minute': float,
			'begin_time': float,
			'beats': list,
		}


import cv2
import numpy as np
import math
from get_reference_frame import get_ref_frame
import os
import csv

def pixel_distance(img1, img2):
    '''
    Get the histogram of an image. For an 8-bit, grayscale image, the
    histogram will be a 256 unit vector in which the nth value indicates
    the percent of the pixels in the image with the given darkness level.
    The histogram's values sum to 1.
    '''
    img1 = img1.astype(np.int16)
    img2 = img2.astype(np.int16)

    return np.sum(np.absolute(np.subtract(img1.flatten(),
        img2.flatten())))

# function for taking frames from video and combining into video to save # images sent to OCR
def combine_images(images, num_images: int, num_horizontal: int):
    '''
    Return a new image with the images glued together
    Arguments:
        images: N x H x W ndarray of images
        num_horizontal: determines the number of images to put next to each other
        on the same row
    '''
    combined_image = None
    for i in range (math.ceil(num_images / num_horizontal)):
        row_image = None
        for j in range (num_horizontal):
            if row_image is None:
                row_image = images[i*num_horizontal + j]
            elif (i*num_horizontal + j >= num_images):
                row_image = cv2.hconcat([row_image, np.zeros((images.shape[1],images.shape[2],3), np.uint8)])
            else:
                row_image = cv2.hconcat([row_image, images[i*num_horizontal + j]])

        if combined_image is None:
            combined_image = row_image
        else:
            combined_image = cv2.vconcat([combined_image, row_image])

    return combined_image

def process_avi_pixels(file_location, time_of_ref=200, window_len=10,
    images_per_combine:int=240, images_per_horizontal:int=12, start_time=None, end_time=None,
    skip=20, threshold=60, fps=25, name=None, out_folder=None, ref_frame_loc=None):
    '''
    Function for taking avi and grabbing cropped frames (fixed) from them
    '''
    ## Getting the reference frame to compare with
    #### !!! This was required for automatically getting ref frame !!!
    # x, y, w, h = get_ref_frame(file_location, time_of_ref, fps, window_len, ref_frame_loc)
    vc = cv2.VideoCapture(file_location)
    ret, frame = vc.read()
    print(frame.shape)
    vc.set(1, 63900)

    ### For cricket matches
    x = 69
    y = 321
    w = 87
    h = 21
    ret, frame = vc.read()
    # print(frame.shape)
    cv2.imwrite(ref_frame_loc + name + '_ref.jpg', frame[y:y+h, x:x+w])

    vc = cv2.VideoCapture(file_location)
    compare_frame = None # this frame is what we compare for skipping
    if vc.isOpened():
        rval, frame = vc.read()
    else:
        rval = False

    timestamps = [] # dictionary mapping from crop to frame
    list_timestamps = [] # list of composite dictionary mappings
    framenum_li = []
    all_framenum_li = []
    frames = None
    time_window = None
    curr_time_window_cnt = 0

    if start_time is not None:
        start = start_time * fps
    else:
        start = 0

    end = end_time * fps

    # frame num, num frames processed, num images generated
    framenum, cnt, cnt_composite = 0, 0, 0

    # loop to start of time window of interest
    while (framenum < start):
        rval, frame = vc.read()
        framenum += 1
    # compare_frame = frame[y:y+h, x:x+w].copy()
    compare_frame = cv2.imread(ref_frame_loc + name + '_ref.jpg')

    while framenum < end:
        # process every 'skip'th frame
        for i in range (skip):
            rval, frame = vc.read()
            framenum += 1

        if frame is None:
            return list_timestamps

        # skip frames disimilar to what we want
        # calibrate treshold for pixel similarity
        while ((pixel_distance(compare_frame, frame[y:y+h, x:x+w]) / (w*h*3)) > threshold):
            rval, frame = vc.read()
            framenum += 1
            if frame is None:
                return list_timestamps

        if frame is None:
            return list_timestamps
        # while (pixel_distance(compare_frame, frame[y:y+h, x:x+w]) > 10):
        #     rval, frame = vc.read()
        #     framenum += 1

        # cv2.imwrite('test.jpg', frame)
        # print(frame.shape)
        # if cnt==21:
        #     cv2.imwrite('test1.jpg', frame[y:y+h, x:x+w])
        #     print(framenum)
        # if cnt==22:
        #     cv2.imwrite('test2.jpg', frame[y:y+h, x:x+w])
        #     print(framenum)
        cropped_frame = frame[y:y+h, x:x+w].copy()
        bordersize = 8
        padded_frame = cv2.copyMakeBorder(
            cropped_frame,
            top=bordersize,
            bottom=bordersize,
            left=bordersize,
            right=bordersize,
            borderType=cv2.BORDER_CONSTANT,
            value=[255, 255, 255]
        )

        cropped_frame = np.expand_dims(padded_frame, axis=0)
        cnt += 1
        timestamps.append(framenum / fps)
        framenum_li.append(framenum)
        # timestamps[cnt % images_per_combine] = framenum # map index to current timestamp

        if frames is None:
            frames = cropped_frame.copy()
        else:
            frames = np.concatenate((frames, cropped_frame), axis=0)

        if (cnt % images_per_combine == 0):
            new_image = combine_images(frames, images_per_combine, images_per_horizontal)
            list_timestamps.append(timestamps)
            all_framenum_li.append(framenum_li)
            timestamps = []
            framenum_li = []

            # send to ocr here
            if not os.path.exists(out_folder):
                os.makedirs(out_folder)

            path = os.path.join(out_folder, name + "_" + str(cnt_composite) + ".jpg")
            cv2.imwrite(path, new_image)
            cnt_composite += 1

            print('Image: ', path, ' SAVED')

            frames = None # reset frames

        # for debugging purposes
        # if (cnt > images_per_combine):
        #     return list_timestamps

    remaining_frames = images_per_combine - frames.shape[0]
    white_frames = np.full((remaining_frames, frames.shape[1], frames.shape[2], frames.shape[3]), 255)

    frames = np.concatenate((frames, white_frames), axis=0)
    new_image = combine_images(frames, images_per_combine, images_per_horizontal)
    list_timestamps.append(timestamps)
    all_framenum_li.append(framenum_li)
    # print(image.shape[0])
    # print(image.shape[1])
    timestamps = {}

    # send to ocr here

    path = os.path.join(out_folder, name + "_" + str(cnt_composite) + ".jpg")
    cv2.imwrite(path, new_image)
    cnt_composite += 1

    frames = None # reset frames
    vc.release()
    return list_timestamps, all_framenum_li, new_image.shape[0], new_image.shape[1]

def save_csv(list_of_timestamps, list_framenum, img_per_comp, name):
    '''
    Save a csv
    Arguments:
        list_of_timestamps: list of dictionaries, with each dictionary mapping
        an image in the crop (row-major order) to a timestamp
        img_per_crop: number of images in each crop
        name: path/name to save the csv to/as
    '''
    if not os.path.exists(name):
        os.makedirs(name)
    num_comps = len(list_of_timestamps)
    with open(name + 'run1.csv', mode='w') as csv_file:
        fieldnames = ['cmpid', 'imgid', 'time', 'frame']
        writer = csv.DictWriter(csv_file, fieldnames=fieldnames)
        writer.writeheader()
        for i in range (num_comps):
            num_imgs = len(list_of_timestamps[i])
            for j in range(num_imgs):
                # print(str(i) + ' , ' + str(j))
                writer.writerow({'cmpid': i, 'imgid': j, 'time': list_of_timestamps[i][j], 'frame': list_framenum[i][j]})

# save timestamp information as JSON
def save_json (list_of_timestamps):
    pass


"""An n x n matrix is valid if every row and every column contains all the integers from 1 to n (inclusive).

Given an n x n integer matrix matrix, return true if the matrix is valid. Otherwise, return false.

"""

class Solution(object):
    def checkValid(self, matrix):
        length=len(matrix)
        rows = [set() for _ in range(length)]
        columns = [set() for _ in range(length)]

        for i in range(length):
            for j in range(length):
                num=matrix[i][j]

                if (num in rows[i]) or (num in columns[j]):
                    return False

                rows[i].add(num)
                columns[j].add(num)

        return True

matrix = [[1,2,3],[3,1,2],[2,3,1]]
print(Solution().checkValid(matrix))