# -*- coding: utf-8 -*-
import os

import numpy as np
import torch as torch

from deepctr_torch.callbacks import EarlyStopping, ModelCheckpoint
from deepctr_torch.inputs import SparseFeat, DenseFeat, VarLenSparseFeat

SAMPLE_SIZE = 64

def gen_sequence(dim, max_len, sample_size):
    return np.array([np.random.randint(0, dim, max_len) for _ in range(sample_size)]), np.random.randint(1, max_len + 1,
                                                                                sample_size)

def get_test_data(sample_size=1000, embedding_size=4, sparse_feature_num=1, dense_feature_num=1,
                  sequence_feature=['sum', 'mean', 'max'], classification=True, include_length=False,
                  hash_flag=False, prefix=''):
    feature_columns = []
    model_input = {}

    if 'weight' in sequence_feature:
        feature_columns.append(
            VarLenSparseFeat(SparseFeat(prefix + "weighted_seq", vocabulary_size=2, embedding_dim=embedding_size),
                             maxlen=3, length_name=prefix + "weighted_seq" + "_seq_length",
                             weight_name=prefix + "weight"))
        s_input, s_len_input = gen_sequence(
            2, 3, sample_size)

        model_input[prefix + "weighted_seq"] = s_input
        model_input[prefix + 'weight'] = np.random.randn(sample_size, 3, 1)
        model_input[prefix + "weighted_seq" + "_seq_length"] = s_len_input
        sequence_feature.pop(sequence_feature.index('weight'))

    for i in range(sparse_feature_num):
        dim = np.random.randint(1, 10)
        feature_columns.append(SparseFeat(prefix + 'sparse_feature_' + str(i), dim, embedding_size, dtype=torch.int32))
    for i in range(dense_feature_num):
        feature_columns.append(DenseFeat(prefix + 'dense_feature_' + str(i), 1, dtype=torch.float32))
    for i, mode in enumerate(sequence_feature):
        dim = np.random.randint(1, 10)
        maxlen = np.random.randint(1, 10)
        feature_columns.append(
            VarLenSparseFeat(SparseFeat(prefix + 'sequence_' + mode, vocabulary_size=dim, embedding_dim=embedding_size),
                             maxlen=maxlen, combiner=mode))

    for fc in feature_columns:
        if isinstance(fc, SparseFeat):
            model_input[fc.name] = np.random.randint(0, fc.vocabulary_size, sample_size)
        elif isinstance(fc, DenseFeat):
            model_input[fc.name] = np.random.random(sample_size)
        else:
            s_input, s_len_input = gen_sequence(
                fc.vocabulary_size, fc.maxlen, sample_size)
            model_input[fc.name] = s_input
            if include_length:
                fc.length_name = prefix + "sequence_" + str(i) + '_seq_length'
                model_input[prefix + "sequence_" + str(i) + '_seq_length'] = s_len_input

    if classification:
        y = np.random.randint(0, 2, sample_size)
    else:
        y = np.random.random(sample_size)

    return model_input, y, feature_columns

def layer_test(layer_cls, kwargs={}, input_shape=None,
               input_dtype=torch.float32, input_data=None, expected_output=None,
               expected_output_shape=None, expected_output_dtype=None, fixed_batch_size=False):
    '''check layer is valid or not

    :param layer_cls:
    :param input_shape:
    :param input_dtype:
    :param input_data:
    :param expected_output:
    :param expected_output_dtype:
    :param fixed_batch_size:

    :return: output of the layer
    '''
    if input_data is None:
        # generate input data
        if not input_shape:
            raise ValueError("input shape should not be none")

        input_data_shape = list(input_shape)
        for i, e in enumerate(input_data_shape):
            if e is None:
                input_data_shape[i] = np.random.randint(1, 4)

        if all(isinstance(e, tuple) for e in input_data_shape):
            input_data = []
            for e in input_data_shape:
                rand_input = (10 * np.random.random(e))
                input_data.append(rand_input)
        else:
            rand_input = 10 * np.random.random(input_data_shape)
            input_data = rand_input

    else:
        # use input_data to update other parameters
        if input_shape is None:
            input_shape = input_data.shape

    if expected_output_dtype is None:
        expected_output_dtype = input_dtype

    # layer initialization
    layer = layer_cls(**kwargs)

    if fixed_batch_size:
        inputs = torch.tensor(input_data.unsqueeze(0), dtype=input_dtype)
    else:
        inputs = torch.tensor(input_data, dtype=input_dtype)

    # calculate layer's output
    output = layer(inputs)

    if not output.dtype == expected_output_dtype:
        raise AssertionError("layer output dtype does not match with the expected one")

    if not expected_output_shape:
        raise ValueError("expected output shape should not be none")

    actual_output_shape = output.shape
    for expected_dim, actual_dim in zip(expected_output_shape, actual_output_shape):
        if expected_dim is not None:
            if not expected_dim == actual_dim:
                raise AssertionError(f"expected_dim:{expected_dim}, actual_dim:{actual_dim}")

    if expected_output is not None:
        # check whether output equals to expected output
        assert_allclose(output, expected_output, rtol=1e-3)

    return output

def check_model(model, model_name, x, y, check_model_io=True):
    '''
    compile model,train and evaluate it,then save/load weight and model file.
    :param model:
    :param model_name:
    :param x:
    :param y:
    :param check_model_io:
    :return:
    '''
    early_stopping = EarlyStopping(monitor='val_acc', min_delta=0, verbose=1, patience=0, mode='max')
    model_checkpoint = ModelCheckpoint(filepath='model.ckpt', monitor='val_acc', verbose=1,
                                       save_best_only=True,
                                       save_weights_only=False, mode='max', period=1)

    model.compile('adam', 'binary_crossentropy',
                  metrics=['binary_crossentropy', 'acc'])
    model.fit(x, y, batch_size=100, epochs=1, validation_split=0.5, callbacks=[early_stopping, model_checkpoint])

    print(model_name + 'test, train valid pass!')
    torch.save(model.state_dict(), model_name + '_weights.h5')
    model.load_state_dict(torch.load(model_name + '_weights.h5'))
    os.remove(model_name + '_weights.h5')
    print(model_name + 'test save load weight pass!')
    if check_model_io:
        torch.save(model, model_name + '.h5')
        model = torch.load(model_name + '.h5')
        os.remove(model_name + '.h5')
        print(model_name + 'test save load model pass!')
    print(model_name + 'test pass!')

def get_device(use_cuda=True):
    device = 'cpu'
    if use_cuda and torch.cuda.is_available():
        print('cuda ready...')
        device = 'cuda:0'
    return device


from glitches.ImageGlitcherInterface import ImageGlitcherInterface
from PIL import Image
from scipy.ndimage import rotate
import random
import numpy as np

class MoveRowSideToSide(ImageGlitcherInterface):

    def __init__(self):
        self.image_glitch_type = "move random row from side to side"

    def glitch_image(self, image_name):
        print(str(image_name))
        im = Image.open("images/" + image_name)

        image_arr = np.asarray(im).copy()
        # print(image_arr)
        im_width, im_height = im.size

        # print(im_width, im_height)

        glitch_start = int(im_height * (3 / 5) + (random.randint(int(im_height * (1 / 8)), int(im_height * (1 / 7))) * (1 if random.random() < 0.5 else -1)))

        for row in range(glitch_start, glitch_start + (random.randint(int(im_height * (1 / 6)), int(im_height * (1 / 5)))), 5):

            row_shift =  random.randint(-50, 50)

            for i in range(0, 5):

                # shifted_arr = image_arr[row + i]
                # shifted_arr = np.roll(shifted_arr, row_shift,  axis=0)
                # image_arr[row + i] = shifted_arr
                image_arr[row + i] = shiftRow(image_arr[row + i], row_shift)

        new_image = Image.fromarray(image_arr, 'RGB')
        self.save_image(image_name, new_image)

def shiftRow(row, shift_amount):
    shifted_arr = np.roll(row, shift_amount,  axis=0)
    return shifted_arr

# def shiftColumn(col, shift_amount):


import os
import copy
import torch
import torch.distributed as dist
import functools
import logging
import pickle
import sys
import signal
import subprocess

def _find_free_port():
    import socket

    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    # Binding to port 0 will cause the OS to find an available port for us
    sock.bind(("", 0))
    port = sock.getsockname()[1]
    sock.close()
    # NOTE: there is still a chance the port could be taken by other processes.
    return port

def init_distributed(cfg):
    os.environ["CUDA_VISIBLE_DEVICES"] = f"{cfg.distributed.dist_local_rank}"
    try:
        dist.init_process_group(
            backend="nccl",
            init_method=cfg.distributed.dist_url,
            world_size=cfg.distributed.world_size,
            rank=cfg.distributed.dist_global_rank,
        )
    except Exception as e:
        logger = logging.getLogger()
        logger.error(f"Distributed Initialization Error URL: {cfg.distributed.dist_url}")
        raise e
    synchronize()

def launch_distributed(args):
    if args.dist_url is None:
        raise ValueError("Multi-node distributed training requires a main node url.")
    print(
        f"=> Distributed running on {args.dist_url} with {args.world_size} GPUs."
    )

    processes = []
    os.environ["OMP_NUM_THREADS"] = "1"

    for local_gpu_id in range(0, args.num_gpus):
        cmd = [sys.executable, "run.py"]
        local_args = copy.deepcopy(args)
        if local_args.dist_local_rank == -1:
            local_args.dist_local_rank = local_gpu_id
        if local_args.dist_global_rank == -1:
            local_args.dist_global_rank = local_gpu_id

        for args_name in local_args.__dict__:
            cmd_string = f"--{args_name}={getattr(local_args, args_name)}"

            if args_name == "opt":
                cmd_string = cmd_string[6:]
            elif args_name == "dev":
                if getattr(local_args, args_name):
                    cmd_string = "--dev"
                else:
                    continue
            elif args_name == "modify":
                modifies = getattr(local_args, args_name)
                if modifies is not None:
                    cmd.append("--modify")
                    cmd.extend(modifies)
                continue
            elif args_name == "launch":
                cmd_string = "--launch"
            elif args_name == "slurm":
                continue
            elif getattr(local_args, args_name) is None:
                continue

            cmd.append(cmd_string)
        current_env = os.environ.copy()
        process = subprocess.Popen(cmd, env=current_env)
        processes.append(process)

    for process in processes:
        process.wait()
        os.kill(process.pid, signal.SIGINT)

        if process.returncode != 0:
            raise subprocess.CalledProcessError(returncode=process.returncode, cmd=cmd)

def get_rank():
    if not dist.is_available():
        return 0
    if not dist.is_initialized():
        return 0
    return dist.get_rank()

def get_world_size():
    if not dist.is_available():
        return 1
    if not dist.is_initialized():
        return 1
    return dist.get_world_size()

def get_local_rank_from_slurm():
    if "SLURM_LOCALID" in os.environ:
        return int(os.environ["SLURM_LOCALID"])
    else:
        raise ValueError("Cannot get local rank from SLURM_LOCALID.")

def get_global_rank_from_slurm():
    if "SLURM_PROCID" in os.environ:
        return int(os.environ["SLURM_PROCID"])
    else:
        raise ValueError("Cannot get global rank from SLURM_PROCID.")

def get_world_size_from_slurm():
    if "SLURM_NTASKS" in os.environ:
        return int(os.environ["SLURM_NTASKS"])
    else:
        raise ValueError("Cannot get world size from SLURM_NTASKS.")

def get_dist_url_from_slurm():
    try:
        with open("./.slurm.hosts") as reader:
            master_node = reader.readlines()[0].strip()
        with open("./.slurm.port") as reader:
            master_port = reader.readlines()[0].strip()
        return f"tcp://{master_node}:{master_port}"
    except Exception as e:
        print(f"Cannot construct dist_url: {e}")

def is_main_process():
    return get_rank() == 0

def is_distributed():
    return dist.is_initialized()

def synchronize():
    """
    Helper function to synchronize (barrier) among all processes when
    using distributed training
    """
    if not dist.is_available():
        return
    if not dist.is_initialized():
        return
    world_size = dist.get_world_size()
    if world_size == 1:
        return
    dist.barrier()

@functools.lru_cache()
def _get_global_gloo_group():
    """
    Return a process group based on gloo backend, containing all the ranks
    The result is cached.
    """
    if dist.get_backend() == "nccl":
        return dist.new_group(backend="gloo")
    else:
        return dist.group.WORLD

def _serialize_to_tensor(data, group):
    backend = dist.get_backend(group)
    assert backend in ["gloo", "nccl"]
    device = torch.device("cpu" if backend == "gloo" else "cuda")

    buffer = pickle.dumps(data.cpu() if isinstance(data, torch.Tensor) else data)
    if len(buffer) > 1024 ** 3:
        logger = logging.getLogger(__name__)
        logger.warning(
            "Rank {} trying to all-gather {:.2f} GB of data on device {}".format(
                get_rank(), len(buffer) / (1024 ** 3), device
            )
        )
    storage = torch.ByteStorage.from_buffer(buffer)
    tensor = torch.ByteTensor(storage).to(device=device)
    return tensor

def _pad_to_largest_tensor(tensor, group):
    """
    Returns:
        list[int]: size of the tensor, on each rank
        Tensor: padded tensor that has the max size
    """
    world_size = dist.get_world_size(group=group)
    assert world_size >= 1, "comm.gather/all_gather must be called from ranks within the given group!"
    local_size = torch.tensor([tensor.numel()], dtype=torch.int64, device=tensor.device)
    size_list = [torch.zeros([1], dtype=torch.int64, device=tensor.device) for _ in range(world_size)]
    dist.all_gather(size_list, local_size, group=group)
    size_list = [int(size.item()) for size in size_list]

    max_size = max(size_list)

    # we pad the tensor because torch all_gather does not support
    # gathering tensors of different shapes
    if local_size != max_size:
        padding = torch.zeros((max_size - local_size,), dtype=torch.uint8, device=tensor.device)
        tensor = torch.cat((tensor, padding), dim=0)
    return size_list, tensor

def gather(data, dst=0, group=None):
    """
    Run gather on arbitrary picklable data (not necessarily tensors).

    Args:
        data: any picklable object
        dst (int): destination rank
        group: a torch process group. By default, will use a group which
            contains all ranks on gloo backend.

    Returns:
        list[data]: on dst, a list of data gathered from each rank. Otherwise,
            an empty list.
    """
    if get_world_size() == 1:
        return [data]
    if group is None:
        group = _get_global_gloo_group()
    if dist.get_world_size(group=group) == 1:
        return [data]
    rank = dist.get_rank(group=group)

    tensor = _serialize_to_tensor(data, group)
    size_list, tensor = _pad_to_largest_tensor(tensor, group)

    # receiving Tensor from all ranks
    if rank == dst:
        max_size = max(size_list)
        tensor_list = [torch.empty((max_size,), dtype=torch.uint8, device=tensor.device) for _ in size_list]
        dist.gather(tensor, tensor_list, dst=dst, group=group)

        data_list = []
        for size, tensor in zip(size_list, tensor_list):
            buffer = tensor.cpu().numpy().tobytes()[:size]
            data_list.append(pickle.loads(buffer))
        return data_list
    else:
        dist.gather(tensor, [], dst=dst, group=group)
        return []

def all_gather(data, group=None):
    """
    Run all_gather on arbitrary picklable data (not necessarily tensors).
    Args:
        data: any picklable object
        group: a torch process group. By default, will use a group which
            contains all ranks on gloo backend.
    Returns:
        list[data]: list of data gathered from each rank
    """
    if get_world_size() == 1:
        return [data]
    if group is None:
        group = _get_global_gloo_group()
    if dist.get_world_size(group) == 1:
        return [data]

    tensor = _serialize_to_tensor(data, group)

    size_list, tensor = _pad_to_largest_tensor(tensor, group)
    max_size = max(size_list)

    # receiving Tensor from all ranks
    tensor_list = [torch.empty((max_size,), dtype=torch.uint8, device=tensor.device) for _ in size_list]
    dist.all_gather(tensor_list, tensor, group=group)

    data_list = []
    for size, tensor in zip(size_list, tensor_list):
        buffer = tensor.cpu().numpy().tobytes()[:size]
        data_list.append(pickle.loads(buffer))

    return data_list


# standard imports
from typing import List, Dict, Tuple
from collections import defaultdict

# custom imports
from graph import Graph
from algos.ttc_utils import find_cycle, update_graph

def ttc(
    employee_preferences: Dict[str, List[str]],
    job_preferences: Dict[str, List[str]]
) -> Tuple[Dict[str, str], Dict[str, str]]:
    matches: Dict[str, str] = {}
    employees = set(employee_preferences.keys())
    jobs = set(job_preferences.keys())
    all_nodes = employees | jobs

    # instantiate a new Graph object with all employee and job nodes
    # initially, this graph will only have nodes, no edges
    G = Graph(all_nodes)

    # queue (counter) to track which job we are currently considering for the current employee
    # i.e., if job_index is 2, then we are considering the 3rd job on the given
    job_queue: Dict = defaultdict(int)
    employee_queue: Dict = defaultdict(int)

    # add edges to build out graph
    for e in employees:
        job_index = job_queue[e]  # new entries start at 0 for defaultdict
        G.add_edge(e, employee_preferences[e][job_index])
    for j in jobs:
        employee_index = employee_queue[j]  # new entries start at 0 for defaultdict
        G.add_edge(j, job_preferences[j][employee_index])

    # Remove top trading cycles until graph is empty
    while G.number_of_nodes_in_graph() > 0:
        # find an arbitrary cycle in the graph
        cycle = find_cycle(G)

        # make assignments of employees to job based on cycle that was found
        for node in cycle:
            if node in employees:
                job: str = G.nodes[node].get_next_node()
                matches[node] = job
                matches[job] = node
                G.delete_node(node)
                G.delete_node(job)

        # update the graph with new edges after cycle was found and matches were made
        G, matches, employee_queue, job_queue = update_graph(
            G,
            matches,
            employee_preferences,
            job_preferences,
            employee_queue,
            job_queue
        )

    return matches, {employee: matches[employee] for employee in employees}


# import numpy as np
# from scipy.spatial.transform import Rotation
#
# class SE3Control(object):
#     """
#
#     """
#     def __init__(self, quad_params):
#         """
#         This is the constructor for the SE3Control object. You may instead
#         initialize any parameters, control gain values, or private state here.
#
#         For grading purposes the controller is always initialized with one input
#         argument: the quadrotor's physical parameters. If you add any additional
#         input arguments for testing purposes, you must provide good default
#         values!
#
#         Parameters:
#             quad_params, dict with keys specified by crazyflie_params.py
#
#         """
#
#         # Quadrotor physical parameters.
#         self.mass = quad_params['mass']  # kg
#         self.Ixx = quad_params['Ixx']  # kg*m^2
#         self.Iyy = quad_params['Iyy']  # kg*m^2
#         self.Izz = quad_params['Izz']  # kg*m^2
#         self.arm_length = quad_params['arm_length']  # meters
#         self.rotor_speed_min = quad_params['rotor_speed_min']  # rad/s
#         self.rotor_speed_max = quad_params['rotor_speed_max']  # rad/s
#         self.k_thrust = quad_params['k_thrust']  # N/(rad/s)**2
#         self.k_drag = quad_params['k_drag']  # Nm/(rad/s)**2
#         self.L = self.arm_length
#         # You may define any additional constants you like including control gains.
#         self.inertia = np.diag(np.array([self.Ixx, self.Iyy, self.Izz]))  # kg*m^2
#         self.gamma = self.k_drag / self.k_thrust
#
#         self.g = 9.81  # m/s^2
#
#         # STUDENT CODE HERE
#
#     def update(self, t, state, flat_output):
#         """
#         This function receives the current time, true state, and desired flat
#         outputs. It returns the command inputs.
#
#         Inputs:
#             t, present time in seconds
#             state, a dict describing the present state with keys
#                 x, position, m
#                 v, linear velocity, m/s
#                 q, quaternion [i,j,k,w]
#                 w, angular velocity, rad/s
#             flat_output, a dict describing the present desired flat outputs with keys
#                 x,        position, m
#                 x_dot,    velocity, m/s
#                 x_ddot,   acceleration, m/s**2
#                 x_dddot,  jerk, m/s**3
#                 x_ddddot, snap, m/s**4
#                 yaw,      yaw angle, rad
#                 yaw_dot,  yaw rate, rad/s
#
#         Outputs:
#             control_input, a dict describing the present computed control inputs with keys
#                 cmd_motor_speeds, rad/s
#                 cmd_thrust, N (for debugging and laboratory; not used by simulator)
#                 cmd_moment, N*m (for debugging; not used by simulator)
#                 cmd_q, quaternion [i,j,k,w] (for laboratory; not used by simulator)
#         """
#         cmd_motor_speeds = np.zeros((4,))
#         cmd_thrust = 0
#         cmd_moment = np.zeros((3,))
#         cmd_q = np.zeros((4,))
#
#         # STUDENT CODE HERE
#         # gain
#
#         # position
#         # kp_x = 3.2
#         # kp_y = 3.3
#         # kp_z = 5
#         kp_x = 8.1
#         kp_y = 7
#         kp_z = 7
#
#         # kd_x = 1.45 * np.sqrt(kp_x)
#         # kd_y = 1.45 * np.sqrt(kp_y)
#         # kd_z = 1.50 * np.sqrt(kp_z)
#         kd_x = 3.4
#         kd_y = 2.8
#         kd_z = 4
#
#         # orientation
#         kR_x = 80
#         kR_y = 80
#         kR_z = 8
#         # kR_x = 90
#         # kR_y = 70
#         # kR_z = 10
#
#         # kw_x = 2 * np.sqrt(40)
#         # kw_y = 2 * np.sqrt(40)
#         # kw_z = 2 * np.sqrt(10)
#         kw_x = 7
#         kw_y = 7
#         kw_z = 0.7
#
#         k_d = np.array([kd_x, kd_y, kd_z])
#         k_p = np.array([kp_x, kp_y, kp_z])
#         k_R = np.array([kR_x, kR_y, kR_z])
#         k_w = np.array([kw_x, kw_y, kw_z])
#         k_d = np.diag(k_d)
#         k_p = np.diag(k_p)
#         k_R = np.diag(k_R)
#         k_w = np.diag(k_w)
#
#         # coefficient
#         # k_F = self.k_thrust
#         # k_M = self.k_drag
#         #
#         # gamma = k_M / k_F
#
#         # copy values from input
#         x_state = state.get('x')
#         v = state.get('v')
#         q = state.get('q')
#         w = state.get('w')
#         x_traj = flat_output.get('x')
#         x_dot = flat_output.get('x_dot')
#         x_ddot = flat_output.get('x_ddot')
#         x_dddot = flat_output.get('x_dddot')
#         x_ddddot = flat_output.get('x_ddddot')
#         yaw = flat_output.get('yaw')
#         yaw_dot = flat_output.get('yaw_dot')
#
#         # calculate control input u1
#         r_ddot_des = x_ddot - np.matmul(k_d, (v - x_dot)) - np.matmul(k_p, (x_state - x_traj))
#         F_des = self.mass * r_ddot_des + np.array([0, 0, self.mass * self.g])
#         R_f = Rotation.from_quat(q)
#         R = R_f.as_matrix()
#
#         b3 = np.matmul(R, np.array([0, 0, 1]))
#         u1 = np.matmul(b3, F_des)
#
#         # calculate control input u2
#         b3_des = F_des / np.linalg.norm(F_des)
#         a_yaw = np.array([np.cos(yaw), np.sin(yaw), 0])
#         b2_des = np.cross(b3_des, a_yaw) / np.linalg.norm(np.cross(b3_des, a_yaw))
#         R_des = np.array([np.cross(b2_des, b3_des), b2_des, b3_des])
#         R_des_transp = np.transpose(R_des)
#         e_R = (np.matmul(np.transpose(R_des_transp), R) - np.matmul(np.transpose(R), R_des_transp)) / 2
#         e_R = np.array(e_R[[2, 0, 1], [1, 2, 0]])
#
#         # since w_des, the desired angular velocities canbe computed from the output of the trajectort generator  zT
#         # and its derivatives, but settiting them to zero will work for the purpose of this project
#         w_des = 0
#         e_w = w - w_des
#
#         ke_R = -np.matmul(k_R, e_R)
#         # print("ke_R")
#         # print(ke_R)
#         ke_w = -np.matmul(k_w, e_w)
#         # print("ke_w")
#         # print(ke_w)
#         error_mat = ke_R + ke_w  # this was typo as ke_R - ke_w, therefore have bug, now resolved
#         u2 = np.matmul(self.inertia, error_mat)
#         # print("u2")
#         # print(u2)
#         u1 = np.array([u1])
#         u = np.concatenate((u1, u2), axis=0)
#         mat = np.array([[1, 1, 1, 1], [0, self.L, 0, -self.L], [-self.L, 0, self.L, 0],
#                         [self.gamma, -self.gamma, self.gamma, -self.gamma]])
#         F_cal = np.matmul(np.linalg.inv(mat), u)
#         F_sign = np.sign(F_cal)
#         F_abs = np.abs(F_cal)
#         # print(F_cal)
#         # print(F_abs)
#         # print(F_abs * F_sign)
#         cmd_motor_speeds = np.sqrt(
#             F_abs / self.k_thrust)  # only want positive speed due to reality and practical manner
#         # print(cmd_motor_speeds)
#         cmd_thrust = u1
#         cmd_moment = u2
#         # cmd_q = R_des_transp
#
#         control_input = {'cmd_motor_speeds': cmd_motor_speeds,
#                          'cmd_thrust': cmd_thrust,
#                          'cmd_moment': cmd_moment,
#                          'cmd_q': cmd_q}
#         return control_input

import numpy as np
from scipy.spatial.transform import Rotation
import math as m

class SE3Control(object):
    """

    """

    def __init__(self, quad_params):
        """
        This is the constructor for the SE3Control object. You may instead
        initialize any parameters, control gain values, or private state here.

        For grading purposes the controller is always initialized with one input
        argument: the quadrotor's physical parameters. If you add any additional
        input arguments for testing purposes, you must provide good default
        values!

        Parameters:
            quad_params, dict with keys specified by crazyflie_params.py

        """

        # Quadrotor physical parameters.
        self.mass = quad_params['mass']  # kg
        self.Ixx = quad_params['Ixx']  # kg*m^2
        self.Iyy = quad_params['Iyy']  # kg*m^2
        self.Izz = quad_params['Izz']  # kg*m^2
        self.arm_length = quad_params['arm_length']  # meters
        self.rotor_speed_min = quad_params['rotor_speed_min']  # rad/s
        self.rotor_speed_max = quad_params['rotor_speed_max']  # rad/s
        self.k_thrust = quad_params['k_thrust']  # N/(rad/s)**2
        self.k_drag = quad_params['k_drag']  # Nm/(rad/s)**2

        # You may define any additional constants you like including control gains.
        self.inertia = np.diag(np.array([self.Ixx, self.Iyy, self.Izz]))  # kg*m^2
        self.g = 9.81  # m/s^2

        # STUDENT CODE HERE
        # Declaring constants:
        Arml = self.arm_length
        gamma = self.k_drag / self.k_thrust
        Gamma_inv = np.array([[1, 1, 1, 1],
                              [0, Arml, 0, -Arml],
                              [-Arml, 0, Arml, 0],
                              [gamma, -gamma, gamma, -gamma]])
        self.Gamma_inv = np.linalg.inv(Gamma_inv)

        # Delcaring the Control Gains
        # self.K_rot           = np.diag([2500,2500,18.75])
        # self.K_omega         = np.diag([300,300,7.55])
        # self.K_d             = np.diag([4.5,4.5,6])
        # self.K_p             = np.diag([8.5,8.5,8])

        # self.K_rot           = np.diag([500,500,18.75])
        # self.K_omega         = np.diag([2.5,2.5,7.55])
        # self.K_d             = np.diag([100,100,6])
        # self.K_p             = np.diag([2.5,2.5,8])
        # self.K_rot           = np.diag([5,5,0])
        # self.K_omega         = np.diag([1,1,0])
        # self.K_d             = np.diag([3,3,20])
        # self.K_p             = np.diag([2,2,5])

        # Tuning from ideal response for linear motion
        zeta = np.array([1, 1, 1])  # desired damping ratio
        percent = 0.01 * np.array([2, 2, 2])  # desired error margin
        ts = np.array([1.2, 1.2, 1.0])  # desired settling time
        wn = np.divide(np.divide(-np.log(percent), zeta), ts)  # natural frequency

        self.K_p = np.multiply(np.identity(3), np.multiply(wn, wn))
        self.K_d = np.multiply(np.identity(3), 2 * wn * zeta)

        print(self.K_p)
        print(self.K_d)
        # Tuning from ideal response for angular motion
        zeta = np.array([1, 1, 1])  # desired damping ratio
        percent = 0.01 * np.array([2, 2, 2])  # desired error margin
        ts = np.array([0.05, 0.05, 0.1])  # desired settling time
        wn = np.divide(np.divide(-np.log(percent), zeta), ts)  # natural frequency

        self.K_rot = np.multiply(np.identity(3), np.multiply(wn, wn))
        self.K_omega = np.multiply(np.identity(3), 2 * wn * zeta) * 2.5

    def update(self, t, state, flat_output):
        """
        This function receives the current time, true state, and desired flat
        outputs. It returns the command inputs.

        Inputs:
            t, present time in seconds
            state, a dict describing the present state with keys
                x, position, m
                v, linear velocity, m/s
                q, quaternion [i,j,k,w]
                w, angular velocity, rad/s
            flat_output, a dict describing the present desired flat outputs with keys
                x,        position, m
                x_dot,    velocity, m/s
                x_ddot,   acceleration, m/s**2
                x_dddot,  jerk, m/s**3
                x_ddddot, snap, m/s**4
                yaw,      yaw angle, rad
                yaw_dot,  yaw rate, rad/s

        Outputs:
            control_input, a dict describing the present computed control inputs with keys
                cmd_motor_speeds, rad/s
                cmd_thrust, N (for debugging and laboratory; not used by simulator)
                cmd_moment, N*m (for debugging; not used by simulator)
                cmd_q, quaternion [i,j,k,w] (for laboratory; not used by simulator)
        """
        cmd_motor_speeds = np.zeros((4,))
        cmd_thrust = 0
        cmd_moment = np.zeros((3,))
        cmd_q = np.zeros((4,))
        # print('Omega: ',state['q'])
        # print('Qaurt: ',state['q'])
        # print('Yaw: ', flat_output['yaw'])

        # STUDENT CODE HERE
        if (flat_output['x'] == np.array([0, 0, 4])).all():
            print('!')
            ## **** Computing U1 *****

        # Rotation
        q = state['q']
        R = Rotation.from_quat(q).as_matrix()  # <---------------------<< Identifying Rotation
        b3 = np.matmul(R, np.array([0, 0, 1]))

        # Desired Acceleration
        err_v = state['v'] - flat_output['x_dot']
        err_x = state['x'] - flat_output['x']
        r_des = flat_output['x_ddot'] - self.K_d * (state['v'] - flat_output['x_dot']) - self.K_p * (
                    state['x'] - flat_output['x'])
        r_des = np.diagonal(r_des)

        # Required Force
        F_des = self.mass * (r_des + np.array([0, 0, self.g]))
        # for i,_ in enumerate(F_des):
        #     if F_des[i] < 0:
        #         F_des[i] = 0
        # F_des = self.mass*(np.array([0,0,self.g]))

        # Required sum of Forces
        u1 = np.dot(b3, F_des)

        ## **** Computing U2 *****

        # Required Yaw Rotation
        o = np.array([0, 0, 0])
        if (F_des == o).all():
            b3_des = np.array([0, 0, 1])
        else:
            b3_des = F_des / np.linalg.norm(F_des)
        a_psi = np.array([m.cos(flat_output['yaw']), m.sin(flat_output['yaw']), 0])
        b2_des = np.cross(b3_des, a_psi)
        b2_des = b2_des / np.linalg.norm(b2_des)
        b1_des = np.cross(b2_des, b3_des)
        # b1_des = b1_des/np.linalg.norm(b1_des)
        R_des = np.array([b1_des, b2_des, b3_des]).T

        # Error in rotation
        err_rot = 0.5 * (np.matmul(R_des.T, R) - np.matmul(R.T, R_des))
        # print(err_rot)
        err_rot = np.array([-err_rot[1, 2], err_rot[0, 2], -err_rot[0, 1]])
        # err_rot = np.array([err_rot[1,0], -err_rot[2,0], err_rot[2,1]])
        # np.where(err_rot < 10e-6, 0, err_rot)

        # Error in angular velocity
        err_omega = -(flat_output['yaw_dot'] - state['w'])
        # np.where(err_omega < 10e-6, 0, err_omega)

        # Computing the Input forces
        # print('err Rot', err_rot)
        U2 = self.inertia * (-self.K_rot * err_rot - self.K_omega * err_omega)  # <---------------------<<<
        U2 = np.array([U2[0, 0], U2[1, 1], U2[2, 2]])

        # Total Input
        U = np.array([u1, U2[0], U2[1], U2[2]])
        forces = np.matmul(self.Gamma_inv, U)
        for i, _ in enumerate(forces):
            if forces[i] < 0:
                forces[i] = 0
        # print(state['x'],'X_state')

        cmd_thrust = u1
        cmd_moment = U2
        cmd_motor_speeds = forces / self.k_thrust
        cmd_motor_speeds = np.sqrt(cmd_motor_speeds)
        for i, _ in enumerate(cmd_motor_speeds):
            if cmd_motor_speeds[i] > self.rotor_speed_max:
                cmd_motor_speeds[i] = self.rotor_speed_max

        cmd_q = Rotation.from_matrix(R_des).as_quat()
        # print('!')

        control_input = {'cmd_motor_speeds': cmd_motor_speeds,
                         'cmd_thrust': cmd_thrust,
                         'cmd_moment': cmd_moment,
                         'cmd_q': cmd_q}
        return control_input


from django.urls import path
from . import views, viewsAbsences, viewsCours, viewsEtudiants, viewsGroupeEtudiants, viewsEnseignants

urlpatterns = [
    path('main/', views.main),

    path('absences/ajout', viewsAbsences.ajout),
    path('absences/affiche/', viewsAbsences.affiche),
    path('absences/modif/<int:id>/', viewsAbsences.modif),
    path('absences/supprimer/<int:id>/', viewsAbsences.supprimer),
    path('absences/updatesauvegarder/<int:id>/', viewsAbsences.updatesauvegarder),

    path('cours/ajout', viewsCours.ajout),
    path('cours/affiche/', viewsCours.affiche),
    path('cours/modif/<int:id>/', viewsCours.modif),
    path('cours/supprimer/<int:id>/', viewsCours.supprimer),
    path('cours/updatesauvegarder/<int:id>/', viewsCours.updatesauvegarder),

    path('enseignants/ajout', viewsEnseignants.ajout),
    path('enseignants/affiche/', viewsEnseignants.affiche),
    path('enseignants/modif/<int:id>/', viewsEnseignants.modif),
    path('enseignants/supprimer/<int:id>/', viewsEnseignants.supprimer),
    path('enseignants/updatesauvegarder/<int:id>/', viewsEnseignants.updatesauvegarder),

    path('etudiants/ajout', viewsEtudiants.ajout),
    path('etudiants/affiche/', viewsEtudiants.affiche),
    path('etudiants/modif/<int:id>/', viewsEtudiants.modif),
    path('etudiants/supprimer/<int:id>/', viewsEtudiants.supprimer),
    path('etudiants/updatesauvegarder/<int:id>/', viewsEtudiants.updatesauvegarder),

    path('grpetudiants/ajout', viewsGroupeEtudiants.ajout),
    path('grpetudiants/affiche/', viewsGroupeEtudiants.affiche),
    path('grpetudiants/modif/<int:id>/', viewsGroupeEtudiants.modif),
    path('grpetudiants/supprimer/<int:id>/', viewsGroupeEtudiants.supprimer),
    path('grpetudiants/updatesauvegarder/<int:id>/', viewsGroupeEtudiants.updatesauvegarder),

]


import unittest
import unittest.mock
import datetime
import io
import sys
from read_files import ReadFiles
from manage_content import ManageContent
from calculate_salary import CalculateSalaryEmploye
from validate import ValidateHour
from main import EmployeesAcme

class TestReadFiles(unittest.TestCase):

    mock_file_content = """RENE=MO10:00-12:00"""

    def test_read_files(self):
        with unittest.mock.patch(
                'builtins.open',
                new=unittest.mock.mock_open(read_data=self.mock_file_content),
                create=True
        ) as _:
            expected = '{"error": "The file should with at least five sets of data"}'
            self.assertEqual(ReadFiles('/').read_files(), expected)
            self.assertEqual(type(ReadFiles('/').read_files()), str)

class TestManageContent(unittest.TestCase):
    data = "PABLO=MO10:00-12:00"
    manage = ManageContent(data)

    def test_get_name(self):
        self.assertEqual(self.manage.get_name_employe(), 'PABLO')

    def test_get_hour_init_hour_end(self):
        self.assertEqual(self.manage.get_hour_init_hour_end(self.data[8::]), ["10:00","12:00"])

    def test_get_turns_employe(self):
        self.assertEqual(type(self.manage.get_turns_employe()), dict)
        self.assertEqual(self.manage.get_turns_employe()['name'], 'PABLO')

class TestCalculateSalary(unittest.TestCase):
    calculate_salary = CalculateSalaryEmploye()

    def test_format_hour(self):
        format_true = self.calculate_salary.format_hour("00:01")
        format_false = self.calculate_salary.format_hour("01")
        self.assertEqual(type(format_true), datetime.time)
        self.assertEqual(format_false, False)

    def test_calculate_time(self):
        start_hour, end_hour = "12", "10"
        c_time = self.calculate_salary.calculate_time(end_hour, start_hour)
        self.assertEqual(c_time, 2)

    def test_calculate_value_hours(self):
        day1 = 'MO'
        day2 = 'SA'
        start_hour = "10:00"
        end_hour = "12:00"
        c_v_h = self.calculate_salary.calculate_value_hours(day1, start_hour, end_hour)
        c_v_h1 = self.calculate_salary.calculate_value_hours(day2, start_hour, end_hour)
        self.assertEqual(c_v_h, 30)
        self.assertEqual(c_v_h1, 40)

    def test_calculate_salary(self):
        data = {'MO': ['10:00', '12:00'], 'TH': ['12:00', '14:00'], 'SU': ['20:00', '21:00']}
        salary = self.calculate_salary.calculate_salary(data)
        self.assertEqual(salary, 85)

class TestValidateHour(unittest.TestCase):

    validate = ValidateHour()

    def test_validate_number(self):
        v_n_true = self.validate.validate_number("12:00")
        v_n_false = self.validate.validate_number("aa:00")
        self.assertEqual(v_n_true, True)
        self.assertEqual(v_n_false, False)

    def test_validate_hour(self):
        v_h_true = self.validate.validate_hour("12:00")
        v_h_false = self.validate.validate_hour("24:00")
        self.assertEqual(v_h_true, True)
        self.assertEqual(v_h_false, False)

class TestEmployeesAcme(unittest.TestCase):

    main = EmployeesAcme()

    def test_get_data(self):
        data_mock = TestReadFiles.mock_file_content
        get_data = self.main.get_data(data_mock)
        self.assertEqual(type(get_data), list)

    def test_caculate_salary(self):
        expected_result = 'The amount to pay DIANA is: 80 USD\nThe amount to pay ASTRID is: 60 USD\n'
        capturedOutput = io.StringIO()
        sys.stdout = capturedOutput
        data = [{'name': 'DIANA', 'SA': ['14:00', '18:00']},
                {'name': 'ASTRID', 'MO': ['10:00', '12:00'],
                'TH': ['12:00', '14:00']}]
        self.main.salary_employes(data)
        sys.stdout = sys.__stdout__
        result = capturedOutput.getvalue()
        self.assertEqual(type(result), str)
        self.assertEqual(expected_result, result)

if __name__ == '__main__':
    unittest.main()


import sequential
import utils
import time
import threaded

def test_battery(op, size):
    print(f"*Op: {op} | Size: {size}*")

    # size of the matrices from method call
    matrix_a = utils.Matrix(size=size)
    matrix_b = utils.Matrix(size=size)

    #-- sequential
    i0 = time.time()
    eval(f"sequential.{op}(matrix_a, matrix_b)")  # op is the operation (add, sub, mult)
    i1 = time.time()

    seq = i1 - i0  # time in seconds it took to run the sequential version of the operation
    print(f"Sequential execution time: {seq} s")

    #-- threaded
    i0 = time.time()
    eval(f"threaded.{op}(matrix_a, matrix_b, threads=2)")
    i1 = time.time()

    thr = i1 - i0  # time in seconds it took to run the threaded version of the operation
    print(f"Threaded execution time: {thr} s")

    print(f"Using threads is {((seq/thr - 1) * 100):.2f}% faster.\n")

test_battery("add", 10)
test_battery("sub", 10)
test_battery("mult", 10)
test_battery("add", 100)
test_battery("sub", 100)
test_battery("mult", 100)
test_battery("add", 1000)
test_battery("sub", 1000)
test_battery("mult", 1000)

# test battery on a generic operation that takes 1 second per iteration
i0 = time.time()
sequential.generic_op(iterations=4)
i1 = time.time()
seq = i1 - i0
print(f"Sequential execution time: {seq} s\n")

# running the same 4 ops on 2 threads should cut the time in half
i0 = time.time()
threaded.generic_op(iterations=4)  # default number of threads is 2
i1 = time.time()
thr = i1 - i0
print(f"Threaded execution time: {thr} s")

print(f"Using threads is {((seq/thr - 1) * 100):.2f}% faster.\n")

# 4 iterations on 4 threads should be the same as running 1 iteration since they all run concurrently
i0 = time.time()
threaded.generic_op(iterations=4, threads=4)  # running with 4 threads this time
i1 = time.time()
thr = i1 - i0
print(f"Threaded execution time: {thr} s")

print(f"Using threads is {((seq/thr - 1) * 100):.2f}% faster.\n")


from io import open

from setuptools import find_packages, setup

with open('requirements.txt', 'r') as fh:
    to_be_installed = [l.rstrip('\n') for l in fh.readlines()]

with open('test-requirements.txt', 'r') as fh:
    test_dev_install = [l.rstrip('\n') for l in fh.readlines()]

version = "0.0.2"
setup(
    name="lrmaCUX",
    version=version,
    description="A python library for interacting with GCS, Cromwell, and Terra",
    url="https://github.com/broadinstitute/lrma-cloud-utils",
    author="Steve Huang",
    author_email="shuang@broadinstitute.org",
    license="BSD 3-Clause",
    long_description=open("README.md").read(),
    long_description_content_type="text/markdown",

    python_requires=">=3.7",
    install_requires=to_be_installed,
    tests_require=test_dev_install,

    packages=find_packages(
        where="src",
        include=['lrmaCU*', 'smrtlink_parser*']
    ),
    package_dir={"": "src"},
    include_package_data=True,

    classifiers=[
        "Development Status :: 1 - preAlpha",
        "Intended Audience :: Science/Research",
        "License :: OSI Approved :: BSD 3-Clause",
        "Natural Language :: English",
        "Operating System :: OS Independent",
        "Programming Language :: Python :: 3.7",
        "Programming Language :: Python :: 3.8",
        "Programming Language :: Python :: 3.9",
        "Programming Language :: Python :: 3.10",
    ],
)


"""create attended_game

Revision ID: 01c0fcdeaa19
Revises: 016103da0466
Create Date: 2021-05-21 16:02:01.591575

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '01c0fcdeaa19'
down_revision = '016103da0466'
branch_labels = None
depends_on = None

def upgrade():
    op.create_table(
    'attended_games',
    sa.Column('id', sa.Integer, primary_key=True),
    sa.Column('user_id', sa.Integer),
    sa.Column('fixture_id', sa.Integer))

def downgrade():
    op.drop_table('attended_games')


from absl import app
from absl import flags
from absl import logging

import os
from os.path import join
import shutil

import cv2
import numpy as np
from tqdm import tqdm

def imread(path, mode="bgr"):
    """
    Helper function to read images
    Mode could be one of 'bgr' or 'rgb'
    """
    img = cv2.imread(path)
    if mode == "rgb":
        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)

    return img

class DirectoryTree:
    """
    A DirectoryTree class using composite pattern.
    A directory tree has a path for itself, a parent
    directory, and child directories. The parent and
    child directories are instances of the DirectoryTree class.
    """

    def __init__(self, path=None, parent=None, depth=0):
        self.parent = parent
        self.path = path
        self.directories = {}
        self.depth = depth
        if depth == 0:
            self.name = self.path
        else:
            self.name = self.path.split('/')[-1]

        if os.path.isfile(self.path):
            raise OSError('Please specify a directory not a file!')

        if not os.path.exists(self.path):
            os.makedirs(self.path)
        else:
            # Iterate through all directories in self.path, and add to self.directories
            for dir in os.listdir(self.path):
                if os.path.isdir(join(self.path, dir)):
                    self.add(dir)

    def add(self, *names):
        """ Add folders to current tree."""
        if not self.exists():
            raise OSError('This directory tree is no longer valid.')
        for name in names:
            if hasattr(self, name):
                raise OSError(
                    'path <%s> already exists in this file structure' % join(self.path, name))

            setattr(self, name, DirectoryTree(
                path=join(self.path, name), parent=self, depth=self.depth + 1))
            self.directories[name] = getattr(self, name)

    def print_all(self):
        """
        A debug function. Prints all directories
        in the tree in an understandable format.
        """
        if not self.exists():
            raise OSError('This directory tree is no longer valid.')
        cur_path = self.path.split('/')[-1]
        s = ''
        if self.depth != 0:
            s = '|'
            for i in range(self.depth):
                s += '___'

        logging.info("%s%s" % (s, cur_path))
        for name, d in self.directories.items():
            d.logging.info_all()

    def remove(self, force=False):
        """
        Delete a directory tree.
        If the directory is not empty
        use the force option for force
        removal.
        """
        if not self.exists():
            raise OSError('This directory tree is no longer valid.')
        if force:
            shutil.rmtree(self.path)
        else:
            os.rmdir(self.path)

        if self.parent is not None:
            delattr(self.parent, self.name)
            del self.parent.directories[self.name]

    def exists(self):
        return os.path.isdir(self.path)

    def get_file_paths(self, exts=[""]):
        """
        Find all files with given extensions.
        This is particularly useful to query
        sepcific files scattered in a directory
        tree of unknown structure.
        """
        res = []
        # Add files in current directory
        for f in os.listdir(self.path):
            file_path = os.path.join(self.path, f)
            if not os.path.isfile(file_path):
                continue
            for ext in exts:
                size_ext = len(ext)
                if size_ext > len(file_path):
                    continue
                if file_path[-size_ext:] == ext:
                    res.append(file_path)

        # Add files in sub directories
        for dt in self.directories.values():
            res.extend(dt.get_file_paths(exts=exts))
        return res

    def get_image_file_paths(self):
        """
        Helper function to all images in
        this directory tree.
        """
        exts = ['.JPG', '.jpg', '.png', '.PNG']
        return self.get_file_paths(exts=exts)

class FeatureExtractor:
    SIFT = 1
    AKAZE = 2
    ORB = 3

class FeatureMatcher:
    BRUTE_FORCE = 1
    BRUTE_FORCE_HAMMING = 2

class Encoder:
    """
    Base class to encode anchor or candidate images.
    """
    feature_extractors = {
        FeatureExtractor.SIFT: cv2.SIFT_create(),
        FeatureExtractor.AKAZE: cv2.AKAZE_create(),
        FeatureExtractor.ORB: cv2.ORB_create()
    }

    def __init__(self, category, path, feature_extractor=FeatureExtractor.ORB):
        self.category = category
        self.path = path
        self.id = hash(self.path)
        self.kp = None
        self.des = None
        self.img = None

        assert feature_extractor in [
            FeatureExtractor.SIFT, FeatureExtractor.AKAZE, FeatureExtractor.ORB], "Feature extractor should be either be sift, orb, or akaze."
        self.feature_extractor = feature_extractor

    def read(self, mode="bgr"):
        """
        Read img in self.path to self.img
        """
        if self.img is None:
            img = imread(self.path, mode=mode)
            np_img = np.array(img)
            self.img = np_img

    def get_image(self):
        """
        Returns self.img if not None
        """
        if not self.img:
            self.read()
        return self.img

    def extract_features(self):
        self.read()
        if (not self.kp) or (not self.des):
            kp, des = Encoder.feature_extractors[self.feature_extractor].detectAndCompute(
                self.img, None)
            self.kp = kp
            self.des = des

    def get_keypoints(self):
        if not self.kp:
            self.extract_features()
        return self.kp

    def get_descriptors(self):
        if not self.des:
            self.extract_features()
        return self.des

class Candidate(Encoder):
    """
    Class to encode candidate images.
    """

    def __init__(self, category, path, feature_extractor=FeatureExtractor.ORB):
        super().__init__(category, path, feature_extractor)

class Anchor(Encoder):
    """
    Class to encode anchor images.
    """

    def __init__(self, category, path, feature_extractor=FeatureExtractor.ORB):
        super().__init__(category, path, feature_extractor)

class Match:
    """
    Class that represents a match between anchors and candidates.
    """

    feature_matchers = {
        FeatureMatcher.BRUTE_FORCE: cv2.BFMatcher(),
        FeatureMatcher.BRUTE_FORCE_HAMMING: cv2.BFMatcher(cv2.NORM_HAMMING, crossCheck=False),
    }

    def __init__(self, anchor, candidate, feature_matcher=FeatureMatcher.BRUTE_FORCE_HAMMING):
        self.anchor = anchor
        self.candidate = candidate
        self.matches = None
        self.good_matches = None
        self.feature_matcher = feature_matcher

    def match_keypoints(self):
        """Match keypoints using the nearest neightbor approach."""
        self.matches = Match.feature_matchers[self.feature_matcher].knnMatch(
            self.anchor.des, self.candidate.des, k=2)

    def ratio_test(self, r=0.5):
        """
        Find good matches with a ratio test.
        If the ratio of two neighboring keypoints
        (in the feature space) is below a specified
        threshold then this is considered to be a
        good match.

        Args:
            r (float, optional): ratio between 0 and 1. Defaults to 0.5.
        """
        assert self.matches, "keypoints should be matched before doing ratio test"
        # Ratio test
        assert r > 0 and r < 1, "ratio should be between 0 and 1 but is %.2f" % r
        good = []
        for m, n in self.matches:
            if m.distance < r * n.distance:
                good.append([m])
        self.good_matches = good

    def drawMatchesKnn(self):
        """A debugging method to visualize matches.

        Returns:
            array: image with matched keypoints from reference and current
        """
        matches = self.good_matches if self.good_matches is not None else self.matches
        drawing = cv2.drawMatchesKnn(
            self.anchor.img,
            self.anchor.kp,
            self.candidate.img,
            self.candidate.kp,
            matches,
            None, flags=2)
        return drawing

def encode(
        root_path: str,
        read_images=False,
        compute_features=False,
        match_keypoints=False,
        feature_extractor=FeatureExtractor.ORB):
    """Helper function to encode images. This
    function takes the data folder structure
    into consideration. The data should be organized
    in the following manner:
        root
        ---- [category name]
        --------------- actual (holds anchor images)
        --------------- proof (holds matching images to the anchor)

    Args:
        root_path (str): root data path
        read_images (bool, optional): Defaults to False.
        compute_features (bool, optional): Defaults to False.
        match_keypoints (bool, optional): Defaults to False.
        feature_extractor (str, optional): . Defaults to "sift".

    All optional arguments support the felxibility of the processing pipeline.
    These options are used to optimize calls to the encode function.
    Ine might make several calls to encode in different stages of processing
    the images.

    Returns:
        tuple: A tuple of anchors, candidates, matches in that order.
    """
    # Select feature matcher based on extractor
    feature_matcher = FeatureMatcher.BRUTE_FORCE
    if feature_extractor in [FeatureExtractor.ORB, FeatureExtractor.AKAZE]:
        feature_matcher = FeatureMatcher.BRUTE_FORCE_HAMMING

    anchors, candidates = [], []
    root_dt = DirectoryTree(root_path)
    logging.info("Encoding anchors and candidates..")
    for dt in tqdm(root_dt.directories.values()):
        category = dt.name

        # Find & encode anchor
        tmp = dt.actual.get_image_file_paths()
        assert len(tmp) == 1, "There should be only single anchor image but found %i in %s" % (
            len(tmp), dt.actual.path)
        anchor_path = tmp[0]
        anchor = Anchor(category, anchor_path, feature_extractor)
        if read_images:
            anchor.read()
        if compute_features:
            anchor.extract_features()
        anchors.append(anchor)

        # Find & encode candidates
        tmp = dt.proof.get_image_file_paths()
        assert len(tmp) > 0, "There should more than zero candidates"
        candidate_paths = tmp
        for candidate_path in candidate_paths:
            candidate = Candidate(category, candidate_path, feature_extractor)
            if read_images:
                candidate.read()
            if compute_features:
                candidate.extract_features()
            candidates.append(candidate)

    # Create a list of possible matches
    # This is a anchor x candidate pairing operation
    # with O(anchors * candidates)
    matches = []
    for anchor in anchors:
        for candidate in candidates:
            matches.append(
                Match(anchor, candidate, feature_matcher=feature_matcher))

    logging.info("Matching anchors and candidates..")
    # Match keypoints in every match
    if match_keypoints:
        assert compute_features, "To do matching use compute_features option."
        for match in tqdm(matches):
            match.match_keypoints()

    return anchors, candidates, matches


# -*- coding: utf-8 -*-
"""
Created on Fri Mar 19 12:44:17 2021

@author: Admin
"""
import torch
import torch.nn.functional as F
from torch import nn

#from ..utils import initialize_weights

class _EncoderBlock(nn.Module):
    def __init__(self, in_channels, out_channels, dropout=False):
        super(_EncoderBlock, self).__init__()
        layers = [
            nn.Conv2d(in_channels, out_channels, kernel_size=3),
            nn.BatchNorm2d(out_channels),
            nn.ReLU(inplace=True),
            nn.Conv2d(out_channels, out_channels, kernel_size=3),
            nn.BatchNorm2d(out_channels),
            nn.ReLU(inplace=True),
        ]
        if dropout:
            layers.append(nn.Dropout())
        layers.append(nn.MaxPool2d(kernel_size=2, stride=2))
        self.encode = nn.Sequential(*layers)

    def forward(self, x):
        return self.encode(x)

class _DecoderBlock(nn.Module):
    def __init__(self, in_channels, middle_channels, out_channels):
        super(_DecoderBlock, self).__init__()
        self.decode = nn.Sequential(
            nn.Conv2d(in_channels, middle_channels, kernel_size=3),
            nn.BatchNorm2d(middle_channels),
            nn.ReLU(inplace=True),
            nn.Conv2d(middle_channels, middle_channels, kernel_size=3),
            nn.BatchNorm2d(middle_channels),
            nn.ReLU(inplace=True),
            nn.ConvTranspose2d(middle_channels, out_channels, kernel_size=2, stride=2),
        )

    def forward(self, x):
        return self.decode(x)

class UNet(nn.Module):
    def __init__(self, num_classes, transform_fn=None):
        super(UNet, self).__init__()
        self.enc1 = _EncoderBlock(3, 64)
        self.enc2 = _EncoderBlock(64, 128)
        self.enc3 = _EncoderBlock(128, 256)
        self.enc4 = _EncoderBlock(256, 512, dropout=True)
        self.center = _DecoderBlock(512, 1024, 512)
        self.dec4 = _DecoderBlock(1024, 512, 256)
        self.dec3 = _DecoderBlock(512, 256, 128)
        self.dec2 = _DecoderBlock(256, 128, 64)
        self.dec1 = nn.Sequential(
            nn.Conv2d(128, 64, kernel_size=3),
            nn.BatchNorm2d(64),
            nn.ReLU(inplace=True),
            nn.Conv2d(64, 64, kernel_size=3),
            nn.BatchNorm2d(64),
            nn.ReLU(inplace=True),
        )
        self.b_final = nn.Conv2d(64, 32, kernel_size=1)
        self.final   = nn.Conv2d(32, num_classes, kernel_size=1)
        self.softmax = nn.Softmax(dim=1)

        self.relu1 = nn.ReLU(inplace=True)
        self.relu2 = nn.ReLU(inplace=True)

        #initialize_weights(self)
        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                nn.init.kaiming_normal_(m.weight, mode="fan_out", nonlinearity="relu")
            elif isinstance(m, nn.BatchNorm2d):
                nn.init.constant_(m.weight, 1)
                nn.init.constant_(m.bias, 0)
            elif isinstance(m, nn.Linear):
                nn.init.xavier_normal_(m.weight)
                nn.init.constant_(m.bias, 0)

        self.transform_fn = transform_fn

    def forward(self, x, trans_flag=False):
        if trans_flag:
            x = self.transform_fn(x)

        enc1 = self.enc1(x)
        enc2 = self.enc2(enc1)
        enc3 = self.enc3(enc2)
        enc4 = self.enc4(enc3)

        center = self.center(enc4)

        dec4 = self.dec4(torch.cat([center, F.interpolate(enc4, center.size()[2:], mode='bilinear')], 1))
        dec3 = self.dec3(torch.cat([dec4, F.interpolate(enc3, dec4.size()[2:], mode='bilinear')], 1))
        dec2 = self.dec2(torch.cat([dec3, F.interpolate(enc2, dec3.size()[2:], mode='bilinear')], 1))
        dec1 = self.dec1(torch.cat([dec2, F.interpolate(enc1, dec2.size()[2:], mode='bilinear')], 1))

        b_final = self.relu1(self.b_final(dec1))
        final   = self.relu2(self.final(b_final))

        out = F.interpolate(final, x.size()[2:], mode='bilinear')
        #out = self.softmax(out)
        return out

    def update_batch_stats(self, flag):
        for m in self.modules():
            if isinstance(m, nn.BatchNorm2d):
                m.update_batch_stats = flag


from config import config
from titanembeds.database import DatabaseInterface, Guilds, Messages
import asyncio
import sys
import logging
import json
import gc
import random
from asyncio_extras import threadpool
logging.basicConfig(filename='titanbot.log',level=logging.INFO,format='%(asctime)s %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p')
logging.getLogger('TitanBot')
logging.getLogger('sqlalchemy')

###########################
#   Cleanup DB Messages   #
#                         #
# Cleans the database     #
# messages store          #
###########################

class TitanCleanupDB:
    def __init__(self):
        super().__init__()
        self.loop = asyncio.get_event_loop()
        self.database = DatabaseInterface(self)
        self.logger = logging.getLogger("titan_cleanupdb")
        self.logger.setLevel(logging.DEBUG)
        fh = logging.FileHandler("titan_cleanupdb.log")
        fh.setLevel(logging.DEBUG)
        session_id = str(random.randrange(100))
        formatter = logging.Formatter("%(asctime)s - {0} - %(levelname)s - %(message)s".format(session_id))
        fh.setFormatter(formatter)
        self.logger.addHandler(fh)
        consoleHandler = logging.StreamHandler()
        consoleHandler.setFormatter(formatter)
        self.logger.addHandler(consoleHandler)
        self.logger.info("Initialized Database Cleaning Class with session id of " + session_id)

    def _cleanup(self):
        try:
            self.loop.run_until_complete(self.logout())
        except: # Can be ignored
            pass
        pending = asyncio.Task.all_tasks()
        gathered = asyncio.gather(*pending)
        try:
            gathered.cancel()
            self.loop.run_until_complete(gathered)
            gathered.exception()
        except: # Can be ignored
            pass

    def run(self):
        try:
            self.loop.run_until_complete(self.start_cleanup())
        except Exception as e:
            print("Error!", e)
        finally:
            try:
                self._cleanup()
            except Exception as e:
                print("Error in cleanup:", e)
            self.loop.close()

    async def start_cleanup(self):
        print('Titan [DiscordBot] [UTILITY: Cleanup database messages]')
        print('------')

        try:
            self.database.connect(config["database-uri"])
        except Exception:
            self.logger.error("Unable to connect to specified database!")
            traceback.print_exc()
            return

        print("working on this...")
        with self.database.get_session() as session:
            guilds = session.query(Guilds).all()
            guilds_new = []
            count = 0
            for guild in guilds:
                guilds_new.append(guild)
            for guild in guilds_new:
                count += 1
                self.logger.info("[{}] snowflake-{} name-{}".format(count, guild.guild_id, guild.name))
                try:
                    channelsjson = json.loads(guild.channels)
                except:
                    continue
                active_channels = []
                for channel in channelsjson:
                    chanid = channel["id"]
                    active_channels.append(chanid)
                    keep_these = session.query(Messages.message_id).filter(Messages.channel_id == chanid).order_by(Messages.timestamp.desc()).limit(50)
                    d = session.query(Messages).filter(Messages.channel_id == chanid, ~Messages.message_id.in_(keep_these)).delete(synchronize_session=False)
                    session.commit()
                    self.logger.info("    --{} [{}]".format(channel["name"], d))
                d = session.query(Messages).filter(Messages.guild_id == guild.guild_id, ~Messages.channel_id.in_(active_channels)).delete(synchronize_session=False)
                session.commit()
                self.logger.info("    INACTIVE {}".format(d))
            self.logger.info("done!")

def main():
    print("Starting...")
    te = TitanCleanupDB()
    te.run()
    gc.collect()

if __name__ == '__main__':
    main()


import copy
import hashlib
import json
import logging
import operator
from datetime import datetime
from functools import reduce
from typing import Any, Dict, List, Optional

from django.conf import settings
from django.core.exceptions import EmptyResultSet
from django.db import models
from django.db.models import Count, Q
from django.db.models.query import QuerySet
from django.utils import timezone
from django.utils.translation import ugettext as _
from humanize import naturaldelta

from bkmonitor.models.bcs_label import BCSLabel
from bkmonitor.utils.common_utils import chunks
from bkmonitor.utils.kubernetes import BcsClusterType, BkmMetricbeatEndpointUpStatus
from core.drf_resource import api

logger = logging.getLogger("kubernetes")

IGNORE_LABEL_KEYS = {
    "pod-template-hash",
    "controller-revision-hash",
    "statefulset.kubernetes.io/pod-name",
    "controller-uid",
    "job-name",
}

class BCSBaseManager(models.Manager):
    def count_service_status_quantity(self, query_set_list):
        """统计每种运行状态的数量 ."""
        if not query_set_list:
            model_items = self.all().values("status").annotate(count=Count("status"))
        else:
            model_items = self.filter(*query_set_list).values("status").annotate(count=Count("status"))
        status_summary = {model["status"]: model["count"] for model in model_items}
        return status_summary

    def count_monitor_status_quantity(self, query_set_list: List) -> Dict:
        """统计每种指标数据状态的数量 ."""
        if not query_set_list:
            model_items = self.all().values("monitor_status").annotate(count=Count("monitor_status"))
        else:
            model_items = self.filter(*query_set_list).values("monitor_status").annotate(count=Count("monitor_status"))
        status_summary = {model["monitor_status"]: model["count"] for model in model_items}
        return status_summary

    def get_cluster_ids(self, bk_biz_id):
        if not isinstance(bk_biz_id, (list, tuple)):
            bk_biz_id = [bk_biz_id]
        cluster_ids = self.filter(bk_biz_id__in=bk_biz_id).values_list("bcs_cluster_id", flat=True)
        return list(cluster_ids)

    def get_bk_biz_ids(self):
        items = self.all().values("bk_biz_id").distinct()
        bk_biz_ids = [item["bk_biz_id"] for item in items]
        return bk_biz_ids

    def filter_by_biz_id(
        self,
        bk_biz_id: int,
    ) -> QuerySet:
        """获得资源查询QuerySet ."""
        query_set = Q(bk_biz_id=bk_biz_id)
        if bk_biz_id < 0:
            clusters = api.kubernetes.get_cluster_info_from_bcs_space({"bk_biz_id": bk_biz_id})
            if not clusters:
                # 业务下没有集群
                raise EmptyResultSet

            shared_q_list = []
            for cluster_id, value in clusters.items():
                cluster_type = value.get("cluster_type")
                namespace_list = value["namespace_list"]
                if cluster_type == BcsClusterType.SHARED and namespace_list and self.model.has_namespace_field():
                    shared_q_list.append(Q(bcs_cluster_id=cluster_id, namespace__in=namespace_list))
                else:
                    shared_q_list.append(Q(bcs_cluster_id=cluster_id))
            if shared_q_list:
                query_set = reduce(operator.or_, shared_q_list)
            else:
                raise EmptyResultSet

        return self.filter(query_set)

class BCSBase(models.Model):
    class Meta:
        abstract = True

    # 运行状态
    STATE_SUCCESS = "success"  # 正常（有数据）
    STATE_FAILURE = "failed"  # 异常
    STATE_DISABLED = "disabled"  # 正常（无数据）

    # 指标数据状态
    METRICS_STATE_STATE_SUCCESS = "success"  # 正常（有数据）
    METRICS_STATE_FAILURE = "failed"  # 异常
    METRICS_STATE_DISABLED = "disabled"  # 正常（无数据）

    api_labels = {}

    labels = models.ManyToManyField(BCSLabel)
    bk_biz_id = models.IntegerField(verbose_name="业务ID", default=0, db_index=True)
    bcs_cluster_id = models.CharField(max_length=32, db_index=True)

    created_at = models.DateTimeField()
    deleted_at = models.DateTimeField(null=True)
    status = models.CharField(max_length=32, default="", db_index=True)
    monitor_status = models.CharField(max_length=32, default="")
    last_synced_at = models.DateTimeField(verbose_name="同步时间")

    unique_hash = models.CharField(max_length=32, null=True, unique=True)

    @classmethod
    def get_resource_usage_columns(cls):
        return [
            {
                "id": "resource_usage_cpu",
                "name": _("CPU使用量"),
                "type": "string",
                "disabled": False,
                "checked": False,
                "sortable": True,
                "sortable_type": "progress",
            },
            {
                "id": "resource_usage_memory",
                "name": _("内存使用量"),
                "type": "string",
                "disabled": False,
                "checked": False,
                "sortable": True,
                "sortable_type": "progress",
            },
            {
                "id": "resource_usage_disk",
                "name": _("磁盘使用量"),
                "type": "string",
                "disabled": False,
                "checked": False,
                "sortable": True,
                "sortable_type": "progress",
            },
        ]

    @classmethod
    def get_resource_usage_ratio_columns(cls):
        return [
            {
                "id": "request_cpu_usage_ratio",
                "name": _("CPU使用率(request)"),
                "type": "progress",
                "disabled": False,
                "checked": False,
                "sortable": True,
                "sortable_type": "progress",
            },
            {
                "id": "limit_cpu_usage_ratio",
                "name": _("CPU使用率(limit)"),
                "type": "progress",
                "disabled": False,
                "checked": True,
                "sortable": True,
                "sortable_type": "progress",
            },
            {
                "id": "request_memory_usage_ratio",
                "name": _("内存使用率(request)"),
                "type": "progress",
                "disabled": False,
                "checked": False,
                "sortable": True,
                "sortable_type": "progress",
            },
            {
                "id": "limit_memory_usage_ratio",
                "name": _("内存使用率(limit) "),
                "type": "progress",
                "disabled": False,
                "checked": True,
                "sortable": True,
                "sortable_type": "progress",
            },
        ]

    @classmethod
    def get_container_resources_columns(cls):
        return [
            {
                "id": "resource_requests_cpu",
                "name": _("cpu request"),
                "type": "string",
                "disabled": False,
                "checked": False,
                "sortable": True,
                "sortable_type": "progress",
            },
            {
                "id": "resource_limits_cpu",
                "name": _("cpu limit"),
                "type": "string",
                "disabled": False,
                "checked": False,
                "sortable": True,
                "sortable_type": "progress",
            },
            {
                "id": "resource_requests_memory",
                "name": _("memory request"),
                "type": "string",
                "disabled": False,
                "checked": False,
                "sortable": True,
                "sortable_type": "progress",
            },
            {
                "id": "resource_limits_memory",
                "name": _("memory limit"),
                "type": "string",
                "disabled": False,
                "checked": False,
                "sortable": True,
                "sortable_type": "progress",
            },
        ]

    @classmethod
    def hash_resource_labels(cls, items: List):
        """获得资源的标签hash_id ."""
        has_ids = []
        for item in items:
            if not item.id:
                continue
            for k, v in item.api_labels.items():
                if k in IGNORE_LABEL_KEYS:
                    continue
                hash_id = cls.md5str(k + ":" + v)
                has_ids.append(hash_id)
        return has_ids

    @classmethod
    def bulk_save_labels(cls, items: List) -> None:
        """批量更新标签 ."""
        bulk_create_label_list = []
        bcs_cluster_id_list = []
        resource_id_map_hash_ids = {}

        # 获得资源的标签hash_id
        has_id_set = set(cls.hash_resource_labels(items))

        # 获取可以新增的标签
        hash_id_existed = []
        for hash_ids in chunks(list(has_id_set), 500):
            hash_id_existed.extend(
                list(BCSLabel.objects.filter(hash_id__in=hash_ids).values_list("hash_id", flat=True))
            )
        hash_id_created = set(has_id_set) - set(hash_id_existed)

        # 批量添加标签
        for item in items:
            if not item.id:
                continue
            for k, v in item.api_labels.items():
                if k in IGNORE_LABEL_KEYS:
                    continue
                bcs_cluster_id_list.append(item.bcs_cluster_id)
                # 获得标签hash值
                hash_id = cls.md5str(k + ":" + v)
                resource_id_map_hash_ids.setdefault((item.id, item.bcs_cluster_id), []).append(hash_id)
                # 需要新增的标签
                if hash_id in hash_id_created:
                    bulk_create_label_list.append(BCSLabel(hash_id=hash_id, key=k, value=v))
        if bulk_create_label_list:
            try:
                BCSLabel.objects.bulk_create(bulk_create_label_list, 200)
            except Exception as exc_info:
                logger.exception(exc_info)

        # 获得中间表远程唯一性数据
        new_resource_label_hash_set = {
            (key[0], key[1], label_hash_id)
            for key, label_hash_ids in resource_id_map_hash_ids.items()
            for label_hash_id in label_hash_ids
        }

        # 获得中间表历史唯一性数据
        old_resource_label_hash_set = {
            (item.resource_id, item.bcs_cluster_id, item.label_id)
            for item in cls.labels.through.objects.filter(bcs_cluster_id__in=bcs_cluster_id_list)
        }

        # 中间表添加数据
        bulk_create_label_relation_list = []
        resource_label_hash_set_inserted = new_resource_label_hash_set - old_resource_label_hash_set
        if resource_label_hash_set_inserted:
            for resource_id, bcs_cluster_id, label_hash_id in resource_label_hash_set_inserted:
                bulk_create_label_relation_list.append(
                    cls.labels.through(
                        bcs_cluster_id=bcs_cluster_id,
                        label_id=label_hash_id,
                        resource_id=resource_id,
                    )
                )
            try:
                cls.labels.through.objects.bulk_create(bulk_create_label_relation_list, 200)
            except Exception as exc_info:
                logger.exception(exc_info)

        # 中间表删除数据
        label_hash_set_inserted_deleted = old_resource_label_hash_set - new_resource_label_hash_set
        if label_hash_set_inserted_deleted:
            for resource_id, bcs_cluster_id, label_hash_id in label_hash_set_inserted_deleted:
                cls.labels.through.objects.filter(
                    bcs_cluster_id=bcs_cluster_id, label_id=label_hash_id, resource_id=resource_id
                ).delete()

    @staticmethod
    def md5str(source: str):
        m = hashlib.md5()
        m.update(source.encode("utf8"))
        return m.hexdigest()

    def get_unique_hash(self):
        raise NotImplementedError

    @staticmethod
    def get_columns(columns_type="list"):
        raise NotImplementedError

    @classmethod
    def count(cls, params):
        if settings.BCS_API_DATA_SOURCE == "db":
            return cls.objects.filter(**params).count()
        return len(cls.load_item_from_api(params))

    @classmethod
    def load_item(cls, params=None):
        params = params or {}
        params = copy.deepcopy(params)
        bk_biz_id = params["bk_biz_id"]
        if bk_biz_id < 0:
            # 如果是研发项目，查询关联的所有集群，包括共享集群
            params.pop("bk_biz_id")
            clusters = api.kubernetes.get_cluster_info_from_bcs_space({"bk_biz_id": bk_biz_id})
            bcs_cluster_id_list = list(clusters.keys())
            params["bcs_cluster_id__in"] = bcs_cluster_id_list
        if settings.BCS_API_DATA_SOURCE == "db":
            return cls.load_item_from_db(params)
        else:
            return cls.load_item_from_api(params)

    @classmethod
    def load_item_from_db(cls, params):
        params = params if params else {}
        try:
            return cls.objects.get(**params)
        except cls.DoesNotExist:
            return

    @classmethod
    def load_item_from_api(cls, params):
        params = params if params else {}
        data = cls.load_list_from_api(params)

        def matched(item, match_params):
            for k, v in match_params.items():
                if v != getattr(item, k):
                    return False
            return True

        for item in data:
            if matched(item, params):
                return item

    @classmethod
    def condition_list_to_conditions(cls, condition_list):
        conditions = {}
        for condition in condition_list:
            for key, value in condition.items():
                if not conditions.get(key):
                    conditions[key] = []
                if not isinstance(value, list):
                    value = [value]
                conditions[key] += value
        return conditions

    @classmethod
    def label_query_set_parser(cls, conditions):
        label_count = 0
        q_list = []
        for key in conditions.keys():
            label_values = conditions.get(key, [])
            if not label_values:
                continue
            q = Q(labels__key=key) & Q(labels__value__in=label_values)
            q_list.append(q)
            label_count += 1

        queryset = reduce(lambda x, y: x | y, q_list)
        data = cls.objects.filter(queryset).annotate(Count("id")).values("id", "id__count")
        id_list = []
        for item in data:
            if item["id__count"] == label_count:
                id_list.append(item["id"])
        if id_list:
            return Q(id__in=id_list)

    @classmethod
    def get_query_set_list_by_params(cls, params):
        q_list = []
        label_conditions = {k.replace("__label_", ""): v for k, v in params.items() if k.find("__label_") == 0}
        else_conditions = {k: v for k, v in params.items() if k.find("__label_") != 0}
        if label_conditions:
            q = cls.label_query_set_parser(label_conditions)
            if not q:
                return []
            else:
                q_list.append(q)
        query_params = {k: v for k, v in params.items()}
        for k, v in else_conditions.items():
            query_params[k] = v

        # 添加bk_biz_id
        bk_biz_id = query_params.get("bk_biz_id")
        if bk_biz_id and hasattr(cls, "bk_biz_id"):
            if isinstance(bk_biz_id, list):
                q_list.append(Q(**{"bk_biz_id__in": bk_biz_id}))
            else:
                q_list.append(Q(**{"bk_biz_id": bk_biz_id}))

        # 添加cluster_id
        bcs_cluster_id = query_params.get("bcs_cluster_id")
        if bcs_cluster_id and hasattr(cls, "bcs_cluster_id"):
            if isinstance(bcs_cluster_id, list):
                q_list.append(Q(**{"bcs_cluster_id__in": bcs_cluster_id}))
            else:
                q_list.append(Q(**{"bcs_cluster_id": bcs_cluster_id}))

        for k, v in query_params.items():
            if hasattr(cls, k) and k not in {"keyword", "bk_biz_id", "bcs_cluster_id", "status"}:
                if isinstance(v, list):
                    q_list.append(Q(**{f"{k}__in": v}))
                else:
                    q_list.append(Q(**{f"{k}": v}))

        # 添加name过滤
        name_value = query_params.get("keyword")
        if name_value and hasattr(cls, "name"):
            for value in name_value:
                q_list.append(Q(name__contains=value))

        return q_list

    @classmethod
    def has_namespace_field(cls):
        return "namespace" in [f.name for f in cls._meta.get_fields()]

    @classmethod
    def has_space_uid_field(cls):
        return "space_uid" in [f.name for f in cls._meta.get_fields()]

    @staticmethod
    def load_list_from_api(params):
        raise NotImplementedError

    def render(self, bk_biz_id, render_type="list"):
        data = {}
        if render_type == "detail":
            data = []
        for column in self.get_columns(render_type):
            render = getattr(self, "render_" + column["id"], "")

            if render and callable(render):
                value = render(bk_biz_id, render_type)
            else:
                value = getattr(self, column["id"], "")
            if render_type == "detail":
                data.append(
                    {
                        "key": column["id"],
                        "name": column["name"],
                        "type": column["type"],
                        "value": value,
                    }
                )
            else:
                data[column["id"]] = value
        return data

    @staticmethod
    def build_link(bk_biz_id, text, dashboard, filter_query):
        filter_query_items = []
        for key, value in filter_query.items():
            if isinstance(value, list):
                for value_item in value:
                    filter_query_items.append(f"filter-{key}={value_item}")
            else:
                filter_query_items.append(f"filter-{key}={value}")
        filter_query_string = "&".join(filter_query_items)
        return {
            "value": text,
            "target": "blank",
            "url": f"?bizId={bk_biz_id}#/k8s?dashboardId={dashboard}&sceneType=detail&{filter_query_string}",
        }

    @staticmethod
    def build_search_link(
        bk_biz_id: int, dashboard_id: str, value: Any, search: Optional[List] = None, scene_type="detail"
    ):
        url = f"?bizId={bk_biz_id}#/k8s?sceneId=kubernetes&dashboardId={dashboard_id}&sceneType={scene_type}"
        if search:
            query_data = json.dumps({"selectorSearch": search})
            url = f"{url}&queryData={query_data}"
        value = {
            "value": value,
            "target": "blank",
            "url": url,
        }
        return value

    def get_label_list(self):
        if settings.BCS_API_DATA_SOURCE == "db":
            return [{"key": label.key, "value": label.value} for label in self.labels.all()]
        if not self.api_labels:
            self.api_labels = {}
        label_list = []
        for k, v in self.api_labels.items():
            label_list.append(
                {
                    "key": k,
                    "value": v,
                }
            )
        return label_list

    def render_age(self, bk_biz_id, render_type="list"):
        if isinstance(self.created_at, timezone.datetime):
            return naturaldelta(datetime.utcnow().replace(tzinfo=timezone.utc) - self.created_at)

    def render_labels(self, bk_biz_id, render_type="list"):
        if self.api_labels:
            return self.api_labels
        elif self.id:
            return {label.key: label.value for label in self.labels.all()}
        else:
            return {}

    def render_label_list(self, bk_biz_id, render_type="list"):
        return self.get_label_list()

    def render_monitor_status(self, bk_biz_id, render_type="list"):
        if self.monitor_status == self.METRICS_STATE_STATE_SUCCESS:
            result = {
                "type": self.METRICS_STATE_STATE_SUCCESS,
                "text": _("正常"),
            }
        elif self.monitor_status == self.METRICS_STATE_FAILURE:
            result = {
                "type": self.METRICS_STATE_FAILURE,
                "text": _("异常"),
            }
        elif self.monitor_status == self.METRICS_STATE_DISABLED:
            result = {
                "type": self.METRICS_STATE_DISABLED,
                "text": _("无数据"),
            }
        else:
            result = {
                "type": self.METRICS_STATE_FAILURE,
                "text": _("异常"),
            }
        return result

    def render_bk_cluster_name(self, bk_biz_id, render_type="list") -> str:
        from bkmonitor.models import BCSCluster

        bcs_cluster = BCSCluster.objects.filter(bcs_cluster_id=self.bcs_cluster_id).first()
        if not bcs_cluster:
            return ""
        bcs_cluster_name = bcs_cluster.name
        return bcs_cluster_name

    @classmethod
    def add_cluster_column(cls, columns: List, columns_type: str) -> List:
        if columns_type == "detail":
            columns.extend(
                [
                    {
                        "id": "bk_cluster_name",
                        "name": _("集群名称"),
                        "type": "string",
                        "disabled": False,
                        "checked": True,
                    },
                ]
            )

        return columns

    @classmethod
    def get_filter_tags(cls, bk_biz_id, field_name, display_name=None, value_map=None, multiable=False):
        value_map = value_map or {}
        try:
            qs = cls.objects.filter_by_biz_id(bk_biz_id)
            items = qs.values_list(field_name, flat=True).distinct()
            display_name = display_name or field_name
            children = sorted(
                [
                    {
                        "name": item,
                        "id": value_map.get(item, item),
                    }
                    for item in items
                    if value_map.get(item, item)
                ],
                key=lambda item: item["id"],
            )
        except EmptyResultSet:
            children = []

        return {
            "name": display_name,
            "id": field_name,
            "multiable": multiable,
            "children": children,
        }

    @classmethod
    def get_filter_cluster(cls, params):
        return cls.get_filter_tags(params["bk_biz_id"], "bcs_cluster_id", display_name="cluster")

    @classmethod
    def get_filter_service_status(cls, params):
        return cls.get_filter_tags(params["bk_biz_id"], "status")

    @classmethod
    def get_filter_namespace(cls, params):
        return cls.get_filter_tags(params["bk_biz_id"], "namespace")

    @classmethod
    def get_filter_workload_name(cls, params):
        return cls.get_filter_tags(params["bk_biz_id"], "workload_name")

    @classmethod
    def get_column_filter_conf(cls, params, field):
        bk_biz_id = params["bk_biz_id"]
        space_related_cluster_ids = params.get("space_related_cluster_ids")
        if bk_biz_id >= 0:
            query = cls.objects.filter(bk_biz_id=bk_biz_id)
        else:
            query = cls.objects.filter(bcs_cluster_id__in=space_related_cluster_ids)
        items = query.values(field).distinct()
        status_list = sorted(list({item[field] for item in items if item[field]}))
        return [
            {
                "text": status,
                "value": status,
            }
            for status in status_list
        ]

    @staticmethod
    def get_cpu_usage_resource(usages: Dict, group_by: List) -> Dict:
        """获得CPU资源使用量 ."""
        data = {}
        for value in usages.values():
            keys = value.get("keys")
            value = value.get("usage", {}).get("resource_usage_cpu")
            key = tuple(keys.get(group_key) for group_key in group_by)
            old_value = data.get(key)
            if old_value is None or value > old_value:
                data[key] = value

        return data

    @staticmethod
    def fetch_container_usage(bk_biz_id: int, bcs_cluster_id: str, group_by: List) -> Dict:
        """获得cpu,memory,disk使用量 ."""
        usage_types = ["cpu", "memory", "disk"]
        bulk_params = [
            {
                "bk_biz_id": bk_biz_id,
                "bcs_cluster_id": bcs_cluster_id,
                "group_by": group_by,
                "usage_type": usage_type,
            }
            for usage_type in usage_types
        ]
        usages = api.kubernetes.fetch_container_usage.bulk_request(bulk_params)
        data = {}
        for usage in usages:
            usage_type = usage["usage_type"]
            usage_data = usage["data"]
            usage_key = "resource_usage_" + usage_type
            for item in usage_data:
                key = ".".join([item[group_key] for group_key in group_by])
                keys = {group_key: item[group_key] for group_key in group_by}
                # 获得旧的值
                data_item = data.get(
                    key,
                    {
                        "keys": keys,
                        "usage": {},
                    },
                )
                data_item_usage = data_item["usage"].get(usage_key)
                # 将新值和旧值进行比较，替换旧值，取最新的资源使用率
                if not data_item_usage or (data_item_usage < item["_result_"]):
                    data_item["usage"][usage_key] = item["_result_"]
                data[key] = data_item

        return data

    @classmethod
    def merge_monitor_status(cls, usage_resource_map: Dict, monitor_operator_status_up_map: Dict) -> Dict:
        """合并两个来源的数据状态 ."""
        # 根据资源使用率设置数据状态
        resource_status_map = {}
        for key, value in usage_resource_map.items():
            if value is not None:
                monitor_status = cls.METRICS_STATE_STATE_SUCCESS
            else:
                monitor_status = cls.METRICS_STATE_DISABLED
            resource_status_map[key] = monitor_status

        # 根据采集器指标设置数据状态
        resource_status_map.update(monitor_operator_status_up_map)

        return resource_status_map

    @staticmethod
    def get_monitor_beat_up_status(
        bk_biz_id: int, bcs_cluster_id: str, monitor_type: Optional[str], group_by: List
    ) -> Dict:
        """获得采集器的采集健康状态 ."""
        params = {
            "bk_biz_id": bk_biz_id,
            "bcs_cluster_id": bcs_cluster_id,
            "group_by": group_by,
        }
        if monitor_type:
            params["monitor_type"] = monitor_type
        metric_value_dict = api.kubernetes.fetch_k8s_bkm_metricbeat_endpoint_up(params)
        return metric_value_dict

    @classmethod
    def update_monitor_status_by_pod(cls, bcs_base_queryset, pod_models) -> None:
        """更新数据状态 ."""
        # 获得已存在的记录
        old_unique_hash_map = {model.unique_hash: (model.monitor_status,) for model in bcs_base_queryset}

        # 获得新的记录
        pod_monitor_status_map = {
            (pod_model.namespace, pod_model.name): pod_model.monitor_status for pod_model in pod_models
        }
        new_unique_hash_map = {}
        for model in bcs_base_queryset:
            monitor_status = cls.METRICS_STATE_FAILURE
            unique_hash = model.unique_hash
            pod_name_str = model.pod_name_list
            if pod_name_str:
                # Get all associated service pods
                pod_name_list = pod_name_str.split(",")
                namespace = model.namespace
                for pod_name in pod_name_list:
                    key = (namespace, pod_name)
                    old_monitor_status = new_unique_hash_map.get(key)
                    new_monitor_status = pod_monitor_status_map.get(key)
                    # pod have no monitor status
                    if new_monitor_status is None:
                        continue
                    if old_monitor_status is None:
                        monitor_status = new_monitor_status
                    else:
                        if old_monitor_status == cls.METRICS_STATE_STATE_SUCCESS:
                            continue
                        elif old_monitor_status == cls.METRICS_STATE_DISABLED:
                            if new_monitor_status == cls.METRICS_STATE_STATE_SUCCESS:
                                monitor_status = new_monitor_status
                        elif old_monitor_status == cls.METRICS_STATE_FAILURE:
                            monitor_status = new_monitor_status
            new_unique_hash_map[unique_hash] = (monitor_status,)

        # 更新资源使用量
        unique_hash_set_for_update = set(old_unique_hash_map.keys()) & set(new_unique_hash_map.keys())
        for unique_hash in unique_hash_set_for_update:
            if new_unique_hash_map[unique_hash] == old_unique_hash_map[unique_hash]:
                continue
            update_kwargs = new_unique_hash_map[unique_hash]
            monitor_status = update_kwargs[0]
            cls.objects.filter(unique_hash=unique_hash).update(
                **{
                    "monitor_status": monitor_status,
                }
            )

    @classmethod
    def convert_up_code_to_monitor_status(cls, code_list: List) -> str:
        code_set = set(code_list)
        if code_set == {BkmMetricbeatEndpointUpStatus.statusOK}:
            # 全部成功
            monitor_status = cls.METRICS_STATE_STATE_SUCCESS
        elif BkmMetricbeatEndpointUpStatus.statusOK in code_set:
            # 部分成功
            monitor_status = cls.METRICS_STATE_FAILURE
        else:
            # 全部失败
            monitor_status = cls.METRICS_STATE_DISABLED

        return monitor_status

    @classmethod
    def convert_up_to_monitor_status(
        cls, metric_value_dict: Dict, group_num: int, key_indexes: List, code_index: int
    ) -> Dict:
        """将up指标转换为采集器的状态 ."""
        result = {}
        # 获得状态码有多少种
        data = {}
        for group_key in metric_value_dict.keys():
            if len(group_key) != group_num:
                continue
            code_value = int(group_key[code_index])
            key = tuple(
                [key_item for index, key_item in enumerate(group_key) if index != code_index and index in key_indexes]
            )
            data.setdefault(key, []).append(code_value)

        for key, code_value_list in data.items():
            monitor_status = cls.convert_up_code_to_monitor_status(code_value_list)
            result[key] = monitor_status

        return result

    def update_monitor_status(self, params: Dict) -> None:
        """更新采集器状态 ."""
        bk_biz_id = params["bk_biz_id"]
        # 判断up指标是否存在
        if not api.kubernetes.has_bkm_metricbeat_endpoint_up({"bk_biz_id": bk_biz_id}):
            return
        # 添加分组
        group_by = ["bk_endpoint_url", "code"]
        params["group_by"] = group_by
        # 获得up指标
        metric_value_dict = api.kubernetes.fetch_k8s_bkm_metricbeat_endpoint_up(params)
        # 将up指标转换为采集器的状态
        if not metric_value_dict:
            # 没有指标返回
            monitor_status = self.METRICS_STATE_DISABLED
        else:
            code_list = [int(key[1]) for key in metric_value_dict.keys() if len(key) == len(group_by)]
            monitor_status = self.convert_up_code_to_monitor_status(code_list)

        self.monitor_status = monitor_status
        self.save()

class BCSBaseUsageResources(models.Model):
    class Meta:
        abstract = True

    resource_usage_cpu = models.FloatField(null=True)
    resource_usage_memory = models.BigIntegerField(null=True)
    resource_usage_disk = models.BigIntegerField(null=True)

    @classmethod
    def get_cpu_human_readable(cls, size) -> str:
        if size is None:
            return ""
        if not size:
            return "0"

        m_size = int(size * 1000)
        return f"{m_size}m"

    @classmethod
    def get_bytes_unit_human_readable(cls, size, precision=0) -> str:
        if size is None:
            return ""
        if not size:
            return "0"
        suffixes = ["B", "KB", "MB", "GB", "TB"]
        suffix_index = 0
        while size >= 1024 and suffix_index < 4:
            suffix_index += 1  # increment the index of the suffix
            size = size / 1024.0  # apply the division
        return "%.*f%s" % (precision, size, suffixes[suffix_index])

    def render_resource_usage_cpu(self, bk_biz_id, render_type="list"):
        return self.get_cpu_human_readable(self.resource_usage_cpu)

    def render_resource_usage_memory(self, bk_biz_id, render_type="list"):
        return self.get_bytes_unit_human_readable(self.resource_usage_memory)

    def render_resource_usage_disk(self, bk_biz_id, render_type="list"):
        return self.get_bytes_unit_human_readable(self.resource_usage_disk)

class BCSBaseResources(BCSBaseUsageResources):
    class Meta:
        abstract = True

    resource_requests_cpu = models.FloatField(null=False, default=0)
    resource_requests_memory = models.BigIntegerField(null=False, default=0)
    resource_limits_cpu = models.FloatField(null=False, default=0)
    resource_limits_memory = models.BigIntegerField(null=False, default=0)

    def render_resource_requests_cpu(self, bk_biz_id, render_type="list"):
        return self.get_cpu_human_readable(self.resource_requests_cpu)

    def render_resource_requests_memory(self, bk_biz_id, render_type="list"):
        return self.get_bytes_unit_human_readable(self.resource_requests_memory)

    def render_resource_limits_cpu(self, bk_biz_id, render_type="list"):
        return self.get_cpu_human_readable(self.resource_limits_cpu)

    def render_resource_limits_memory(self, bk_biz_id, render_type="list"):
        return self.get_bytes_unit_human_readable(self.resource_limits_memory)

    def render_resources(self, bk_biz_id, render_type="list"):
        resources = []
        if self.resource_requests_cpu:
            resources.append(
                {
                    "key": "requests cpu",
                    "value": self.get_cpu_human_readable(self.resource_requests_cpu),
                }
            )
        if self.resource_requests_memory:
            resources.append(
                {
                    "key": "requests memory",
                    "value": self.get_bytes_unit_human_readable(self.resource_requests_memory),
                }
            )
        if self.resource_limits_cpu:
            resources.append(
                {
                    "key": "limits cpu",
                    "value": self.get_cpu_human_readable(self.resource_limits_cpu),
                }
            )
        if self.resource_limits_memory:
            resources.append(
                {
                    "key": "limits memory",
                    "value": self.get_bytes_unit_human_readable(self.resource_limits_memory),
                }
            )
        return resources