import logging
import os
import os.path as osp
import time

import numpy as np
import torch
from detectron2.checkpoint import PeriodicCheckpointer
from detectron2.data import MetadataCatalog
from detectron2.data.common import AspectRatioGroupedDataset
from detectron2.utils.events import EventStorage
from loguru import logger as loguru_logger
from pytorch_lightning import seed_everything

import core.utils.my_comm as comm
import ref
from core.gdrn_modeling.data_loader import (build_gdrn_test_loader,
                                            build_gdrn_train_loader)
from core.gdrn_modeling.dataset_factory import register_datasets_in_cfg
from core.gdrn_modeling.engine import GDRN_Lite
from core.gdrn_modeling.engine_utils import (batch_data, get_out_coor,
                                             get_out_mask)
from core.gdrn_modeling.main_gdrn import setup
from core.gdrn_modeling.models import GDRN, GDRNT
from core.utils import solver_utils
from core.utils.data_utils import denormalize_image
from core.utils.default_args_setup import (my_default_argument_parser,
                                           my_default_setup)
from core.utils.my_checkpoint import MyCheckpointer
from core.utils.my_setup import setup_for_distributed
from core.utils.my_writer import (MyCommonMetricPrinter, MyJSONWriter,
                                  MyPeriodicWriter, MyTensorboardXWriter)
from core.utils.utils import get_emb_show
from lib.utils.time_utils import get_time_str
from lib.utils.utils import dprint, get_time_str, iprint

logger = logging.getLogger("detectron2")
import datetime
from collections import OrderedDict

from detectron2.evaluation import (DatasetEvaluator, DatasetEvaluators,
                                   inference_context)
from detectron2.utils.logger import log_every_n_seconds
from torch.cuda.amp import autocast

from core.gdrn_modeling.gdrn_evaluator import (GDRN_Evaluator,
                                               gdrn_inference_on_dataset,
                                               save_result_of_dataset)
from core.utils.my_comm import (all_gather, get_world_size, is_main_process,
                                synchronize)
import cv2

def gdrn_inference_feat_on_dataset(cfg, model, data_loader, evaluator, amp_test=False):
    """Run model on the data_loader and evaluate the metrics with evaluator.
    Also benchmark the inference speed of `model.forward` accurately. The model
    will be used in eval mode.

    Args:
        model (nn.Module): a module which accepts an object from
            `data_loader` and returns some outputs. It will be temporarily set to `eval` mode.

            If you wish to evaluate a model in `training` mode instead, you can
            wrap the given model and override its behavior of `.eval()` and `.train()`.
        data_loader: an iterable object with a length.
            The elements it generates will be the inputs to the model.
        evaluator (DatasetEvaluator): the evaluator to run. Use `None` if you only want
            to benchmark, but don't want to do any evaluation.

    Returns:
        The return value of `evaluator.evaluate()`
    """
    num_devices = get_world_size()
    logger = logging.getLogger(__name__)
    logger.info("Start inference on {} images".format(len(data_loader)))

    total = len(data_loader)  # inference data loader must have a fixed length
    if evaluator is None:
        # create a no-op evaluator
        evaluator = DatasetEvaluators([])
    evaluator.reset()

    num_warmup = min(5, total - 1)
    start_time = time.perf_counter()
    total_compute_time = 0
    total_process_time = 0
    eval_objs = []

    with inference_context(model), torch.no_grad():
        for idx, inputs in enumerate(data_loader):
            if idx == num_warmup:
                start_time = time.perf_counter()
                total_compute_time = 0
                total_process_time = 0

            start_compute_time = time.perf_counter()
            #############################
            # process input
            if not isinstance(inputs, list):  # bs=1
                inputs = [inputs]
            batch = batch_data(cfg, inputs, phase="test")
            if evaluator.train_objs is not None:
                roi_labels = batch["roi_cls"].cpu().numpy().tolist()
                obj_names = [evaluator.obj_names[_l] for _l in roi_labels]
                if all(_obj not in evaluator.train_objs for _obj in obj_names):
                    continue

            if len(eval_objs) == 0 or obj_names[0] not in eval_objs:

                eval_objs.append(obj_names[0])
            else:

                continue

            with autocast(enabled=amp_test):

                if cfg.MODEL.WEIGHTS == '/home/khiemphi/GDR-Net/gdrn_lm_resnet.pth':
                    features = model(
                        batch["roi_img"],
                    )
                else:
                    features = model(
                        batch["roi_img"],
                    )["last_hidden_state"].squeeze(0)

                features-= features.mean()
                features/= features.std ()
                features*=  64
                features+= 128

                if cfg.MODEL.WEIGHTS == '/home/khiemphi/GDR-Net/gdrn_lm_resnet.pth':
                    features_np = features.cpu().numpy()
                    features_np = np.resize( features_np,  (batch["roi_img"].shape[2:]))
                else:
                    features_np = features.cpu().numpy().reshape((batch["roi_img"].shape[2:]))

                features_np = np.clip(features_np, 0, 255).astype('uint8')

                blur = cv2.GaussianBlur(features_np,(5,5), 2).astype(np.uint8)
                heatmap_img = cv2.applyColorMap(blur, cv2.COLORMAP_JET)

                img = batch["roi_img"].squeeze(0).cpu().numpy().transpose(1, 2, 0)
                img *= 255
                img = img.astype(np.uint8)
                super_imposed_img = cv2.addWeighted(heatmap_img, 0.5, img, 0.5, 0)

                if cfg.MODEL.WEIGHTS == '/home/khiemphi/GDR-Net/gdrn_lm_resnet.pth':
                    filename = "features_resnet_" + obj_names[0] + "_" + os.path.basename(inputs[0]["file_name"][0])
                    folder_path = "features_vis/resnet"
                else:
                    filename = "features_swin_" + obj_names[0] + "_" + os.path.basename(inputs[0]["file_name"][0])
                    folder_path = "features_vis/swin"
                full_path = os.path.join(folder_path, filename)

                cv2.imwrite(full_path, super_imposed_img)

            if torch.cuda.is_available():
                torch.cuda.synchronize()
            cur_compute_time = time.perf_counter() - start_compute_time
            total_compute_time += cur_compute_time
            # NOTE: added
            # TODO: add detection time here
            outputs = [{} for _ in range(len(inputs))]
            for _i in range(len(outputs)):
                outputs[_i]["time"] = cur_compute_time

            start_process_time = time.perf_counter()

            cur_process_time = time.perf_counter() - start_process_time
            total_process_time += cur_process_time

            iters_after_start = idx + 1 - num_warmup * int(idx >= num_warmup)
            seconds_per_img = total_compute_time / iters_after_start
            if idx >= num_warmup * 2 or seconds_per_img > 5:
                total_seconds_per_img = (time.perf_counter() - start_time) / iters_after_start
                eta = datetime.timedelta(seconds=int(total_seconds_per_img * (total - idx - 1)))
                log_every_n_seconds(
                    logging.INFO, f"Inference done {idx+1}/{total}. {seconds_per_img:.4f} s / img. ETA={str(eta)}", n=5
                )

    # Measure the time only for this worker (before the synchronization barrier)
    total_time = time.perf_counter() - start_time
    total_time_str = str(datetime.timedelta(seconds=total_time))
    # NOTE this format is parsed by grep
    logger.info(
        f"Total inference time: {total_time_str} "
        f"({total_time / (total - num_warmup):.6f} s / img per device, on {num_devices} devices)"
    )
    # pure forward time
    total_compute_time_str = str(datetime.timedelta(seconds=int(total_compute_time)))
    logger.info(
        "Total inference pure compute time: {} ({:.6f} s / img per device, on {} devices)".format(
            total_compute_time_str, total_compute_time / (total - num_warmup), num_devices
        )
    )
    # post_process time
    total_process_time_str = str(datetime.timedelta(seconds=int(total_process_time)))
    logger.info(
        "Total inference post process time: {} ({:.6f} s / img per device, on {} devices)".format(
            total_process_time_str, total_process_time / (total - num_warmup), num_devices
        )
    )

class Lite(GDRN_Lite):
    def set_my_env(self, args, cfg):
        my_default_setup(cfg, args)  # will set os.environ["PYTHONHASHSEED"]
        seed_everything(int(os.environ["PYTHONHASHSEED"]), workers=True)
        setup_for_distributed(is_master=self.is_global_zero)

    def do_test_feat_map(self, cfg, model, epoch=None, iteration=None):
        results = OrderedDict()
        model_name = osp.basename(cfg.MODEL.WEIGHTS).split(".")[0]
        for dataset_name in cfg.DATASETS.TEST:
            if epoch is not None and iteration is not None:
                evaluator = self.get_evaluator(
                    cfg,
                    dataset_name,
                    osp.join(cfg.OUTPUT_DIR, f"inference_epoch_{epoch}_iter_{iteration}", dataset_name),
                )
            else:
                evaluator = self.get_evaluator(
                    cfg, dataset_name, osp.join(cfg.OUTPUT_DIR, f"inference_{model_name}", dataset_name)
                )
            data_loader = build_gdrn_test_loader(cfg, dataset_name, train_objs=evaluator.train_objs)
            data_loader = self.setup_dataloaders(data_loader, replace_sampler=False, move_to_device=False)

            backbone = model.module.backbone

            # Now we do forward with the backbone to get feature maps
            gdrn_inference_feat_on_dataset(cfg, backbone, data_loader, evaluator, amp_test=cfg.TEST.AMP_TEST)

            results_i = gdrn_inference_on_dataset(cfg, model, data_loader, evaluator, amp_test=cfg.TEST.AMP_TEST)

            results[dataset_name] = results_i

        if len(results) == 1:
            results = list(results.values())[0]
        return results

    def run(self, args, cfg):
        self.set_my_env(args, cfg)

        logger.info(f"Used GDRN module name: {cfg.MODEL.CDPN.NAME}")
        model, optimizer = eval(cfg.MODEL.CDPN.NAME).build_model_optimizer(cfg)

        #Let's load weight with the pytorch-way

        logger.info("Model:\n{}".format(model))

        # don't forget to call `setup` to prepare for model / optimizer for distributed training.
        # the model is moved automatically to the right device.
        model, optimizer = self.setup(model, optimizer) # look into setup to swap out features

        if True:
            # sum(p.numel() for p in model.parameters() if p.requires_grad)
            params = sum(p.numel() for p in model.parameters()) / 1e6
            logger.info("{}M params".format(params))

        if args.eval_only:
            MyCheckpointer(model, save_dir=cfg.OUTPUT_DIR).resume_or_load(cfg.MODEL.WEIGHTS, resume=args.resume)
            self.do_test_feat_map(cfg, model)
            return self.do_test(cfg, model)

        self.do_train(cfg, args, model, optimizer, resume=args.resume)
        return self.do_test(cfg, model)

    def do_train(self, cfg, args, model, optimizer, resume=False):

        model.train()

        # some basic settings =========================
        dataset_meta = MetadataCatalog.get(cfg.DATASETS.TRAIN[0])
        data_ref = ref.__dict__[dataset_meta.ref_key]
        obj_names = dataset_meta.objs

        # load data ===================================
        train_dset_names = cfg.DATASETS.TRAIN
        data_loader = build_gdrn_train_loader(cfg, train_dset_names)
        data_loader_iter = iter(data_loader)

        # load 2nd train dataloader if needed
        train_2_dset_names = cfg.DATASETS.get("TRAIN2", ())
        train_2_ratio = cfg.DATASETS.get("TRAIN2_RATIO", 0.0)
        if train_2_ratio > 0.0 and len(train_2_dset_names) > 0:
            data_loader_2 = build_gdrn_train_loader(cfg, train_2_dset_names)
            data_loader_2_iter = iter(data_loader_2)
        else:
            data_loader_2 = None
            data_loader_2_iter = None

        images_per_batch = cfg.SOLVER.IMS_PER_BATCH
        if isinstance(data_loader, AspectRatioGroupedDataset):
            dataset_len = len(data_loader.dataset.dataset)
            if data_loader_2 is not None:
                dataset_len += len(data_loader_2.dataset.dataset)
            iters_per_epoch = dataset_len // images_per_batch
        else:
            dataset_len = len(data_loader.dataset)
            if data_loader_2 is not None:
                dataset_len += len(data_loader_2.dataset)
            iters_per_epoch = dataset_len // images_per_batch
        max_iter = cfg.SOLVER.TOTAL_EPOCHS * iters_per_epoch
        dprint("images_per_batch: ", images_per_batch)
        dprint("dataset length: ", dataset_len)
        dprint("iters per epoch: ", iters_per_epoch)
        dprint("total iters: ", max_iter)

        data_loader = self.setup_dataloaders(data_loader, replace_sampler=False, move_to_device=False)
        if data_loader_2 is not None:
            data_loader_2 = self.setup_dataloaders(data_loader_2, replace_sampler=False, move_to_device=False)

        scheduler = solver_utils.build_lr_scheduler(cfg, optimizer, total_iters=max_iter)

        # resume or load model ===================================
        extra_ckpt_dict = dict(
            optimizer=optimizer,
            scheduler=scheduler,
        )
        if hasattr(self._precision_plugin, "scaler"):
            extra_ckpt_dict["gradscaler"] = self._precision_plugin.scaler

        checkpointer = MyCheckpointer(
            model,
            cfg.OUTPUT_DIR,
            save_to_disk=self.is_global_zero,
            **extra_ckpt_dict,
        )
        start_iter = checkpointer.resume_or_load(cfg.MODEL.WEIGHTS, resume=resume).get("iteration", -1) + 1

        if cfg.SOLVER.CHECKPOINT_BY_EPOCH:
            ckpt_period = cfg.SOLVER.CHECKPOINT_PERIOD * iters_per_epoch
        else:
            ckpt_period = cfg.SOLVER.CHECKPOINT_PERIOD
        periodic_checkpointer = PeriodicCheckpointer(
            checkpointer, ckpt_period, max_iter=max_iter, max_to_keep=cfg.SOLVER.MAX_TO_KEEP
        )

        # build writers ==============================================
        tbx_event_writer = self.get_tbx_event_writer(cfg.OUTPUT_DIR, backup=not cfg.get("RESUME", False))
        tbx_writer = tbx_event_writer._writer  # NOTE: we want to write some non-scalar data
        writers = (
            [MyCommonMetricPrinter(max_iter), MyJSONWriter(osp.join(cfg.OUTPUT_DIR, "metrics.json")), tbx_event_writer]
            if self.is_global_zero
            else []
        )

        # compared to "train_net.py", we do not support accurate timing and
        # precise BN here, because they are not trivial to implement
        logger.info("Starting training from iteration {}".format(start_iter))
        iter_time = None
        with EventStorage(start_iter) as storage:
            for iteration in range(start_iter, max_iter):
                storage.iter = iteration
                epoch = iteration // dataset_len + 1

                if np.random.rand() < train_2_ratio:
                    data = next(data_loader_2_iter)
                else:
                    data = next(data_loader_iter)

                if iter_time is not None:
                    storage.put_scalar("time", time.perf_counter() - iter_time)
                iter_time = time.perf_counter()

                # forward ============================================================

                batch = batch_data(cfg, data)

                out_dict, loss_dict = model(
                    batch["roi_img"],
                    gt_xyz=batch.get("roi_xyz", None),
                    gt_xyz_bin=batch.get("roi_xyz_bin", None),
                    gt_mask_trunc=batch["roi_mask_trunc"],
                    gt_mask_visib=batch["roi_mask_visib"],
                    gt_mask_obj=batch["roi_mask_obj"],
                    gt_region=batch.get("roi_region", None),
                    gt_allo_quat=batch.get("allo_quat", None),
                    gt_ego_quat=batch.get("ego_quat", None),
                    gt_allo_rot6d=batch.get("allo_rot6d", None),
                    gt_ego_rot6d=batch.get("ego_rot6d", None),
                    gt_ego_rot=batch.get("ego_rot", None),
                    gt_trans=batch.get("trans", None),
                    gt_trans_ratio=batch["roi_trans_ratio"],
                    gt_points=batch.get("roi_points", None),
                    sym_infos=batch.get("sym_info", None),
                    roi_classes=batch["roi_cls"],
                    roi_cams=batch["roi_cam"],
                    roi_whs=batch["roi_wh"],
                    roi_centers=batch["roi_center"],
                    resize_ratios=batch["resize_ratio"],
                    roi_coord_2d=batch.get("roi_coord_2d", None),
                    roi_extents=batch.get("roi_extent", None),
                    do_loss=True,
                )
                losses = sum(loss_dict.values())
                assert torch.isfinite(losses).all(), loss_dict

                loss_dict_reduced = {k: v.item() for k, v in comm.reduce_dict(loss_dict).items()}
                losses_reduced = sum(loss for loss in loss_dict_reduced.values())
                if self.is_global_zero:
                    storage.put_scalars(total_loss=losses_reduced, **loss_dict_reduced)

                optimizer.zero_grad(set_to_none=True)
                self.backward(losses)
                optimizer.step()

                storage.put_scalar("lr", optimizer.param_groups[0]["lr"], smoothing_hint=False)
                scheduler.step()

                if (
                    cfg.TEST.EVAL_PERIOD > 0
                    and (iteration + 1) % cfg.TEST.EVAL_PERIOD == 0
                    and iteration != max_iter - 1
                ):
                    self.do_test(cfg, model, epoch=epoch, iteration=iteration)
                    # Compared to "train_net.py", the test results are not dumped to EventStorage
                    self.barrier()
                    torch.save(model, "best_current_model.pth")
                # iteration - start_iter > 5 and (
                    (iteration + 1) % cfg.TRAIN.PRINT_FREQ == 0 or iteration == max_iter - 1 or iteration < 100
                #)

                if True:
                    for writer in writers:
                        writer.write()
                    # visualize some images ========================================
                    if False:
                        with torch.no_grad():
                            vis_i = 0
                            roi_img_vis = batch["roi_img"][vis_i].cpu().numpy()
                            roi_img_vis = denormalize_image(roi_img_vis, cfg).transpose(1, 2, 0).astype("uint8")
                            tbx_writer.add_image("input_image", roi_img_vis, iteration)

                            out_coor_x = out_dict["coor_x"].detach()
                            out_coor_y = out_dict["coor_y"].detach()
                            out_coor_z = out_dict["coor_z"].detach()
                            out_xyz = get_out_coor(cfg, out_coor_x, out_coor_y, out_coor_z)

                            out_xyz_vis = out_xyz[vis_i].cpu().numpy().transpose(1, 2, 0)
                            out_xyz_vis = get_emb_show(out_xyz_vis)
                            tbx_writer.add_image("out_xyz", out_xyz_vis, iteration)

                            gt_xyz_vis = batch["roi_xyz"][vis_i].cpu().numpy().transpose(1, 2, 0)
                            gt_xyz_vis = get_emb_show(gt_xyz_vis)
                            tbx_writer.add_image("gt_xyz", gt_xyz_vis, iteration)

                            out_mask = out_dict["mask"].detach()
                            out_mask = get_out_mask(cfg, out_mask)
                            out_mask_vis = out_mask[vis_i, 0].cpu().numpy()
                            tbx_writer.add_image("out_mask", out_mask_vis, iteration)

                            gt_mask_vis = batch["roi_mask"][vis_i].detach().cpu().numpy()
                            tbx_writer.add_image("gt_mask", gt_mask_vis, iteration)

                if (iteration + 1) % periodic_checkpointer.period == 0 or (
                    periodic_checkpointer.max_iter is not None and (iteration + 1) >= periodic_checkpointer.max_iter
                ):
                    if hasattr(optimizer, "consolidate_state_dict"):  # for ddp_sharded
                        optimizer.consolidate_state_dict()
                periodic_checkpointer.step(iteration, epoch=epoch)

@loguru_logger.catch
def main(args):
    #1. Let's set up configs
    cfg = setup(args)
    cfg.DATALOADER.NUM_WORKERS = 1 # decrease cpu usage here, now we need to modify the model later
    Lite(
        accelerator="gpu",
        devices=args.num_gpus,
        num_nodes=args.num_machines,
        precision=16 if cfg.SOLVER.AMP.ENABLED else 32,
    ).run(args, cfg)

if __name__ == "__main__":
    parser = my_default_argument_parser()
    args = parser.parse_args()
    main(args)


import sys
from collections import defaultdict

def doParseLog():
    feat_id_dict = defaultdict(lambda: 0)
    for line in sys.stdin:
        # recv = recv + 1
        try:
            splits = line.strip().split()
            '''split_len = len(splits)
            # common_feature_index|feat_num|feat_list
            if split_len == 3:
                feat_strs = splits[2]
                for fstr in feat_strs.split('\x01'):
                    filed, feat_val = fstr.split('\x02')
                    feat, val = feat_val.split('\x03')
                    feat_id_dict["{}:{}".format(filed, feat)] += 1
            # sample_id|click_y|conversion_y|common_feature_index|feat_num|feat_list
            elif split_len == 6:
                # click_y=0 & conversion_y=1过滤
                if splits[1] == '0' and splits[2] == '1':
                    continue
                feat_strs = splits[5]
                for fstr in feat_strs.split('\x01'):
                    filed, feat_val = fstr.split('\x02')
                    feat, val = feat_val.split('\x03')
                    feat_id_dict["{}:{}".format(filed, feat)] += 1'''
            for fstr in splits[4:]:
                filed, feat, val = fstr.split(':')
                feat_id_dict["{}:{}".format(filed, feat)] += 1
        except:
            continue
    for feat_id, cnts in feat_id_dict.items():
        print("{0}\t{1}".format(feat_id, cnts))

if __name__ == '__main__':
    doParseLog()


class CarNotEnoughFuel(Exception):
    def __init__(self, message="Not Enough Fuel"):
        Exception.__init__(self, message)

class CarNegativeDistance(Exception):
    def __init__(self, message="You Can't Break The Laws Of Physics"):
        Exception.__init__(self, message)

class Car:
    fuel = 20
    rate = 0.1
    traveled = 0

    def __init__(self, fuel=20, rate=0.1):
        self.fuel = fuel
        self.rate = rate

    def __str__(self):
        return f"Fuel = {self.fuel}\n " \
               f"Rate = {self.rate}\n " \
               f"Traveled = {self.traveled}"

    def drive(self, distance):
        if self.fuel < distance * self.rate:
            raise CarNotEnoughFuel("Not Enough Fuel")
        else:
            if distance < 0:
                raise CarNegativeDistance()
            else:
                self.traveled += distance
                self.fuel -= distance * self.rate

car1 = Car(30, 5)
print(car1)
try:
    car1.drive(-3)
except CarNotEnoughFuel:
    print("You Don't Have Enough Fuel")
except CarNegativeDistance:
    print("You Can't Travel Negative Distance")
print(car1)


from keras.models import Sequential
from keras.layers import Dense, Dropout, BatchNormalization, Flatten
from keras.applications import MobileNetV3Large
from tensorflow.keras.applications.mobilenet_v3 import preprocess_input
from NetUtils import NetUtils

class MobileNet(NetUtils):

    def __init__(self, input_shape, num_classes, load_weights = None):
        self.input_shape = input_shape
        self.mobilenet_base = MobileNetV3Large(weights='imagenet',
                                        include_top=False,
                                        input_shape=input_shape)
        self.model = Sequential()
        self.model.add(self.mobilenet_base)
        self.model.add(Flatten())
        self.model.add(Dense(16, activation='relu'))
        self.model.add(Dense(num_classes, activation='softmax'))
        self.model.layers[0].trainable = False

        self.model.compile(loss='categorical_crossentropy',
                           optimizer='adam',
                           metrics=['accuracy'])

        if load_weights:
            self.model.load_weights(load_weights)

    def create_generators(self, train_dir, val_dir, test_dir, batch_size, data_augm=False, preprocessing_function=None):
        super().create_generators(train_dir, val_dir, test_dir, batch_size, data_augm,
                                  preprocessing_function=preprocess_input)


#%% 12 - Valide e Corrija o telefone

import sys

"""
Valida e corrige número de telefone. Faça um programa que leia um número
de telefone, e corrija o número no caso deste conter somente 7 dígitos,
acrescentando o '3' na frente. O usuário pode informar o número com ou
sem o traço separador.

Valida e corrige número de telefone
Telefone: 461-0133
Telefone possui 7 dígitos. Vou acrescentar o digito três na frente.
Telefone corrigido sem formatação: 34610133
Telefone corrigido com formatação: 3461-0133
"""

telefone = input('Digite um telefone no formato XXXX-XXXX: ')

def validacao_telefone(telefone):
    telefone_lista = telefone.split('-')
    if len(telefone_lista[0]) not in [3,4]: return False, telefone_lista
    if len(telefone_lista[1]) != 4: return False, telefone_lista
    for num in telefone_lista:
        try: num = int(num)
        except: return False, telefone_lista
    return True, telefone_lista

telefone_valido, telefone_lista = validacao_telefone(telefone)
novo_telefone = ''

if len(telefone_lista[0]) == 3 and telefone_valido:
    novo_telefone = '3' + telefone_lista[0] + telefone_lista[1]

if novo_telefone:
    print(f'Telefone corrigido sem formatação: {novo_telefone}')
    print(f'Telefone corrigido com formatação: {novo_telefone[:4]}-{novo_telefone[4:]}')
    sys.exit()

if telefone_valido: print(f'Telefone válido {telefone}')
else: print('Telefone Inválido')


import music21
import numpy as np
import matplotlib.pyplot as plt
import mido
import csv
import IPython.display as ipd
import midi2audio
import glob
from convertVector import convertVector

class xmlLoader(convertVector):
    DIR = ""
    def __init__(self, dir) :
        print("Welcome to XMLoader")
        self.DIR = dir

    def convert(self, root, mode):
        self.setKeyMode(mode)
        self.setKeyRoot(root)
        x_all = []
        y_all = []
        print("-----------XML LOADING -----------------")
        for f in glob.glob(self.DIR + "/*.xml"):
            print(f)
            score = music21.converter.parse(f)
            key = score.analyze("key")
            if key.mode == self.KEY_MODE:
                inter = music21.interval.Interval(key.tonic, music21.pitch.Pitch(self.KEY_ROOT))
                score = score.transpose(inter)
                note_seq, chord_seq = self.make_note_and_chord_seq_from_musicxml(score)
                onehot_seq = self.add_rest_nodes(self.note_seq_to_onehot(note_seq))
                chroma_seq = self.chord_seq_to_chroma(chord_seq)
                self.divide_seq(onehot_seq, chroma_seq, x_all, y_all)

        x_all = np.array(x_all)
        y_all = np.array(y_all)
        print("----------------------------------------")
        return x_all, y_all

# MusicXMLデータからNote列とChordSymbol列を生成
# 時間分解能は BEAT_RESO にて指定
    def make_note_and_chord_seq_from_musicxml(self, score):
        note_seq = [None] * (self.TOTAL_MEASURES * self.N_BEATS * self.BEAT_RESO)
        chord_seq = [None] * (self.TOTAL_MEASURES * self.N_BEATS * self.BEAT_RESO)

        for element in score.parts[0].elements:
            if isinstance(element, music21.stream.Measure):
                measure_offset = element.offset
                for note in element.notes:
                    if isinstance(note, music21.note.Note):
                        onset = measure_offset + note._activeSiteStoredOffset
                        offset = onset + note._duration.quarterLength
                        for i in range(int(onset * self.BEAT_RESO), int(offset * self.BEAT_RESO + 1)):
                            note_seq[i] = note
                    if isinstance(note, music21.harmony.ChordSymbol):
                        chord_offset = measure_offset + note.offset
                        for i in range(int(chord_offset * self.BEAT_RESO),
                            int((measure_offset + self.N_BEATS) * self.BEAT_RESO + 1)):
                            chord_seq[i] = note
        return note_seq, chord_seq


import uuid

import matplotlib as mpl
import pydot
from matplotlib import pyplot as plt

mpl.use('Agg')

def get_root(tree):
    for s in tree.keys():
        if s.parent is None:
            return s

def s2node(s, tree):
    if s in tree.keys():
        # return pydot.Node(str(uuid.uuid4()),label='n='+str(tree[s].n))
        return pydot.Node(str(uuid.uuid4()), label=str(tree[s].v))
    else:
        return None

def add_children(s, s_node, tree, graph, d):
    if d > 0:
        for a in tree[s].a.keys():
            tree[s].a[a].n
            tree[s].a[a].q
            assert len(tree[s].a[a].s.keys()) == 1
            for ns in tree[s].a[a].s.keys():
                ns_node = s2node(ns, tree)
                if ns_node is not None:
                    graph.add_node(ns_node)
                    # graph.add_edge(pydot.Edge(s_node, ns_node, label="n="+str(n)+" q="+str(q)))
                    graph.add_edge(pydot.Edge(s_node, ns_node, label=str(ns.action[0])))
                    add_children(ns, ns_node, tree, graph, d - 1)

def get_node_num_next(s, tree, depths, nodeNums, d):
    d = d + 1
    if (len(depths) <= d) and (len(tree[s].a.keys()) > 0):
        depths.append(d)
        nodeNums.append(0)

    for a in tree[s].a.keys():
        for ns in tree[s].a[a].s.keys():
            nodeNums[d] += 1
            if ns in tree.keys():
                get_node_num_next(ns, tree, depths, nodeNums, d)

def plot_tree(tree, d, path, format="svg"):
    graph = pydot.Dot(graph_type='digraph')
    root = get_root(tree)
    root_node = s2node(root, tree)
    graph.add_node(root_node)
    add_children(root, root_node, tree, graph, d)
    filename = path + "." + format
    if format == "svg":
        graph.write_svg(filename)
    elif format == "png":
        graph.write_png(filename)

def plot_node_num(tree, path, format="svg"):
    root = get_root(tree)
    depths = [0]
    nodeNums = [1]
    d = 0
    get_node_num_next(root, tree, depths, nodeNums, d)
    filename = path + "." + format
    plt.plot(depths, nodeNums)
    plt.xlabel('Depth')
    plt.ylabel('Node Number')
    plt.savefig(filename)


LOAD_AHEAD = 10000
VISIBLE_TIME = 2000
SCREEN_WIDTH = 640
SCREEN_HEIGHT = 480

MAX_SPRITES = 500

KEYS = 8
TOLERANCE = 10
TIME_TO_RELOAD = 1

ENEMY_SPRITES = ['A', 'B', 'C', 'D', 'D', 'C', 'B', 'A'] # mirrored sprites for easy glancing for key
NOTES = ['c', 'd', 'e', 'f', 'g', 'a', 'b', 'co']
key_map = {97: 0, 115: 1, 100: 2, 102: 3, 106: 4, 107: 5, 108: 6, 59: 7}

PLAYER_DIFFICULTY = 2

global tempo
global media
global earned

global score

global enemy_list

def init():
    global score
    score = 0
    global earned

    global media
    media = {}

    global tempo
    tempo = 1000 * 60 / 4 / 150  # 120 bpm


"""
Game server manager for Phagocytes

This is used to handle the creation on the fly of new server instance
to allow multiple games to be run at the same time
"""

import multiprocessing
import subprocess

import atexit
import os

import requests
import sys
from flask import Flask, request, jsonify

__author__ = "Benjamin Schubert <ben.c.schubert@gmail.com>"

class FullCapacityException(Exception):
    """
    Error raised when a server is operating at full capacity
    """
    pass

class NotifierFlask(Flask):
    """
    This is a Flask server that is able of sending a token to another server before launch
    """
    token = None
    capacity = multiprocessing.cpu_count()
    ports_used = None

    def setup_ports(self):
        """
        initializes which ports are available for this server
        """
        self.ports_used = [False] * self.capacity

    def next_available_port(self) -> int:
        """
        gets the next available port usable by the server

        :return: the index of the next available port
        """
        for index in range(len(self.ports_used)):
            if not self.ports_used[index]:
                self.ports_used[index] = True
                return index

        raise FullCapacityException()

    def get_token(self, host: str, port: int):
        """
        Get the server authentication token against the authentication server

        :param host: flask server host
        :param port: port of the flask server
        """
        r = requests.post(
            "http://{}:{}/games/manager".format(app.config["AUTH_SERVER"], app.config["AUTH_SERVER_PORT"]),
            json={"port": port, "host": host, "capacity": multiprocessing.cpu_count()}
        )

        if r.status_code == requests.codes.ok:
            self.token = r.json()["token"]
        else:
            print("Couldn't contact authentication server")
            exit(2)

    def run(self, host: str=None, port: int=None, debug: bool=None, **options):
        """
        Gets the token from the authentication server before launching Flask

        :param host: host on which to listen
        :param port: port on which to listen
        :param debug: whether to enable debugging options or not
        :param options: additional options sent to the server
        """
        self.get_token(host, port)
        self.debug = debug
        super().run(host, port, debug, **options)

    def create_game_server(self, **kwargs):
        """
        Creates a new game server

        :param kwargs: arguments to pass to the executable
        """
        cmd = [sys.executable]
        if not getattr(sys, 'frozen', False):
            cmd.append("manage.py")
        cmd.extend(
            [
                "node", "-p", str(self.config["PORT_GAMESERVER"] + self.next_available_port()),
                "-a", str(app.config["AUTH_SERVER"]), "--auth-port", str(app.config["AUTH_SERVER_PORT"]),
            ] + [entry for entries in [["--" + key, item] for key, item in kwargs.items()] for entry in entries])

        if self.debug:
            cmd.append("-d")

        subprocess.Popen(cmd, cwd=os.path.dirname(os.path.abspath(__file__)), stderr=sys.stderr, stdout=sys.stdout)

app = NotifierFlask("phagocytes_game_manager")

# Flask configuration
if getattr(sys, 'frozen', False):
    application_path = os.path.dirname(sys.executable)
else:
    application_path = os.path.dirname(__file__)

config_path = os.environ.get("PHAGOCYTE_GAMEMANAGER_SERVER", os.path.join(application_path, "config.cfg"))
app.config.from_pyfile(config_path)
app.setup_ports()

@atexit.register
def unregister_app():
    """
    unregisters the manager from the server
    """
    requests.delete(
        "http://{}:{}/games/manager".format(app.config["AUTH_SERVER"], app.config["AUTH_SERVER_PORT"]),
        json={"token": app.token}
    )

@app.route("/games", methods=["POST"])
def create():
    """
    Creates a new game server

    :return: 200 on ok, 401 if token is not valid, 412 if capacity is exceeded
    """
    if request.json["token"] != app.token:
        return (jsonify({"error": "unauthorized"}), 401)

    try:
        app.create_game_server(**request.json)
    except FullCapacityException:
        return jsonify({"error": "maximum capacity exceeded"}), 412
    return "", 200

@app.route("/games", methods=["DELETE"])
def remove():
    """
    Marks the port as unused and available for a new server
    """
    if request.json["token"] != app.token:
        return jsonify({"error": "unauthorized"}), 401

    for index in range(len(app.ports_used)):
        if app.config["PORT_GAMESERVER"] + index == request.json["port"]:
            app.ports_used[index] = False
            return "", 200

    return jsonify({"error": "port not found"}), 404

def runserver(host, port, debug=False):
    """
    runs the server

    :param debug: whether to enable debugging or not
    :param host: host on which to listen
    :param port: port on which to listen
    """
    app.run(host=host, port=port, debug=debug)


"""
test Data Loaders
"""
import os
from collections import namedtuple
from AGCN.utils.data_loader import PointcloudLoader, SMILESLoader, ImageCIFARLoader, ImageFashionLoader
import time

class LoaderTester(object):
    """
    abstract class for tester, include basic functions and members
    """
    def __init__(self, loader_job):
        self.loader_job = loader_job

    def try_load(self):
        raise NotImplementedError

class PointcloudLoaderTester(LoaderTester):

    def try_load(self):

        try:
            loader = PointcloudLoader(self.loader_job.dataset_class,
                                  self.loader_job.dataset_name,
                                  self.loader_job.file_name,
                                  self.loader_job.tasks)
        except 'failed to create instance of PointcloudLoader':
            raise Exception('FAiled')
        assert isinstance(loader, PointcloudLoader)
        print('Loader Object Created!')

        try:
            dataset = loader.load()
        except "loading failed":
            raise Exception('Data loading failed')

        print("Loading Successful!")

class SmilesLoaderTester(LoaderTester):

    def try_load(self):

        try:
            loader = SMILESLoader(self.loader_job.dataset_class,
                                  self.loader_job.dataset_name,
                                  self.loader_job.file_name,
                                  self.loader_job.tasks,
                                  self.loader_job.smiles_filed,
                                  feature='MolGraph',
                                  splitter='index',
                                  download_url=self.loader_job.download_url)
            loader.load()
        except 'failed to create instance of SMILESLoader':
            raise Exception('FAiled')
        assert isinstance(loader, SMILESLoader)

class ImageCIFALoaderTester(LoaderTester):

    def try_load(self):
        try:
            loader = ImageCIFARLoader(
                self.loader_job.dataset_class,
                self.loader_job.dataset_name,
                self.loader_job.file_name,
                self.loader_job.tasks)
        except 'failed to create instance of ImageCIFARLoader':
            raise Exception('FAiled')

        assert isinstance(loader, ImageCIFARLoader)
        start_t = time.time()
        xx = loader.load()
        end_t = time.time() - start_t
        print ("time used for featurization of 50000 32*32 image: %f second", end_t)

class ImageFashionLoaderTester(LoaderTester):

    def try_load(self):
        try:
            loader = ImageFashionLoader(
                self.loader_job.dataset_class,
                self.loader_job.dataset_name,
                self.loader_job.file_name,
                self.loader_job.tasks)
        except 'failed to create instance of ImageFashionLoader':
            raise Exception('FAiled')

        assert isinstance(loader, ImageFashionLoader)
        start_t = time.time()
        loader.featurize(shard_size=2048)
        end_t = time.time() - start_t
        print ("time used for featurization of 60000 28*28 greyscale image: %f second", end_t)

def test_cifarloader():
    cifarimage_tasks = [
        'airplane', 'auto', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck'
    ]

    cifar_loader = namedtuple('cifarimage_loader', 'dataset_class, dataset_name, file_name, tasks')
    cifar_loader.tasks = cifarimage_tasks
    cifar_loader.dataset_class = 'images'
    cifar_loader.dataset_name = 'cifar'
    cifar_loader.file_name = 'cifar10'

    cifar_tester = ImageCIFALoaderTester(cifar_loader)
    try:
        cifar_tester.try_load()
        print("Testing succeed!")
    except "failed":
        raise Exception('FAiled to create ImageCIFARLoader')

def test_fashionloader():
    fashionimage_tasks = [
        'T-shirt', 'Trouser', 'Pullover', 'Dress', 'Coat', 'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot'
    ]

    loader = namedtuple('fashionimage_loader', 'dataset_class, dataset_name, file_name, tasks')
    loader.tasks = fashionimage_tasks
    loader.dataset_class = 'images'
    loader.dataset_name = 'fashion'
    loader.file_name = 'fashion'

    tester = ImageFashionLoaderTester(loader)
    try:
        tester.try_load()
        print("Testing succeed!")
    except "failed":
        raise Exception('FAiled to create ImageCIFARLoader')

def test_smiles():
    tox21_tasks = [
        'NR-AR', 'NR-AR-LBD', 'NR-AhR', 'NR-Aromatase', 'NR-ER', 'NR-ER-LBD',
        'NR-PPAR-gamma', 'SR-ARE', 'SR-ATAD5', 'SR-HSE', 'SR-MMP', 'SR-p53'
    ]
    smiles_loader_1 = namedtuple('smiles_loader',
                                 'dataset_class, dataset_name, file_name, tasks, smiles_filed, download_url')

    smiles_loader_1.tasks = tox21_tasks
    smiles_loader_1.dataset_class = 'chemistry'
    smiles_loader_1.dataset_name = 'tox21'
    smiles_loader_1.file_name = 'tox21.csv.gz'
    smiles_loader_1.smiles_filed = 'smiles'
    smiles_loader_1.download_url = 'http://deepchem.io.s3-website-us-west-1.amazonaws.com/datasets/tox21.csv.gz'

    smile_tester = SmilesLoaderTester(smiles_loader_1)
    try:
        smile_tester.try_load()
        print('Testing succeed!')
    except "failed":
        raise Exception("Testing Failed")

def test_pc():

    pc_loader_1 = namedtuple('pc_loader', 'dataset_class, dataset_name, file_name, tasks')

    pc_tasks = ['bicycle', 'pedestrian', 'biker', 'van', 'excavator', 'traffic_sign', 'scooter', 'bench',
                'trash', 'vegetation', 'cyclist', 'umbrella', 'bus', 'ticket_machine', 'trunk', 'post', 'building',
                'traffic_lights', '4wd', 'ute', 'car', 'pillar', 'tree', 'pole', 'truck', 'trailer']

    pc_loader_1.tasks = pc_tasks
    pc_loader_1.dataset_class = 'pointcloud'
    pc_loader_1.dataset_name = 'sydney'
    pc_loader_1.file_name = 'objects'

    pc_loader_test = PointcloudLoaderTester(pc_loader_1)
    try:
        pc_loader_test.try_load()
        print("Testing Succeed!")
    except "failed":
        raise Exception("Testing Failed")

if __name__ =="__main__":
    # test_pc()
    # test_smiles()
    # test_cifarloader()
    test_fashionloader()


import requests
from html import unescape
from bs4 import BeautifulSoup
from tkinter import *
from tkinter import ttk
import os

def review_button():
    tp = Toplevel()
    tp.title("Flashcard_Studio/Sets")
    tp.geometry("800x350")
    global setnames
    global setnamesStrVar
    setnames = []
    label2 = Label(tp, text = "Sets:")
    label2.pack()
    # listdir collects set names to be displayed in OptionMenu() and which when one is selected and the
    # setselbut Button is clicked will run selectsetoptions()
    for path in os.listdir("flashdata/"):
        setnames.append(path)
    setnamesStrVar = StringVar()
    setnamesStrVar.set(setnames[0])
    dropdownsetsel = ttk.OptionMenu(tp, setnamesStrVar, *setnames)
    setselbut = Button(tp, text = "Set Options:", command = selectsetoptions)
    dropdownsetsel.pack()
    setselbut.pack()

def selectsetoptions():
    global selectsetdata
    filenames = setnamesStrVar.get()
    # Gets the name of the set picked to be edited
    with open(f"flashdata/{filenames}", "r") as file:
        selectsetdata = []
        file.seek(0)
        selectsetdata = file.read().splitlines()
        file.close()
    # Opens the slected set file and puts all the data into a 1D list formatted [word, defintion]
    tpp = Toplevel()
    tpp.title("Flashcard_Studio/Sets/Mode")
    flashcardmodebutton = Button(tpp, text = "Flashcards", command = flashcardmode)
    flashcardmodebutton.pack()
    # Currently there is only one option for flashcards available, and that is flashcardmode()

def flashcardmode():
    global dataindex
    global flipped
    global flashcardflip
    flipped = False
    dataindex = 0
    fm = Toplevel()
    fm.geometry("1200x625")
    fm.title("Flashcard_Studio/Sets/Mode/Flashcards")
    flashcardflip = Button(fm, text = selectsetdata[dataindex], height = 10, width = 120, command = flip)
    # The flashcards are just a massive button that uses flip() to change the text on the button.
    prevflashbutton = Button(fm, text = "<", command = prevflash)
    nextflashbutton = Button(fm, text = ">", command = nextflash)
    flashcardflip.pack()
    prevflashbutton.pack()
    nextflashbutton.pack()

def flip():
    global flipped
    if flipped == False:
        flashcardflip.config(text = selectsetdata[dataindex+1])
        # Because of the way the list of flashcards is formatted ([word, definition]), the
        # flashcards are in groups of 2.
        flipped = True
    else:
        flashcardflip.config(text = selectsetdata[dataindex])
        flipped = False

def prevflash():
    global dataindex
    dataindex = dataindex - 2
    print(dataindex)

def nextflash():
    global dataindex
    dataindex = dataindex + 2
    print(dataindex)

def scrape():
    HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36"
    }
    soup = BeautifulSoup(requests.get(entry_text.get(), headers=HEADERS).text, "html.parser")
    # Gets the actual HTML file of the Quizlet that you would like to scrape.
    flashcard_title = soup.find("h1").text

    file = open(f'flashdata/{flashcard_title}.txt', 'w')
    # Creates and opens a new file or opens an old one ready for re/writing

    setdata = soup.find_all("div", class_ = "SetPageTerms-term")
    setdata += soup.find_all("div", class_ = "SetPageTerms-term SetPageTerms-term--beyondSignupThreshold")
    # Collecting the setdata divs, not the raw data.

    for index, flashcards in enumerate(setdata):
        flashlist = setdata[index]
        flashword = flashlist.find("a", class_ = "SetPageTerm-wordText").text
        flashdefinition = flashlist.find("a", class_ = "SetPageTerm-definitionText").text
        with open(f"flashdata/{flashcard_title}.txt", "a") as file:
            file.write(f"w: {flashword}\n")
            file.write(f"def: {flashdefinition}\n")
    file.close()

root = Tk()

root.title("Flashcard_Studio/Scrape")
root.minsize(700,100)
root.maxsize(700,100)

label = Label(text = "Quizlet Link:")
entry_text = StringVar()
entry = Entry(textvariable = entry_text, width = 75)
button1 = Button(text = "Get Flashcards", command = scrape)
button2 = Button(root, text = "Review Sets", command = review_button)
label.pack()
entry.pack()
button1.pack()
button2.pack()

root.mainloop()


import torch
import torch.nn as nn
import torch.nn.functional as F

class DeviationLoss(nn.Module):

    def __init__(self):
        super().__init__()

    def forward(self, y_pred, y_true):
        confidence_margin = 2.
        # confidence_margin = 0.5
        # print(y_pred,y_true)
        # size=5000 is the setting of l in algorithm 1 in the paper

        # contain_one = torch.nonzero(y_true>0)
        # if contain_one.shape[0] > 1:
        #     # contain_one = 0
        #     print(contain_one.shape[0])
        #     for i in range(len(contain_one)):
        #         print(y_pred[contain_one[i][0]])

        # ref = torch.abs(torch.normal(mean=0., std=torch.full([5000], 1.))).cuda()
        ref = torch.normal(mean=0.1, std=torch.full([5000], 2.).cuda())
        dev = (y_pred - torch.mean(ref)) / torch.std(ref)
        # dev = (y_pred - torch.mean(ref)) / torch.std(ref)
        # ????????????????????????????????????????
        inlier_loss = torch.pow(torch.tanh(dev), 8)
        # ??????????
        outlier_loss = torch.pow(torch.tanh((confidence_margin - dev)), 2)
        # outlier_loss = torch.abs(torch.tanh((confidence_margin - dev)))
        # return torch.mean((1-alpha)*(1 - y_true) * inlier_loss ** self.gamma + alpha*y_true * outlier_loss ** self.gamma)
        # return torch.mean(y_pred ** self.gamma * (1 - y_true) * inlier_loss + (1-y_pred) ** self.gamma * y_true * outlier_loss)
        loss = (1 - y_true) * inlier_loss + y_true * outlier_loss
        # loss_normal = F.normalize(loss,p=1,dim=1)
        return torch.mean(loss)

# 这个损失函数可以用于二分类问题的深度异常检测。其基本思路是通过标准化模型的输出，
# 将正常样本的得分分布映射到一个均值为0，方差为1的标准正态分布上，再利用一个置信度阈值对异常样本进行分类。
# 在损失函数的计算中，使用了正常样本得分的均值和标准差来标准化模型输出，
# 然后使用两个不同的损失项来惩罚正常样本和异常样本，以便在分类中更好地区分它们。
# 具体而言，对于一个输入样本，将其输出值进行标准化，得到样本的偏差值（dev），
# 然后使用tanh函数将其映射到(-1,1)之间。对于正常样本，使用偏差值的平方乘以一个较大的gamma作为损失项；
# 对于异常样本，使用置信度阈值（confidence_margin）和偏差值之间的差的平方作为损失项。
# 最终的损失函数是两个损失项的加权和，权重是一个二元变量y_true，表示输入样本是否为异常样本。
# 这个损失函数的优点是简单易懂，易于实现，并且对于大多数深度异常检测问题具有很好的效果。
# 但是，对于一些特定的异常检测问题，可能需要根据实际情况进行调整。

class TransDLossBinary(nn.Module):

    def __init__(self, gamma=2 ,c_margin=1.4):
        super().__init__()
        self.gamma = gamma
        self.c_margin = c_margin

    def forward(self, y_pred, y_true):
        confidence_margin = self.c_margin
        gamma = self.gamma
        # 离散正态分布中抽取随机数

        # 查看异常数据评分
        # contain_one = torch.nonzero(y_true>0)
        # if contain_one.shape[0] > 1:
        #     # contain_one = 0
        #     print(contain_one.shape[0])
        #     for i in range(len(contain_one)):
        #         print(y_pred[contain_one[i][0]])

        # 较宽的分布能够更好地适应数据的变化和噪声，因此模型的训练效果可能会更好。
        ref = torch.normal(mean=0.1, std=torch.full([5000], 2.).cuda())
        # std_ref = torch.std(ref)
        # mean_ref = torch.mean(ref)
        dev = (y_pred - torch.mean(ref)) / torch.std(ref)
        # 默认正常点分布都在定义的混合高斯区间之中
        inlier_loss = torch.pow(torch.tanh(dev), 2*gamma)
        # 离群点损失
        outlier_loss = torch.pow(torch.tanh(confidence_margin - dev), 2)
        return torch.mean((1 - y_true) * inlier_loss + y_true * outlier_loss)

class DeviationEmbedF1Loss(nn.Module):
    def __init__(self,gamma):
        super().__init__()
        self.gamma = gamma

    def forward(self, y_pred, y_true):
        confidence_margin = 1.2
        # print(y_pred,y_true)
        # size=5000 is the setting of l in algorithm 1 in the paper
        # 离散正态分布中抽取随机数

        # 查看异常数据评分
        # contain_one = torch.nonzero(y_true>0)
        # if contain_one.shape[0] > 1:
        #     # contain_one = 0
        #     print(contain_one.shape[0])
        #     for i in range(len(contain_one)):
        #         print(y_pred[contain_one[i][0]])

        # 标准化
        # ref = torch.abs(torch.normal(mean=0.1, std=torch.full([5000], 0.7))).cuda()
        ref = torch.normal(mean=0.1, std=torch.full([5000], 1.).cuda())
        dev = (y_pred - torch.mean(ref)) / torch.std(ref)
        # dev = (y_pred - torch.mean(ref)) / torch.std(ref)
        # 默认正常点分布都在定义的混合高斯区间之中
        inlier_loss = torch.pow(torch.tanh(dev), self.gamma*2)
        # 离群点损失
        outlier_loss = torch.pow(torch.tanh((confidence_margin - dev)), 2)
        loss = (1 - y_true) * inlier_loss + y_true * outlier_loss
        # loss_normal = F.normalize(loss,p=1,dim=1)

        f1_score = self.f1_score_loss(y_true, y_pred)

        # return torch.mean(loss)

        return torch.mean(0.5 * loss+0.5 * f1_score)
    def f1_score_loss(self, y_true, y_pred):
        alpha = 0.75
        pt = y_pred
        loss = - alpha * (1 - pt) ** self.gamma * y_true * torch.log(pt) - \
               (1 - alpha) * pt ** self.gamma * (1 - y_true) * torch.log(1 - pt)
        # if self.reduction == 'elementwise_mean':
        loss = torch.mean(loss)
        # elif self.reduction == 'sum':
        #     loss = torch.sum(loss)
        return loss

class BCEFocalLoss(torch.nn.Module):
    """
    二分类的Focalloss alpha 固定
    """
    def __init__(self, gamma=2, alpha=0.75, reduction='elementwise_mean'):
        super().__init__()
        self.gamma = gamma
        self.alpha = alpha
        self.reduction = reduction

    def forward(self, _input, target):
        # pt = torch.sigmoid(_input)
        pt = _input
        alpha = self.alpha
        loss = - alpha * (1 - pt) ** self.gamma * target * torch.log(pt) - \
               (1 - alpha) * pt ** self.gamma * (1 - target) * torch.log(1 - pt)
        if self.reduction == 'elementwise_mean':
            loss = torch.mean(loss)
        elif self.reduction == 'sum':
            loss = torch.sum(loss)
        return loss

class BCEFocalLossFew(torch.nn.Module):
    """
    二分类的Focalloss alpha 固定
    """
    def __init__(self, gamma=4, alpha=0.8, reduction='elementwise_mean'):
        super().__init__()
        self.gamma = gamma
        self.alpha = alpha
        self.reduction = reduction

    def forward(self, _input, target):
        # pt = torch.sigmoid(_input)
        pt = _input
        alpha = self.alpha
        # loss = - alpha * (1 - pt) ** self.gamma * target * torch.log(pt) - \
        #        (1 - alpha) * pt ** self.gamma * (1 - target) * torch.log(1 - pt)
        loss = - alpha * (1 - pt) ** 1 * target * torch.log(pt) - \
               (1 - alpha) * pt ** 8 * (1 - target) * torch.log(1 - pt)
        if self.reduction == 'elementwise_mean':
            loss = torch.mean(loss)
        elif self.reduction == 'sum':
            loss = torch.sum(loss)
        return loss

class BinaryFocalLoss(nn.Module):
    def __init__(self, alpha=1, gamma=2, logits=True, reduce=True):
        super(BinaryFocalLoss, self).__init__()
        self.alpha = alpha
        self.gamma = gamma
        self.logits = logits
        self.reduce = reduce

    def forward(self, inputs, targets):
        if self.logits:
            BCE_loss = F.binary_cross_entropy_with_logits(inputs, targets, reduction='none')
        else:
            BCE_loss = F.binary_cross_entropy(inputs, targets, reduction='none')
        pt = torch.exp(-BCE_loss)
        F_loss = self.alpha * (1-pt)**self.gamma * BCE_loss

        if self.reduce:
            return torch.mean(F_loss)
        else:
            return F_loss

# 构造损失
def build_criterion(criterion, gamma=2,c_margin=1.4):
    if criterion == "deviation":
        # 自定义偏差损失
        print("Loss : Deviation")
        return DeviationLoss()
    elif criterion == "TransDLossBinary":
        # 自定义偏差损失
        print("Loss : Transformer deviation Binary")
        return TransDLossBinary(gamma,c_margin)
    elif criterion == "BCE":
        print("Loss : Binary Cross Entropy")
        return torch.nn.BCEWithLogitsLoss()
    elif criterion == "BCEfocal":
        print("Loss : BCEfocal")
        return BCEFocalLoss()
    elif criterion == "deviEmbedfocal":
        # 偏差损失+focal loss
        print("Loss : deviation Embedding focal loss")
        return DeviationEmbedF1Loss(gamma)
    elif criterion == "BCEFew":
        print("Loss : Binary Cross Entropy Few")
        return BCEFocalLossFew()
    else:
        raise NotImplementedError


github_storage = "https://raw.githubusercontent.com/FacultadInformatica-LinkedData/Curso2020-2021/master/Assignment4"

"""Leemos el fichero RDF de la forma que lo hemos venido haciendo"""

from rdflib import Graph, Namespace, Literal
from rdflib.namespace import RDF, RDFS
g = Graph()
g.namespace_manager.bind('ns', Namespace("http://somewhere#"), override=False)
g.namespace_manager.bind('vcard', Namespace("http://www.w3.org/2001/vcard-rdf/3.0#"), override=False)
g.parse(github_storage+"/resources/example6.rdf", format="xml")

ns = Namespace("http://somewhere#")

for s, p, o in g:
    print(s, p, o)

""" Task 7.1 """
print("Task 7.1 SPARQL")
print("------------------------------")
sparql_query = """ select distinct ?subClass where
{
    ?subClass <http://www.w3.org/2000/01/rdf-schema#subClassOf> <http://somewhere#Person>.
}
"""
result = g.query(sparql_query)
for row in result:
    print(f"{row.subClass}")

print("------------------------------\n")

print("Task 7.1 RDFlib")
print("------------------------------")

for s, p, o in g.triples((None, RDFS.subClassOf, ns.Person)):
    print(s)

print("------------------------------\n")

""" Task 7.2 """
print("Task 7.2")
print("------------------------------")
sparql_query = """ select distinct ?instance where
{
    ?class <http://www.w3.org/2000/01/rdf-schema#subClassOf>* <http://somewhere#Person>.
    ?instance <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> ?class.
}
"""
result = g.query(sparql_query)
for row in result:
    print(f"{row.instance}")

print("------------------------------\n")

print("Task 7.2 RDFlib")
print("------------------------------")

def get_instance_nested_classes(class_):
    for s, p, o in g.triples((None, RDF.type, class_)):
        print(s)
    for s, p, o in g.triples((None, RDFS.subClassOf, class_)):
        get_instance_nested_classes(s)

get_instance_nested_classes(ns.Person)

print("------------------------------\n")

""" Task 7.3 """
print("Task 7.3")
print("------------------------------")
sparql_query = """ select distinct ?instance ?property ?value ?instanceClass where
{
    ?class <http://www.w3.org/2000/01/rdf-schema#subClassOf>* <http://somewhere#Person>.
    ?instance <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> ?class;
    ?property ?value.
}
"""
result = g.query(sparql_query)
for row in result:
    print(f"Individual: {row.instance}, property: {row.property}, value: {row.value}")

print("------------------------------\n")

print("Task 7.3 RDFlib")
print("------------------------------")

def get_properties_nested_classes(class_):
    for s, p, o in g.triples((None, RDF.type, class_)):
        for s, p, o in g.triples((s, None, None)):
            print(f"Individual: {s}, property: {p}, value: {o}")
    for s, p, o in g.triples((None, RDFS.subClassOf, class_)):
        get_properties_nested_classes(s)

get_properties_nested_classes(ns.Person)

print("------------------------------\n")


import amulet
from amulet.api.selection import SelectionGroup, SelectionBox
import os

def schematics_to_world():

    level = amulet.load_level( os.path.realpath(__file__) + "\..\city_map")
    baseFolder = (os.path.realpath(__file__) + "/../")
    schematics = []
    #Goes throuhg every .schematic in the result folder, and adds them to the city_map minecraft world at the coordinates specified in the filenames
    for filename in os.listdir(baseFolder + "result"):
        parsed_filename = filename.split(".")[0].split("-")
        if (len(parsed_filename) < 2):
            continue
        x = int(parsed_filename[1])*-250
        y = int(parsed_filename[0])*-250
        y_origin = int(parsed_filename[2])
        structure = amulet.load_level(os.path.join(baseFolder + "result/"+filename))
        water_fill = amulet.load_level(os.path.join(baseFolder + "settings/water_fill.construction"))
        boundingbox = structure.bounds(structure.dimensions[0])
        selection_box = boundingbox.selection_boxes[0]
        #If the tile has some coast in it, remove the the bottom layer of blocks to make way for actual minecraft water
        if (y_origin == 0):
            boundingbox = SelectionGroup([SelectionBox((selection_box.point_1[0],1,selection_box.point_1[2]),(selection_box.point_2[0], selection_box.point_2[1], selection_box.point_2[2]))])
        #Copy the city model .schematic file into the world in the correct position and rotation
        level.paste(
        structure,
        structure.dimensions[0],
        boundingbox,
        level.dimensions[0],
        (x,10+(selection_box.point_2[1])/2+y_origin,y),
        scale=(1.0, 1.0, -1.0),
        rotation=(0.0, 90.0, 0.0),
        include_blocks=True,
        include_entities=False,
        skip_blocks=(),
        copy_chunk_not_exist=False)

        #If coast line, paste water under
        if (y_origin == 0):
            level.paste(
            water_fill,
            water_fill.dimensions[0],
            water_fill.bounds(water_fill.dimensions[0]),
            level.dimensions[0],
            (x,water_fill.bounds(water_fill.dimensions[0]).selection_boxes[0].point_2[1]/2,y),
            scale=(1.0, 1.0, 1.0),
            rotation=(0.0, 0.0, 0.0),
            include_blocks=True,
            include_entities=False,
            skip_blocks=(),
            copy_chunk_not_exist=False
            )

    # save the changes to the world
    level.save()

    # close the world
    level.close()
    for filename in os.listdir(baseFolder + "result"):
        if os.path.exists(baseFolder+ "result/"+filename):
            os.remove(baseFolder+"result/"+filename)


import numpy as np
from gym import spaces
import tensorflow as tf

high = [np.finfo(np.float32).max, np.finfo(np.float32).max]
low = [0, 0]
high = np.array(high)
low = np.array(low)
observation_space = spaces.Box(low, high)
print(observation_space.shape[-1])
a = [[128, 10]]
print(np.array(a).reshape([-1, observation_space.shape[-1]]))


import sys
sys.stdin = open('12764.txt')
import heapq

# 1. 시작 순서가 빠른대로 정렬한다.
# 2. 한명씩 빈자리로 출발.
# 3. 전부다 사용 중이라면 새로운 컴퓨터 생성
N = int(input())
soldiers = [list(map(int, input().split())) for _ in range(N)]
soldiers.sort()

computer = dict()
computer_num = 1
pq = []
possible_use = []
for start, end in soldiers:

    if not pq:
        heapq.heappush(pq, (end, computer_num))
        computer[computer_num] = 1
        computer_num += 1

    else:
        while pq:
            finish, cn = heapq.heappop(pq)
            if finish > start:
                heapq.heappush(pq, (finish, cn))
                break
            heapq.heappush(possible_use, cn)

        if possible_use:
            num = heapq.heappop(possible_use)
            heapq.heappush(pq, (end, num))
            computer[num] += 1
        else:
            heapq.heappush(pq, (end, computer_num))
            computer[computer_num] = 1
            computer_num += 1

print(computer_num - 1)
for i in computer.values():
    print(i, end=' ')


import datetime
import time

from django.utils import translation
from django.utils.translation import ugettext as _

from dataflow.flow.exceptions import NodeValidError
from dataflow.flow.handlers.node_utils import NodeUtils
from dataflow.flow.models import FlowNodeRelation
from dataflow.flow.node_types import NodeTypes
from dataflow.flow.utils.concurrency import concurrent_call_func
from dataflow.pizza_settings import BATCH_MODULE_NAME, STREAM_MODULE_NAME
from dataflow.shared.datamanage.datamanage_helper import DatamanageHelper
from dataflow.shared.log import flow_logger as logger
from dataflow.shared.meta.processing.data_processing_helper import DataProcessingHelper
from dataflow.shared.meta.result_table.result_table_helper import ResultTableHelper

"""
监控相关
数据流监控：
1. 获取整个 flow 的状态
2. 获取 flow 中单个节点的详细状态
"""
DMONITOR_ALERTS = "dmonitor_alerts"
DATA_LOSS_IO_TOTAL = "data_loss_io_total"
RESULT_TABLE_STATUS = "rt_status"
DATA_DELAY_MAX = "data_delay_max"
DATA_TREND = "data_trend"
STREAM_UNIT = _("条/分钟")
# 间隔1天
INTERVAL = 1
# 1 小时，60 分钟
START_TIME_ONE_HOUR = "now() - 1h"
# 1 天
START_TIME_ONE_DAY = "now() - 1d"
# 7 天，一周
START_TIME_ONE_WEEK = "now() - 7d"
# 30 天，一个月
START_TIME_ONE_MONTH = "now() - 30d"
# 90 天，三个月
START_TIME_THREE_MONTH = "now() - 90d"
# 常量
ONE_DAY_SECOND = 24 * 60 * 60
PERIOD_HOUR = "hour"
PERIOD_DAY = "day"
PERIOD_WEEK = "week"
PERIOD_MONTH = "month"

class MonitorHandler(object):
    # 各种节点支持的打点类型
    DEFAULT_NODE_MONITOR_MAP = {
        # 实时类别埋点
        NodeTypes.STREAM_SOURCE: [DATA_DELAY_MAX, DATA_TREND],
        NodeTypes.STREAM: [DATA_DELAY_MAX, DATA_TREND],
        NodeTypes.DATA_MODEL_APP: [DATA_DELAY_MAX, DATA_TREND],
        NodeTypes.DATA_MODEL_STREAM_INDICATOR: [
            DATA_DELAY_MAX,
            DATA_TREND,
        ],
        # 离线类别埋点
        NodeTypes.BATCH_SOURCE: [RESULT_TABLE_STATUS, DATA_TREND],
        NodeTypes.BATCH: [RESULT_TABLE_STATUS, DATA_TREND],
        NodeTypes.BATCHV2: [RESULT_TABLE_STATUS, DATA_TREND],
        NodeTypes.DATA_MODEL_BATCH_INDICATOR: [
            RESULT_TABLE_STATUS,
            DATA_TREND,
        ],
        NodeTypes.MODEL_APP: [RESULT_TABLE_STATUS, DATA_TREND],
        # 存储节点
        NodeTypes.MYSQL_STORGAE: [DATA_DELAY_MAX, DATA_TREND],
        NodeTypes.POSTGRESQL_STORAGE: [DATA_DELAY_MAX, DATA_TREND],
        NodeTypes.HDFS_STORAGE: [DATA_DELAY_MAX, DATA_TREND],
        NodeTypes.HERMES_STORAGE: [DATA_DELAY_MAX, DATA_TREND],
        NodeTypes.DRUID_STORAGE: [DATA_DELAY_MAX, DATA_TREND],
        NodeTypes.TREDIS_STORAGE: [DATA_DELAY_MAX, DATA_TREND],
        NodeTypes.IGNITE_STORAGE: [DATA_DELAY_MAX, DATA_TREND],
        NodeTypes.TSPIDER_STORAGE: [DATA_DELAY_MAX, DATA_TREND],
        NodeTypes.TCAPLUS_STORAGE: [DATA_DELAY_MAX, DATA_TREND],
        NodeTypes.CLICKHOUSE_STORAGE: [DATA_DELAY_MAX, DATA_TREND],
        NodeTypes.QUEUE_STORAGE: [DATA_DELAY_MAX, DATA_TREND],
        NodeTypes.QUEUE_PULSAR_STORAGE: [DATA_DELAY_MAX, DATA_TREND],
        NodeTypes.ES_STORAGE: [DATA_DELAY_MAX, DATA_TREND],
        NodeTypes.TSDB_STORAGE: [DATA_DELAY_MAX, DATA_TREND]
        # 暂无埋点
        # NodeTypes.KV_SOURCE: [DATA_DELAY_MAX, DATA_TREND],
        # NodeTypes.UNIFIED_KV_SOURCE: [DATA_DELAY_MAX, DATA_TREND],
        # NodeTypes.TDW_BATCH: [RESULT_TABLE_STATUS, DATA_TREND],
        # NodeTypes.TDW_JAR_BATCH: [RESULT_TABLE_STATUS, DATA_TREND],
        # NodeTypes.TDW_STORAGE: [DATA_DELAY_MAX, DATA_TREND],
        # NodeTypes.PROCESS_MODEL: [DATA_DELAY_MAX, DATA_TREND, RESULT_TABLE_STATUS],
        # NodeTypes.MODEL_TS_CUSTOM: [DATA_DELAY_MAX, DATA_TREND, RESULT_TABLE_STATUS]
    }

    # 实时类埋点节点
    STREAM_NODE_MONITOR = [
        NodeTypes.STREAM_SOURCE,
        NodeTypes.STREAM,
        NodeTypes.DATA_MODEL_APP,
        NodeTypes.DATA_MODEL_STREAM_INDICATOR,
    ]

    # 离线类埋点
    BATCH_NODE_MONITOR = [
        NodeTypes.BATCH_SOURCE,
        NodeTypes.BATCH,
        NodeTypes.BATCHV2,
        NodeTypes.DATA_MODEL_BATCH_INDICATOR,
        NodeTypes.MODEL_APP,
    ]

    @staticmethod
    def get_monitor_config_by_node(node):
        node_type = node.node_type
        if node_type in NodeTypes.SOURCE_CATEGORY:
            # 获取rt结果表在其它flow的节点类型
            origin_rt = FlowNodeRelation.objects.filter(
                result_table_id=node.result_table_ids[0],
                node_type__in=NodeTypes.CALC_CATEGORY,
            )
            if origin_rt is None or origin_rt.count() == 0:
                # 若数据源节点非RT数据源
                return {}
            else:
                node_type = origin_rt[0].node_type
        if node_type in [
            NodeTypes.STREAM,
            NodeTypes.DATA_MODEL_APP,
            NodeTypes.DATA_MODEL_STREAM_INDICATOR,
        ]:
            return {"module": STREAM_MODULE_NAME, "component": "*"}
        elif node_type in NodeTypes.BATCH_CATEGORY:
            return {
                "module": BATCH_MODULE_NAME,
                "component": "*",
            }
        else:
            return {}

    def __init__(self):
        pass

    def list_monitor_data(self):
        """
        获得监控打点集合下指定类型的数据
        """
        raise NotImplementedError("`list_monitor_data()` must be implemented.")

    def get_input_output_loss_node_handler(self, node_handler, input_output_tag, data_format="series"):
        """
        根据节点，获取它的 input、output 指标
        input_output_tag = input_count/output_count
        format = 'series' : 取序列值，时间戳 + 字段
        format = 'value' : 取求和后的值，没有时间戳
        """
        node_type = node_handler.node_type
        if node_type in self.STREAM_NODE_MONITOR:
            return MonitorUtil.get_stream_input_output_metrics(node_handler, input_output_tag, data_format)
        elif node_type in self.BATCH_NODE_MONITOR:
            # 离线计算打点
            return MonitorUtil.get_batch_input_output_metrics(node_handler, input_output_tag, data_format)
        else:
            # 存储节点，看它的上游是实时还是离线类型节点，再定指标
            from_process_node = NodeUtils.get_storage_upstream_node_handler(node_handler.node_id)
            if from_process_node.node_type in self.STREAM_NODE_MONITOR:
                return MonitorUtil.get_stream_input_output_metrics(node_handler, input_output_tag, data_format)
            elif from_process_node.node_type in self.BATCH_NODE_MONITOR:
                return MonitorUtil.get_batch_input_output_metrics(node_handler, input_output_tag, data_format)
            else:
                return {}

class MonitorUtil(object):
    # -------- 组装 json 工具类
    @staticmethod
    def merge_input_and_output_data(input_dict, output_dict, result_table_id):
        if not input_dict:
            return {}
        input_dict["output"] = output_dict["output"]
        input_dict["data"][result_table_id]["output"] = output_dict["output"]
        return input_dict

    @staticmethod
    def get_input_or_output_metrics_data(processing_data_list, input_output_tag, data_format):
        """
        [
            {
                "data_set_id": "591_abc_123",
                "storage_cluster_type": "None",
                "node_id": "26016",
                "value": {
                    "input_count": 958327,
                    "time": 1617699342
                },
                "flow_id": "8514"
            },
            {
                "data_set_id": "591_abc_123",
                "storage_cluster_type": "['None', None]",
                "node_id": "26016",
                "value": {
                    "input_count": 0,
                    "time": 1617699342
                },
                "flow_id": "8514"
            },
            {
                "data_set_id": "591_abc_123",
                "storage_cluster_type": "kafka",
                "node_id": "26016",
                "value": {
                    "input_count": 0,
                    "time": 1617699342
                },
                "flow_id": "8514"
            }
        ]
        无论是 input/output，查询出来有多个值。取 input_count/output_count 最大的那个 dict
        """
        # 有多条，input_count/output_count 按条件取值，只有实时类有多条，内存或者kafka
        # 1. input_count 取内存中的值 storage_cluster_type: None
        # 2. output_count 取组件中的值 storage_cluster_type: kafka
        # processing_data_list 可能为空
        if data_format == "value":
            input_or_output_value = 0
            if not processing_data_list:
                return 0
            processing_data_len = len(processing_data_list)
            if processing_data_len > 1:
                # 有内存和 kafka
                for item in processing_data_list:
                    if input_output_tag == "input_count" and item["storage_cluster_type"] == "None":
                        return item["value"][input_output_tag]
                    if input_output_tag == "output_count" and item["storage_cluster_type"] == "kafka":
                        return item["value"][input_output_tag]
            elif processing_data_len == 1:
                # 只有一个值
                return processing_data_list[0]["value"][input_output_tag]
            return input_or_output_value
        else:
            # series
            result_dict = {}
            processing_data_len = len(processing_data_list)
            if processing_data_len > 1:
                for item in processing_data_list:
                    if input_output_tag == "input_count" and item["storage_cluster_type"] == "None":
                        return item
                    if input_output_tag == "output_count" and item["storage_cluster_type"] == "kafka":
                        return item
            elif processing_data_len == 1:
                # 只有一个值
                return processing_data_list[0]
            return result_dict

    @staticmethod
    def build_input_output_sum_dict(input_output_tag, result_table_id, input_output_tag_sum, node_type, unit):
        if input_output_tag == "input_count":
            return_dict = {
                "data": {
                    result_table_id: {
                        "output": 0,
                        "unit": unit,
                        "input": input_output_tag_sum,
                    }
                },
                "node_type": node_type,
                "output": 0,
                "unit": unit,
                "input": input_output_tag_sum,
            }
        else:
            return_dict = {
                "data": {
                    result_table_id: {
                        "output": input_output_tag_sum,
                        "unit": unit,
                        "input": 0,
                    }
                },
                "node_type": node_type,
                "output": input_output_tag_sum,
                "unit": unit,
                "input": 0,
            }
        return return_dict

    @staticmethod
    def build_input_output_series_dict(input_output_tag, result_table_id, result_dict):
        # 格式转化成前端的格式，返回
        data_dict = {}
        input_list = []
        time_list = []
        if result_dict and result_dict["series"]:
            for item in result_dict["series"]:
                input_list.append(item[input_output_tag])
                time_list.append(item["time"])
            if input_output_tag == "input_count":
                data_dict[result_table_id] = {"input": input_list, "time": time_list}
            else:
                data_dict[result_table_id] = {"output": input_list, "time": time_list}
        return data_dict

    @staticmethod
    def get_batch_monitor_unit(scheduler_period):
        """
        获取监控显示的数据吞吐量单位
        """
        unit = _("条/小时")
        if scheduler_period == PERIOD_HOUR:
            unit = _("条/小时")
        elif scheduler_period == PERIOD_DAY:
            unit = _("条/天")
        elif scheduler_period == PERIOD_WEEK:
            unit = _("条/周")
        elif scheduler_period == PERIOD_MONTH:
            unit = _("条/月")
        return unit

    @staticmethod
    def get_stream_input_output_metrics(node_handler, input_output_tag, data_format="series"):
        """
        实时类的指标，计算单位：条/分钟
        计算策略：计算最近 10 分钟的节点，除以 10 得到条/分钟
        包括：
        实时数据源
        实时计算，数据模型，实时指标
        """
        node_type = node_handler.node_type
        result_table_id = node_handler.result_table_ids[0]
        if node_type == NodeTypes.STREAM_SOURCE:
            # 区别：数据源传参 data_set_ids，计算节点传参 node_ids
            stream_source_data_list = DatamanageHelper.get_result_table_metric_v2(
                {
                    "measurement": input_output_tag,
                    "data_set_ids": result_table_id,
                    "start_time": START_TIME_ONE_HOUR,
                    "fill": "none",
                    "format": data_format,
                }
            )
        else:
            stream_source_data_list = DatamanageHelper.get_result_table_metric_v2(
                {
                    "measurement": input_output_tag,
                    "node_ids": [node_handler.node_id],
                    "start_time": START_TIME_ONE_HOUR,
                    "fill": "none",
                    "format": data_format,
                }
            )
        if data_format == "value":
            input_output_sum = (
                MonitorUtil.get_input_or_output_metrics_data(stream_source_data_list, input_output_tag, data_format)
                / 60
            )
            return MonitorUtil.build_input_output_sum_dict(
                input_output_tag,
                result_table_id,
                input_output_sum,
                node_type,
                STREAM_UNIT,
            )
        else:
            # series
            input_output_result_dict = MonitorUtil.get_input_or_output_metrics_data(
                stream_source_data_list, input_output_tag, data_format
            )
            return MonitorUtil.build_input_output_series_dict(
                input_output_tag, result_table_id, input_output_result_dict
            )

    @staticmethod
    def get_batch_metrics_start_time(schedule_period):
        if schedule_period == PERIOD_HOUR:
            # 24 小时取平均
            start_time = START_TIME_ONE_DAY
            ave_num = 24
        elif schedule_period == PERIOD_DAY:
            # 7 天取平均
            start_time = START_TIME_ONE_WEEK
            ave_num = 7
        elif schedule_period == PERIOD_WEEK:
            # 4 周取平均
            start_time = START_TIME_ONE_MONTH
            ave_num = 4
        else:
            # PERIOD_MONTH
            # 3 月取平均
            start_time = START_TIME_THREE_MONTH
            ave_num = 3
        return start_time, ave_num

    @staticmethod
    def get_batch_input_output_metrics(node_handler, input_output_tag, data_format="series"):
        """
        离线类的指标，计算单位：跟窗口保持一致
        计算策略：
        包括：
        离线数据源，离线计算等
        """
        node_type = node_handler.node_type
        result_table_id = node_handler.result_table_ids[0]
        if node_type == NodeTypes.BATCH_SOURCE:
            from_batch_node = node_handler.get_rt_generated_node()
            if from_batch_node:
                schedule_period = from_batch_node.get_schedule_period_from_batch_node()
            else:
                # 清洗表，上游无离线计算
                schedule_period = PERIOD_HOUR
            start_time, ave_num = MonitorUtil.get_batch_metrics_start_time(schedule_period)
            batch_source_data_list = DatamanageHelper.get_result_table_metric_v2(
                {
                    "measurement": input_output_tag,
                    "data_set_ids": result_table_id,
                    "start_time": start_time,
                    "fill": "none",
                    "format": data_format,
                }
            )
        else:
            if node_type in NodeTypes.STORAGE_CATEGORY:
                schedule_period = NodeUtils.get_storage_upstream_node_handler(
                    node_handler.node_id
                ).get_schedule_period_from_batch_node()
            else:
                schedule_period = node_handler.get_schedule_period_from_batch_node()
            start_time, ave_num = MonitorUtil.get_batch_metrics_start_time(schedule_period)
            batch_source_data_list = DatamanageHelper.get_result_table_metric_v2(
                {
                    "measurement": input_output_tag,
                    "node_ids": [node_handler.node_id],
                    "start_time": start_time,
                    "fill": "none",
                    "format": data_format,
                }
            )
        if data_format == "value":
            input_output_sum = (
                MonitorUtil.get_input_or_output_metrics_data(batch_source_data_list, input_output_tag, data_format)
                / ave_num
            )
            return MonitorUtil.build_input_output_sum_dict(
                input_output_tag,
                result_table_id,
                input_output_sum,
                node_type,
                MonitorUtil.get_batch_monitor_unit(schedule_period),
            )
        else:
            # series
            input_output_result_dict = MonitorUtil.get_input_or_output_metrics_data(
                batch_source_data_list, input_output_tag, data_format
            )
            return MonitorUtil.build_input_output_series_dict(
                input_output_tag, result_table_id, input_output_result_dict
            )

class FlowMonitorHandlerSet(MonitorHandler):
    """
    Flow监控打点集合
    """

    DEFAULT_MONITOR_TYPES = [DMONITOR_ALERTS, DATA_LOSS_IO_TOTAL, RESULT_TABLE_STATUS]

    def __init__(self, flow):
        self.flow = flow
        super(FlowMonitorHandlerSet, self).__init__()

    def list_monitor_data(self):
        monitor_types = self.DEFAULT_MONITOR_TYPES
        supported_monitor = {
            DMONITOR_ALERTS: self.get_flow_monitor_alerts,
            DATA_LOSS_IO_TOTAL: self.get_flow_data_loss_io_total,
            RESULT_TABLE_STATUS: self.get_flow_rt_status,
        }
        func_info = []
        monitor_data = {}
        for monitor_type in monitor_types:
            func_info.append([supported_monitor[monitor_type], {}])
        threads_res = concurrent_call_func(func_info)
        for idx, monitor_type in enumerate(monitor_types):
            monitor_data.update({monitor_type: threads_res[idx]})

        return monitor_data

    def get_flow_monitor_alerts(self):
        """
        告警信息监控打点
        @return:
        [
            {
                'message': 'xxx',
                'message_en': 'xxx',
                'full_message': 'xxx',
                'full_message_en': 'xxx',
                'msg': 'xxx',
                'node_id': 'xxx',
                'time': 1544174262
            }
        ]
        """
        current_language = translation.get_language()
        lang = "" if current_language in ["zh-hans", "zh-cn"] else "_%s" % current_language
        # 界面实际要读取的监控信息，只取这个监控信息
        msg_key = "full_message%s" % lang
        flow_id = self.flow.flow_id
        now = datetime.datetime.now()
        end_time = now.strftime("%Y-%m-%d %H:%M:%S")
        start_time = (now + datetime.timedelta(days=-1 * INTERVAL)).strftime("%Y-%m-%d %H:%M:%S")
        data = DatamanageHelper.get_alert_details(flow_id, start_time, end_time)
        for _d in data:
            if "alert_time" in _d:
                time_array = time.strptime(_d["alert_time"], "%Y-%m-%d %H:%M:%S")
                _d["time"] = int(time.mktime(time_array))
                _d["msg"] = _d[msg_key] if msg_key in _d else _("获取指定监控信息错误.")
            else:
                logger.error("日志格式错误: %s" % _d)
        return data

    def get_flow_data_loss_io_total(self):
        """
        1. 数据源节点，通过参数 data_set_ids 获取。静态数据源不用获取指标
        2. 实时计算节点，展示单位：条/分钟
        3. 离线计算节点，根据窗口配置 schedule_period 进行展示。函数：monitor_unit
           埋点只支持 7 天，周任务和月任务可能看不到指标数据
        返回 json 格式：
        result_node_map = {
            "224955": {
                "data": {
                    "615_mdnf_online_datadeal_tyf": {
                        "output": 10,
                        "unit": "条/分钟",
                        "input": 10
                    }
                },
                "node_type": "realtime",
                "output": 10,
                "unit": "条/分钟",
                "input": 10
            }
        }
        目前 data 内外值都是一样的，因为只有一个 rt 输出
        """
        result_node_map = {}
        flow_node_handler_list = self.flow.nodes
        for node_handler in flow_node_handler_list:
            if node_handler.node_type in list(MonitorHandler.DEFAULT_NODE_MONITOR_MAP.keys()):
                result_table_id = node_handler.result_table_ids[0]
                input_dict = self.get_input_output_loss_node_handler(node_handler, "input_count", "value")
                output_dict = self.get_input_output_loss_node_handler(node_handler, "output_count", "value")
                merged_result = MonitorUtil.merge_input_and_output_data(input_dict, output_dict, result_table_id)
                if merged_result:
                    result_node_map[node_handler.node_id] = merged_result
        return result_node_map

    def get_flow_rt_status(self):
        data = NodeMonitorHandlerSet.list_node_result_table_status(self.flow.nodes)
        return data

class NodeMonitorHandlerSet(MonitorHandler):
    def __init__(self, node_handler, monitor_type):
        self.node_handler = node_handler
        self.monitor_type = monitor_type
        super(NodeMonitorHandlerSet, self).__init__()

    def list_monitor_data(self):
        monitor_data = {}
        # 该节点未配置监控
        if self.node_handler.node_type not in list(self.DEFAULT_NODE_MONITOR_MAP.keys()):
            return monitor_data
        monitor_types = self.DEFAULT_NODE_MONITOR_MAP[self.node_handler.node_type]
        if self.monitor_type:
            if self.monitor_type not in monitor_types:
                raise NodeValidError("当前节点仅支持监控类型%s" % monitor_types)
            monitor_types = [self.monitor_type]
        # 各种监控类型对应的监控查询方式
        supported_monitor = {
            DMONITOR_ALERTS: self.get_node_monitor_alerts,
            DATA_LOSS_IO_TOTAL: self.get_node_data_loss_io_total,
            RESULT_TABLE_STATUS: self.get_node_result_table_status,
            DATA_DELAY_MAX: self.get_node_data_delay_max,
            DATA_TREND: self.get_node_data_trend,
        }
        for monitor_type in monitor_types:
            monitor_data.update({monitor_type: supported_monitor[monitor_type]()})
        return monitor_data

    def get_node_result_table_status(self):
        """
        获取离线节点各个rt的调度情况，仅离线节点需要统计该信息
        @return: node_monitors
        {
            'rt_id1': {
                'interval': xx,
                'start_time': xx,
                'node_type': xx
            }
        }
        """
        # 非离线 RT 不可获取调度信息
        data = {}
        processing_id = None
        processing_type = None
        if self.node_handler.node_type in NodeTypes.BATCH_SOURCE_CATEGORY:
            processing_id = self.node_handler.result_table_ids[0]
            processing_type = ResultTableHelper.get_processing_type(processing_id)
        elif self.node_handler.node_type in NodeTypes.BATCH_CATEGORY:
            processing_id = self.node_handler.processing_id
            processing_type = DataProcessingHelper.get_data_processing_type(processing_id)
        if processing_id and processing_type and processing_type in ["batch"]:
            data = DatamanageHelper.get_batch_executions([processing_id])
        return data

    @staticmethod
    def list_node_result_table_status(nodes):
        """
        获取离线rt的调度情况，仅离线节点需要统计该信息
        @param nodes:
        @return: node_monitors
        {
            'node_id1': {
                'rt_id1': {
                    'interval': xx,
                    'start_time': xx,
                    'node_type': xx
                }
            }
        }
        """
        batch_node_info_dict = {}
        for _n in nodes:
            # 暂不考虑离线数据源
            if _n.node_type in [
                NodeTypes.BATCH,
                NodeTypes.BATCHV2,
                NodeTypes.MODEL_APP,
                NodeTypes.DATA_MODEL_BATCH_INDICATOR,
            ]:
                for result_table_id in _n.result_table_ids:
                    batch_node_info_dict[result_table_id] = {
                        "node_id": _n.node_id,
                        "node_type": _n.node_type,
                    }
        # 批量获取flow中所有离线rt对应的错误状态信息
        data = DatamanageHelper.get_batch_executions(list(batch_node_info_dict.keys()))
        node_monitors = {}
        for _k, _v in list(data.items()):
            if _v:
                if _k not in batch_node_info_dict:
                    logger.warning(_("离线表执行状态返回信息错误，包含不存在的rt(%s)，节点可能已被删除") % _k)
                    continue
                node_monitors_detail = node_monitors.setdefault(batch_node_info_dict[_k]["node_id"], {})
                node_monitors_detail[_k] = {
                    "interval": _v.get("interval"),
                    "start_time": (_v["execute_history"][0]["start_time"] if _v.get("execute_history") else "-"),
                }
                # TODO：目前一个节点只有一个RT，为兼容旧接口，将RT相关信息放到节点层之下
                node_monitors_detail.update(node_monitors_detail[_k])
        return node_monitors

    def get_node_data_delay_max(self):
        node_type = self.node_handler.node_type
        result_table_id = self.node_handler.result_table_ids[0]
        if node_type == NodeTypes.STREAM_SOURCE or node_type == NodeTypes.BATCH_SOURCE:
            # 若为数据源节点，通过 result_table_id 获取打点信息
            source_delay_list = DatamanageHelper.get_result_table_metric_v2(
                {
                    "measurement": "process_time_delay",
                    "start_time": START_TIME_ONE_DAY,
                    "fill": "none",
                    "data_set_ids": [result_table_id],
                }
            )
            # 有多条，取 storage_cluster_type = kafka 或者 hdfs 的一条
            result_dict = None
            for item in source_delay_list:
                if item["storage_cluster_type"] == NodeTypes.KAFKA or item["storage_cluster_type"] == NodeTypes.HDFS:
                    result_dict = item
        else:
            processing_delay_list = DatamanageHelper.get_result_table_metric_v2(
                {
                    "measurement": "process_time_delay",
                    "start_time": START_TIME_ONE_DAY,
                    "fill": "none",
                    "node_ids": [self.node_handler.node_id],
                }
            )
            # 有多条，取 process_time_delay 最大的一个
            max_process_time_delay = -1
            result_dict = None
            for item in processing_delay_list:
                if item["series"][0]["process_time_delay"] > max_process_time_delay:
                    result_dict = item
                    max_process_time_delay = item["series"][0]["process_time_delay"]
        # 格式转化成前端的格式，返回
        data_delay_max_dict = {}
        delay_max_list = []
        time_list = []
        if result_dict and result_dict["series"]:
            for item in result_dict["series"]:
                delay_max_list.append(item["process_time_delay"])
                time_list.append(item["time"])
            data_delay_max_dict[result_table_id] = {
                "delay_max": delay_max_list,
                "time": time_list,
            }
        return data_delay_max_dict

    def get_node_monitor_alerts(self):
        # node_type = self.node_handler.node_type
        node_id = self.node_handler.node_id
        result_table_id = self.node_handler.result_table_ids[0]
        time_now = int(time.time())
        # 最近一天的告警
        start_time = time_now - ONE_DAY_SECOND
        # TODO 请求数据源告警列表返回都是 []
        alert_detail_list = DatamanageHelper.get_alert_detail_v2(
            {"start_time": start_time, "fill": "none", "node_id": node_id}
        )
        data_dict = {}
        alert_list = []
        for alert in alert_detail_list:
            item = {
                "time": alert["id"],
                "last_full_message_en": alert["full_message_en"],
                "last_message_en": alert["message_en"],
                "last_alert_status": alert["alert_status"],
                "last_recover_time": alert["alert_recover_time"],
                "last_message": alert["message"],
                "msg": alert["message"],
                "last_full_message": alert["full_message"],
            }
            alert_list.append(item)
        data_dict[result_table_id] = {"alert_list": alert_list}
        return data_dict

    def get_node_data_loss_io_total(self):
        data_loss_io_total = {}
        result_table_id = self.node_handler.result_table_ids[0]
        data_loss_io_total[result_table_id] = {"input": [], "output": [], "time": []}
        # input_dict，output_dict 都有可能是空
        input_dict = self.get_input_output_loss_node_handler(self.node_handler, "input_count", "series")
        output_dict = self.get_input_output_loss_node_handler(self.node_handler, "output_count", "series")
        if input_dict and input_dict[result_table_id]:
            data_loss_io_total[result_table_id]["input"] = input_dict[result_table_id]["input"]
            data_loss_io_total[result_table_id]["time"] = input_dict[result_table_id]["time"]
        # 把 output 移到 input_dict，有些节点没有输出
        if output_dict and output_dict[result_table_id]:
            data_loss_io_total[result_table_id]["output"] = output_dict[result_table_id]["output"]
        return data_loss_io_total

    def get_node_data_trend(self):
        """
        get_node_data_trend = get_node_data_loss_io_total + get_node_monitor_alerts
        """
        data_loss_io_total = self.get_node_data_loss_io_total()
        # 补充告警打点
        monitor_alert = self.get_node_monitor_alerts()
        result_table_id = self.node_handler.result_table_ids[0]
        data_loss_io_total[result_table_id]["alert_list"] = monitor_alert[result_table_id]["alert_list"]
        return data_loss_io_total


R = 256
class TrieNode:
    __slots__ = ('val', 'next')
    def __init__(self, val=None):
        self.val = val
        self.next = [None] * R

class Trie:
    __slots__ = ('root', 'size')

    def __init__(self, key_vals=[]):
        self.root = None
        self.size = 0

        for k, v in key_vals:
            self.put(k, v)

    def put(self, key, val=0):
        self.root =  self._put(self.root, key, val)

        return self

    def _put(self, TrieNode, key, val, d=0):
        if TrieNode is None:
            TrieNode = TrieNode()
        if d == len(key):
            if TrieNode.val is None: self.size += 1
            TrieNode.val = val
        else:
            c = ord(key[d])
            TrieNode.next[c] = self._put(TrieNode.next[c], key, val, d+1)

        return TrieNode

    def isEmpty(self):
        return self.size == 0

    def contains(self, key):
        return self.get(key) != None

    def get(self, key):
        TrieNode = self._search(self.root, key)

        if TrieNode is None:
            return None
        else:
            return  TrieNode.val

    def _search(self, TrieNode, key, d=0):
        if TrieNode is None:
            return None
        elif d == len(key):
            return TrieNode
        else:
            c = ord(key[d])
            return self._search(TrieNode.next[c], key, d+1)

    def delete(self, key):
        self.root = self._delete(self.root, key)

        return self

    def _delete(self, TrieNode, key, d=0):
        if TrieNode is None:
            return None
        elif d == len(key) and TrieNode.val is not None:
            TrieNode.val = None
            self.size -= 1
        else:
            c = ord(key[d])
            TrieNode.next[c] = self._delete(TrieNode.next[c], key, d+1)

        if TrieNode.val is not None:
            return TrieNode

        for n in TrieNode.next:
            if n is not None:
                return TrieNode

        return None

    def prefix(self, pre):
        TrieNode = self._search(self.root, pre)

        return self._collect(TrieNode, pre)

    def _collect(self, TrieNode, pre):
        if TrieNode is None:
            return []
        elif TrieNode.val is not None:
            result = [pre]
        else:
            result = []

        for i in range(len(TrieNode.next)):
            result += [s for s in self._collect(TrieNode.next[i], pre + chr(i))]

        return result

    def keys(self):
        return self.prefix('')

    def __iter__(self):
        return self.keys().__iter__()

    def items(self):
        raise NotImplementedError


from typing import Optional
from enum import Enum

from pydantic import BaseModel, Field

from fastapi import FastAPI
from fastapi import Body, Query, Path

app = FastAPI()

# Models

class HairColor(Enum):
    white = "White"
    black = "Black"
    red = "Red"

class Person(BaseModel):
    first_name: str = Field(
        ...,
        min_length=1,
        max_length=50
        )
    last_name: str = Field(
        ...,
        min_length=1,
        max_length=50
        )
    age: int = Field(
        ...,
        gt=0,
        le=115
        )
    hair_color: Optional[HairColor] = Field(default=None)
    is_married: Optional[bool] = Field(default=None)

class Location(BaseModel):
    city: str
    state: str
    country: str

@app.get("/")
def home():
    return {"hello": "World"}

@app.post("/person/new")
def create_person(person: Person = Body(...)):
    return person

@app.get("/person/detail")
def person_detail(
        name: Optional[str] = Query(
            default=None,
            min_length=1,
            max_length=50,
            title="Person Name",
            description="This is a person name. It's between 1 and 50 character"),
        age: int = Query(...)

):
    return {
        "name": name,
        "age": age
    }

@app.get("/person/detail/{person_id}")
def person_detail(person_id: int = Path(..., gt=0)):
    return {
        "person_id": person_id
    }

@app.put("/person/{person_id}")
def update_person(
        person_id: int = Path(..., title="Person ID", description="This is the person ID", gt=0),
        person: Person = Body(...),
        location: Location = Body(...)
):
    # results = person.dict()
    # results.update(location.dict())
    results = dict()
    results.update(dict(person))
    results.update(dict(location))
    return results


def heapifyUp(nums):
    ind = len(nums)-1
    par = (ind-1)/2

    while par >=0 and nums[par] < nums[ind]:
        #swap
        temp = nums[ind]
        nums[ind] = nums[par]
        nums[par] = temp

        ind = par
        par = (ind-1)/2

def extract_max(nums):
    ind = len(nums)
    if ind!=0:
        temp = nums[0]
        nums[0] = nums[ind-1]
        del nums[ind-1]
        heapifyDown(nums)

        return temp

def heapifyDown(nums):
    ind = 0
    lf = ind*2 + 1
    n = len(nums)

    while lf < n:
        bg = lf
        rt = ind*2 + 2
        if rt < n and nums[rt] > nums[lf]:
            bg = rt

        if nums[ind] > nums[bg]:
            break
        else:
            temp= nums[ind]
            nums[ind] = nums[bg]
            nums[bg] = temp

        ind = bg
        lf = ind*2 + 1

def kthLargestElement(nums, k):
    heap = []
    for x in nums:
        heap.append(x)
        heapifyUp(heap)
    num = 0
    for x in range(k):
        num = extract_max(heap)
    return num