#!/usr/bin/python3

import time
start = time.time()
# time.sleep(2)

import apex
# import cv2

# import math
# import numpy as np
# import os
# import pdb
# import random

# from taxonomy.utils_flat import *

# end = time.time()
# print(end - start)

"""
TODO: GET THE MODELS TRAINING, THEN GO BACK LATER AND WRITE THE
UNIT TESTS FOR TAXONOMY CONVERTER

Should have fixed ratios --> then experiment with it.

Train w/ MGDA
Train w/o MGDA.
Get results on the training set as well.

Submit the jobs first -- for all training/test sets.

fix the max_iters -- 1.2 Million examples

make sure we have the right flags to evaluate on the train dataset.
"""

"""
NVIDIA Apex has 4 optimization levels:

    O0 (FP32 training): basically a no-op. Everything is FP32 just as before.
    O1 (Conservative Mixed Precision): only some whitelist ops are done in FP16.
    O2 (Fast Mixed Precision): this is the standard mixed precision training.
        It maintains FP32 master weights and optimizer.step acts directly on the FP32 master weights.
    O3 (FP16 training): full FP16. Passing keep_batchnorm_fp32=True can speed
        things up as cudnn batchnorm is faster anyway.
"""

class ToFlatLabel(object):
    def __init__(self, tc_init, dataset):
        self.dataset = dataset
        self.tc = tc_init

    def __call__(self, image, label):
        return image, self.tc.transform_label(label, self.dataset)

# cv2.ocl.setUseOpenCL(False)
# cv2.setNumThreads(0)

def get_parser():
    import argparse
    from util import config

    parser = argparse.ArgumentParser(description='PyTorch Semantic Segmentation')
    parser.add_argument('--config', type=str, default='config/ade20k/ade20k_pspnet50.yaml', help='config file')
    parser.add_argument('opts', help='see config/ade20k/ade20k_pspnet50.yaml for all options', default=None, nargs=argparse.REMAINDER)
    args = parser.parse_args()
    assert args.config is not None
    cfg = config.load_cfg_from_cfg_file(args.config)
    if args.opts is not None:
        cfg = config.merge_cfg_from_list(cfg, args.opts)
    return cfg

def get_logger():
    import logging
    logger_name = "main-logger"
    logger = logging.getLogger(logger_name)
    logger.setLevel(logging.INFO)
    handler = logging.StreamHandler()
    fmt = "[%(asctime)s %(levelname)s %(filename)s line %(lineno)d %(process)d] %(message)s"
    handler.setFormatter(logging.Formatter(fmt))
    logger.addHandler(handler)
    return logger

def worker_init_fn(worker_id):
    import random
    random.seed(args.manual_seed + worker_id)

def main_process():
    return not args.multiprocessing_distributed or (args.multiprocessing_distributed and args.rank % args.ngpus_per_node == 0)

def main():
    """
    """
    # with open('test_2.txt', 'a') as f:
    #     f.write('test')
    #     f.close()
    import torch, os, math
    import torch.backends.cudnn as cudnn
    import torch.nn as nn
    import torch.nn.functional as F
    import torch.nn.parallel
    import torch.optim
    import torch.utils.data

    import torch.multiprocessing as mp
    import torch.distributed as dist
# from tensorboardX import SummaryWriter
    from util.dataset_config import infos

    from util import config
    from util.verification_utils import verify_architecture
    from util.avg_meter import AverageMeter, SegmentationAverageMeter
    from taxonomy.utils_flat import TaxonomyConverter
    from taxonomy.utils_baseline import StupidTaxonomyConverter
    import pickle

    print('Using PyTorch version: ', torch.__version__)
    args = get_parser()
    os.environ["CUDA_VISIBLE_DEVICES"] = ','.join(str(x) for x in args.train_gpu)

    ###### FLAT-MIX CODE #######################
    print(os.environ["CUDA_VISIBLE_DEVICES"])

    # Randomize args.dist_url too avoid conflicts on same machine
    args.dist_url = args.dist_url[:-2] + str(os.getpid() % 100).zfill(2)

    from util.dataset_config import infos

    if args.use_multiple_datasets and args.universal: # multiple datasets training, must be on universal taxononmy
        if args.tax_version == 0:
            args.tc = StupidTaxonomyConverter(version=args.tax_version)
        else:
            if args.finetune:
                args.tc = TaxonomyConverter(version=args.tax_version, finetune=True, finetune_dataset=args.finetune_dataset)
            else:
                args.tc = TaxonomyConverter(version=args.tax_version) #, train_datasets=args.dataset, test_datasets=args.test_dataset) #, train_datasets=args.dataset, test_datasets=args.test_dataset)

        args.data_root = {dataset:infos[dataset].dataroot for dataset in args.dataset}
        args.train_list = {dataset:infos[dataset].trainlist for dataset in args.dataset}
        args.classes = args.tc.classes
        # args.save_path = args.save_path.replace("{}", '-'.join([infos[dataset].shortname for dataset in args.dataset]))

    elif (not args.use_multiple_datasets) and args.universal: # single dataset on universal taxonomy training
        args.tc = TaxonomyConverter(version=args.tax_version, train_datasets=[args.dataset], test_datasets=args.test_dataset)
        args.data_root = infos[args.dataset].dataroot
        args.train_list = infos[args.dataset].trainlist
        args.classes = args.tc.classes
        # args.save_path = args.save_path.replace("{}", info[args.dataset].shortname)

    elif (not args.use_multiple_datasets) and (not args.universal): # single dataset on self taxnonmy training
        args.data_root = infos[args.dataset].dataroot
        args.train_list = infos[args.dataset].trainlist
        args.classes = infos[args.dataset].num_classes
        # args.save_path = args.save_path.replace("{}", infos[args.dataset].shortname)
    else:
        print('wrong mode, please check')
        exit()

    # verify arch after args.classes is populated
    verify_architecture(args)

    if args.manual_seed is not None:
        cudnn.benchmark = False
        cudnn.deterministic = True
        torch.manual_seed(args.manual_seed)
        np.random.seed(args.manual_seed)
        torch.manual_seed(args.manual_seed)
        torch.cuda.manual_seed_all(args.manual_seed)
    if args.dist_url == "env://" and args.world_size == -1:
        args.world_size = int(os.environ["WORLD_SIZE"])
    args.distributed = args.world_size > 1 or args.multiprocessing_distributed
    args.ngpus_per_node = len(args.train_gpu)
    if len(args.train_gpu) == 1:
        args.sync_bn = False
        args.distributed = False
        args.multiprocessing_distributed = False
    if args.multiprocessing_distributed:
        args.world_size = args.ngpus_per_node * args.world_size
        mp.spawn(main_worker, nprocs=args.ngpus_per_node, args=(args.ngpus_per_node, args))
    else:
        main_worker(args.train_gpu, args.ngpus_per_node, args)

def get_train_transform_list(args, split, dataset_name):
    """
        Args:
        -   args:
        -   split

        Return:
        -   List of transforms
    """
    from util.normalization_utils import get_imagenet_mean_std
    from util import transform

    mean, std = get_imagenet_mean_std()
    if split == 'train':
        transform_list = [
            transform.RandScale([args.scale_min, args.scale_max]),
            transform.RandRotate([args.rotate_min, args.rotate_max], padding=mean, ignore_label=args.ignore_label),
            transform.RandomGaussianBlur(),
            transform.RandomHorizontalFlip(),
            transform.Crop([args.train_h, args.train_w], crop_type='rand', padding=mean, ignore_label=args.ignore_label),
            transform.ToTensor(),
            transform.Normalize(mean=mean, std=std)
        ]
    elif split == 'val':
        transform_list = [
            transform.Crop([args.train_h, args.train_w], crop_type='center', padding=mean, ignore_label=args.ignore_label),
            transform.ToTensor(),
            transform.Normalize(mean=mean, std=std)
        ]
    else:
        print('Unknown split. Quitting ...')
        quit()

    transform_list += [ToFlatLabel(args.tc, dataset_name)]

    return transform.Compose(transform_list)

def load_pretrained_weights(args, model, optimizer):
    """
        Args:
        -   args
        -   model: Passed by reference

        Returns:
        -   None
    """
    import torch, os, math

    resume_iter = 0

    if args.weight:
        if os.path.isfile(args.weight):
            if main_process():
                logger.info("=> loading weight '{}'".format(args.weight))
            checkpoint = torch.load(args.weight)
            model.load_state_dict(checkpoint['state_dict'])
            if main_process():
                logger.info("=> loaded weight '{}'".format(args.weight))
        else:
            if main_process():
                logger.info("=> no weight found at '{}'".format(args.weight))

    if args.resume:
        if os.path.isfile(args.resume):
            if main_process():
                logger.info("=> loading checkpoint '{}'".format(args.resume))
            # checkpoint = torch.load(args.resume)
            checkpoint = torch.load(args.resume, map_location=lambda storage, loc: storage.cuda())
            # args.start_epoch = checkpoint['epoch']
            args.start_epoch = 0 # we don't really rely on this, but on resume_iter
            if args.finetune:
                args.start_epoch = 0
            model.load_state_dict(checkpoint['state_dict'])
            optimizer.load_state_dict(checkpoint['optimizer'])
            resume_iter = checkpoint['current_iter']
            if main_process():
                logger.info("=> loaded checkpoint '{}' (epoch {})".format(args.resume, checkpoint['epoch']))
        else:
            if main_process():
                logger.info("=> no checkpoint found at '{}'".format(args.resume) + ' Please check')
                exit()

    return model, optimizer, resume_iter

            # optimizer = get_optimizer(args.model)

def get_model(args, criterion, BatchNorm):
    """
        Args:
        -

        Returns:
        -
    """
    if args.arch == 'psp':
        from ccsa.ccsa_pspnet import CCSA_PSPNet
        model = CCSA_PSPNet(layers=args.layers, classes=args.classes, zoom_factor=args.zoom_factor, criterion=criterion, BatchNorm=BatchNorm, network_name=args.network_name)

    elif args.arch == 'psa':
        model = PSANet(layers=args.layers, classes=args.classes, zoom_factor=args.zoom_factor, psa_type=args.psa_type,
                       compact=args.compact, shrink_factor=args.shrink_factor, mask_h=args.mask_h, mask_w=args.mask_w,
                       normalization_factor=args.normalization_factor, psa_softmax=args.psa_softmax,
                       criterion=criterion,
                       BatchNorm=BatchNorm)
    return model

def get_optimizer(args, model):
    """
    Create a parameter list, where first 5 entries (ResNet backbone) have low learning rate
    to not clobber pre-trained weights, and later entries (PPM derivatives) have high learning rate.

        Args:
        -   args
        -   model

        Returns:
        -   optimizer
    """
    import torch, os, math

    if args.arch == 'psp':
        modules_ori = [model.layer0, model.layer1, model.layer2, model.layer3, model.layer4]
        modules_new = [model.ppm, model.cls, model.aux]
    elif args.arch == 'psa':
        modules_ori = [model.layer0, model.layer1, model.layer2, model.layer3, model.layer4]
        modules_new = [model.psa, model.cls, model.aux]
    params_list = []
    for module in modules_ori:
        params_list.append(dict(params=module.parameters(), lr=args.base_lr))

    for module in modules_new:
        if args.finetune:
            params_list.append(dict(params=module.parameters(), lr=args.base_lr))
        else:
            params_list.append(dict(params=module.parameters(), lr=args.base_lr * 10))
    args.index_split = 5
    optimizer = torch.optim.SGD(params_list, lr=args.base_lr, momentum=args.momentum, weight_decay=args.weight_decay)
    return optimizer

def get_rank_to_dataset_map(args):
    """
        Obtain a mapping from GPU rank (index) to the name of the dataset residing on this GPU.

        Args:
        -   args

        Returns:
        -   rank_to_dataset_map
    """
    rank_to_dataset_map = {}
    for dataset, gpu_idxs in args.dataset_gpu_mapping.items():
        for gpu_idx in gpu_idxs:
            rank_to_dataset_map[gpu_idx] = dataset
    print('Rank to dataset map: ', rank_to_dataset_map)
    return rank_to_dataset_map

def main_worker(gpu, ngpus_per_node, argss):
    """
    Consider if a dataset has size 18,000 and is placed on a single GPU, of 4 gpus.
    Batch size 32. In this case, len(train_data) = 18,000 but len(train_loader) = 2250
    Because effective batch size is 8.

    Consider if a dataset has size 118287. If placed on 2/4 gpus with batch size 32.
    In this case, len(train_data) = 118287 and len(train_loader) = 7393.
    """

    # with open('test_3.txt', 'a') as f:
    #     f.write('test')
    #     f.close()
    global args
    args = argss

    from ccsa.ccsa_data import CCSA_Data
    from util import dataset
    from taxonomy.utils_flat import TaxonomyConverter
    from multiobjective_opt.dist_mgda_utils import scale_loss_and_gradients
    import apex
    import torch, os, math
    import torch.backends.cudnn as cudnn
    import torch.nn as nn
    import torch.nn.functional as F
    import torch.nn.parallel
    import torch.optim
    import torch.utils.data

    import torch.multiprocessing as mp
    import torch.distributed as dist
    from tensorboardX import SummaryWriter
    from util.dataset_config import infos

    from util import config
    from util.verification_utils import verify_architecture
    from util.avg_meter import AverageMeter, SegmentationAverageMeter
    from util.util import poly_learning_rate

    # with open('test_mainworker.txt', 'a') as f:
    #     f.write('test\t')
    #     f.close()
# os.sleep
    # time.sleep(30)
    if args.sync_bn:
        if args.multiprocessing_distributed:
            # BatchNorm = torch.nn.SyncBatchNorm
            BatchNorm = apex.parallel.SyncBatchNorm
        else:
            from lib.sync_bn.modules import BatchNorm2d
            BatchNorm = BatchNorm2d
    else:
        BatchNorm = nn.BatchNorm2d
    print('Using batchnorm variant: ', BatchNorm)
    if args.distributed:
        if args.dist_url == "env://" and args.rank == -1:
            args.rank = int(os.environ["RANK"])
        if args.multiprocessing_distributed:
            args.rank = args.rank * ngpus_per_node + gpu
        dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url, world_size=args.world_size, rank=args.rank)

    criterion = nn.CrossEntropyLoss(ignore_index=args.ignore_label)
    model = get_model(args, criterion, BatchNorm)
    optimizer = get_optimizer(args, model)

    if True:
        global logger, writer
        logger = get_logger()
        writer = SummaryWriter(args.save_path)
        args.logger = logger

        if main_process():
            logger.info(args)
            logger.info("=> creating model ...")
            logger.info("Classes: {}".format(args.classes))
            logger.info(model)
    if args.distributed:
        torch.cuda.set_device(gpu)
        args.batch_size = int(args.batch_size / ngpus_per_node)
        args.batch_size_val = int(args.batch_size_val / ngpus_per_node)
        args.batch_size_val = max(1, args.batch_size_val)
        args.workers = int(args.workers / ngpus_per_node)
        if args.use_apex:
            model, optimizer = apex.amp.initialize(model.cuda(), optimizer, opt_level=args.opt_level, keep_batchnorm_fp32=args.keep_batchnorm_fp32, loss_scale=args.loss_scale)
            model = apex.parallel.DistributedDataParallel(model)
        else:
            model = torch.nn.parallel.DistributedDataParallel(model.cuda(), device_ids=[gpu])

    else:
        model = torch.nn.DataParallel(model.cuda())

    model, optimizer, args.resume_iter = load_pretrained_weights(args, model, optimizer)

    domain_idx_map = {
        'coco-panoptic-v1-qvga': 0,
        'mapillary_vistas_comm-qvga': 1,
        'ade20k-v1-qvga': 2
    }

    train_transform_dict = {}
    for dname, domain_idx in domain_idx_map.items():
        train_transform_dict[domain_idx] = get_train_transform_list(args, split='train', dataset_name=dname)

    # FLATMIX ADDITION
    train_data = CCSA_Data(split='train', data_roots=args.data_root, data_lists=args.train_list, transform_dict=train_transform_dict)

    from util.txt_utils import read_txt_file
    num_examples = len(train_data)

    num_examples_total = args.num_examples
    args.epochs = math.ceil(num_examples_total / num_examples)
    args.max_iters = math.floor(num_examples_total / (args.batch_size * args.ngpus_per_node))

    # avoid too frequent saving to waste time, on small datasets
    if args.epochs > 200:
        args.save_freq = args.epochs // 100

    logger.info(f'Train data has len {len(train_data)} on {args.rank}')
    if args.distributed:

        train_sampler = torch.utils.data.distributed.DistributedSampler(train_data, num_replicas=args.ngpus_per_node, rank=args.rank)
        logger.info(f"rank: {args.rank}, actual_replica: {train_sampler.num_replicas}, length of sampler, {len(train_sampler)}")

    else:
        train_sampler = None
    train_loader = torch.utils.data.DataLoader(train_data, batch_size=args.batch_size, shuffle=(train_sampler is None), num_workers=args.workers, pin_memory=True, sampler=train_sampler, drop_last=True)
    logger.info(f'Train loader has len {len(train_loader)} on {args.rank}')

    if args.evaluate:
        val_transform = get_train_transform_list(args, split='val')
        # val_transform = transform.Compose(val_transform_list)
        val_data = dataset.SemData(split='val', data_root=args.data_root, data_list=args.val_list, transform=val_transform)
        if args.distributed:
            val_sampler = torch.utils.data.distributed.DistributedSampler(val_data)
        else:
            val_sampler = None
        val_loader = torch.utils.data.DataLoader(val_data, batch_size=args.batch_size_val, shuffle=False, num_workers=args.workers, pin_memory=True, sampler=val_sampler)

    for epoch in range(args.start_epoch, args.epochs):
        epoch_log = epoch + 1
        logger.info(f'New epoch {epoch_log} starts on rank {args.rank}')

        if args.distributed:
            train_sampler.set_epoch(epoch)
        print(f'On training epoch {epoch} in GPU {args.rank}')
        loss_train, mIoU_train, mAcc_train, allAcc_train = train(train_loader, model, optimizer, epoch)
        # if main_process():
        #     writer.add_scalar('loss_train', loss_train, epoch_log)
        #     writer.add_scalar('mIoU_train', mIoU_train, epoch_log)
        #     writer.add_scalar('mAcc_train', mAcc_train, epoch_log)
        #     writer.add_scalar('allAcc_train', allAcc_train, epoch_log)

        if ((epoch_log % args.save_freq == 0)) and main_process():
            filename = args.save_path + '/train_epoch_' + str(epoch_log) + '.pth'
            logger.info('Saving checkpoint to: ' + filename)
            torch.save({'epoch': epoch_log, 'state_dict': model.state_dict(), 'optimizer': optimizer.state_dict(),
                'current_iter': (epoch + 1) * len(train_loader), 'max_iter': args.max_iters}, filename)
            if epoch_log / args.save_freq > 2:
                # if (epoch_log - 3) % 10 != 0:
                if not args.finetune:
                    deletename = args.save_path + '/train_epoch_' + str(epoch_log - args.save_freq * 2) + '.pth'
                    os.remove(deletename)

        if (epoch == args.epochs - 1) and main_process():
            filename = args.save_path + '/train_epoch_final.pth'
            logger.info('Saving checkpoint to: ' + filename)
            torch.save({'epoch': epoch_log, 'state_dict': model.state_dict(), 'optimizer': optimizer.state_dict(),
                'current_iter': (epoch + 1) * len(train_loader), 'max_iter': args.max_iters}, filename)
            exit()

        # if args.evaluate:
        #     loss_val, mIoU_val, mAcc_val, allAcc_val = validate(val_loader, model, criterion)
        #     if main_process():
        #         writer.add_scalar('loss_val', loss_val, epoch_log)
        #         writer.add_scalar('mIoU_val', mIoU_val, epoch_log)
        #         writer.add_scalar('mAcc_val', mAcc_val, epoch_log)
        #         writer.add_scalar('allAcc_val', allAcc_val, epoch_log)

def train(train_loader, model, optimizer, epoch):
    """
    No MGDA -- whole iteration takes 0.31 sec.
    0.24 sec to run typical backward pass (with no MGDA)

    With MGDA -- whole iteration takes 1.10 sec.
    1.05 sec to run backward pass w/ MGDA subroutine -- scale_loss_and_gradients() in every iteration.

    TODO: Profile which part of Frank-Wolfe is slow

    """

    from util.avg_meter import AverageMeter, SegmentationAverageMeter
    from util.util import poly_learning_rate

    import torch.distributed as dist
    from multiobjective_opt.dist_mgda_utils import scale_loss_and_gradients

    import torch, os, math, time

    batch_time = AverageMeter()
    data_time = AverageMeter()
    main_loss_meter = AverageMeter()
    aux_loss_meter = AverageMeter()
    loss_meter = AverageMeter()
    sam = SegmentationAverageMeter()

    model.train()
    # set bn to be eval() and see the norm
    # def set_bn_eval(m):
    #     classname = m.__class__.__name__
    #     if classname.find('BatchNorm') != -1:
    #         m.eval()
    # model.apply(set_bn_eval)
    end = time.time()
    max_iter = args.max_iters

    for i, (input, target, batch_domain_idxs) in enumerate(train_loader):
        # pass
        # if main_process():
        data_time.update(time.time() - end)
        if args.zoom_factor != 8:
            h = int((target.size()[1] - 1) / 8 * args.zoom_factor + 1)
            w = int((target.size()[2] - 1) / 8 * args.zoom_factor + 1)
            # 'nearest' mode doesn't support align_corners mode and 'bilinear' mode is fine for downsampling
            target = F.interpolate(target.unsqueeze(1).float(), size=(h, w), mode='bilinear', align_corners=True).squeeze(1).long()
        input = input.cuda(non_blocking=True)
        target = target.cuda(non_blocking=True)
        batch_domain_idxs = batch_domain_idxs.cuda(non_blocking=True)

        if args.use_mgda:
            output, loss, main_loss, aux_loss, scales = forward_backward_mgda(input, target, model, optimizer, args)
        else:
            #print('Batch domain idxs: ', batch_domain_idxs.shape, batch_domain_idxs.device, batch_domain_idxs)
            output, loss, main_loss, aux_loss = forward_backward_full_sync(input, target, model, optimizer, args, batch_domain_idxs)

        optimizer.step()

        n = input.size(0)
        if args.multiprocessing_distributed:
            main_loss, aux_loss, loss = main_loss.detach() * n, aux_loss * n, loss * n  # not considering ignore pixels
            count = target.new_tensor([n], dtype=torch.long)
            dist.all_reduce(main_loss), dist.all_reduce(aux_loss), dist.all_reduce(loss), dist.all_reduce(count)
            n = count.item()
            main_loss, aux_loss, loss = main_loss / n, aux_loss / n, loss / n

        sam.update_metrics_gpu(output, target, args.classes, args.ignore_label, args.multiprocessing_distributed)

        main_loss_meter.update(main_loss.item(), n)
        aux_loss_meter.update(aux_loss.item(), n)
        loss_meter.update(loss.item(), n)
        # if main_process():
        if i > 0:
            batch_time.update(time.time() - end)
        end = time.time()

        current_iter = epoch * len(train_loader) + i + 1 + args.resume_iter
        current_lr = poly_learning_rate(args.base_lr, current_iter, max_iter, power=args.power)
        # current_lr = 0
        for index in range(0, args.index_split):
            optimizer.param_groups[index]['lr'] = current_lr
        for index in range(args.index_split, len(optimizer.param_groups)):
            if args.finetune:
                optimizer.param_groups[index]['lr'] = current_lr
            else:
                optimizer.param_groups[index]['lr'] = current_lr * 10

        remain_iter = max_iter - current_iter
        remain_time = remain_iter * batch_time.avg
        t_m, t_s = divmod(remain_time, 60)
        t_h, t_m = divmod(t_m, 60)
        remain_time = '{:02d}:{:02d}:{:02d}'.format(int(t_h), int(t_m), int(t_s))

        if (i + 1) % args.print_freq == 0 and main_process():
        # if True:
            logger.info('Epoch: [{}/{}][{}/{}] '
                        'Data {data_time.val:.3f} ({data_time.avg:.3f}) '
                        'Batch {batch_time.val:.3f} ({batch_time.avg:.3f}) '
                        'Remain {remain_time} '
                        'MainLoss {main_loss_meter.val:.4f} '
                        'AuxLoss {aux_loss_meter.val:.4f} '
                        'Loss {loss_meter.val:.4f} '
                        'Accuracy {accuracy:.4f}.'.format(epoch+1, args.epochs, i + 1, len(train_loader),
                                                          batch_time=batch_time,
                                                          data_time=data_time,
                                                          remain_time=remain_time,
                                                          main_loss_meter=main_loss_meter,
                                                          aux_loss_meter=aux_loss_meter,
                                                          loss_meter=loss_meter,
                                                          accuracy=sam.accuracy) + f'current_iter: {current_iter}' + f' rank: {args.rank} ')
            if args.use_mgda and main_process():
                # Scales identical in each process, so print out only in main process.
                scales_str = [f'{d}: {scale:.2f}' for d,scale in scales.items()]
                scales_str = ' , '.join(scales_str)
                logger.info(f'Scales: {scales_str}')

        if main_process() and current_iter == max_iter - 5: # early exit to prevent iter number not matching between gpus
            break
        # if main_process():
        #     writer.add_scalar('loss_train_batch', main_loss_meter.val, current_iter)
        #     writer.add_scalar('mIoU_train_batch', np.mean(intersection / (union + 1e-10)), current_iter)
        #     writer.add_scalar('mAcc_train_batch', np.mean(intersection / (target + 1e-10)), current_iter)
        #     writer.add_scalar('allAcc_train_batch', accuracy, current_iter)

    iou_class, accuracy_class, mIoU, mAcc, allAcc = sam.get_metrics()
    # if main_process():
    logger.info('Train result at epoch [{}/{}]: mIoU/mAcc/allAcc {:.4f}/{:.4f}/{:.4f}.'.format(epoch+1, args.epochs, mIoU, mAcc, allAcc))
    return main_loss_meter.avg, mIoU, mAcc, allAcc

def forward_backward_full_sync(input, target, model, optimizer, args, batch_domain_idxs):
    """
        Args:
        -   input: Tensor of size (?) representing
        -   target: Tensor of size (?) representing
        -   model
        -   optimizer
        -   args

        Returns:
        -   output: Tensor of size (?) representing
        -   loss: Tensor of size (?) representing
        -   main_loss: Tensor of size (?) representing
        -   aux_loss: Tensor of size (?) representing
    """
    output, main_loss, aux_loss = model(input, target, batch_domain_idxs)
    if not args.multiprocessing_distributed:
        main_loss, aux_loss = torch.mean(main_loss), torch.mean(aux_loss)
    loss = main_loss + args.aux_weight * aux_loss

    optimizer.zero_grad()
    if args.use_apex and args.multiprocessing_distributed:
        with apex.amp.scale_loss(loss, optimizer) as scaled_loss:
            scaled_loss.backward()
    else:
        loss.backward()
    return output, loss, main_loss, aux_loss

def forward_backward_mgda(input, target, model, optimizer, args):
    from multiobjective_opt.dist_mgda_utils import scale_loss_and_gradients
    """
        We rely upon the ddp.no_sync() of gradients:
        https://github.com/pytorch/pytorch/blob/master/torch/nn/parallel/distributed.py

        Args:
        -   input: Tensor of size (?) representing
        -   target: Tensor of size (?) representing
        -   model
        -   optimizer
        -   args

        Returns:
        -   output: Tensor of size (?) representing
        -   loss: Tensor of size (?) representing
        -   main_loss: Tensor of size (?) representing
        -   aux_loss: Tensor of size (?) representing
    """
    with model.no_sync():
        output, main_loss, aux_loss = model(input, target)
        loss = main_loss + args.aux_weight * aux_loss
        loss, scales = scale_loss_and_gradients(loss, optimizer, model, args)

    return output, loss, main_loss, aux_loss, scales

def validate(val_loader, model, criterion):
    if main_process():
        logger.info('>>>>>>>>>>>>>>>> Start Evaluation >>>>>>>>>>>>>>>>')
    batch_time = AverageMeter()
    data_time = AverageMeter()
    loss_meter = AverageMeter()
    sam = SegmentationAverageMeter()

    model.eval()
    if main_process():
        end = time.time()
    for i, (input, target) in enumerate(val_loader):
        if main_process():
            data_time.update(time.time() - end)
        input = input.cuda(non_blocking=True)
        target = target.cuda(non_blocking=True)
        output = model(input)
        if args.zoom_factor != 8:
            output = F.interpolate(output, size=target.size()[1:], mode='bilinear', align_corners=True)
        loss = criterion(output, target)

        n = input.size(0)
        if args.multiprocessing_distributed:
            loss = loss * n  # not considering ignore pixels
            count = target.new_tensor([n], dtype=torch.long)
            dist.all_reduce(loss), dist.all_reduce(count)
            n = count.item()
            loss = loss / n
        else:
            loss = torch.mean(loss)

        output = output.max(1)[1]
        sam.update_metrics_gpu(output, target, args.classes, args.ignore_label, args.multiprocessing_distributed)
        loss_meter.update(loss.item(), input.size(0))
        if main_process():
            batch_time.update(time.time() - end)
            end = time.time()
        if ((i + 1) % args.print_freq == 0) and main_process():
            logger.info('Test: [{}/{}] '
                        'Data {data_time.val:.3f} ({data_time.avg:.3f}) '
                        'Batch {batch_time.val:.3f} ({batch_time.avg:.3f}) '
                        'Loss {loss_meter.val:.4f} ({loss_meter.avg:.4f}) '
                        'Accuracy {accuracy:.4f}.'.format(i + 1, len(val_loader),
                                                          data_time=data_time,
                                                          batch_time=batch_time,
                                                          loss_meter=loss_meter,
                                                          accuracy=sam.accuracy))

    iou_class, accuracy_class, mIoU, mAcc, allAcc = sam.get_metrics()
    if main_process():
        logger.info('Val result: mIoU/mAcc/allAcc {:.4f}/{:.4f}/{:.4f}.'.format(mIoU, mAcc, allAcc))
        for i in range(args.classes):
            logger.info('Class_{} Result: iou/accuracy {:.4f}/{:.4f}.'.format(i, iou_class[i], accuracy_class[i]))
        logger.info('<<<<<<<<<<<<<<<<< End Evaluation <<<<<<<<<<<<<<<<<')
    return loss_meter.avg, mIoU, mAcc, allAcc

end = time.time()
print(end-start)
if __name__ == '__main__':
    print('main')

    main()


import requests

from bs4 import BeautifulSoup
from sqlalchemy.orm import Session
from sqlalchemy import create_engine

from urllib import parse
from urllib.parse import urlparse

from base import Base
from website import Website
from page import Page

class Crawler():
    def __init__(self, domain):
        self.domain = domain
        self.scanned_list = []

    def scan_page(self, url, session):
        print(url)
        self.scanned_list.append(url)
        request = requests.get(url)
        request.encoding = 'utf-8'
        html = request.text
        soup = BeautifulSoup(html)

        try:
            self.save_soup(session, soup, url)
        except Exception as e:
            print(e)

        links = soup.find_all('a')
        for link in links:
            href = link.get('href')
            next_page = self.href_to_url(url, href)
            if not self.is_outgoing(next_page) and next_page not in self.scanned_list:
                if self.is_valid(next_page):
                    self.scan_page(next_page, session)

    def scan_website(self, session):
        url = "http://" + self.domain
        self.scan_page(url, session)

    def href_to_url(self, url, href):
        return parse.urljoin(url, href)

    def is_outgoing(self, url):
        url = urlparse(url)
        if self.domain == url.netloc:
            return False
        if url.netloc[:4] == "www." and self.domain in url.netloc:
            return False
        return True

    def is_valid(self, url):
        if "#" in url:
            return False
        if ".pdf" in url:
            return False
        if ".jpg" in url or ".gif" in url:
            return False
        return True

    def save_soup(self, session, soup, page_url):
        try:
            soup_description = soup.find(
                attrs={"property": "og:description"}).get("content")
        except Exception:
            soup_description = ""

        session.add(Page(title=soup.title.string,
                         description=soup_description,
                         url=page_url,
                         website_id=self.domain))
        session.commit()

    def save_website(self, session):
        session.add(Website(url=self.domain,
                            html=True))

def main():
    engine = create_engine("sqlite:///searchengine.db")
    Base.metadata.create_all(engine)
    session = Session(bind=engine)
    crawler = Crawler("24chasa.bg")
    crawler.save_website(session)
    crawler.scan_website(session)

if __name__ == '__main__':
    main()


"""
Program to build bioactivity prediction tool streamlit app
"""

# importing necessary libraries
import streamlit as st
import pandas as pd
from PIL import Image
from padelpy import padeldescriptor
import glob
import base64
import pickle

# molecular descriptor calculator
def desc_calc():
    """
    function to initiate PaDEL software via padelpy library to calculate molecular descriptors
    :return: calculated descriptors of input molecules
    """
    # list and sort fingerprint XML files to initiate padelpy/padel software
    xml_files = glob.glob('*.xml')
    xml_files.sort()

    # create list of "keys" for all padel fingerprint types to use for dictionary
    fp_list = ['AtomPairs2DCount',
               'AtomPairs2D',
               'EState',
               'CDKextended',
               'CDK',
               'CDKgraphonly',
               'KlekotaRothCount',
               'KlekotaRoth',
               'MACCS',
               'PubChem',
               'SubstructureCount',
               'Substructure']

    # create dictionary of keys and corresponding values (xml files)
    fp = dict(zip(fp_list, xml_files))

    # initiate padelpy and set fingerprint to "PubChem" type to calculate PubChem fingerprints
    fingerprint = 'PubChem'
    fingerprint_output_file = ''.join([fingerprint,'.csv'])     # output: Pubchem.csv
    fingerprint_descriptortypes = fp[fingerprint]

    padeldescriptor(mol_dir='molecule.smi',
                    d_file=fingerprint_output_file,  # 'PubChem.csv'
                    descriptortypes=fingerprint_descriptortypes,     # descriptortypes='PubChem.xml'
                    detectaromaticity=True,
                    standardizenitro=True,
                    standardizetautomers=True,
                    threads=2,
                    removesalt=True,
                    log=True,
                    fingerprints=True)

# file download
def filedownload(df):
    """
    function to enable users to download prediction results from webapp
    :param df:dataframe of predicted pIC50 values of input molecules
    :return: csv file
    """
    csv = df.to_csv(index=False)
    b64 = base64.b64encode(csv.encode()).decode()  # strings <-> bytes conversions
    href = f'Download Predictions'
    return href

# model building
def build_model(input_data):
    """
    function to read in Random Forest model built and saved as a pickle object. Function then uses model to predict
    pIC50 values from the molecular descriptors calculated from the input molecules.
    :param input_data: dataframe containing molecular descriptors
    :return:
    """
    # reads in saved Random Forest model
    load_model = pickle.load(open('model.pkl', 'rb'))
    # apply model to make predictions
    prediction = load_model.predict(input_data)
    st.header('**Prediction output**')
    prediction_output = pd.Series(prediction, name='pIC50')
    molecule_name = pd.Series(load_data[1], name='molecule_name')
    df = pd.concat([molecule_name, prediction_output], axis=1)
    st.write(df)
    st.markdown(filedownload(df), unsafe_allow_html=True)

# logo image for app
image = Image.open('logo.png')
st.image(image, use_column_width=True)

# formatting streamlit app
# page title
st.markdown("""
# Bioactivity Prediction App for target protein, Hepatocyte Growth Factor Receptor (HGFR)

Abnormal expression of hepatocyte growth factor receptor (HGFR) has been associated with tumors with high cell proliferation rate,
cell dissociation, tumor aggressiveness, and invasiveness. Literature has supported it as being a promising target in cancer therapy, however the protein's
characteristic of therapeutic resistance is still a barrier for developing effective drugs that can inhibit it. Discovery of novel compounds/molecules with
promising bioactivity with HGFR is needed.

This app allows users to determine if a molecule of interest has the potential to be a drug candidate by predicting the
bioactivity towards inhibiting the HGFR. The app accepts .txt files containing the canonical simplified molecular-input line-entry system (SMILES)
notation of the molecule(s) of interest and respective ChEMBLID. Results will return predicted pIC50 value(s) of the molecule(s)
of interest.

A note on predicted pIC50 values: the prediction model was trained with consideration of compounds with pIC50 values >6 as **active** while values <5 as **inactive**.

---
**Acknowledgement**
- App built in 'python' + 'streamlit' by **Sara Then** with project inspiration and guidance from [Chanin Nantasenamat/Data Professor](http://youtube.com/dataprofessor)'s Bioinformatics series.
- Descriptor calculated using [PaDEL-Descriptor](http://www.yapcwsoft.com/dd/padeldescriptor/)
- Details of the project and code are available on the github repo (https://github.com/sara-then/HGF-drugdiscovery-project)
---
""")

# sidebar
with st.sidebar.header('Upload your molecule .txt file'):
    uploaded_file = st.sidebar.file_uploader("Upload your input file", type=['txt'])
    st.sidebar.markdown("""
[Example input file](https://raw.githubusercontent.com/dataprofessor/bioactivity-prediction-app/main/example_acetylcholinesterase.txt)
""")

if st.sidebar.button('Predict'):
    load_data = pd.read_table(uploaded_file, sep=' ', header=None)
    load_data.to_csv('molecule.smi', sep='\t', header=False, index=False)

    st.header('**Original input data**')
    st.write(load_data)

    with st.spinner("Calculating descriptors..."):
        desc_calc()

    # Read in calculated descriptors and display the dataframe
    st.header('**Calculated molecular descriptors**')
    desc = pd.read_csv('PubChem.csv')
    st.write(desc)
    st.write(desc.shape)

    # Read descriptor list used in previously built model
    st.header('**Subset of descriptors from built model**')
    Xlist = list(pd.read_csv('descriptor_list.csv').columns)
    desc_subset = desc[Xlist]
    st.write(desc_subset)
    st.write(desc_subset.shape)

    # Apply trained model to make prediction on query compounds
    build_model(desc_subset)
else:
    st.info('Upload input data in the sidebar to start!')


import json

json_fn = "../mentoriasudea2022/data/marvel.json"   #   Ruta al archivo json con los datos.

clave_gen = "Cinemateca Marvel"   #   Clave principal de búsqueda por cinemateca
clave_pel = 'Iron Man'            #   Clave de búsqueda de película

def f_trazo(texto):
    print(f"{'<'*int(len(texto)/2)} {'>'*int(len(texto)/2)}")
    print(f"{texto}")
    print(f"{'<'*int(len(texto)/2)} {'>'*int(len(texto)/2)}")

with open(json_fn) as marvel:

    data=json.load(marvel)
    texto = "01. Datos cargados de json"
    f_trazo("BEGIN " + texto)
    print(f"{type(data)} >> {data[clave_gen][clave_pel]}")
    f_trazo("END " + texto)

texto = " 02. Datos de peliculas"
f_trazo("BEGIN " + texto)
#for peli in data[clave_gen]:
for peli in data:

    print(f" 02. Datos {type(peli)} de peliculas {peli}")

f_trazo("END " + texto)

texto = " 03. Datos de peli {type(peli)} contienen {peli}"
f_trazo("BEGIN " + texto)

for peli in data[clave_gen]:
  print(f" 03. Datos de peli {type(peli)} >> {peli}")

f_trazo("END " + texto)

datos_peli = data[clave_gen][clave_pel]
texto=" 04. Detalle de {clave_gen} : {clave_pel} {type(datos_peli)}"
f_trazo("BEGIN " + texto)
print(f" 04. Detalle de {clave_gen} : {clave_pel} {type(datos_peli)}")
f_trazo("END " + texto)

texto=" 05. {det} : {datos_peli[det]} es  {type(det)}"
f_trazo("BEGIN " + texto)

for det in datos_peli:

    print(f" 05. {det} : {datos_peli[det]} es {type(det)}")
f_trazo("END " + texto)

texto=" 06. Otro detalle {type(datos_peli)} = {datos_peli}"
f_trazo("BEGIN " + texto)
print(f" 06. Otro detalle {type(datos_peli)} = {datos_peli}")
f_trazo("END " + texto)

texto=" 07. Claves de "
f_trazo("BEGIN " + texto )

for det_peli in data[clave_gen][clave_pel]:
    print(f" 07. {det_peli} : {data[clave_gen][clave_pel][det_peli]}")
f_trazo("END " + texto)

with open(json_fn) as marvel:

    data=json.load(marvel)
    for cin in data:
      print("Cin " + cin)
      for pel in data[cin]:
        texto=cin + " ****" + " Pelicula: " + pel
        f_trazo(texto)
        for det in data[cin][pel]:
            print(f' >>>> {det} = {data[cin][pel][det]}')
        f_trazo( " Fin de >>> " + texto)
        f_trazo(texto="")


#!/usr/bin/python3
#--coding: utf-8; mode:python--

from curses.ascii import isdigit
from json import *
from os import stat
from classes.Bottle import *
from functionalities.Import import *
from classes.State import *
from classes.Problem import *
from functionalities.Export import*
from classes.Frontier_Deque import*
from classes.Frontier_Heapq import*
from classes.Frontier_List import*
from classes.State_space import *
import resource
import time

def main():
    menu_strategies()

def print_Menu():
    print("Welcome to our water sort agent solver\nSelect the option you want to:")
    print("0. BREADTH ALGORITHM")
    print("1. DEPTH ALGORITHM")
    print("2. UNIFORM COST ALGORITHM")
    print("3. A* ALGORITHM")
    print("4. GREEDY  ALGORITHM")
    print("5. Exit")
    print("----------------------------------------------------------------------\n")

def print_menu_funcionalities():
    print("0. Import json")
    print("1. Test frontier")
    print("2. Export the solution")
    print("3. Back")
    print("----------------------------------------------------------------------\n")

def menu_strategies():

    while True:
        while True:
            print_Menu()
            try:
                user_option = int(input("Select the strategy you want to analyze: "))
                break
            except:
                print("\nERROR: You have to choose an option between 0 and 5.\n")
        if user_option not in range(6) and isdigit(user_option) == False:
            print("\nERROR: You have to choose an option between 0 and 5.\n")
            continue
        else:
            if user_option ==0:
                print("BREADTH ALGORITHM")
                menu_main_functionality('BREADTH')
            elif user_option==1:
                print("DEPTH ALGORITHM")
                menu_main_functionality('DEPTH')
            elif user_option == 2:
                print("UNIFORM COST ALGORITHM")
                menu_main_functionality('UNIFORM')
            elif user_option == 3:
                print("A* ALGORITHM")
                menu_main_functionality('A')
            elif user_option == 4:
                print("GREEDY ALGORITHM")
                menu_main_functionality('GREEDY')
            elif user_option==5:
                print("Bye!")
                break

def menu_main_functionality(strategy):
    problem = Problem(0, [], State([]))
    already_imported = False
    while True:
        while True:
            print_menu_funcionalities()
            try:
                user_option = int(input("Select a functionality: "))
                break
            except:
                print("\nERROR: You have to choose an option between 0 and 3.\n")
        if user_option not in range(4) and isdigit(user_option) == False:
            print("\nERROR: You have to choose an option between 0 and 3.\n")
            continue
        elif (user_option == 1 or user_option == 2) and already_imported == False:
            print("\nERROR: You have to import a problem before selecting option 1 or 2.\n")
        else:
            if user_option == 0:

                path_json = input("Introduce the path of the problem:")
                problem = read_States_Txt(path_json, 1)
                if problem is not None:
                    print("Problem imported successfully!")
                    already_imported = True
                else:
                    already_imported = False

            elif user_option==1:
                random_heuristic = "{0:.2f}".format(uniform(0.0, 1000.0))
                random_value = "{0:.2f}".format(uniform(0.0, 1000.0))
                initial_node = Node(0, 0.0, problem.initState, None, None, 0, random_heuristic, random_value)
                if(initial_node == None):
                    continue
                else:
                    dq_frontier = Frontier_Deque()
                    hq_frontier = Frontier_Heapq()
                    list_frontier = Frontier_List()
                    dq_frontier.push(initial_node)
                    hq_frontier.push(initial_node)
                    list_frontier.push(initial_node)
                    #Test deque frontier
                    #test_frontier(dq_frontier)
                    #Test heapque frontier
                    test_frontier(hq_frontier)
                    #Test list frontier
                    #test_frontier(list_frontier)
            elif user_option == 2:
                state_space = State_space(strategy, problem)
                node_solution = state_space.search()
                if node_solution == 1:
                    print(f"\nERROR: Maximum depth ({problem.max_depth}) reached.\n")
                else:
                    state_space.export(node_solution)
            elif user_option == 3:
                print("Bye!")
                break

def test_frontier(frontier):
    number_nodes = 1
    id_node = 1
    minimum_time = 0
    maximum_time = 0
    sum_times = 0
    average_time = 0
    while resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/10**6 < 0.1:
        try:
            node = frontier.pop()
            number_nodes -= 1
            successors = State().successor_fn(node.state)
            depth = node.depth + 1
            for successor in successors:
                random_heuristic = "{0:.2f}".format(uniform(0.0, 1000.0))
                random_value = "{0:.2f}".format(uniform(0.0, 1000.0))
                node = Node(id_node, node.cost+successor[2], successor[1], node.ID, successor[0], depth, random_heuristic, random_value)
                pre_time = time.time()
                frontier.push(node)
                post_time = time.time()
                current_time = post_time-pre_time
                sum_times += current_time
                if(id_node == 1):
                    minimum_time = current_time
                    maximum_time = current_time
                else:
                    if(current_time < minimum_time):
                        minimum_time = current_time
                    if(current_time > maximum_time):
                        maximum_time = current_time
                number_nodes += 1
                id_node += 1
                average_time = sum_times/(id_node-1)
        except MemoryError:
            print("Memory consumed: " + str(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/10**6) + " GB" + "\nNumber of elements stored: " + str(number_nodes) + " elements" + "\nMinimum insertion time: " + str(minimum_time) + " seconds" + "\nMaximum insertion time: " + str(maximum_time) + " seconds" + "\nAverage insertion time: " + str(average_time) + " seconds\n")
    print("Memory consumed: " + str(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/10**6) + " GB" + "\nNumber of elements stored: " + str(number_nodes) + " elements" + "\nMinimum insertion time: " + str(minimum_time) + " seconds" + "\nMaximum insertion time: " + str(maximum_time) + " seconds" + "\nAverage insertion time: " + str(average_time) + " seconds\n")

if __name__ =='__main__':
    main()


import os.path
from pathlib import Path
from datetime import datetime
import socket
import selectors
import uuid
from Crypto.Cipher import AES, PKCS1_OAEP
from Crypto.PublicKey import RSA
from Crypto.Random import get_random_bytes
from Crypto.Util.Padding import unpad
from base64 import b64decode, b64encode
from checksum import checksum
import database
import protocol

class Server:
    """
    Server code for mmn15 - Defensive System Programing
    course number 20937

    this is a simple server-client code for exchanging encrypted files based on users
    files are kept inside folders named as the user id of the client that sent them

    @author yonatan tzukerman
    @date 23/03/2023
    """
    DATABASE = "server.db"
    PACKET_SIZE = 1024
    BLOCK_FLAG = False

    def __init__(self, port, host=''):
        """
        set up server parameters
        :param host: hosting ip
        :param port: port number for server to be hosted on
        """
        self.host = host
        self.port = port
        self.users = []  # stores UUID of all users
        self.sel = selectors.DefaultSelector()

        # connect to the database and restore any previous data
        self.backup_db = database.Database(Server.DATABASE)
        self.load_backup_data()

        # keeps track of all files pending crc approval
        self.pending_crc = []

        # handlers for the protocols
        self.requestHandler = {
            protocol.RequestCodes.REQUEST_REGISTRATION.value: self.registration,
            protocol.RequestCodes.REQUEST_PUBLIC_KEY.value: self.public_key_request,
            protocol.RequestCodes.REQUEST_LOGIN.value: self.login_request,
            protocol.RequestCodes.REQUEST_SEND_FILE.value: self.file_request,
            protocol.RequestCodes.REQUEST_VALID_CRC.value: self.valid_crc,
            protocol.RequestCodes.REQUEST_WARNING_CRC.value: self.wrong_crc,
            protocol.RequestCodes.REQUEST_ERROR_CRC.value: self.failed_crc
        }

    def load_backup_data(self):
        """
        load backup user data from database
        """
        data = self.backup_db.fetch_data("clients", "ID, Name")
        self.users = data

        print("Printing user list:")
        for user in self.users:
            print(f"User name - {user[1]}, \nID - {user[0]}\n")

    def start(self):
        """
        Start up the server
        contains an infinite loop for listening to connections
        """
        try:
            # just some socket code to setup the server
            sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            sock.bind((self.host, self.port))
            sock.listen()
            sock.setblocking(Server.BLOCK_FLAG)
            # register the accept connection function for the selector
            self.sel.register(sock, selectors.EVENT_READ, self.accept_connection)
        except Exception as e:
            print(f"Error while setting up server: {e}")
            return

        print(f"Server up and listening in {self.port}!")

        # the main loop
        while True:
            try:
                events = self.sel.select()
                for key, mask in events:
                    try:
                        callback = key.data
                        callback(key.fileobj)
                    except:
                        self.sel.unregister(key.fileobj)
            except Exception as e:
                print(f"Error while listening: {e}")

    def accept_connection(self, sock):
        """
        This function is called whenever there's a new connection
        :param sock: the socket setup in start
        """
        # accept the connection, log it and send to read()
        conn, addr = sock.accept()
        conn.setblocking(Server.BLOCK_FLAG)
        print(f"connection from {addr}")
        self.sel.register(conn, selectors.EVENT_READ, self.read)

    def read(self, conn: socket.socket):
        """
        this is called to read the data from any connection
        :param conn: socket connection
        :return: communicates back to the client if needed
        """
        data = conn.recv(Server.PACKET_SIZE)
        if data:
            try:
                # read the code from the header and send to the proper handler function
                req = protocol.RequestHeader()
                req.unpack(data)
                print(f"Received code: {req.code}")
                if req.code in self.requestHandler.keys():
                    result = self.requestHandler[req.code](conn, data)

                    # try to respond with 2107 if an error happened
                    if not result:
                        try:
                            response = protocol.ErrorResponse()
                            self.write(conn, response.pack())
                        except Exception as e:
                            print(f"failed to deliver exception message - {e}")
            except Exception as e:
                print(f"Exception while reading data from request - {e}")
        else:
            print("No data in connection")

        # closing things
        self.sel.unregister(conn)
        conn.close()

    def write(self, conn: socket.socket, data):
        """
        Write data to client
        :param conn: connection to write back to
        :param data: (bytes) a packed request based on the assignment protocol
        :return: True if sent, False if failed
        """
        size = len(data)
        sent = 0

        # in case data is somehow bigger than packet size (it shouldn't be), we send it in chunks
        while sent < size:
            send_size = min(size - sent, Server.PACKET_SIZE)
            send_data = data[sent:sent + send_size]

            # padding data with 0 until reached packet size
            if len(send_data) < Server.PACKET_SIZE:
                send_data += bytearray(Server.PACKET_SIZE - len(send_data))

            # try writing to client
            try:
                conn.send(send_data)
                sent += len(send_data)
            except:
                print(f"Failed to respond to {conn}")
                return False
        print("Response sent")
        return True

    def registration(self, conn: socket.socket, data):
        """
        CODE = 1100

        client wants to register for the server
        payload is the client name
        if client name is already taken, respond with 2101
        otherwise, register the client as a user with his own UUID and respond with 2100

        :param conn: connection to write back to
        :param data: Registration Request header + payload in bytes (packed)
        :return: (bool) succeeded?
        """
        try:
            request = protocol.RegistrationRequest()
            request.unpack(data)
            print(f"client trying to register as {request.name}")

            # check if name already registered:
            name_used = False
            for user in self.users:
                if user[1] == request.name:
                    name_used = True

            if not name_used:
                # username isn't taken. generate UUID and write him down
                user_id = uuid.uuid4()
                print(f"registration accepted. Generated UUID: {user_id.hex}")

                if not self.backup_db.query(f"INSERT INTO clients (ID, Name, LastSeen) VALUES (?, ?, ?)",
                                            [user_id.hex, request.name, str(datetime.now())]):
                    return False
                self.users.append((user_id.hex, request.name))

                # load data into a response and send it
                response = protocol.RegistrationResponse()  # 2100
                response.client_id = user_id.bytes
                response.header.payload_size = len(response.client_id)

                return self.write(conn, response.pack())
            else:
                # username already taken. send 2101
                response = protocol.RegistrationFailedResponse()
                return self.write(conn, response.pack())

        except Exception as e:
            # print and fall back on exception
            print(f"Exception in registration: {e}")
            return False

    def generate_keys(self, public_key, user_id):
        """
        inner function to generate AES session key
        and encrypt it using a public RSA key and

        :param public_key: public RSA key in base 64
        :param user_id: user id in hex
        :return: tuple - (encrypted session key, session key)
        """
        try:
            print(f"Generating session key for user {user_id}")

            # using Crypto (pycrypto) generate a random session key and encrypt it with public RSA key
            session_key = get_random_bytes(protocol.SYMMETRIC_KEY_SIZE)
            rsa_key = RSA.importKey(b64decode(public_key))
            rsa_cipher = PKCS1_OAEP.new(rsa_key)
            enc_session_key = rsa_cipher.encrypt(session_key)

            print(f"Public key is {public_key} \nSession key is {session_key.hex()}")
            return enc_session_key, session_key

        except Exception as e:
            print(f"Exception while generating keys {e}")
            return None

    def public_key_request(self, conn, data):
        """
        CODE = 1101

        a user is trying to start a new session with a new public key
        this can either be a user that wants to replace his public key, or just a new user
        payload is the client name + public key

        respond with 2102 containing the encrypted session key

        :param conn: connection to write back to
        :param data: Registration Request header + payload in bytes (packed)
        :return: (bool) succeeded?
        """
        try:
            # unpack the data from user
            request = protocol.PublicKeyRequest()
            request.unpack(data)

            user_id = request.header.client_id.hex()
            user_name = request.name
            pub_key = b64encode(request.public_key)

            # make sure user_id and username match the registered data
            if (user_id, user_name) not in self.users:
                return False

            # create an AES session key
            enc_session_key, session_key = self.generate_keys(pub_key, request.header.client_id.hex())

            # delete the old user data and insert the new data
            # this is basically like an update... I decided to divide the "half registered"
            # users from the fully registered ones so delete and insert seems more fitting
            self.backup_db.query(f"DELETE FROM clients WHERE ID = ? AND name = ?", [user_id, user_name])
            if not self.backup_db.query(f"INSERT INTO clients (ID, Name, PublicKey, LastSeen, AESKey) "
                                        f"VALUES (?, ?, ?, ?, ?)",
                                        [user_id, user_name, request.public_key, str(datetime.now()), session_key]):
                return False

            # load data into response and send it
            response = protocol.PublicKeyResponse()  # 2102
            response.client_id = request.header.client_id
            response.symmetric_key = enc_session_key
            response.header.payload_size = len(response.client_id) + len(response.symmetric_key)

            return self.write(conn, response.pack())

        except Exception as e:
            # print and fall back on exception
            print(f"Exception in public key request: {e}")
            return False

    def login_request(self, conn, data):
        """
        CODE = 1102

        a registered user is trying to log in
        need to make sure we have their details in the database before responding

        respond with 2105 containing the encrypted session key if all is good
        respond with 2106 with UUID if data is missing and user needs to register

        :param conn: connection to write back to
        :param data: Registration Request header + payload in bytes (packed)
        :return: (bool) succeeded?
        """
        try:
            # unpacking the data
            request = protocol.LoginRequest()
            request.unpack(data)
            user_id = request.header.client_id.hex()
            user_name = request.name

            # check if user is registered in the database
            query = self.backup_db.query_with_result(f"SELECT PublicKey FROM clients WHERE ID = ? AND Name = ?",
                                                     [user_id, user_name])
            if query:
                # if user is in database, we're all good to send 2105
                public_key = b64encode(query[0][0])

                # create session key
                enc_session_key, session_key = self.generate_keys(public_key, user_id)
                if not self.backup_db.query(f"UPDATE clients SET AESKey = ?, LastSeen = ? WHERE ID = ? AND Name = ?",
                                            [session_key, str(datetime.now()), user_id, user_name]):
                    return False

                # load data into response and send
                response = protocol.LoginResponse()  # 2105
                response.client_id = request.header.client_id
                response.symmetric_key = enc_session_key
                response.header.payload_size = protocol.CLIENT_ID_SIZE + len(response.symmetric_key)

                return self.write(conn, response.pack())
            else:
                # if no data found, we send 2106
                print(f"No data found for user {user_name}, {user_id} during login attempt. sending 2106")
                response = protocol.LoginFailedResponse()  # 2106
                response.client_id = request.header.client_id

                return self.write(conn, response.pack())

        except Exception as e:
            # print and fall back on exception
            print(f"Exception in login request: {e}")
            return False

    def send_checksum(self, conn, file_path, client_id, file_name):
        """
        "private" function to calculate and send the checksum
        this is seperated just in case we ever want to calculate and send the checksum in other functions

        :param conn: connection to write back to
        :param file_path: path to the file we're doing the checksum for
        :param client_id: client id that owns the file
        :param file_name: name of the actual file
        :return: (bool) succeeded?
        """
        try:
            # check the checksum
            cksum, content_size = checksum(str(file_path))
            print(f"check sum is {cksum}")

            # create and pack the response
            response = protocol.CRCResponse()  # 2103
            response.client_id = client_id
            response.content_size = content_size
            response.file_name = bytearray(file_name, 'utf-8')
            response.cksum = cksum

            response.header.payload_size = protocol.CLIENT_ID_SIZE + protocol.CONTENT_SIZE_SIZE
            response.header.payload_size += protocol.NAME_SIZE + protocol.CHECK_SUM_SIZE

            return self.write(conn, response.pack())
        except:
            # if an error happened, we fall back
            return False

    def file_request(self, conn, data):
        """
        CODE = 1103

        a registered user is trying to send a file
        the file should be encrypted using the most recent AES session key
        we want to decrypt it, save and return the details + CRC using 2103

        CRC is the error detecting code. a hashed version of the file data to make sure we (by very high probability)
        got the same file that the client sent us
        CRC is calculated in the same way it does in linux cksum command

        important note:
        this function can be very memory heavy
        that's because we're storing file data and there isn't really a limit on file size...
        I try to minimize it by writing to hard disk and not storing in memory for further use
        yet still the entire request is heavy...
        this can be optimized by decrypting and writing the file while reading the packets
        however, the code is already fairly messy and this will only complicate if further
        so for the current use, this should be fine.

        :param conn: connection to write back to
        :param data: Registration Request header + payload in bytes (packed)
        :return: (bool) succeeded?
        """
        try:
            # unpacking the data. because the file size can be very large,
            # we block other connections while reading and unpacking
            conn.setblocking(True)
            request = protocol.FileSendRequest()  # 1103
            request.unpack(conn, data)
            conn.setblocking(False)
            user_id_hex = request.header.client_id.hex()

            print(f"1103 request from user id is = {user_id_hex} \nFile name is {request.file_name} and size is {request.content_size}")

            # get AES key from db
            query = self.backup_db.query_with_result(f"SELECT AESKey FROM clients WHERE ID = ?",
                                                     [user_id_hex])
            if not query:
                print("Missing user session key for file decryption")
                return False

            session_key = query[0][0]

            # create a directory named after the user id (if non is there)
            path = os.path.join(Path().resolve(), user_id_hex)
            file_path = os.path.join(path, request.file_name)
            if not os.path.exists(user_id_hex):
                os.makedirs(path)

            # check if it's a file that's pending checksum result. in which case we need to check again
            if file_path not in self.pending_crc:
                # make sure user doesn't already have a file by this name another way to handle duplicate files is by
                # adding (1), (2) and so on.. but in that way we'll need to return the new name with the protocol
                # which as per my understanding of the assignment means changing the protocol,
                # and we're not allowed to do that
                if os.path.exists(file_path):
                    print(f"User already has a file by the name {request.file_name}")
                    return False
                self.pending_crc.append(file_path)
            else:
                # remove the file to rewrite it again before the checksum
                print(f"file path is {file_path}")
                os.remove(file_path)

            # decrypt file content
            iv = AES.block_size * b'\0'
            dec_bytes = b''
            for chunk in request.content:
                base64_chunk = b64encode(chunk)
                cipher = AES.new(session_key, AES.MODE_CBC, iv)
                dec_bytes += unpad(cipher.decrypt(b64decode(base64_chunk)), AES.block_size)

            # write the file
            # I'm writing it here and not after making sure the CRC is valid to not store everything in memory
            with open(file_path, "wb") as f:
                f.write(dec_bytes)

            # writing the file in database
            self.backup_db.query(f"INSERT INTO files (ID, FileName, FilePath, Verified) VALUES (?, ?, ?, ?)",
                                 [user_id_hex, request.file_name, file_path, 0])
            self.backup_db.query(f"UPDATE clients SET LastSeen = ? WHERE ID = ?",
                                 [str(datetime.now()), request.header.client_id.hex()])

            return self.send_checksum(conn, file_path, request.header.client_id, request.file_name)

        except Exception as e:
            print(f"Error while receiving file - {e}")
            return False

    def valid_crc(self, conn, data):
        """
        CODE = 1104

        the CRC we sent is valid!
        meaning we can change the status inside the database and remove from pending list

        respond with 2104

        :param conn: connection to write back to
        :param data: Registration Request header + payload in bytes (packed)
        :return: (bool) succeeded?
        """
        try:
            request = protocol.ValidCRCRequest()  # 1104
            request.unpack(data)
            file_name = request.file_name
            user_id = request.header.client_id.hex()

            print(f"CRC is VALID for file {file_name} by user {user_id}")

            # update db (and make sure it worked)
            db_flag_1 = self.backup_db.query(f"UPDATE clients SET LastSeen = ? WHERE ID = ?",
                                             [str(datetime.now()), user_id])
            db_flag_2 = self.backup_db.query(f"UPDATE files SET Verified = ? WHERE ID = ? AND fileName = ?",
                                             [1, user_id, file_name])
            if not db_flag_1 or not db_flag_2:
                return False

            # remove from pending list
            file_path = os.path.join(Path().resolve(), user_id, file_name)
            self.pending_crc.remove(file_path)

            # create and pack the response
            response = protocol.GenericResponse()  # 2104
            response.client_id = request.header.client_id
            return self.write(conn, response.pack())

        except Exception as e:
            print(f"Exception in valid CRC - {e}")
            return False

    def wrong_crc(self, conn, data):
        """
        CODE = 1105

        the CRC we sent is wrong.
        that means we should get a new file request from the user

        not responding with anything (unless 2107), but note that this should only happen 3 times
        I didn't add any checks for that from server side
        mainly because there's nothing stopping the client from sending more files or requests

        :param conn: connection to write back to
        :param data: Registration Request header + payload in bytes (packed)
        :return: (bool) succeeded?
        """
        try:
            request = protocol.WarningCRCRequest()  # 1105
            request.unpack(data)
            user_id = request.header.client_id.hex()
            file_name = request.file_name

            print(f"CRC is WRONG for file {file_name} by user {user_id}")

            # update last seen
            if not self.backup_db.query(f"UPDATE clients SET LastSeen = ? WHERE ID = ?",
                                        [str(datetime.now()), user_id]):
                return False
            return True
        except Exception as e:
            print(f"Exception in failed CRC - {e}")
            return False

    def failed_crc(self, conn, data):
        """
        CODE = 1106

        the CRC we sent is wrong, and we failed the 3 checks
        meaning we should remove the file from directory and pending list

        respond with 2104

        :param conn: connection to write back to
        :param data: Registration Request header + payload in bytes (packed)
        :return: (bool) succeeded?
        """
        try:
            request = protocol.ErrorCRCRequest()  # 1106
            request.unpack(data)

            file_name = request.file_name
            user_id = request.header.client_id.hex()

            print(f"CRC is WRONG. Checksum failed for file {file_name} by user {user_id}")

            db_flag_1 = self.backup_db.query(f"UPDATE clients SET LastSeen = ? WHERE ID = ?",
                                             [str(datetime.now()), user_id])
            query = self.backup_db.query_with_result(f"SELECT FilePath FROM files WHERE ID = ? AND fileName = ?",
                                                     [user_id, file_name])
            # no need to keep the file
            db_flag_2 = self.backup_db.query(f"DELETE FROM files WHERE ID = ? AND fileName = ?",
                                             [user_id, file_name])

            if not db_flag_1 or not db_flag_2 or not query:
                return False
            file_path = query[0][0]

            # TODO: validate that this is safe and we're not deleting anything important
            # it should be because it's ID based but it's always good to double check
            print(f"file path is {file_path}")
            os.remove(file_path)
            self.pending_crc.remove(file_path)

            # pack and send response
            response = protocol.GenericResponse()  # 2104
            response.client_id = request.header.client_id
            return self.write(conn, response.pack())

        except Exception as e:
            print(f"Exception in failed CRC - {e}")
            return False


from django.urls import path
from .views import HomePageView, QuoteDetailView, QuoteCreateView, QuoteUpdateView, QuoteDeleteView, CollectionsList, CollectionDetailView, CollectionCreateView, CollectionAddView, QuoteList

urlpatterns = [
    path('quote/<int:pk>/edit', QuoteUpdateView.as_view(), name = 'quote_edit'),
    path('', HomePageView.as_view(), name='home'),
    path('quote/<int:pk>/', QuoteDetailView.as_view(), name="quote_detail"),
    path('quote/new/', QuoteCreateView.as_view(), name = 'quote_new'),
    path('quote/<int:pk>/delete', QuoteDeleteView.as_view(), name='quote_delete'),
    path('collections', CollectionsList.as_view(), name='collections'),
    path('collections/<int:pk>/', CollectionDetailView.as_view(), name='collection_detail'),
    path('collections/new', CollectionCreateView.as_view(), name='collection_new'),
    path('collections/<int:pk>/add', CollectionAddView.as_view(), name='collection_add'),
    path('quotelist', QuoteList.as_view(), name='quote_list')
]


import difflib
import errno
import glob
import os
import re
import shutil
import stat
import subprocess
import sys
import tempfile
from os import environ
from subprocess import PIPE, Popen
from threading import Thread, Timer

SUCCESS = 0
FAILURE = 1

class Options(object):
    def __init__(self):
        self.binary = ""  # this program's name
        self.cleanupTmp = True  # if false do not remote tmp folder created
        self.p4filename = ""  # file that is being compiled
        self.compilerSrcDir = ""  # path to compiler source tree
        self.verbose = False
        self.replace = False  # replace previous outputs
        self.dumpToJson = False
        self.compilerOptions = []
        self.runDebugger = False
        self.runDebugger_skip = 0
        self.generateP4Runtime = False
        self.generateBfRt = False

def usage(options):
    name = options.binary
    print(name, "usage:")
    print(name, "rootdir [options] file.p4")
    print("Invokes compiler on the supplied file, possibly adding extra arguments")
    print("`rootdir` is the root directory of the compiler source tree")
    print("options:")
    print("          -b: do not remove temporary results for failing tests")
    print("          -v: verbose operation")
    print("          -f: replace reference outputs with newly generated ones")
    print('          -a "args": pass args to the compiler')
    print("          --p4runtime: generate P4Info message in text format")
    print("          --bfrt: generate BfRt message in text format")

def isError(p4filename):
    # True if the filename represents a p4 program that should fail
    return "_errors" in p4filename

def ignoreStderr(options):
    for line in open(options.p4filename):
        if "P4TEST_IGNORE_STDERR" in line:
            return True
    return False

class Local(object):
    # object to hold local vars accessable to nested functions
    pass

def run_timeout(options, args, timeout, stderr):
    if options.verbose:
        print(args[0], args[len(args) - 1])  # handy for manual cut-and-paste
    print(" ".join(args))
    local = Local()
    local.process = None
    local.filter = None

    def target():
        procstderr = None
        if stderr is not None:
            # copy stderr to the specified file, stripping file path prefixes
            # from the start of lines
            outfile = open(stderr, "w")
            # This regex is ridiculously verbose; it's written this way to avoid
            # features that are not supported on both GNU and BSD (i.e., macOS)
            # sed. BSD sed's character class support is not great; for some
            # reason, even some character classes that the man page claims are
            # available don't seem to actually work.
            local.filter = Popen(
                [
                    "sed",
                    "-E",
                    r"s|^[-[:alnum:][:punct:][:space:]_/]*/([-[:alnum:][:punct:][:space:]_]+\.[ph]4?[:(][[:digit:]]+)|\1|",
                ],
                stdin=PIPE,
                stdout=outfile,
            )
            procstderr = local.filter.stdin
        local.process = Popen(args, stderr=procstderr)
        local.process.wait()
        if local.filter is not None:
            local.filter.stdin.close()
            local.filter.wait()

    thread = Thread(target=target)
    thread.start()
    thread.join(timeout)
    if thread.is_alive():
        print("Timeout ", " ".join(args), file=sys.stderr)
        local.process.terminate()
        thread.join()
    if local.process is None:
        # never even started
        if options.verbose:
            print("Process failed to start")
        return -1
    if options.verbose:
        print("Exit code ", local.process.returncode)
    return local.process.returncode

timeout = 10 * 60

def compare_files(options, produced, expected, ignore_case):
    if options.replace:
        if options.verbose:
            print("Saving new version of ", expected)
        shutil.copy2(produced, expected)
        return SUCCESS

    if options.verbose:
        print("Comparing", expected, "and", produced)

    args = "-B -u -w"
    if ignore_case:
        args = args + " -i"
    cmd = "diff " + args + " " + expected + " " + produced + " >&2"
    if options.verbose:
        print(cmd)
    exitcode = subprocess.call(cmd, shell=True)
    if exitcode == 0:
        return SUCCESS
    else:
        return FAILURE

def check_generated_files(options, tmpdir, expecteddir):
    files = os.listdir(tmpdir)
    for file in files:
        if options.verbose:
            print("Checking", file)
        produced = os.path.join(tmpdir, file)
        expected = os.path.join(expecteddir, file)
        if options.replace:
            # Only create files when explicitly asked to do so
            if options.verbose:
                print("Expected file does not exist; creating", expected)
            shutil.copy2(produced, expected)
        elif not os.path.isfile(expected):
            # The file is missing and we do not replace. This is an error.
            print(
                'Missing reference for file %s. Please rerun the test with the -f option turned on'
                ' or rerun all tests using "P4TEST_REPLACE=True make check".' % expected
            )
            return FAILURE
        result = compare_files(options, produced, expected, file[-6:] == "-error")
        # We do not want to compare stderr output generated by p4c-dpdk
        if result != SUCCESS and (file[-6:] == "-error"):
            return SUCCESS
        if result != SUCCESS and not ignoreStderr(options):
            return result
        if produced.endswith(".spec") and environ.get("DPDK_PIPELINE") is not None:
            clifile = os.path.splitext(produced)[0] + ".cli"
            with open(clifile, "w", encoding="utf-8") as f:
                f.write("; SPDX-License-Identifier: BSD-3-Clause\n")
                f.write("; Copyright(c) 2020 Intel Corporation\n")
                f.write("\n")
                f.write("mempool MEMPOOL0 buffer 9304 pool 32K cache 256 cpu 0\n")
                f.write("\n")
                f.write("link LINK0 dev 0000:00:04.0 rxq 1 128 MEMPOOL0 txq 1 512 promiscuous on\n")
                f.write("link LINK1 dev 0000:00:05.0 rxq 1 128 MEMPOOL0 txq 1 512 promiscuous on\n")
                f.write("link LINK2 dev 0000:00:06.0 rxq 1 128 MEMPOOL0 txq 1 512 promiscuous on\n")
                f.write("link LINK3 dev 0000:00:07.0 rxq 1 128 MEMPOOL0 txq 1 512 promiscuous on\n")
                f.write("\n")
                f.write("pipeline PIPELINE0 create 0\n")
                f.write("\n")
                f.write("pipeline PIPELINE0 port in 0 link LINK0 rxq 0 bsz 32\n")
                f.write("pipeline PIPELINE0 port in 1 link LINK1 rxq 0 bsz 32\n")
                f.write("pipeline PIPELINE0 port in 2 link LINK2 rxq 0 bsz 32\n")
                f.write("pipeline PIPELINE0 port in 3 link LINK3 rxq 0 bsz 32\n")
                f.write("\n")
                f.write("pipeline PIPELINE0 port out 0 link LINK0 txq 0 bsz 32\n")
                f.write("pipeline PIPELINE0 port out 1 link LINK1 txq 0 bsz 32\n")
                f.write("pipeline PIPELINE0 port out 2 link LINK2 txq 0 bsz 32\n")
                f.write("pipeline PIPELINE0 port out 3 link LINK3 txq 0 bsz 32\n")
                f.write("\n")
                f.write("pipeline PIPELINE0 build ")
                f.write(str(expected))
                f.write("\n")
                f.write("pipeline PIPELINE0 commit\n")
                f.write("thread 1 pipeline PIPELINE0 enable\n")
                f.write("pipeline PIPELINE0 abort\n")
                f.close()
            dpdk_log = os.path.splitext(produced)[0] + ".log"

            def kill(process):
                process.kill()

            print(
                "exec "
                + environ.get("DPDK_PIPELINE")
                + " -n 4 -c 0x3 -- -s "
                + str(clifile)
                + " 1>"
                + dpdk_log
            )
            pipe = subprocess.Popen(
                "exec "
                + environ.get("DPDK_PIPELINE")
                + " -n 4 -c 0x3 -- -s "
                + str(clifile)
                + " 1>"
                + dpdk_log,
                cwd=".",
                shell=True,
            )
            timer = Timer(5, kill, [pipe])
            try:
                timer.start()
                out, err = pipe.communicate()
            finally:
                timer.cancel()
            with open(dpdk_log, "r") as f:
                readfile = f.read()
                if "Error" in readfile:
                    print(readfile)
                    return FAILURE
    return SUCCESS

def file_name(tmpfolder, base, suffix, ext):
    return os.path.join(tmpfolder, base + "-" + suffix + ext)

def process_file(options, argv):
    assert isinstance(options, Options)

    tmpdir = tempfile.mkdtemp(dir=".")
    basename = os.path.basename(options.p4filename)
    base, ext = os.path.splitext(basename)
    dirname = os.path.dirname(options.p4filename)
    if "_samples/" in dirname:
        expected_dirname = dirname.replace("_samples/", "_samples_outputs/", 1)
    elif "_errors/" in dirname:
        expected_dirname = dirname.replace("_errors/", "_errors_outputs/", 1)
    elif "p4_14/" in dirname:
        expected_dirname = dirname.replace("p4_14/", "p4_14_outputs/", 1)
    elif "p4_16/" in dirname:
        expected_dirname = dirname.replace("p4_16/", "p4_16_outputs/", 1)
    else:
        expected_dirname = dirname + "_outputs"  # expected outputs are here
    if not os.path.exists(expected_dirname):
        os.makedirs(expected_dirname)

    if options.verbose:
        print("Writing temporary files into ", tmpdir)
    stderr = os.path.join(tmpdir, basename + "-error")
    spec = os.path.join(tmpdir, basename + ".spec")
    p4runtimeFile = os.path.join(tmpdir, basename + ".p4info.txt")
    p4runtimeEntriesFile = os.path.join(tmpdir, basename + ".entries.txt")
    bfRtSchemaFile = os.path.join(tmpdir, basename + ".bfrt.json")

    def getArch(path):
        v1Pattern = re.compile("include.*v1model\.p4")
        pnaPattern = re.compile("include.*pna\.p4")
        psaPattern = re.compile("include.*psa\.p4")
        ubpfPattern = re.compile("include.*ubpf_model\.p4")
        with open(path, "r", encoding="utf-8") as f:
            for line in f:
                if v1Pattern.search(line):
                    return "v1model"
                elif psaPattern.search(line):
                    return "psa"
                elif pnaPattern.search(line):
                    return "pna"
                elif ubpfPattern.search(line):
                    return "ubpf"
            return None

    if not os.path.isfile(options.p4filename):
        raise Exception("No such file " + options.p4filename)
    args = ["./p4c-dpdk", "--dump", tmpdir, "-o", spec] + options.compilerOptions
    arch = getArch(options.p4filename)
    if arch is not None:
        args.extend(["--arch", arch])
        if options.generateP4Runtime:
            args.extend(["--p4runtime-files", p4runtimeFile])
            args.extend(["--p4runtime-entries-files", p4runtimeEntriesFile])
        if options.generateBfRt:
            args.extend(["--bf-rt-schema", bfRtSchemaFile])

    if "p4_14" in options.p4filename or "v1_samples" in options.p4filename:
        args.extend(["--std", "p4-14"])
    args.extend(argv)
    if options.runDebugger:
        if options.runDebugger_skip > 0:
            options.runDebugger_skip = options.runDebugger_skip - 1
        else:
            args[0:0] = options.runDebugger.split()
            os.execvp(args[0], args)
    result = run_timeout(options, args, timeout, stderr)

    if result != SUCCESS:
        print("Error compiling")
        print("".join(open(stderr).readlines()))
        # If the compiler crashed fail the test
        if "Compiler Bug" in open(stderr).readlines():
            return FAILURE

    # invert result
    expected_error = isError(options.p4filename)
    if expected_error:
        # invert result
        if result == SUCCESS:
            result = FAILURE
        else:
            result = SUCCESS

    if result == SUCCESS:
        result = check_generated_files(options, tmpdir, expected_dirname)

    if options.cleanupTmp:
        if options.verbose:
            print("Removing", tmpdir)
        shutil.rmtree(tmpdir)
    return result

def isdir(path):
    try:
        return stat.S_ISDIR(os.stat(path).st_mode)
    except OSError:
        return False

######################### main

def main(argv):
    options = Options()

    options.binary = argv[0]
    if len(argv) <= 2:
        usage(options)
        sys.exit(FAILURE)

    options.compilerSrcdir = argv[1]
    argv = argv[2:]
    if not os.path.isdir(options.compilerSrcdir):
        print(options.compilerSrcdir + " is not a folder", file=sys.stderr)
        usage(options)
        sys.exit(FAILURE)

    while argv[0][0] == "-":
        if argv[0] == "-b":
            options.cleanupTmp = False
        elif argv[0] == "-v":
            options.verbose = True
        elif argv[0] == "-f":
            options.replace = True
        elif argv[0] == "-j":
            options.dumpToJson = True
        elif argv[0] == "-a":
            if len(argv) == 0:
                print("Missing argument for -a option")
                usage(options)
                sys.exit(FAILURE)
            else:
                options.compilerOptions += argv[1].split()
                argv = argv[1:]
        elif argv[0][1] == "D" or argv[0][1] == "I" or argv[0][1] == "T":
            options.compilerOptions.append(argv[0])
        elif argv[0][0:4] == "-gdb":
            options.runDebugger = "gdb --args"
            if len(argv[0]) > 4:
                options.runDebugger_skip = int(argv[0][4:]) - 1
        elif argv[0] == "--p4runtime":
            options.generateP4Runtime = True
        elif argv[0] == "--bfrt":
            options.generateBfRt = True
        else:
            print("Unknown option ", argv[0], file=sys.stderr)
            usage(options)
            sys.exit(FAILURE)
        argv = argv[1:]

    if "P4TEST_REPLACE" in os.environ:
        options.replace = True

    options.p4filename = argv[-1]
    options.testName = None
    if options.p4filename.startswith(options.compilerSrcdir):
        options.testName = options.p4filename[len(options.compilerSrcdir) :]
        if options.testName.startswith("/"):
            options.testName = options.testName[1:]
        if options.testName.endswith(".p4"):
            options.testName = options.testName[:-3]

    result = process_file(options, argv)
    if isError(options.p4filename) and result == FAILURE:
        print("Program was expected to fail")

    sys.exit(result)

if __name__ == "__main__":
    main(sys.argv)


"""
a module that creates ascii art from images
"""

# Import an image processing tool
from PIL import Image
import numpy as np
from colorize import bold

def main():
    """
    main process that converts image and prints ascii art
    """
    # initalise images
    # initalize color numpy arrays
    red = np.array((255, 0, 0))
    black = np.array((0, 0, 0))
    green = np.array((0, 255, 0))
    yellow = np.array((255, 255, 0))
    blue = np.array((0, 0, 255))
    purple = np.array((255, 0, 255))
    cyan = np.array((0, 255, 255))
    white = np.array((255, 255, 255))

    # initalize list of numpy array colors
    colors = [red, green, blue, cyan, yellow, purple, white, black]
    # initalize list of color strings
    str_clrs = ['red', 'green', 'blue', 'cyan', 'yellow', 'purple', 'white', 'black']

    images = {
        "python": Image.open("python.jpg"),
    }
    # enter choice
    choice = input("enter name of character: ")
    image = images[choice]
    # Define the ASCII characters
    asci = ['\u1D77', '\u25AE', u'\u213B']
    # initialize
    text = ''
    len_chars = len(asci)
    step = 255/len_chars
    value, value2 = 0, step
    x_pos, y_pos, z_pos = 0, 0, 0
    # open and resize the image
    scale = image.height/350
    image = image.resize((int(image.width / scale), int(image.height / scale)))
    image = image.resize((int(image.width / 1), int(image.height / 2.3)))

    # For every pixel in the image...
    while y_pos < image.height:
        # Find its brightness
        brightness = sum(image.getpixel((x_pos, y_pos))) / len_chars
        # find its color
        color = image.getpixel((x_pos, y_pos))
        # make value1 numpy array out of the color rgb
        color_array = np.array(color)
        # initalise the subtracted arrays list
        subtracted_arrays = []

        # find the color the current pixel is closest to
        for item in colors:
            difference = np.abs(item - color_array)
            difference = sum(difference)
            subtracted_arrays.append(difference)

        minpos = subtracted_arrays.index(min(subtracted_arrays))
        color = str_clrs[minpos]

        # Depending on its brightness, assign it an ASCII character
        while z_pos < len_chars:
            #print(value1, brightness, value2)
            if value <= brightness <= value2:
                #print(characters[z_pos])
                text += bold(color, asci[z_pos])
                break
            # increment variables
            value += step
            value2 += step
            z_pos += 1
        # reset window and idx
        z_pos = 0
        value = 0
        value2 = step
        # next pixel same line
        x_pos += 1
        # new line
        if x_pos == image.width - 1:
            text += '\n'
            y_pos += 1
            x_pos = 0
    # finally print the output
    print("\033c")
    print(text)

if __name__ == '__main__':
    main()


import pyautogui
import time
import logging
from itertools import compress

logger = logging.getLogger(__name__)

def press_key(keyval=None):
    """
        Presses the given key and releases the other keys in 'wasd'
    """

    stop_press = ['w', 'a', 'd', 's']
    if keyval:
        stop_press.remove(keyval)
        pyautogui.keyDown(keyval)

    for i in stop_press:
        pyautogui.keyUp(i)

    if keyval:
        pyautogui.keyUp(keyval)

def array_press(key_array: dict):
    """
        Presses the keys based on the input
        [0, 0, 0, 0] & ['w', 'a', 's', 'd'] = all keys released
        [1, 0, 0, 1] & ['w', 'a', 's', 'd'] = 'w', 'd' pressed, others released

    """
    base_array = ['w', 'a' , 's', 'd']
    to_press = list(compress(base_array, key_array))

    for key in to_press:
        pyautogui.keyDown(key)
        base_array.remove(key)

    # print(f'press: {to_press} \t\t release: {base_array}')

    for key in base_array:
        pyautogui.keyUp(key)

def cont_straight():
    press_key('w')

def cont_left():
    press_key('a')

def cont_right():
    press_key('d')

def cont_slow():
    press_key('s')

def stop_all():
    press_key()

def control_test():
    """
        Run test
    """

    array_press([0,0,0,0])
    array_press([1,0,0,0])
    array_press([1,1,0,0])
    array_press([1,0,0,1])
    array_press([0,0,0,0])

if __name__ == "__main__":
    logging.basicConfig(level=logging.INFO)
    control_test()