import csv
import os
import logging
import argparse
import random
import collections
from tqdm import tqdm, trange
import json

import pdb

import numpy as np
import torch
from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler
from torch.utils.data.distributed import DistributedSampler

from pytorch_pretrained_bert.tokenization import BertTokenizer
from pytorch_pretrained_bert.optimization import BertAdam

from tensorboardX import SummaryWriter

logging.basicConfig(format='%(asctime)s - %(levelname)s - %(name)s -   %(message)s',
                    datefmt='%m/%d/%Y %H:%M:%S',
                    level=logging.INFO)
logger = logging.getLogger(__name__)

class InputExample(object):
    """A single training/test example for simple sequence classification."""

    def __init__(self, guid, text_a, text_b=None, label=None):
        """Constructs a InputExample.

        Args:
            guid: Unique id for the example.
            text_a: string. The untokenized text of the first sequence. For single
            sequence tasks, only this sequence must be specified.
            text_b: (Optional) string. The untokenized text of the second sequence.
            Only must be specified for sequence pair tasks.
            label: (Optional) string. The label of the example. This should be
            specified for train and dev examples, but not for test examples.
        """
        self.guid = guid
        self.text_a = text_a
        self.text_b = text_b
        self.label = label

class InputFeatures(object):
    """A single set of features of data."""

    def __init__(self, input_ids, input_len, label_id):
        self.input_ids = input_ids
        self.input_len = input_len
        self.label_id = label_id

class DataProcessor(object):
    """Base class for data converters for sequence classification data sets."""
    def get_train_examples(self, data_dir):
        """Gets a collection of `InputExample`s for the train set."""
        raise NotImplementedError()

    def get_dev_examples(self, data_dir):
        """Gets a collection of `InputExample`s for the dev set."""
        raise NotImplementedError()

    def get_labels(self):
        """Gets the list of labels for this data set."""
        raise NotImplementedError()

    @classmethod
    def _read_tsv(cls, input_file, quotechar=None):
        """Reads a tab separated value file."""
        with open(input_file, "r", encoding='utf-8') as f:
            reader = csv.reader(f, delimiter="\t", quotechar=quotechar)
            lines = []
            for line in reader:
                if len(line) > 0 and line[0][0] == '#':     # ignore comments (starting with '#')
                    continue
                lines.append(line)
            return lines

class Processor(DataProcessor):
    """Processor for the belief tracking dataset (GLUE version)."""

    def __init__(self, config):
        super(Processor, self).__init__()

        # WOZ2.0 dataset
        if config.data_dir == "data/woz" or config.data_dir=="data/woz-turn":
            fp_ontology = open(os.path.join(config.data_dir, "ontology_dstc2_en.json"), "r")
            ontology = json.load(fp_ontology)
            ontology = ontology["informable"]
            del ontology["request"]
            for slot in ontology.keys():
                ontology[slot].append("do not care")
                ontology[slot].append("none")
            fp_ontology.close()

        # MultiWOZ dataset
        elif config.data_dir == "data/multiwoz":
            fp_ontology = open(os.path.join(config.data_dir, "ontology.json"), "r")
            ontology = json.load(fp_ontology)
            for slot in ontology.keys():
                ontology[slot].append("none")
            fp_ontology.close()

            if not config.target_slot == 'all':
                slot_idx = {'attraction':'0:1:2', 'bus':'3:4:5:6', 'hospital':'7', 'hotel':'8:9:10:11:12:13:14:15:16:17',\
                            'restaurant':'18:19:20:21:22:23:24', 'taxi':'25:26:27:28', 'train':'29:30:31:32:33:34'}
                target_slot =[]
                for key, value in slot_idx.items():
                    if key != config.target_slot:
                        target_slot.append(value)
                config.target_slot = ':'.join(target_slot)

        else:
            raise NotImplementedError()

        # sorting the ontology according to the alphabetic order of the slots
        ontology = collections.OrderedDict(sorted(ontology.items()))

        # select slots to train
        nslots = len(ontology.keys())
        target_slot = list(ontology.keys())
        if config.target_slot == 'all':
            self.target_slot_idx = [*range(0, nslots)]
        else:
            self.target_slot_idx = sorted([ int(x) for x in config.target_slot.split(':')])

        for idx in range(0, nslots):
            if not idx in self.target_slot_idx:
                del ontology[target_slot[idx]]

        self.ontology = ontology
        self.target_slot = list(self.ontology.keys())
        for i, slot in enumerate(self.target_slot):
            if slot == "pricerange":
                self.target_slot[i] = "price range"

        logger.info('Processor: target_slot')
        logger.info(self.target_slot)

    def get_train_examples(self, data_dir, accumulation=False):
        """See base class."""
        return self._create_examples(
            self._read_tsv(os.path.join(data_dir, "train.tsv")), "train", accumulation)

    def get_dev_examples(self, data_dir, accumulation=False):
        """See base class."""
        return self._create_examples(
            self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev", accumulation)

    def get_test_examples(self, data_dir, accumulation=False):
        """See base class."""
        return self._create_examples(
            self._read_tsv(os.path.join(data_dir, "test.tsv")), "test", accumulation)

    def get_labels(self):
        """See base class."""
        return [ self.ontology[slot] for slot in self.target_slot]

    def _create_examples(self, lines, set_type, accumulation=False):
        """Creates examples for the training and dev sets."""
        prev_dialogue_index = None
        examples = []
        for (i, line) in enumerate(lines):
            guid = "%s-%s-%s" % (set_type, line[0], line[1])  # line[0]: dialogue index, line[1]: turn index
            if accumulation:
                if prev_dialogue_index is None or prev_dialogue_index != line[0]:
                    text_a = line[2]
                    text_b = line[3]
                    prev_dialogue_index = line[0]
                else:
                    # The symbol '#' will be replaced with '[SEP]' after tokenization.
                    text_a = line[2] + " # " + text_a
                    text_b = line[3] + " # " + text_b
            else:
                text_a = line[2]  # line[2]: user utterance
                text_b = line[3]  # line[3]: system response

            label = [line[4+idx] for idx in self.target_slot_idx]

            examples.append(
                InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label))
        return examples

def convert_examples_to_features(examples, label_list, max_seq_length, tokenizer, max_turn_length):
    """Loads a data file into a list of `InputBatch`s."""

    label_map = [{label: i for i, label in enumerate(labels)} for labels in label_list]
    slot_dim = len(label_list)

    features = []
    prev_dialogue_idx = None
    all_padding = [0] * max_seq_length
    all_padding_len = [0, 0]

    max_turn = 0
    for (ex_index, example) in enumerate(examples):
        if max_turn < int(example.guid.split('-')[2]):
            max_turn = int(example.guid.split('-')[2])
    max_turn_length = min(max_turn+1, max_turn_length)
    logger.info("max_turn_length = %d" % max_turn)

    for (ex_index, example) in enumerate(examples):
        tokens_a = [x if x != '#' else '[SEP]' for x in tokenizer.tokenize(example.text_a)]
        tokens_b = None
        if example.text_b:
            tokens_b = [x if x != '#' else '[SEP]' for x in tokenizer.tokenize(example.text_b)]
            # Modifies `tokens_a` and `tokens_b` in place so that the total
            # length is less than the specified length.
            # Account for [CLS], [SEP], [SEP] with "- 3"
            _truncate_seq_pair(tokens_a, tokens_b, max_seq_length - 3)
        else:
            # Account for [CLS] and [SEP] with "- 2"
            if len(tokens_a) > max_seq_length - 2:
                tokens_a = tokens_a[:(max_seq_length - 2)]

        # The convention in BERT is:
        # (a) For sequence pairs:
        #  tokens:   [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]
        #  type_ids: 0   0  0    0    0     0       0 0    1  1  1  1   1 1
        # (b) For single sequences:
        #  tokens:   [CLS] the dog is hairy . [SEP]
        #  type_ids: 0   0   0   0  0     0 0
        #
        # Where "type_ids" are used to indicate whether this is the first
        # sequence or the second sequence. The embedding vectors for `type=0` and
        # `type=1` were learned during pre-training and are added to the wordpiece
        # embedding vector (and position vector). This is not *strictly* necessary
        # since the [SEP] token unambigiously separates the sequences, but it makes
        # it easier for the model to learn the concept of sequences.
        #
        # For classification tasks, the first vector (corresponding to [CLS]) is
        # used as as the "sentence vector". Note that this only makes sense because
        # the entire model is fine-tuned.

        tokens = ["[CLS]"] + tokens_a + ["[SEP]"]
        input_len = [len(tokens), 0]

        if tokens_b:
            tokens += tokens_b + ["[SEP]"]
            input_len[1] = len(tokens_b) + 1

        input_ids = tokenizer.convert_tokens_to_ids(tokens)

        # Zero-pad up to the sequence length.
        padding = [0] * (max_seq_length - len(input_ids))
        input_ids += padding
        assert len(input_ids) == max_seq_length

        label_id = []
        label_info = 'label: '
        for i, label in enumerate(example.label):
            if label == 'dontcare':
                label = 'do not care'
            label_id.append(label_map[i][label])
            label_info += '%s (id = %d) ' % (label, label_map[i][label])

        if ex_index < 5:
            logger.info("*** Example ***")
            logger.info("guid: %s" % example.guid)
            logger.info("tokens: %s" % " ".join(
                [str(x) for x in tokens]))
            logger.info("input_ids: %s" % " ".join([str(x) for x in input_ids]))
            logger.info("input_len: %s" % " ".join([str(x) for x in input_len]))
            logger.info("label: " + label_info)

        curr_dialogue_idx = example.guid.split('-')[1]
        curr_turn_idx = int(example.guid.split('-')[2])

        if prev_dialogue_idx is not None and prev_dialogue_idx != curr_dialogue_idx:
            if prev_turn_idx < max_turn_length:
                features += [InputFeatures(input_ids=all_padding,
                                           input_len=all_padding_len,
                                           label_id=[-1]*slot_dim)]\
                            *(max_turn_length - prev_turn_idx - 1)
            assert len(features) % max_turn_length == 0

        if prev_dialogue_idx is None or prev_turn_idx < max_turn_length:
            features.append(
                InputFeatures(input_ids=input_ids,
                              input_len=input_len,
                              label_id=label_id))

        prev_dialogue_idx = curr_dialogue_idx
        prev_turn_idx = curr_turn_idx

    if prev_turn_idx < max_turn_length:
        features += [InputFeatures(input_ids=all_padding,
                                   input_len=all_padding_len,
                                   label_id=[-1]*slot_dim)]\
                    * (max_turn_length - prev_turn_idx - 1)
    assert len(features) % max_turn_length == 0

    all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long)
    all_input_len= torch.tensor([f.input_len for f in features], dtype=torch.long)
    all_label_ids = torch.tensor([f.label_id for f in features], dtype=torch.long)

    # reshape tensors to [#batch, #max_turn_length, #max_seq_length]
    all_input_ids = all_input_ids.view(-1, max_turn_length, max_seq_length)
    all_input_len = all_input_len.view(-1, max_turn_length, 2)
    all_label_ids = all_label_ids.view(-1, max_turn_length, slot_dim)

    return all_input_ids, all_input_len, all_label_ids

def get_label_embedding(labels, max_seq_length, tokenizer, device):
    features = []
    for label in labels:
        label_tokens = ["[CLS]"] + tokenizer.tokenize(label) + ["[SEP]"]
        label_token_ids = tokenizer.convert_tokens_to_ids(label_tokens)
        label_len = len(label_token_ids)

        label_padding = [0] * (max_seq_length - len(label_token_ids))
        label_token_ids += label_padding
        assert len(label_token_ids) == max_seq_length

        features.append((label_token_ids, label_len))

    all_label_token_ids = torch.tensor([f[0] for f in features], dtype=torch.long).to(device)
    all_label_len = torch.tensor([f[1] for f in features], dtype=torch.long).to(device)

    return all_label_token_ids, all_label_len

def _truncate_seq_pair(tokens_a, tokens_b, max_length):
    """Truncates a sequence pair in place to the maximum length."""

    # This is a simple heuristic which will always truncate the longer sequence
    # one token at a time. This makes more sense than truncating an equal percent
    # of tokens from each, since if one sequence is very short then each token
    # that's truncated likely contains more information than a longer sequence.
    while True:
        total_length = len(tokens_a) + len(tokens_b)
        if total_length <= max_length:
            break
        if len(tokens_a) > len(tokens_b):
            tokens_a.pop()
        else:
            tokens_b.pop()

def warmup_linear(x, warmup=0.002):
    if x < warmup:
        return x / warmup
    return 1.0 - x

def main():
    parser = argparse.ArgumentParser()

    ## Required parameters
    parser.add_argument("--data_dir",
                        default=None,
                        type=str,
                        required=True,
                        help="The input data dir. Should contain the .tsv files (or other data files) for the task.")
    parser.add_argument("--bert_model", default='bert-base-uncased', type=str, required=True,
                        help="Bert pre-trained model selected in the list: bert-base-uncased, "
                             "bert-large-uncased, bert-base-cased, bert-large-cased, bert-base-multilingual-uncased, "
                             "bert-base-multilingual-cased, bert-base-chinese.")
    parser.add_argument("--bert_dir", default='/home/.pytorch_pretrained_bert',
                        type=str, required=False,
                        help="The directory of the pretrained BERT model")
    parser.add_argument("--task_name",
                        default=None,
                        type=str,
                        required=True,
                        help="The name of the task to train: bert-gru-sumbt, bert-lstm-sumbt"
                             "bert-label-embedding, bert-gru-label-embedding, bert-lstm-label-embedding")
    parser.add_argument("--output_dir",
                        default=None,
                        type=str,
                        required=True,
                        help="The output directory where the model predictions and checkpoints will be written.")
    parser.add_argument("--target_slot",
                        default='all',
                        type=str,
                        required=True,
                        help="Target slot idx to train model. ex. 'all', '0:1:2', or an excluding slot name 'attraction'" )
    parser.add_argument("--tf_dir",
                        default='tensorboard',
                        type=str,
                        required=False,
                        help="Tensorboard directory")
    parser.add_argument("--nbt",
                        default='rnn',
                        type=str,
                        required=True,
                        help="nbt type: rnn or transformer" )
    parser.add_argument("--fix_utterance_encoder",
                        action='store_true',
                        help="Do not train BERT utterance encoder")

    ## Other parameters
    parser.add_argument("--max_seq_length",
                        default=64,
                        type=int,
                        help="The maximum total input sequence length after WordPiece tokenization. \n"
                             "Sequences longer than this will be truncated, and sequences shorter \n"
                             "than this will be padded.")
    parser.add_argument("--max_label_length",
                        default=32,
                        type=int,
                        help="The maximum total input sequence length after WordPiece tokenization. \n"
                             "Sequences longer than this will be truncated, and sequences shorter \n"
                             "than this will be padded.")
    parser.add_argument("--max_turn_length",
                        default=22,
                        type=int,
                        help="The maximum total input turn length. \n"
                             "Sequences longer than this will be truncated, and sequences shorter \n"
                             "than this will be padded.")
    parser.add_argument('--hidden_dim',
                        type=int,
                        default=100,
                        help="hidden dimension used in belief tracker")
    parser.add_argument('--num_rnn_layers',
                        type=int,
                        default=1,
                        help="number of RNN layers")
    parser.add_argument('--zero_init_rnn',
                        action='store_true',
                        help="set initial hidden of rnns zero")
    parser.add_argument('--attn_head',
                        type=int,
                        default=4,
                        help="the number of heads in multi-headed attention")
    parser.add_argument("--do_train",
                        action='store_true',
                        help="Whether to run training.")
    parser.add_argument("--do_eval",
                        action='store_true',
                        help="Whether to run eval on the test set.")
    parser.add_argument("--do_lower_case",
                        action='store_true',
                        help="Set this flag if you are using an uncased model.")
    parser.add_argument("--distance_metric",
                        type=str,
                        default="cosine",
                        help="The metric for distance between label embeddings: cosine, euclidean.")
    parser.add_argument("--train_batch_size",
                        default=4,
                        type=int,
                        help="Total dialog batch size for training.")
    parser.add_argument("--dev_batch_size",
                        default=1,
                        type=int,
                        help="Total dialog batch size for validation.")
    parser.add_argument("--eval_batch_size",
                        default=16,
                        type=int,
                        help="Total dialog batch size for evaluation.")
    parser.add_argument("--learning_rate",
                        default=5e-5,
                        type=float,
                        help="The initial learning rate for BertAdam.")
    parser.add_argument("--num_train_epochs",
                        default=3.0,
                        type=float,
                        help="Total number of training epochs to perform.")
    parser.add_argument("--patience",
                        default=10.0,
                        type=float,
                        help="The number of epochs to allow no further improvement.")
    parser.add_argument("--warmup_proportion",
                        default=0.1,
                        type=float,
                        help="Proportion of training to perform linear learning rate warmup for. "
                             "E.g., 0.1 = 10%% of training.")
    parser.add_argument("--no_cuda",
                        action='store_true',
                        help="Whether not to use CUDA when available")
    parser.add_argument("--local_rank",
                        type=int,
                        default=-1,
                        help="local_rank for distributed training on gpus")
    parser.add_argument('--seed',
                        type=int,
                        default=42,
                        help="random seed for initialization")
    parser.add_argument('--gradient_accumulation_steps',
                        type=int,
                        default=1,
                        help="Number of updates steps to accumulate before performing a backward/update pass.")
    parser.add_argument('--fp16',
                        action='store_true',
                        help="Whether to use 16-bit float precision instead of 32-bit")
    parser.add_argument('--loss_scale',
                        type=float, default=0,
                        help="Loss scaling to improve fp16 numeric stability. Only used when fp16 set to True.\n"
                             "0 (default value): dynamic loss scaling.\n"
                             "Positive power of 2: static loss scaling value.\n")
    parser.add_argument("--do_not_use_tensorboard",
                        action='store_true',
                        help="Whether to run eval on the test set.")

    args = parser.parse_args()

    # check output_dir
    if os.path.exists(args.output_dir) and os.listdir(args.output_dir) and args.do_train:
        raise ValueError("Output directory ({}) already exists and is not empty.".format(args.output_dir))
    os.makedirs(args.output_dir, exist_ok=True)

    if not args.do_train and not args.do_eval and not args.do_analyze:
        raise ValueError("At least one of `do_train` or `do_eval` must be True.")

    # Tensorboard logging
    if not args.do_not_use_tensorboard:
        tb_file_name = args.output_dir.split('/')[1]
        summary_writer = SummaryWriter("./%s/%s" % (args.tf_dir, tb_file_name))
    else:
        summary_writer = None

    # Logger
    fileHandler = logging.FileHandler(os.path.join(args.output_dir, "%s.txt"%(tb_file_name)))
    logger.addHandler(fileHandler)
    logger.info(args)

    # CUDA setup
    if args.local_rank == -1 or args.no_cuda:
        device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu")
        n_gpu = torch.cuda.device_count()
    else:
        torch.cuda.set_device(args.local_rank)
        device = torch.device("cuda", args.local_rank)
        n_gpu = 1
        # Initializes the distributed backend which will take care of sychronizing nodes/GPUs
        torch.distributed.init_process_group(backend='nccl')
    logger.info("device: {} n_gpu: {}, distributed training: {}, 16-bits training: {}".format(
        device, n_gpu, bool(args.local_rank != -1), args.fp16))

    if args.gradient_accumulation_steps < 1:
        raise ValueError("Invalid gradient_accumulation_steps parameter: {}, should be >= 1".format(
            args.gradient_accumulation_steps))

    args.train_batch_size = int(args.train_batch_size / args.gradient_accumulation_steps)

    # Set random seed
    random.seed(args.seed)
    np.random.seed(args.seed)
    torch.manual_seed(args.seed)
    if n_gpu > 0:
        torch.cuda.manual_seed_all(args.seed)

    ###############################################################################
    # Load data
    ###############################################################################

    # Get Processor
    processor = Processor(args)
    label_list = processor.get_labels()
    num_labels = [len(labels) for labels in label_list] # number of slot-values in each slot-type

    # tokenizer
    vocab_dir = os.path.join(args.bert_dir, '%s-vocab.txt' % args.bert_model)
    if not os.path.exists(vocab_dir):
        raise ValueError("Can't find %s " % vocab_dir)
    tokenizer = BertTokenizer.from_pretrained(vocab_dir, do_lower_case=args.do_lower_case)

    num_train_steps = None
    accumulation = False

    if args.do_train:
        train_examples = processor.get_train_examples(args.data_dir, accumulation=accumulation)
        dev_examples = processor.get_dev_examples(args.data_dir, accumulation=accumulation)

        ## Training utterances
        all_input_ids, all_input_len, all_label_ids = convert_examples_to_features(
            train_examples, label_list, args.max_seq_length, tokenizer, args.max_turn_length)

        num_train_batches = all_input_ids.size(0)
        num_train_steps = int(num_train_batches / args.train_batch_size / args.gradient_accumulation_steps * args.num_train_epochs)

        logger.info("***** Running training *****")
        logger.info("  Num examples = %d", len(train_examples))
        logger.info("  Batch size = %d", args.train_batch_size)
        logger.info("  Num steps = %d", num_train_steps)

        all_input_ids, all_input_len, all_label_ids = all_input_ids.to(device), all_input_len.to(device), all_label_ids.to(device)

        train_data = TensorDataset(all_input_ids, all_input_len, all_label_ids)
        if args.local_rank == -1:
            train_sampler = RandomSampler(train_data)
        else:
            train_sampler = DistributedSampler(train_data)

        train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=args.train_batch_size)

        ## Dev utterances
        all_input_ids_dev, all_input_len_dev, all_label_ids_dev = convert_examples_to_features(
            dev_examples, label_list, args.max_seq_length, tokenizer, args.max_turn_length)

        logger.info("***** Running validation *****")
        logger.info("  Num examples = %d", len(dev_examples))
        logger.info("  Batch size = %d", args.dev_batch_size)

        all_input_ids_dev, all_input_len_dev, all_label_ids_dev = \
            all_input_ids_dev.to(device), all_input_len_dev.to(device), all_label_ids_dev.to(device)

        dev_data = TensorDataset(all_input_ids_dev, all_input_len_dev, all_label_ids_dev)
        dev_sampler = SequentialSampler(dev_data)
        dev_dataloader = DataLoader(dev_data, sampler=dev_sampler, batch_size=args.dev_batch_size)

    logger.info("Loaded data!")

    ###############################################################################
    # Build the models
    ###############################################################################

    # Prepare model
    if args.nbt =='rnn':
        from BeliefTrackerSlotQueryMultiSlot import BeliefTracker
    elif args.nbt == 'transformer':
        from BeliefTrackerSlotQueryMultiSlotTransformer import BeliefTracker
    else:
        raise ValueError('nbt type should be either rnn or transformer')

    model = BeliefTracker(args, num_labels, device)
    if args.fp16:
        model.half()
    model.to(device)
    save_configure(args, num_labels, processor.ontology)

    ## Get slot-value embeddings
    label_token_ids, label_len = [], []
    for labels in label_list:
        token_ids, lens = get_label_embedding(labels, args.max_label_length, tokenizer, device)
        label_token_ids.append(token_ids)
        label_len.append(lens)

    ## Get domain-slot-type embeddings
    slot_token_ids, slot_len = \
        get_label_embedding(processor.target_slot, args.max_label_length, tokenizer, device)

    ## Initialize slot and value embeddings
    model.initialize_slot_value_lookup(label_token_ids, slot_token_ids)

    # Data parallelize when use multi-gpus
    if args.local_rank != -1:
        try:
            from apex.parallel import DistributedDataParallel as DDP
        except ImportError:
            raise ImportError(
                "Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training.")

        model = DDP(model)
    elif n_gpu > 1:
        model = torch.nn.DataParallel(model)

    # Prepare optimizer
    if args.do_train:
        def get_optimizer_grouped_parameters(model):
            param_optimizer = [(n, p) for n, p in model.named_parameters() if p.requires_grad]
            no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
            optimizer_grouped_parameters = [
                {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01,
                 'lr': args.learning_rate},
                {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0,
                 'lr': args.learning_rate},
            ]
            return optimizer_grouped_parameters

        if n_gpu == 1:
            optimizer_grouped_parameters = get_optimizer_grouped_parameters(model)
        else:
            optimizer_grouped_parameters = get_optimizer_grouped_parameters(model.module)

        t_total = num_train_steps

        if args.local_rank != -1:
            t_total = t_total // torch.distributed.get_world_size()
        if args.fp16:
            try:
                from apex.optimizers import FP16_Optimizer
                from apex.optimizers import FusedAdam
            except ImportError:
                raise ImportError(
                    "Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training.")

            optimizer = FusedAdam(optimizer_grouped_parameters,
                                  lr=args.learning_rate,
                                  bias_correction=False,
                                  max_grad_norm=1.0)
            if args.loss_scale == 0:
                optimizer = FP16_Optimizer(optimizer, dynamic_loss_scale=True)
            else:
                optimizer = FP16_Optimizer(optimizer, static_loss_scale=args.loss_scale)

        else:
            optimizer = BertAdam(optimizer_grouped_parameters,
                                 lr=args.learning_rate,
                                 warmup=args.warmup_proportion,
                                 t_total=t_total)
        logger.info(optimizer)

    ###############################################################################
    # Training code
    ###############################################################################

    if args.do_train:
        logger.info("Training...")

        global_step = 0
        last_update = None
        best_loss = None

        for epoch in trange(int(args.num_train_epochs), desc="Epoch"):
            # Train
            model.train()
            tr_loss = 0
            nb_tr_examples = 0
            nb_tr_steps = 0

            for step, batch in enumerate(tqdm(train_dataloader, desc="Iteration")):
                batch = tuple(t.to(device) for t in batch)
                input_ids, input_len, label_ids = batch

                # Forward
                if n_gpu == 1:
                    loss, loss_slot, acc, acc_slot, _ = model(input_ids, input_len, label_ids, n_gpu)
                else:
                    loss, _, acc, acc_slot, _ = model(input_ids, input_len, label_ids, n_gpu)

                    # average to multi-gpus
                    loss = loss.mean()
                    acc = acc.mean()
                    acc_slot = acc_slot.mean(0)

                if args.gradient_accumulation_steps > 1:
                    loss = loss / args.gradient_accumulation_steps

                # Backward
                if args.fp16:
                    optimizer.backward(loss)
                else:
                    loss.backward()

                # tensrboard logging
                if summary_writer is not None:
                    summary_writer.add_scalar("Epoch", epoch, global_step)
                    summary_writer.add_scalar("Train/Loss", loss, global_step)
                    summary_writer.add_scalar("Train/JointAcc", acc, global_step)
                    if n_gpu == 1:
                        for i, slot in enumerate(processor.target_slot):
                            summary_writer.add_scalar("Train/Loss_%s" % slot.replace(' ','_'), loss_slot[i], global_step)
                            summary_writer.add_scalar("Train/Acc_%s" % slot.replace(' ','_'), acc_slot[i], global_step)

                tr_loss += loss.item()
                nb_tr_examples += input_ids.size(0)
                nb_tr_steps += 1
                if (step + 1) % args.gradient_accumulation_steps == 0:
                    # modify lealrning rate with special warm up BERT uses
                    lr_this_step = args.learning_rate * warmup_linear(global_step / t_total, args.warmup_proportion)
                    if summary_writer is not None:
                        summary_writer.add_scalar("Train/LearningRate", lr_this_step, global_step)
                    for param_group in optimizer.param_groups:
                        param_group['lr'] = lr_this_step
                    optimizer.step()
                    optimizer.zero_grad()
                    global_step += 1

            # Perform evaluation on validation dataset
            model.eval()
            dev_loss = 0
            dev_acc = 0
            dev_loss_slot, dev_acc_slot = None, None
            nb_dev_examples, nb_dev_steps = 0, 0

            for step, batch in enumerate(tqdm(dev_dataloader, desc="Validation")):
                batch = tuple(t.to(device) for t in batch)
                input_ids, input_len, label_ids = batch
                if input_ids.dim() == 2:
                    input_ids = input_ids.unsqueeze(0)
                    input_len = input_len.unsqueeze(0)
                    label_ids = label_ids.unsuqeeze(0)

                with torch.no_grad():
                    if n_gpu == 1:
                        loss, loss_slot, acc, acc_slot, _ = model(input_ids, input_len, label_ids, n_gpu)
                    else:
                        loss, _, acc, acc_slot, _= model(input_ids, input_len, label_ids, n_gpu)

                        # average to multi-gpus
                        loss = loss.mean()
                        acc = acc.mean()
                        acc_slot = acc_slot.mean(0)

                num_valid_turn = torch.sum(label_ids[:,:,0].view(-1) > -1, 0).item()
                dev_loss += loss.item() * num_valid_turn
                dev_acc += acc.item() * num_valid_turn

                if n_gpu == 1:
                    if dev_loss_slot is None:
                        dev_loss_slot = [ l * num_valid_turn for l in loss_slot]
                        dev_acc_slot = acc_slot * num_valid_turn
                    else:
                        for i, l in enumerate(loss_slot):
                            dev_loss_slot[i] = dev_loss_slot[i] + l * num_valid_turn
                        dev_acc_slot += acc_slot * num_valid_turn

                nb_dev_examples += num_valid_turn

            dev_loss = dev_loss / nb_dev_examples
            dev_acc = dev_acc / nb_dev_examples

            if n_gpu == 1:
                dev_acc_slot = dev_acc_slot / nb_dev_examples

            # tensorboard logging
            if summary_writer is not None:
                summary_writer.add_scalar("Validate/Loss", dev_loss, global_step)
                summary_writer.add_scalar("Validate/Acc", dev_acc, global_step)
                if n_gpu == 1:
                    for i, slot in enumerate(processor.target_slot):
                        summary_writer.add_scalar("Validate/Loss_%s" % slot.replace(' ','_'), dev_loss_slot[i]/nb_dev_examples, global_step)
                        summary_writer.add_scalar("Validate/Acc_%s" % slot.replace(' ','_'), dev_acc_slot[i], global_step)

            dev_loss = round(dev_loss, 6)
            if last_update is None or dev_loss < best_loss:
                # Save a trained model
                output_model_file = os.path.join(args.output_dir, "pytorch_model.bin")
                if args.do_train:
                    if n_gpu == 1:
                        torch.save(model.state_dict(), output_model_file)
                    else:
                        torch.save(model.module.state_dict(), output_model_file)

                last_update = epoch
                best_loss = dev_loss
                best_acc = dev_acc

                logger.info("*** Model Updated: Epoch=%d, Validation Loss=%.6f, Validation Acc=%.6f ***" % (last_update, best_loss, best_acc))
            else:
                logger.info("*** Model NOT Updated: Epoch=%d, Validation Loss=%.6f, Validation Acc=%.6f  ***" % (epoch, dev_loss, dev_acc))

            if last_update + args.patience <= epoch:
                break

    ###############################################################################
    # Evaluation
    ###############################################################################
    # Load a trained model that you have fine-tuned
    output_model_file = os.path.join(args.output_dir, "pytorch_model.bin")
    model = BeliefTracker(args, num_labels, device)

    if args.local_rank != -1:
        try:
            from apex.parallel import DistributedDataParallel as DDP
        except ImportError:
            raise ImportError(
                "Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training.")

        model = DDP(model)
    elif n_gpu > 1:
        model = torch.nn.DataParallel(model)

    # in the case that slot and values are different between the training and evaluation
    ptr_model = torch.load(output_model_file)

    if n_gpu == 1:
        state = model.state_dict()
        state.update(ptr_model)
        model.load_state_dict(state)
    else:
        print("Evaluate using only one device!")
        model.module.load_state_dict(ptr_model)

    model.to(device)

    # Evaluation
    if args.do_eval and (args.local_rank == -1 or torch.distributed.get_rank() == 0):

        eval_examples = processor.get_test_examples(args.data_dir, accumulation=accumulation)
        all_input_ids, all_input_len, all_label_ids = convert_examples_to_features(
            eval_examples, label_list, args.max_seq_length, tokenizer, args.max_turn_length)
        all_input_ids, all_input_len, all_label_ids = all_input_ids.to(device), all_input_len.to(device), all_label_ids.to(device)
        logger.info("***** Running evaluation *****")
        logger.info("  Num examples = %d", len(eval_examples))
        logger.info("  Batch size = %d", args.eval_batch_size)

        eval_data = TensorDataset(all_input_ids, all_input_len, all_label_ids)

        # Run prediction for full data
        eval_sampler = SequentialSampler(eval_data)
        eval_dataloader = DataLoader(eval_data, sampler=eval_sampler, batch_size=args.eval_batch_size)

        model.eval()
        eval_loss, eval_accuracy = 0, 0
        eval_loss_slot, eval_acc_slot = None, None
        nb_eval_steps, nb_eval_examples = 0, 0

        accuracies = {'joint7':0, 'slot7':0, 'joint5':0, 'slot5':0, 'joint_rest':0, 'slot_rest':0,
                      'num_turn':0, 'num_slot7':0, 'num_slot5':0, 'num_slot_rest':0}

        for input_ids, input_len, label_ids in tqdm(eval_dataloader, desc="Evaluating"):
            if input_ids.dim() == 2:
                input_ids = input_ids.unsqueeze(0)
                input_len = input_len.unsqueeze(0)
                label_ids = label_ids.unsuqeeze(0)

            with torch.no_grad():
                if n_gpu == 1:
                    loss, loss_slot, acc, acc_slot, pred_slot = model(input_ids, input_len, label_ids, n_gpu)
                else:
                    loss, _, acc, acc_slot, pred_slot = model(input_ids, input_len, label_ids, n_gpu)
                    nbatch = label_ids.size(0)
                    nslot = pred_slot.size(3)
                    pred_slot = pred_slot.view(nbatch, -1, nslot)

            accuracies = eval_all_accs(pred_slot, label_ids, accuracies)

            nb_eval_ex = (label_ids[:,:,0].view(-1) != -1).sum().item()
            nb_eval_examples += nb_eval_ex
            nb_eval_steps += 1

            if n_gpu == 1:
                eval_loss += loss.item() * nb_eval_ex
                eval_accuracy += acc.item() * nb_eval_ex
                if eval_loss_slot is None:
                    eval_loss_slot = [ l * nb_eval_ex for l in loss_slot]
                    eval_acc_slot = acc_slot * nb_eval_ex
                else:
                    for i, l in enumerate(loss_slot):
                        eval_loss_slot[i] = eval_loss_slot[i] + l * nb_eval_ex
                    eval_acc_slot += acc_slot * nb_eval_ex
            else:
                eval_loss += sum(loss) * nb_eval_ex
                eval_accuracy += sum(acc) * nb_eval_ex

        eval_loss = eval_loss / nb_eval_examples
        eval_accuracy = eval_accuracy / nb_eval_examples
        if n_gpu == 1:
            eval_acc_slot = eval_acc_slot / nb_eval_examples

        loss = tr_loss / nb_tr_steps if args.do_train else None

        if n_gpu == 1:
            result = {'eval_loss': eval_loss,
                      'eval_accuracy': eval_accuracy,
                      'loss': loss,
                      'eval_loss_slot':'\t'.join([ str(val/ nb_eval_examples) for val in eval_loss_slot]),
                      'eval_acc_slot':'\t'.join([ str((val).item()) for val in eval_acc_slot])
                        }
        else:
            result = {'eval_loss': eval_loss,
                      'eval_accuracy': eval_accuracy,
                      'loss': loss
                      }

        out_file_name = 'eval_results'
        if args.target_slot=='all':
            out_file_name += '_all'
        output_eval_file = os.path.join(args.output_dir, "%s.txt" % out_file_name)

        if n_gpu == 1:
            with open(output_eval_file, "w") as writer:
                logger.info("***** Eval results *****")
                for key in sorted(result.keys()):
                    logger.info("  %s = %s", key, str(result[key]))
                    writer.write("%s = %s\n" % (key, str(result[key])))

        out_file_name = 'eval_all_accuracies'
        with open(os.path.join(args.output_dir, "%s.txt" % out_file_name), 'w') as f:
            f.write('joint acc (7 domain) : slot acc (7 domain) : joint acc (5 domain): slot acc (5 domain): joint restaurant : slot acc restaurant \n')
            f.write('%.5f : %.5f : %.5f : %.5f : %.5f : %.5f \n' % (
                (accuracies['joint7']/accuracies['num_turn']).item(),
                (accuracies['slot7']/accuracies['num_slot7']).item(),
                (accuracies['joint5']/accuracies['num_turn']).item(),
                (accuracies['slot5'] / accuracies['num_slot5']).item(),
                (accuracies['joint_rest']/accuracies['num_turn']).item(),
                (accuracies['slot_rest'] / accuracies['num_slot_rest']).item()
            ))

def eval_all_accs(pred_slot, labels, accuracies):

    def _eval_acc(_pred_slot, _labels):
        slot_dim = _labels.size(-1)
        accuracy = (_pred_slot == _labels).view(-1, slot_dim)
        num_turn = torch.sum(_labels[:, :, 0].view(-1) > -1, 0).float()
        num_data = torch.sum(_labels > -1).float()
        # joint accuracy
        joint_acc = sum(torch.sum(accuracy, 1) / slot_dim).float()
        # slot accuracy
        slot_acc = torch.sum(accuracy).float()
        return joint_acc, slot_acc, num_turn, num_data

    # 7 domains
    joint_acc, slot_acc, num_turn, num_data = _eval_acc(pred_slot, labels)
    accuracies['joint7'] += joint_acc
    accuracies['slot7'] += slot_acc
    accuracies['num_turn'] += num_turn
    accuracies['num_slot7'] += num_data

    # restaurant domain
    joint_acc, slot_acc, num_turn, num_data = _eval_acc(pred_slot[:,:,18:25], labels[:,:,18:25])
    accuracies['joint_rest'] += joint_acc
    accuracies['slot_rest'] += slot_acc
    accuracies['num_slot_rest'] += num_data

    pred_slot5 = torch.cat((pred_slot[:,:,0:3], pred_slot[:,:,8:]), 2)
    label_slot5 = torch.cat((labels[:,:,0:3], labels[:,:,8:]), 2)

    # 5 domains (excluding bus and hotel domain)
    joint_acc, slot_acc, num_turn, num_data = _eval_acc(pred_slot5, label_slot5)
    accuracies['joint5'] += joint_acc
    accuracies['slot5'] += slot_acc
    accuracies['num_slot5'] += num_data

    return accuracies

def save_configure(args, num_labels, ontology):
    with open(os.path.join(args.output_dir, "config.json"),'w') as outfile:
        data = { "hidden_dim": args.hidden_dim,
                "num_rnn_layers": args.num_rnn_layers,
                "zero_init_rnn": args.zero_init_rnn,
                "max_seq_length": args.max_seq_length,
                "max_label_length": args.max_label_length,
                "num_labels": num_labels,
                "attn_head": args.attn_head,
                 "distance_metric": args.distance_metric,
                 "fix_utterance_encoder": args.fix_utterance_encoder,
                 "task_name": args.task_name,
                 "bert_dir": args.bert_dir,
                 "bert_model": args.bert_model,
                 "do_lower_case": args.do_lower_case,
                 "ontology": ontology}
        json.dump(data, outfile, indent=4)

if __name__ == "__main__":
    main()


from datetime import datetime
import os
from collections import defaultdict
import csv
import argparse
from utils import *

# Multiprocessing
from multiprocessing import Pool

parser = argparse.ArgumentParser()
parser.add_argument('-s', help='stations')
parser.add_argument('-areas', help='areas')
parser.add_argument('-n_hours', help='n_hours')
args = parser.parse_args()

features = ['','PM2.5','O3','AMB_TEMP','CH4','CO','NMHC','NO','NO2','NOx','PM10','RAINFALL','RH','SO2','THC','WD_HR','WIND_DIREC','WIND_SPEED','WS_HR']

data_root_folder = "./EPA_Station_rawdata"
train_begin_year = 2015  # default value
train_end_year = 2018  # default value
test_begin_year = 2019  # default value
test_end_year = 2019  # default value
target_variable = "PM2.5"

def main(cfg):
    global missing_value_filling_n_hours_avg
    global data_root_folder
    global train_begin_year
    global train_end_year
    global test_begin_year
    global test_end_year
    missing_value_filling_n_hours_avg = cfg['missing_value_filling_n_hours_avg'] if cfg['missing_value_filling_n_hours_avg'] else 24
    data_root_folder = cfg['data_root_folder']
    train_begin_year = cfg['train_begin_year']
    train_end_year = cfg['train_end_year']
    test_begin_year = cfg['test_begin_year']
    test_end_year = cfg['test_end_year']
    global target_variable
    target_variable = cfg['variable']

    multiprcossing_number = cfg["multiprcossing_number"] if cfg and "multiprcossing_number" in cfg else 10

    if not args.areas and not cfg['areas']:
        areas = ["North", "South", "Central", "East", "Other"]
    elif args.areas:
        areas = args.areas.split(',')
    elif cfg['areas']:
        areas = cfg['areas']

    for area in areas:
        print('################ Processing area: ' + area + ' ################')
        if not args.s and not cfg['stations']:
            stations = os.listdir('/'.join([data_root_folder, area]))
        elif args.s:
            stations = args.s.split(',')
        elif cfg['stations']:
            stations = cfg['stations']

        # Make the Pool of workers
        pool = Pool(multiprcossing_number)

        pool.map(station_multiprocess, zip(stations, [area] * len(stations)))

def station_multiprocess(station_input):
    station, area = station_input
    global data_root_folder
    global train_begin_year
    global train_end_year
    global test_begin_year
    global test_end_year

    if '.' in station: return
    if station not in os.listdir('/'.join([data_root_folder, area])): return
    csv_files = os.listdir('/'.join([data_root_folder, area, station]))
    print('Converting station to LSTM format: ' + station)
    for year in list(range(train_begin_year, train_end_year+1)) + list(range(test_begin_year, test_end_year+1)):
        for csv_file in csv_files:
            if str(int(year) - 1911) == csv_file[:3]:
                raw_csv_path = '/'.join([data_root_folder, area, station, csv_file])
                lstm_csv_path = '/'.join([data_root_folder, area, station, str(year) + '.csv'])
                raw_csv_to_lstm_csv(raw_csv_path, lstm_csv_path, str(year), area, station)

def gen_day_empty(date_str, day_of_year, month):
    day, mon, year = [int(s) for s in date_str.split('.')]
    return {i: [date_str + ' ' + format(i, '02d') + ':00:00'] + [0 for xx in range(len(features)-1)] + [str((day_of_year) % 365), str(i), str(datetime(year, mon, day).weekday()), month] for i in range(1, 25)}

def raw_csv_to_lstm_csv(raw_csv, lstm_csv, year, area, station):
    global missing_value_filling_n_hours_avg
    # try:
    # TODO train test begin end year
    if year == '2019':
        output = open(lstm_csv, "w")
    else:
        lstm_csv = '/'.join([data_root_folder, area, station]) + '/' + str(train_begin_year) + '_' + str(train_end_year) + '.csv'
        if year == '2015':
            output = open(lstm_csv, "w")
        else:
            output = open(lstm_csv, "a")
    wr = csv.writer(output)
    if year == '2015' or year == '2019':
        wr.writerow(['Date Time','PM2.5','O3','AMB_TEMP','CH4','CO','NMHC','NO','NO2','NOx','PM10','RAINFALL','RH','SO2','THC','WD_HR','WIND_DIREC','WIND_SPEED','WS_HR','DAY_OF_YEAR','HOUR','WEEKDAY','MONTH'])

    day_ctr = 0
    d = gen_day_empty('01.01.' + year, 0, '1')
    with open(raw_csv, newline='') as f:
        # try:
        rows = csv.reader(f)
        cur_date = year + '/01/01'

        last_n_hours_rows = defaultdict(list)
        if not args.n_hours and not missing_value_filling_n_hours_avg:
            n_hours = 24
        elif args.n_hours:
            n_hours = args.n_hours
        elif missing_value_filling_n_hours_avg:
            n_hours = missing_value_filling_n_hours_avg

        for row in rows:
            if '測項' in row: continue  # pass first header row

            date_str = row[0]
            if cur_date != date_str:
                cur_date = date_str
                orig_day = str(day)
                year, mon, day = date_str.split('/')
                if int(day) > 1 :
                    day_ctr += (int(day) - int(orig_day)) % 30
                else:
                    day_ctr += 1
                for hr in range(1, 25):
                    wr.writerow(d[hr])
                d = gen_day_empty(day + '.' + mon + '.' + year, day_ctr, mon.lstrip('0'))
            year, mon, day = date_str.split('/')

            try:
                feature_ind = features.index(row[2])
            except ValueError:
                pass
            else:
                if len(last_n_hours_rows[feature_ind]) < n_hours + 24:
                    extend_list = []
                    for s in row[3:]:
                        try:
                            extend_list.append(int(s))
                        except:
                            extend_list.append(round(sum(last_n_hours_rows[feature_ind])/len(last_n_hours_rows[feature_ind]), 3)) if len(last_n_hours_rows[feature_ind]) != 0 else extend_list.append(0)
                    last_n_hours_rows[feature_ind].extend(extend_list)
                else:
                    extend_list = []
                    for s in row[3:]:
                        try:
                            extend_list.append(int(s))
                        except:
                            extend_list.append(round(sum(last_n_hours_rows[feature_ind])/len(last_n_hours_rows[feature_ind]), 3))
                    last_n_hours_rows[feature_ind] = last_n_hours_rows[feature_ind][24:] + extend_list

                for hour in range(1, 25):
                    value = row[2+hour]
                    if value == 'NR':
                        d[hour][feature_ind] = 0
                    elif ('#' in value or '*' in value or 'x' in value or 'A' in value or value == ''):
                        try:
                            fill_value = round(sum(last_n_hours_rows[feature_ind][-(24-hour+1)-n_hours:-(24-hour+1)])/len(last_n_hours_rows[feature_ind][-(24-hour+1)-n_hours:-(24-hour+1)]), 3) if len(last_n_hours_rows[feature_ind][-(24-hour+1)-n_hours:-(24-hour+1)]) > 0 else 0
                            d[hour][feature_ind] = float(fill_value)
                        except Exception as e:
                            print(last_n_hours_rows[feature_ind])
                            print(repr(e))
                    elif (feature_ind == 15 or feature_ind == 16) and (value == '888' or value == '999'):
                        d[hour][feature_ind] = -1
                    else:
                        d[hour][feature_ind] = float(value)
        for hr in range(1, 25):
            wr.writerow(d[hr])

        output.close()

if __name__ == '__main__':
    cfg = read_config()
    main(cfg)


# -*- coding: utf-8 -*-

# by digiteng...(digiteng@gmail.com)
# https://github.com/digiteng/
# 06.2020 - 11.2020(v2.0) - 11.2021(v3.x)
from __future__ import absolute_import
from Plugins.Plugin import PluginDescriptor
from Components.config import config
from threading import Timer
from datetime import datetime
from six.moves import reload_module
from . import xtra
from . import download
from enigma import addFont
from Tools.Directories import resolveFilename, SCOPE_FONTS

addFont(resolveFilename(SCOPE_FONTS, "DejaVuSans.ttf"), "xtraRegular", 100, 1)

with open("/var/lib/opkg/info/enigma2-plugin-extensions-xtraevent.control") as origin:
	for version in origin:
		if not "Version: " in version:
			continue
		try:
			versionnumber = version.split('+')[1]
		except IndexError:
			print("[xtraEvent] can't detect version!")

try:
	if config.plugins.xtraEvent.timerMod.value == "Clock":
		tc = config.plugins.xtraEvent.timerClock.value
		dt = datetime.today()
		setclk = dt.replace(day=dt.day + 1, hour=tc[0], minute=tc[1], second=0, microsecond=0)
		ds = setclk - dt
		secs = ds.seconds + 1

		def startDownload():
			from . import download
			download.downloads("").save()

		t = Timer(secs, startDownload)
		t.start()

except Exception as err:
	with open("/tmp/xtraEvent.log", "a+") as f:
		f.write("plugin timer clock, %s\n" % (err))

def ddwn():
	try:
		if config.plugins.xtraEvent.timerMod.value == "Period":
			download.downloads("").save()
			tmr = config.plugins.xtraEvent.timer.value
			t = Timer(3600 * int(tmr), ddwn) # 1h=3600
			t.start()
	except Exception as err:
		with open("/tmp/xtra_error.log", "a+") as f:
			f.write("xtra plugin ddwn, %s\n\n" % err)

try:
	if config.plugins.xtraEvent.timerMod.value == "Period":
		Timer(30, ddwn).start()
except Exception as err:
	with open("/tmp/xtra_error.log", "a+") as f:
		f.write("xtra plugin timer start, %s\n\n" % err)

def main(session, **kwargs):
	try:
		reload_module(xtra)
		reload_module(download)
		session.open(xtra.xtra)
	except:
		import traceback
		traceback.print_exc()

def Plugins(**kwargs):
	return [PluginDescriptor(name=_("xtraEvent"), description=_("Poster, backdrop, banner, info and more") + " git-" + versionnumber, where=PluginDescriptor.WHERE_PLUGINMENU, icon="plugin.png", fnc=main)]


from typing import Tuple

import tensorflow as tf
from dependency_injector.providers import Callable

class PdfNormalization(Callable):
    """
    https://stats.stackexchange.com/questions/66616/converting-normalizing-very-small-likelihood-values-to-probability
    """

    def __init__(self, features_cnt: int, precision=16):
        self.pdf_in, self.normalize_pdf = self.__build_model(features_cnt, precision)

    def __build_model(self, features_cnt, precision=16):
        pdf_in, x_out = self.__transform_input(features_cnt)
        threshold = self.__calculate_threshold(precision, features_cnt)
        x_out = self.__normalize(x_out, threshold, features_cnt)

        return pdf_in, x_out

    def __transform_input(self, features_cnt: Tuple[int, int]):
        pdf_in = tf.placeholder(tf.float64, (None, features_cnt), name="pdf")
        ln = tf.log(pdf_in, "log")
        max_ln = tf.reshape(tf.reduce_max(ln, 1, name="max_pdf"), (-1, 1))
        x_out = tf.subtract(ln, max_ln, "sub")

        return pdf_in, x_out

    def __calculate_threshold(self, precision: int, feats_cnt: int):
        epsilon = tf.constant(10 ^ -precision, dtype=tf.float64, name="epsilon")
        n = tf.constant(feats_cnt, tf.float64, name="n")
        threshold = tf.log(epsilon / n, "threshold")

        return threshold

    def __normalize(self, x_out, threshold, features_cnt: int):
        x_out = tf.exp(x_out, "exp")

        def replace_with_zero_values_below_threshold(x_row):
            less_than_threshold = tf.less(x_row, threshold)
            return tf.where(less_than_threshold, tf.zeros((features_cnt,), dtype=tf.float64), x_row)

        x_out = tf.map_fn(replace_with_zero_values_below_threshold, x_out)
        x_sum = tf.reshape(tf.reduce_sum(x_out, 1, name="sum"), (-1, 1))
        x_out = tf.divide(x_out, x_sum, name="divide")

        return x_out


import rdflib, requests
from flask import Flask, render_template, request, jsonify

#######################################################################################################################################################
#######################################################################################################################################################
########################################################## Environment ###############################################33###########################
#######################################################################################################################################################

app = Flask(__name__)

# Local database
g = rdflib.Graph()
g.parse("data/rdf-with-links.ttl", format="ttl")

#######################################################################################################################################################
#######################################################################################################################################################
########################################################## SPARQL Queries ###############################################33###########################
#######################################################################################################################################################
def run_sparql_query(district_name, year="2021"):
    g = rdflib.Graph()
    g.parse("data/rdf-with-links.ttl", format="ttl")

    query = f"""
            PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
            PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
            PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
            PREFIX owl: <http://www.w3.org/2002/07/owl#>
            PREFIX dbo: <http://dbpedia.org/ontology/>
            PREFIX nso: <http://madridwastemanagement.org/group01/ontology/>
            PREFIX wst: <http://www.disit.org/km4city/schema#>
            PREFIX time: <http://www.w3.org/2006/time#>

            SELECT ?wasteName ?wikidataLink ?m (SUM(?val) AS ?totalAmount)
            WHERE {{
              ?districtInstance a dbo:District ;
                    rdfs:label "{district_name}";
                    wst:hasResidue ?wasteinstance.
              ?wasteinstance rdfs:label ?wasteName;
                 nso:hasTotal ?totalinstance.
              ?totalinstance nso:value ?val;
                 time:month ?m;
                 time:year "{year}"^^xsd:gYear.
              OPTIONAL {{ ?wasteinstance owl:sameAs ?wikidataLink. }}
            }}
            GROUP BY ?wasteName ?wikidataLink ?m
            ORDER BY ?m ?wasteName
            """
    results = g.query(query)
    output = []
    for row in results:
        output.append({
            "wasteName": str(row[0]),
            "wikidataLink": str(row[1]) if row[1] is not None else "",
            "month": str(row[2]),
            "totalAmount": float(row[3])
        })
    return output

def format_waste_type(waste_type):
    waste_type_map = {
        "CDW": "Construction and Demolition Waste",
        "biomedicalWaste": "Biomedical Waste",
        "clothing": "Clothing",
        "glass": "Glass",
        "non-recyclable-waste": "Non Recyclable Waste",
        "organicFood": "Organic Food",
        "packagingProductsOfPlastics": "Packaging Products of Plastic",
        "wastePaper": "Waste Paper",
        "wasteContainer": "Waste Container",
        "horseBed": "Horse Bed"
    }
    return waste_type_map.get(waste_type, "Unknown Waste Type")

app.jinja_env.filters['format_waste'] = format_waste_type

def reverse_format_waste_type(formatted_waste_type):
    reverse_waste_type_map = {
        "Construction and Demolition Waste": "CDW",
        "Biomedical Waste": "biomedicalWaste",
        "Clothing": "clothing",
        "Glass": "glass",
        "Non Recyclable Waste": "non-recyclable-waste",
        "Organic Food": "organicFood",
        "Packaging Products of Plastic": "packagingProductsOfPlastics",
        "Waste Paper": "wastePaper",
        "Waste Container": "wasteContainer",
        "Horse Bed": "horseBed"
    }
    return reverse_waste_type_map.get(formatted_waste_type, formatted_waste_type)

def fetch_yearly_district_waste(year="2021"):
    """
    Fetches the total amount of waste for each district for a specified year.

    :param year: The year for which the waste data is to be fetched.
    """
    g = rdflib.Graph()
    g.parse("data/rdf-with-links.ttl", format="ttl")

    query = f"""
            PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
            PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
            PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
            PREFIX owl: <http://www.w3.org/2002/07/owl#>
            PREFIX dbo: <http://dbpedia.org/ontology/>
            PREFIX nso: <http://madridwastemanagement.org/group01/ontology/>
            PREFIX wst: <http://www.disit.org/km4city/schema#>
            PREFIX time: <http://www.w3.org/2006/time#>

            SELECT ?districtName (SUM(xsd:float(?val)) AS ?totalAmount) ?year
            WHERE {{
              ?districtInstance a dbo:District ;
                                rdfs:label ?districtName;
                                wst:hasResidue ?wasteinstance.
              ?wasteinstance nso:hasTotal ?totalinstance.
              ?totalinstance nso:value ?val;
                             time:year ?year.
            FILTER (?year = "{year}"^^xsd:gYear)

            }}

            GROUP BY ?districtName
            ORDER BY ?year
            """
    results = g.query(query)
    output = []
    for row in results:
        output_dict = {
            "districtName": str(row[0]),
            "totalAmount": float(row[1]),
            "year": str(row[2]),
        }
        output.append(output_dict)
    return output

def run_district_details_query(district_id):
    endpoint_url = "https://query.wikidata.org/sparql"
    """
           Given a district id it retrieves the details of the district from wikidata
           :param district_name:
           :return:
           """
    query = f"""
            PREFIX schema: <http://schema.org/>
            PREFIX wdt: <http://www.wikidata.org/prop/direct/>
            PREFIX wd: <http://www.wikidata.org/entity/>

            SELECT ?population ?description ?area ?coordinates ?label
            WHERE {{
              BIND(wd:{district_id} AS ?district)
              ?district wdt:P1082 ?population.
              OPTIONAL {{ ?district wdt:P2046 ?area}}.
              ?district schema:description ?description.
              ?district wdt:P625 ?coordinates.
              ?district rdfs:label ?label .
              FILTER (langMatches( lang(?label), "ES" ) )
              FILTER(LANG(?description) = "en")
            }}
            """
    try:
        response = requests.get(endpoint_url, params={'query': query, 'format': 'json'})
        data = response.json()
        results = data['results']['bindings']
        output = []
        for result in results:
            output_dict = {
                'population': result['population']['value'],
                'description': result['description']['value'],
                'area': result.get('area', {'value': '--'})['value'],
                'coordinates': result['coordinates']['value'],
                'name': result['label']['value']
            }
            output.append(output_dict)
            return output
    except Exception as e:
        print(f"An error occurred: {e}")
        return []

def run_wasteType_query(wikidata_id):
    endpoint_url = "https://query.wikidata.org/sparql"
    query = f"""
            PREFIX schema: <http://schema.org/>
            SELECT ?description
            WHERE {{
              wd:{wikidata_id} schema:description ?description.
              FILTER(LANG(?description) = "en")
            }}
            LIMIT 1
            """
    try:
        response = requests.get(endpoint_url, params={'query': query, 'format': 'json'})
        data = response.json()
        results = data['results']['bindings']
        if results:
            output = results[0]['description']['value']
            return output
        else:
            return "No description found"
    except Exception as e:
        print(f"An error occurred: {e}")
        return "An error occurred while fetching the description"

def fetch_waste_type_distribution(waste_type="CDW", year="2021", month="1"):
    """
    Fetches distribution data of a specific waste type (default: CDW) across all districts for a specified year (default: 2021).

    :param waste_type: The waste type to fetch data for, default is 'CDW'.
    :param year: The year for which the data is to be fetched, default is '2021'.
    """
    g = rdflib.Graph()
    g.parse("data/rdf-with-links.ttl", format="ttl")

    query = f"""
        PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
        PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
        PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
        PREFIX owl: <http://www.w3.org/2002/07/owl#>
        PREFIX dbo: <http://dbpedia.org/ontology/>
        PREFIX nso: <http://madridwastemanagement.org/group01/ontology/>
        PREFIX wst: <http://www.disit.org/km4city/schema#>
        PREFIX time: <http://www.w3.org/2006/time#>

        SELECT ?districtName ?amount
        WHERE {{
            ?district a dbo:District ;
                      rdfs:label ?districtName ;
                      wst:hasResidue ?wasteinstance.
            ?wasteinstance rdfs:label "{waste_type}" ;
                           nso:hasTotal ?total.
            ?total nso:value ?amount ;
                   time:year "{year}"^^xsd:gYear;
                   time:month  "{month}"^^xsd:gMonth .
        }}
        """
    print(f"Fetching distribution data for waste type: {waste_type}, year: {year}, month: {month}")
    results = g.query(query)
    distribution_data = []
    for row in results:
        distribution_data.append({
            "districtName": str(row[0]),
            "amount": float(row[1])
        })
    print(f"Distribution Data: {distribution_data}")
    return distribution_data

#######################################################################################################################################################
#######################################################################################################################################################
########################################################## Rooter ######### ###############################################33###########################
#######################################################################################################################################################

@app.route('/')
def index():
    initial_year = "2021"  # You can change this to the current year or any default year
    waste_data = fetch_yearly_district_waste(initial_year)
    return render_template('index.html', waste_data=waste_data)

@app.route('/updateIndexYear')
def update_year():
    selected_year = request.args.get('year', '2021')
    print(selected_year)
    waste_data = fetch_yearly_district_waste(selected_year)
    return render_template('index.html', waste_data=waste_data, selected_year=selected_year)

@app.route("/district")
def district():
    district_name = request.args.get("name")
    year = request.args.get("year", "2021")
    district_id = request.args.get("wikidataID")
    waste_results = run_sparql_query(district_name, year)
    district_details = run_district_details_query(district_id)
    return render_template("district.html", waste_results=waste_results, district_details=district_details,
                           selected_year=year)

@app.route("/wasteType")
def waste_type():
    wikidata_id = request.args.get("wikidata_id")
    waste_type_name = request.args.get("name", "Unknown Waste Type")
    formatted_waste_type_name = format_waste_type(waste_type_name)
    query_format_waste_type_name = reverse_format_waste_type(waste_type_name)
    waste_type_result = run_wasteType_query(wikidata_id)
    distribution_data = fetch_waste_type_distribution(query_format_waste_type_name)
    return render_template("wasteType.html", wasteType=formatted_waste_type_name, result=waste_type_result,
                           distribution=distribution_data)

@app.route("/updateWasteType")
def update_wasteType():
    wikidata_id = request.args.get("wikidata_id")
    waste_type_name = request.args.get("name", "Unknown Waste Type")
    year = request.args.get("year", "2021")
    month = request.args.get("month", "1")
    formatted_waste_type_name = format_waste_type(waste_type_name)
    query_format_waste_type_name = reverse_format_waste_type(waste_type_name)
    waste_type_result = run_wasteType_query(wikidata_id)
    distribution_data = fetch_waste_type_distribution(query_format_waste_type_name, year, month)
    return render_template("wasteType.html", wasteType=formatted_waste_type_name, result=waste_type_result,
                           distribution=distribution_data, year=year, month=month)

if __name__ == "__main__":
    app.run(debug=True)


#!/usr/bin/python3
""""0-subs module"""
import requests

def number_of_subscribers(subreddit):
    """
    args:
        subreddit - variable for subreddit to querry
    return:
        returns number of subscribers for a given subreddit
        or 0 if invalid subreddit is given
    """

    user_agent = "0-subs"
    url = "https://www.reddit.com/r/{}/about.json".format(subreddit)

    headers = {"User-Agent": user_agent}
    try:
        response = requests.get(url, headers=headers, allow_redirects=False)
        if response.status_code == 200:
            data = response.json()
            if "data" in data and "subscribers" in data["data"]:
                return data["data"]["subscribers"]
            else:
                return 0
        else:
            return 0
    except requests.exceptions.RequestException:
        return 0


import os
import openai
import streamlit as st
from dotenv import load_dotenv

load_dotenv()
openai.api_type = "azure"
openai.api_version = "2023-05-15"
openai.api_key  = os.environ['OPENAI_API_KEY']
openai.api_base = os.getenv("OPENAI_API_BASE")

response = openai.ChatCompletion.create(
    engine="gpt-4",
    messages=[
        {"role": "system", "content": "Assistant is a large language model trained by OpenAI."},
        {"role": "user", "content": "Where are the headquarters of Just Eat Takeaway.com?"}
    ]
)

st.title('Hackathon 2023')
st.write(response['choices'][0]['message']['content'])
# print(response['choices'][0]['message']['content'])


from DependencyAnlyzer.P4ProgramGraph import P4ProgramGraph
from P4ProgramParser.P416JsonParser import ParsedP416ProgramForV1ModelArchitecture
from P4ProgramParser.P4ProgramParserFactory import P4ProgramParserFactory
from ParserMapper.make_tcam import buildParserMapper
from RMTHardwareSimulator import RMTHardwareFactory
from util import loadP416JsonUsingAutoGeneratedJsonParser
from ParserMapper import HeaderLib
# from ParserMapper import *
# from ParserMapper import RefCount
# from P416JsonParser import  ParserOpOp
from utils import Util
import time

#64, 96, and 64 words of 8, 16, and 32b
p4ProgramParserFactory = P4ProgramParserFactory()
hw = RMTHardwareFactory.createRmtHardware(rmtHardwaRemodelName = "RMT_V1",
    instructionSetConfigurationJsonFile= "../Resources/HardwareConfigs/RMTV1ModelInstructions.json",
    hardwareSpecConfigurationJsonFile = "../Resources/HardwareConfigs/RMTV1model32Stages.json")
# p4program = p4ProgramParserFactory.getParsedP4Program(p4JsonFile="../Resources/P4ProgramsForPaper/QoSModifier/qos_modifier.json",p4VersionAndArchitecture="P416_V1_Model")
# p4program = p4ProgramParserFactory.getParsedP4Program(p4JsonFile="../Resources/P4ProgramsForPaper/L2L3Simple/l2l3Simple.json",p4VersionAndArchitecture="P416_V1_Model")
# p4program = p4ProgramParserFactory.getParsedP4Program(p4JsonFile="../Resources/P4ProgramsForPaper/L2L3Complex/l2l3complex.json",p4VersionAndArchitecture="P416_V1_Model")
p4program = p4ProgramParserFactory.getParsedP4Program(p4JsonFile="../Resources/P4ProgramsForPaper/P4anony/p4anony.json",p4VersionAndArchitecture="P416_V1_Model")

p4program.buildHeaderVector(hw)
p4ProgramGraph = P4ProgramGraph(p4program)
p4ProgramGraph.loadPipelines(hw)
headerFieldSpecsInP4ProgramToBeUsedForParserMapper, totalRawBitwidth = p4ProgramGraph.headeranalyzer(hw)
parseGraphHeaderList, parsedGraphHeaders, initHeader = HeaderLib.loadParseGraph(parserObject = p4program.parsers[0], p4ProgramGraph = p4ProgramGraph) # There is only one parser in v1model
start = int(round(time.time() * 1000))
buildParserMapper(parseGraphHeaderList, parsedGraphHeaders, hw, initHeader)
end = int(round(time.time() * 1000))
print("Total time required for parser TCAM entry generation is "+str(end-start)+" miliseconds")
print("==============================================================================")

p4ProgramGraph.loadPipelines(hw)
headerFieldSpecsInP4Program,totalRawBitwidth = p4ProgramGraph.headeranalyzer(hw)
# mappedPacketHeaderVector1 = hw.mapHeaderFieldsUsingGoogleOR(headerFieldSpecsInP4Program)
# headerFieldSpecsInP4Program = {33: 1, 16: 1} # this line is for test purpose
start = int(round(time.time() * 1000))
mappedPacketHeaderVector = hw.mapHeaderFields(headerFieldSpecsInP4Program)
end = int(round(time.time() * 1000))
print(mappedPacketHeaderVector)
print("Total time required for header vector mapping is "+str(end-start)+" miliseconds")
print("==============================================================================")
p4ProgramGraph.storePHVFieldMappingForHeaderFields(mappedPacketHeaderVector=mappedPacketHeaderVector)
Util.calculatePHVWaste(headerFieldSpecsInP4Program, mappedPacketHeaderVector,totalRawBitwidth)
start = int(round(time.time() * 1000))
p4ProgramGraph.embedPipelines(hw)
end = int(round(time.time() * 1000))
print("Total time required for TDG mapping is "+str(end-start)+" miliseconds")
print("Total Nodes in the TDG is: "+str(p4ProgramGraph.rawTDGNodeCount) )
print("Total Edges in the TDG is: "+str(p4ProgramGraph.rawTDGEdgeCount) )
hw.calculateTotalLatency(p4ProgramGraph, hw)

#===================== Upto here header mappign is done


# coding: utf-8

"""
    IONOS Cloud - DNS API

    Cloud DNS service helps IONOS Cloud customers to automate DNS Zone and Record management.   # noqa: E501

    The version of the OpenAPI document: 1.12.0
    Contact: support@cloud.ionos.com
    Generated by: https://openapi-generator.tech
"""

import pprint
import re  # noqa: F401

import six

from ionoscloud_dns.configuration import Configuration

class KeyData(object):
    """NOTE: This class is auto generated by OpenAPI Generator.
    Ref: https://openapi-generator.tech

    Do not edit the class manually.
    """

    """
    Attributes:
      openapi_types (dict): The key is attribute name
                            and the value is attribute type.
      attribute_map (dict): The key is attribute name
                            and the value is json key in definition.
    """
    openapi_types = {

        'flags': 'int',

        'protocol': 'int',

        'alg': 'int',

        'pub_key': 'str',
    }

    attribute_map = {

        'flags': 'flags',

        'protocol': 'protocol',

        'alg': 'alg',

        'pub_key': 'pubKey',
    }

    def __init__(self, flags=None, protocol=None, alg=None, pub_key=None, local_vars_configuration=None):  # noqa: E501
        """KeyData - a model defined in OpenAPI"""  # noqa: E501
        if local_vars_configuration is None:
            local_vars_configuration = Configuration()
        self.local_vars_configuration = local_vars_configuration

        self._flags = None
        self._protocol = None
        self._alg = None
        self._pub_key = None
        self.discriminator = None

        if flags is not None:
            self.flags = flags
        if protocol is not None:
            self.protocol = protocol
        if alg is not None:
            self.alg = alg
        if pub_key is not None:
            self.pub_key = pub_key

    @property
    def flags(self):
        """Gets the flags of this KeyData.  # noqa: E501

        Represents the key's metadata and usage information.  # noqa: E501

        :return: The flags of this KeyData.  # noqa: E501
        :rtype: int
        """
        return self._flags

    @flags.setter
    def flags(self, flags):
        """Sets the flags of this KeyData.

        Represents the key's metadata and usage information.  # noqa: E501

        :param flags: The flags of this KeyData.  # noqa: E501
        :type flags: int
        """

        self._flags = flags

    @property
    def protocol(self):
        """Gets the protocol of this KeyData.  # noqa: E501

        Represents the protocol associated with the key (3 represents DNSSEC).  # noqa: E501

        :return: The protocol of this KeyData.  # noqa: E501
        :rtype: int
        """
        return self._protocol

    @protocol.setter
    def protocol(self, protocol):
        """Sets the protocol of this KeyData.

        Represents the protocol associated with the key (3 represents DNSSEC).  # noqa: E501

        :param protocol: The protocol of this KeyData.  # noqa: E501
        :type protocol: int
        """

        self._protocol = protocol

    @property
    def alg(self):
        """Gets the alg of this KeyData.  # noqa: E501

        Represents the signing algorithm.  # noqa: E501

        :return: The alg of this KeyData.  # noqa: E501
        :rtype: int
        """
        return self._alg

    @alg.setter
    def alg(self, alg):
        """Sets the alg of this KeyData.

        Represents the signing algorithm.  # noqa: E501

        :param alg: The alg of this KeyData.  # noqa: E501
        :type alg: int
        """

        self._alg = alg

    @property
    def pub_key(self):
        """Gets the pub_key of this KeyData.  # noqa: E501

        Represents the public key data in Base64 encoding.  # noqa: E501

        :return: The pub_key of this KeyData.  # noqa: E501
        :rtype: str
        """
        return self._pub_key

    @pub_key.setter
    def pub_key(self, pub_key):
        """Sets the pub_key of this KeyData.

        Represents the public key data in Base64 encoding.  # noqa: E501

        :param pub_key: The pub_key of this KeyData.  # noqa: E501
        :type pub_key: str
        """

        self._pub_key = pub_key
    def to_dict(self):
        """Returns the model properties as a dict"""
        result = {}

        for attr, _ in six.iteritems(self.openapi_types):
            value = getattr(self, attr)
            if isinstance(value, list):
                result[attr] = list(map(
                    lambda x: x.to_dict() if hasattr(x, "to_dict") else x,
                    value
                ))
            elif hasattr(value, "to_dict"):
                result[attr] = value.to_dict()
            elif isinstance(value, dict):
                result[attr] = dict(map(
                    lambda item: (item[0], item[1].to_dict())
                    if hasattr(item[1], "to_dict") else item,
                    value.items()
                ))
            else:
                result[attr] = value

        return result

    def to_str(self):
        """Returns the string representation of the model"""
        return pprint.pformat(self.to_dict())

    def __repr__(self):
        """For `print` and `pprint`"""
        return self.to_str()

    def __eq__(self, other):
        """Returns true if both objects are equal"""
        if not isinstance(other, KeyData):
            return False

        return self.to_dict() == other.to_dict()

    def __ne__(self, other):
        """Returns true if both objects are not equal"""
        if not isinstance(other, KeyData):
            return True

        return self.to_dict() != other.to_dict()


from decimal import Decimal
import itertools as it

from datetime import datetime

from app import db
import click
from .models import Ticket, Route

import xml.etree.ElementTree as ET

class FlightType:
    ONE_WAY = "ONE_WAY"
    TWO_WAY = "TWO_WAY"

class PriceType:
    ADULT = "SingleAdult"
    CHILD = "SingleChild"
    INFANT = "SingleInfant"

def get_flights(itinerary):
    """
    Returns all flights info for "OnwardPricedItinerary" and "ReturnPricedItinerary" tags
    """
    dt_format = "%Y-%m-%dT%H%M"
    onward_flights = []
    for flight in itinerary[0]:  # Flights
        extracted_data = {item.tag: item.text for item in flight}
        onward_flights.append(Ticket(
            carrier_id=extracted_data["Carrier"],
            flight_number=extracted_data["FlightNumber"],
            source=extracted_data["Source"],
            destination=extracted_data["Destination"],
            departure_time=datetime.strptime(extracted_data["DepartureTimeStamp"], dt_format),
            arrival_time=datetime.strptime(extracted_data["ArrivalTimeStamp"], dt_format),
            class_=extracted_data["Class"],
            number_of_stops=extracted_data["NumberOfStops"],
            fare_basis=extracted_data["FareBasis"],
            ticket_type=extracted_data["TicketType"],
            warning_text=extracted_data["WarningText"]
            ))
    return sorted(onward_flights, key=lambda ticket: ticket.departure_time)

def get_prices(pricing):
    """
    Extracts extracts all prices from "Pricing" tag
    """
    prices = {
        PriceType.ADULT: 0,
        PriceType.CHILD: 0,
        PriceType.INFANT: 0
    }
    for price in pricing:
        if price.attrib.get("ChargeType") == "TotalAmount":
            prices[price.attrib.get("type")] = Decimal(price.text)
    return prices

def extract(two_way=True):
    """
        extracts data from via3.xml, viaow.xml and saves it to DB
    """
    file_name = "via3.xml" if two_way else "viaow.xml"
    tree = ET.parse(file_name)
    root = tree.getroot()
    flight_type = FlightType.TWO_WAY if two_way else FlightType.ONE_WAY

    for flight in root.findall("./PricedItineraries/Flights"):
        onwards = get_flights(flight[0])  # onward flights
        prices_child_pos = 2 if two_way else 1
        currency = flight[prices_child_pos].attrib.get("currency")
        backwards = get_flights(flight[1]) if two_way else []  # backward flights
        prices = get_prices(flight[prices_child_pos])
        source = onwards[0]
        final_destination = onwards[-1]
        duration = (final_destination.arrival_time - source.departure_time).total_seconds()
        route = Route(
            carrier_id=source.carrier_id,
            source=source.source,
            destination=final_destination.destination,
            departure_time=source.departure_time,
            arrival_time=final_destination.arrival_time,
            duration=duration,
            adult_price=prices[PriceType.ADULT],
            child_price=prices[PriceType.CHILD],
            infant_price=prices[PriceType.INFANT],
            currency=currency,
            flight_type=flight_type
        )
        if two_way:
            route.back_departure_time = backwards[0].departure_time
            route.back_arrival_time = backwards[-1].arrival_time
        db.session.add(route)
        for ticket in it.chain(onwards, backwards):
            ticket = db.session.merge(ticket)
            route.tickets.append(ticket)
    db.session.commit()

@click.command()
def populate_db():
    """
    Extracts data from xml files and puts into db
    """
    db.create_all()
    extract()
    extract(two_way=False)


from pymongo import MongoClient
from lxml import html
import requests

__author__ = 'Garrett Johnston'

client = MongoClient()
db = client.football

baseURL = 'https://web.archive.org/web/20140905214405/http://games.espn.go.com/ffl/tools/projections?'

for weekNum in range(1, 18):
    print(weekNum)
    for positionId in range(0, 7, 2):
        url = baseURL + '&scoringPeriodId=' + str(weekNum) + '&seasonId=2014&slotCategoryId=' + str(positionId)

        page = requests.get(url)
        tree = html.fromstring(page.content)

        playerNames = tree.xpath('//td[@class="playertablePlayerName"]/a/text()')
        projections = tree.xpath(
            '//td[contains(@class,\'playertableStat\') and contains(@class,\'appliedPoints\')]/text()')
        teams = tree.xpath('//td[@class="playertablePlayerName"]/text()')

        # Clean up team name data
        for i in range(0, len(teams)):
            teams[i] = teams[i].split()[1]

        # Gather data from second page
        if positionId != 0:
            # Get second page using this url
            page = requests.get(url + '&startIndex=40')
            tree = html.fromstring(page.content)

            playerNames.extend(tree.xpath('//td[@class="playertablePlayerName"]//a/text()'))
            projections.extend(tree.xpath(
                '//td[contains(@class,\'playertableStat\') and contains(@class,\'appliedPoints\') and contains(@class,\'sortedCell\')]/text()'))
            teamsPage2 = tree.xpath('//td[@class="playertablePlayerName"]/text()')

            for i in range(0, len(teamsPage2)):
                teamsPage2[i] = teamsPage2[i].split()[1]

            teams.extend(teamsPage2)

        print(len(playerNames), playerNames)
        print(len(projections), projections)
        print(len(teams), teams)

        for playerName, team, projection in zip(playerNames, teams, projections):
            db['espnProjections'].insert_one(
                {
                    "name": playerName,
                    "team": team,
                    "week": weekNum,
                    "projectedPoints": projection
                }
            )


from django.shortcuts import render
from django.http import HttpResponse
def home(request):
    if request.method=="POST":
            a = request.POST["txtnum1"]
            b = request.POST["txtnum2"]
            c = int(a)+int(b)
            return HttpResponse("Result is "+str(c))
    else:
      return render(request,"addition.html")
def prime(request):
    if request.method=="POST":
      num=int(request.POST["num"])
      s=''
      for i in range(2,num):
        if num%i==0:
            s='not prime'
            break
      else:
        s='prime'
      return HttpResponse(s)
    else:
        return render(request,"prime.html")

def fibonacci(request):
    a=-1
    b=1
    res=''
    for i in range(10):
        c=a+b
        res+=str(c) + " "
        a=b
        b=c
    return HttpResponse(res)


from netModel import Net
import torch
from constants import Constants
from trainModel import TrainModel

def runExamples(neuralNetwork):
    inputTensorList = [
        torch.tensor([5.0, 6.0]),  # 0.5864
        torch.tensor([0.0, 0.0]),  # 0
        torch.tensor([-1.0, 1.0]),  # -0.6301
        torch.tensor([3.0, -1.0]),  # 0.4438
        torch.tensor([5.0, 0.0]),  # −0.9589
        torch.tensor([0.0, 1.0]),  # 0.3129
        torch.tensor([1, 3.1415926]),  # 0.9092
        torch.tensor([0, -6.0]),  # −0.9430
        torch.tensor([0, -3.1415926])  # −0.8414
    ]
    for inputTensor in inputTensorList:
        print(neuralNetwork(inputTensor).item())

def main():
    x = TrainModel()
    x.train()
    x.saveToFile()

    neuralNetwork = Net(Constants.INPUT_LAYER_SIZE, Constants.HIDDEN_LAYER_SIZE, Constants.HIDDEN_LAYER_SIZE, Constants.OUTPUT_LAYER_SIZE)
    neuralNetwork.load_state_dict(torch.load(Constants.NEURAL_NETWORK_FILE_NAME))
    neuralNetwork.eval()

    runExamples(neuralNetwork)

if __name__ == "__main__":
    main()


from flask import Flask, request, url_for, Blueprint, jsonify
from project.models import Music, musics_schema, music_schema
from project import db

musics_blueprint = Blueprint('musics', __name__, template_folder='template')

# endpoint to show all musics
@musics_blueprint.route("/", methods=["GET"])
def get_musics():
    all_musics = Music.query.all()
    result = musics_schema.dump(all_musics)
    return jsonify(result)

# endpoint to get a music for selected person
@musics_blueprint.route("/person/<person_id>", methods=["GET"])
def get_musics_for_person(person_id):

    musics = Music.query.all()
    result = musics_schema.dump(musics)
    # for music in result:
    #     if str(music['person_id']) == person_id:
    #         return jsonify(result)
    #     else: return jsonify([])
    return jsonify([music for music in result if str(music['person_id']) == person_id])

# endpoint to create new music
@musics_blueprint.route("", methods=["POST"])
def add_music():
    title = request.json['title']
    person_id = request.json['person_id']

    new_music = Music(title, person_id)
    db.session.add(new_music)
    db.session.commit()

    return music_schema.jsonify(new_music)

# endpoint to get music detail by id
@musics_blueprint.route("<id>", methods=["GET"])
def get_music(id):

    music = Music.query.get(id)
    return music_schema.jsonify(music)

# endpoint to update music detail by id
@musics_blueprint.route("/<id>", methods=["PUT"])
def update_music(id):

    music = Music.query.get(id)
    title = request.json["title"]

    music.title = title
    db.session.commit()

    return music_schema.jsonify(music)

# endpoint to delete music
@musics_blueprint.route("/<id>", methods=["DELETE"])
def delete_music(id):

    music = Music.query.get(id)
    db.session.delete(music)
    db.session.commit()

    return music_schema.jsonify(music)


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import ctypes
import json
import sys
sys.path.append('./')
from queue import Queue
from threading import Event, Thread
from typing import Dict, Iterable, List

import cv2
import numpy as np
import torch

import vacl_stream
import vaststream
from vaststream import *

class StreamInfo(ctypes.Structure):
    _fields_ = [
                ("stream_id", ctypes.c_uint32),
                ("device_id", ctypes.c_uint32),
                ("model_op_context", ctypes.c_uint32),
                ("model_name", ctypes.c_wchar_p)
                ]

class Bert:
    def __init__(
        self,
        model_info: str,
        bytes_size: int = 1536,
        device_id: int = 0,
        batch_size: int = 1,
        **kwargs
        ):
        # get model path and hw_config path
        if isinstance(model_info, str):
            with open(model_info) as f:
                model_info = json.load(f)

        self.vacl_stream = vacl_stream.VaclStream()
        self.input_id = 0
        self.input_dict = {}
        self.event_dict = {}
        self.result_dict = {}
        self.balance_mode = 0
        self.device_id = device_id
        self.batch_size = batch_size
        self.bytes_size = bytes_size
        self.vast_stream = vast_stream()

        self.vast_stream.init(self.device_id)
        init_context, stream_id = self.vast_stream.create_stream(self.device_id, self.balance_mode)
        assert stream_id != 0, 'Fail to create stream.'

        self.stream_id = stream_id
        self.init_op_context = init_context

        def callback(output_description, ulOutPointerArray, ulArraySize, user_data_ptr):
            user_data = ctypes.cast(user_data_ptr, ctypes.POINTER(vacl_stream.StreamInfo))
            input_id = output_description.contents.input_id
            device_ddr = self.input_dict.pop(input_id)
            if isinstance(device_ddr, list):
                for ddr in device_ddr:
                    free_reuslt = self.vast_stream.free_data_on_device(ddr, self.device_id)
                    if free_reuslt != 0:
                        raise "free error.\n"
            else:
                self.vast_stream.free_data_on_device(device_ddr, self.device_id)

            model_name = user_data.contents.model_name
            stream_output_list = self._get_stream_output(ulOutPointerArray, ulArraySize)
            heatmap = np.squeeze(stream_output_list[0])
            _, shape = self.vast_stream.get_output_shape_by_index(model_name, 0)
            ndims = shape.ndims
            heatmap_shape = []
            for i in range(ndims):
                heatmap_shape.append(shape.shapes[i])
            self.result_dict[input_id] = (heatmap_shape, heatmap)
            self.event_dict[input_id].set()
            print("callback count:", input_id)

        # create stream and build model
        op_context_id = self.vast_stream.create_stream_op(self.stream_id, 20001)
        self.vast_stream.invoke_stream_op(op_context_id, self.init_op_context)
        self.vdsp_op_context = op_context_id
        model_name = self._register_model_info(model_info)
        self.model_name = model_name
        model_op_context = self.vast_stream.invoke_run_model_op(self.stream_id, self.vdsp_op_context, model_name)
        self.model_op_context = model_op_context
        self.vast_stream.build_stream(self.stream_id)
        self.callback_output = vaststream.output_callback_type(callback)
        self.user_data = StreamInfo(self.stream_id, self.device_id, self.model_op_context, self.model_name)
        user_data_ptr = cast(POINTER(StreamInfo)(self.user_data), c_void_p)
        self.vast_stream.subscribe_device_report(self.stream_id, self.callback_output, user_data_ptr)

    def _register_model_info(self, model_info_dict: Dict):
        assert self.stream_id != 0
        model_name = model_info_dict.get('model_name')
        model_hw_cfg_file = model_info_dict.get('hw_config_file')
        model_dynamic_json = model_info_dict.get('model_dynamic_json')

        # dynamic model description has higher priority here.
        if model_dynamic_json != None:
            if self.vast_stream.register_dynamic_model(model_name, model_dynamic_json, self.device_id, model_hw_cfg_file, self.batch_size):
                raise ValueError("Failed to register dynamic model.")
        else:
            model_lib_path = model_info_dict.get('model_lib_path')
            model_graph_path = model_info_dict.get('model_graph_path')
            model_params_path = model_info_dict.get('model_params_path')
            if self.vast_stream.register_static_model(model_name, model_lib_path, model_graph_path, model_params_path, self.device_id, model_hw_cfg_file, self.batch_size):
                raise ValueError("Failed to register model.")

        return model_name

    def _get_stream_output(self, ulOutPointerArray, ulArraySize, **kwargs):
        output_count_per_batch = self.vast_stream.get_output_num_per_batch(self.model_name)
        assert output_count_per_batch != 0
        count = int(ulArraySize / output_count_per_batch)
        result_list = []
        index = 0
        for i in range(count):
            data_list = []
            for j in range(output_count_per_batch):
                size = self.vast_stream.get_output_size_by_index(self.model_name, j) # bytes
                data = np.zeros(int(size / 2), dtype=np.float16) # fp16
                output_data = data.ctypes.data_as(ctypes.c_void_p)
                self.vast_stream.get_model_output(self.model_name, j, c_uint64(ulOutPointerArray[index]), output_data)
                data_list.append(data)
                index += 1
            result_list.append(data_list)
        return result_list

    def _copy_to_device(self, data, bytes_size):
        host_addr = data.ctypes.data_as(ctypes.c_void_p)
        device_addr = self.vast_stream.copy_data_to_device(host_addr, bytes_size, self.device_id)
        return device_addr

    def _run_stream(self, inputs: List[np.ndarray]):
        # copy data to device and get input addr on device
        addrs_list = [self._copy_to_device(input, self.bytes_size) for input in inputs]
        data_array_size = len(addrs_list)
        data_array = (c_uint64 * data_array_size) (*addrs_list)
        input_id = self.input_id
        self.input_dict[input_id] = addrs_list
        self.event_dict[input_id] = Event()
        print(" Testing:", input_id, end=',')

        # run model
        ret = self.vast_stream.run_stream(self.stream_id, self.input_id, data_array, len(addrs_list))
        self.input_id += 1
        return input_id

    def run(self, inputs: List[np.ndarray]):
        inputs_len = len(inputs)
        if inputs_len < 1:
            raise ValueError('The number of inputs should be greater than 0.')

        input_id = self._run_stream(inputs)
        self.event_dict[input_id].wait()
        result = self.result_dict.pop(input_id)
        del self.event_dict[input_id]
        return result

    def run_batch(self, datasets: Iterable[List[np.ndarray]]):
        queue = Queue(20)

        def input_thread():
            for data in datasets:
                input_id = self._run_stream(data)
                queue.put(input_id)
            queue.put(None)

        thread = Thread(target=input_thread)
        thread.start()

        while True:
            input_id = queue.get()
            if input_id is None:
                break
            self.event_dict[input_id].wait()
            result = self.result_dict.pop(input_id)
            del self.event_dict[input_id]
            yield result