from collections import Counter
from typing import NoReturn

from nltk import word_tokenize
import numpy as np
import torch
from torch.utils.data.dataset import Dataset

from lib.data_utils import Vocab, partial_class
from lib.datasets.loaders import SNIPSLoader, ROSTDLoader, CLINC150Loader, SSTLoader

SUBSETS = ("train", "val", "test")

class OODDataset(Dataset):
    """Defines a dataset for out-of-domain detection."""
    def __init__(self, loader, tok_fn, return_intent_labels=True, to_lower_case=True):
        """
        Create OODDataset given a data loader and a tokenization function.

        Args:
            loader: dataset loader
            tok_fn: tokenization function
            return_intent_labels: whether to return intent labels with instance (default: True)
            to_lower_case: whether to lowercase text data
        """
        super().__init__()
        self.loader = loader
        for attr in ["raw_texts", "raw_labels", "ood_labels"]:
            setattr(self, attr, getattr(self.loader, attr))
        self.n_ood = sum(self.ood_labels)
        self.n_indomain = len(self) - self.n_ood

        if to_lower_case:
            self.raw_texts = [t.lower() for t in self.raw_texts]
        self.tokenized_texts = [tok_fn(t) for t in self.raw_texts]
        self.vectorized_texts = None
        self.return_intent_labels = return_intent_labels
        self.label_vocab, self.vectorized_labels, self.label_cnts = self.vectorize_labels()
        self.encoder = None

    def __len__(self):
        return len(self.raw_texts)

    def __getitem__(self, idx):
        if self.return_intent_labels:
            return self.vectorized_texts[idx], self.vectorized_labels[idx], self.ood_labels[idx]
        return self.vectorized_texts[idx], self.ood_labels[idx]

    def vectorize_labels(self):
        """
        Map raw labels onto their numerical representation.

        Returns:
            - label vocabulary, i.e mapping from labels to indexes
            - vectorized labels
            - label counts, i.e. number of instances in each class
        """
        label_counter = Counter(self.raw_labels)
        if 'oos' in label_counter:
            label_counter.pop('oos')
        unique_labels, label_cnts = zip(*sorted(label_counter.items()))
        unique_labels, label_cnts = list(unique_labels), list(label_cnts)
        label_vocab = {label: index for index, label in enumerate(unique_labels)}
        vectorized_labels = [label_vocab.get(label, -1) for label in self.raw_labels]
        return label_vocab, vectorized_labels, label_cnts

    def vectorize_texts(self, encoder) -> NoReturn:
        """
        Map tokenized texts into respective numerical sequences.

        Args:
            encoder: mapping from tokens to integer values
        """
        self.encoder = encoder
        self.vectorized_texts = [self.encoder.encode(t) for t in self.tokenized_texts]

# Some helper methods for faster build of common dataset setup
def get_simple_splits(loader_cls, add_valid_to_vocab=False, add_test_to_vocab=False,
                      return_intent_labels=True, tok_fn=word_tokenize, to_lower_case=True):
    """
    Get train/dev/test split

    Args:
        loader_cls: class that loads OOD dataset from the raw files.
        add_valid_to_vocab: whether to add words from the validation split to the vocabulary (default: False)
        add_test_to_vocab: whether to add words from the test split to the vocabulary (default: False)
        return_intent_labels: whether to return intent labels with instances (default: True)
        tok_fn: tokenization function
        to_lower_case: whether to lowercase text instances

    Returns:
        (tuple):
            mapping from split names (train/val/test) to Dataset instances
            vocabulary
    """
    datasets = {}
    for subset in SUBSETS:
        dataset = OODDataset(loader=loader_cls(subset=subset), tok_fn=tok_fn,
                             return_intent_labels=return_intent_labels,
                             to_lower_case=to_lower_case)
        datasets[subset] = dataset

    vocab = Vocab()
    texts = datasets['train'].tokenized_texts
    if add_valid_to_vocab:
        texts = texts + datasets['val'].tokenized_texts
    if add_test_to_vocab:
        texts = texts + datasets['test'].tokenized_texts
    vocab.build(texts=texts)
    for subset in SUBSETS:
        datasets[subset].vectorize_texts(vocab)
    return datasets, vocab

def get_transformer_splits(loader_cls, tokenizer, return_intent_labels=True):
    """
    Get train/dev/test split of the OOD dataset in the form suitable for transformer models.

    Args:
        loader_cls: class for loading OOD dataset from raw files.
        tokenizer: tokenizer from the `transformers` library
        return_intent_labels: whether to return intent labels with instances (default: True)

    Returns:
        list of dataset splits in order train/dev/test
    """
    datasets = []
    for subset in SUBSETS:
        dataset = OODDataset(loader_cls(subset=subset), tokenizer.tokenize,
                             return_intent_labels)
        dataset.vectorize_texts(tokenizer)
        datasets.append(dataset)
    return datasets

def get_loader(dataset_name, **kwargs):
    """
    Get loader class for a dataset.

    Args:
        dataset_name: name of the dataset
        **kwargs: additional arguments for specific datasets

    Returns:
        dataset loader class
    """
    if dataset_name == "rostd":
        loader = partial_class(ROSTDLoader,
                               data_root_dir="data/rostd",
                               use_coarse_labels=False)
    elif dataset_name == "rostd_coarse":
        loader = partial_class(ROSTDLoader,
                               data_root_dir="data/rostd",
                               use_coarse_labels=True)
    elif dataset_name == "snips_75":
        loader = partial_class(SNIPSLoader,
                               data_root_dir="data/snips",
                               K=75, version=kwargs['version'])
    elif dataset_name == "snips_25":
        loader = partial_class(SNIPSLoader,
                               data_root_dir="data/snips",
                               K=25, version=kwargs['version'])
    elif dataset_name == "clinc":
        loader = partial_class(CLINC150Loader,
                               data_path="data/clinc/data_full.json",
                               unsupervised=True)
    elif dataset_name == "clinc_sup":
        loader = partial_class(CLINC150Loader,
                               data_path="data/clinc/data_full.json",
                               unsupervised=False)
    elif dataset_name == 'sst':
        loader = partial_class(SSTLoader,
                               data_root_dir="data/sst",
                               ood_type=kwargs['ood_type'])
    else:
        raise RuntimeError(f"Bad dataset: {dataset_name}")
    return loader

def get_dataset_transformers(tokenizer, dataset_name, **kwargs):
    """
    Get OOD dataset splits.

    Args:
        tokenizer: tokenizer from `transformers` library related to a specific transformer
        dataset_name: name of the dataset
        **kwargs: additional arguments for a specific dataset

    Returns:
        train/dev/test split of the OOD dataset
    """
    loader = get_loader(dataset_name, **kwargs)
    return get_transformer_splits(loader, tokenizer)

def get_dataset_simple(dataset_name,
                       add_valid_to_vocab=False,
                       add_test_to_vocab=False,
                       to_lower_case=True,
                       tok_fn=word_tokenize,
                       **kwargs):
    """
    Get OOD dataset splits for an OOD dataset.

    Args:
        dataset_name: name of the OOD dataset
        add_valid_to_vocab: whether to add valid words to the vocabulary
        add_test_to_vocab: whether to add test words to the vocabulary
        to_lower_case: whether to transform text to lowercase
        tok_fn: tokenization function
        **kwargs: additional arguments for ta specific dataset

    Returns:
        (tuple):
            mapping from split names (train/val/test) to Dataset instances
            vocabulary
    """
    loader = get_loader(dataset_name, **kwargs)
    return get_simple_splits(loader,
                             add_valid_to_vocab=add_valid_to_vocab,
                             add_test_to_vocab=add_test_to_vocab,
                             tok_fn=tok_fn,
                             to_lower_case=to_lower_case)

def collate_fn_simple(data, pad_idx: int = 0, bos_idx: int = 1, eos_idx: int = 2):
    """
    Collate function for baseline models: LLR, CNN, LSTM.

    Args:
        data: batch of data to be collated
        pad_idx: padding index
        bos_idx: begin of sequence index
        eos_idx: end of sequence index

    Returns:
        (tuple):
            torch.Tensor: batch of numericalized sentences
            torch.Tensor: batch of numericalized labels
            torch.Tensor: batch of bool values: true if sentence is OOD end false otherwise
    """
    max_len = max(len(datum[0]) + 2 for datum in data)
    labels = torch.zeros(len(data), dtype=torch.long)
    ood_labels = torch.zeros(len(data), dtype=torch.long)
    batch = []
    for idx, (numerical_sent, label, is_ood) in enumerate(data):
        labels[idx] = label
        ood_labels[idx] = is_ood
        batch.append([bos_idx] + numerical_sent + [eos_idx] + [pad_idx]*(max_len - len(numerical_sent) - 2))
    return torch.tensor(batch), labels, ood_labels

def collate_fn_bow(data, vocab_size):
    """
    Collate function for BoW method.

    Args:
        data: batch of data to be collated

    Returns:
        tuple of torch.Tensors: numericalized sentence, labels, ood_labels
    """
    labels = torch.zeros(len(data), dtype=torch.long)
    ood_labels = torch.zeros(len(data), dtype=torch.long)
    rows, cols = [], []
    values = []
    for idx, (numerical_sent, label, is_ood) in enumerate(data):
        labels[idx] = label
        ood_labels[idx] = is_ood
        for num, cnt in zip(*np.unique(numerical_sent, return_counts=True)):
            rows.append(idx)
            cols.append(num)
            values.append(cnt)
    indices = np.vstack((rows, cols))

    i = torch.LongTensor(indices)
    v = torch.FloatTensor(values)
    batch = torch.sparse.FloatTensor(i, v, torch.Size((len(data), vocab_size)))
    return batch, labels, ood_labels


'''Generator functions allow us to write a function that can send back a value and then later resume to pick up where it left off.
This type of function is a generator in Python, allowing us to generate a sequence of values over time. The main difference in syntax will be the use of a yield statement.

when a generator function is compiled they become an object that supports an iteration protocol. That means when they are called in your code they don't actually return a value and then exit.
Instead, generator functions will automatically suspend and resume their execution and state around the last point of value generation.

The main advantage here is that instead of having to compute an entire series of values up front, the generator computes one value and then suspends its activity awaiting the next instruction.
This feature is known as state suspension and makes it memory efficient due to all set of values not being stored upfront as is done for e.g using .append() for existing list in memory.'''

#Example:

#normal function:

def create_cubes(n):
    result = []
    for x in range(n):
        result.append(x**3)
    return result

create_cubes(3)

# generator:

def create_squares(n):
    for x in range(n):
        yield x**2
for x in create_squares(5):
    print(x)
# can cast as list too - list(create_squares(5))

def gen_fibon(n):
    a = 1
    b = 1
    for i in range(n):
        yield a
        #tuple matching
        a,b=b,a+b
for number in gen_fibon(5):
    print(number)

#next() and iter() built-in functions:
# The next() function allows us to access the next element in a sequence.

def simple_gen():
    for x in range(3):
        yield x
for num in simple_gen():
    print(num)
# Assign simple_gen
g = simple_gen()

print(next(g))
print(next(g))

# iter():

s = 'hello'

#Iterate over string
#for let in s:
#    print(let)

s_iter = iter(s)

# Now we can use next() to access next values in iterable.

next(s_iter)
next(s_iter)

# Create a generator that generates the squares of numbers up to some number N.

def create_squares(n):
    for x in range(n):
        yield x**2
for x in create_squares(10):
    print(x)

# Create a generator that yields "n" random numbers between a low and high number (that are inputs).
# Note: Use the random library.

import random

def rand_num(low,high,n):
    for i in range(n):
        yield random.randint(low,high)

for num in rand_num(1,10,12):
    print(num)

# Explain a use case for a generator using a yield statement where you would not want to use a normal function with a return statement.
'''If the output has the potential of taking up a large amount of memory and you only intend to iterate through it, you would want to use a generator.
 '''
#Can you explain what gencomp is in the code below?

my_list = [1,2,3,4,5]

gencomp = (item for item in my_list if item > 3)

for item in gencomp:
    print(item)

# generator comprehension using list comprehension.


import json
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable
from torch.distributions import Categorical
import numpy as np
from net_utils_rl import run_lstm
# This piece of code is for prediction of the WHERE conditions
class Seq2SQLCondPredictor(nn.Module):
    def __init__(self, N_word, N_h, N_depth, max_col_num, max_tok_num):
        super(Seq2SQLCondPredictor, self).__init__()
        self.N_h = N_h
        self.max_tok_num = max_tok_num # Taken 200
        self.max_col_num = max_col_num # Taken 45
        self.cond_lstm = nn.LSTM(input_size=N_word, hidden_size=N_h/2,
                num_layers=N_depth, batch_first=True,
                dropout=0.3, bidirectional=True)
        # Here we are using an encoder - decoder LSTM using pointer networks
        # Short read on Pointer Networks is available in the reference paper sections
        # of my intern folder
        # Or it is mentioned in the reference section in the Paper
        self.cond_decoder = nn.LSTM(input_size=self.max_tok_num,
                hidden_size=N_h, num_layers=N_depth,
                batch_first=True, dropout=0.3)

        self.cond_out_g = nn.Linear(N_h, N_h)
        self.cond_out_h = nn.Linear(N_h, N_h)
        self.cond_out = nn.Sequential(nn.Tanh(), nn.Linear(N_h, 1))

        self.softmax = nn.Softmax(dim=1)

    def gen_gt_batch(self, tok_seq, gen_inp=True):
        """
        :param tok_seq: (list) List of indices of tokens of WHERE clauses within the token-space of the query (='all_toks') for each question
                        Start of each list is 7, end of each list is 1, any token not present in the set union(SQL_TOK, q, col) is 0
                        E.g.[[7, 2, 27, 4, 34, 1], [7, 2, 14, 15, 16, 17, 4, 38, 1]]
        param: gen_inp: (bool)
        If gen_inp: generate the input token sequence (removing <END>)
        Otherwise: generate the output token sequence (removing <BEG>)

        :return ret_inp_var: (Variable) a one-hot-tensor with (x,y,z)=1 if example x contains token z in position y
        :return ret_len: (array) Lengths of INPUT token sequences for the decoder (without the <BEG>/<END> token)
        """
        B = len(tok_seq)
        ret_len = np.array([len(one_tok_seq)-1 for one_tok_seq in tok_seq])
        max_len = max(ret_len) #Longest input sequence
        ret_array = np.zeros((B, max_len, self.max_tok_num), dtype=np.float32)
        for b, one_tok_seq in enumerate(tok_seq): # for each training example
            out_one_tok_seq = one_tok_seq[:-1] if gen_inp else one_tok_seq[1:] #Don't need <END> if generating input sequence
            for t, tok_id in enumerate(out_one_tok_seq): # for each token_id in the example condition
                ret_array[b, t, tok_id] = 1 # Flag to use for decoding

        ret_inp = torch.from_numpy(ret_array)

        ret_inp_var = Variable(ret_inp) #[B, max_len, max_tok_num]

        return ret_inp_var, ret_len

    def forward(self, x_emb_var, x_len, col_inp_var, col_name_len, col_len,
            col_num, gt_where, gt_cond, reinforce):
        """
        :param x_emb_var: (Variable) embedded versions of ALL tokens in the batch. shape=[batch_size, max(x_len), embedding_vector_len]
        :param x_len: (array) number of tokens for each training example (including SQL_TOKEN, question_tokens, column_tokens). shape=(batch_size,)
        :param col_inp_var: (Variable) embedding of each token in each column. shape=(col_name_len, max_tokens_in_a_column_name, embedding_vector_len)
        :param col_name_len: (array) number of tokens for each column name (across the whole batch). Length = sum(col_num)
        :param col_len: (array) number of columns for each example; TODO: remove the argument, it is redundant
        :param col_num: (list) Number of columns for each example
        :param gt_where: List of indices of tokens of WHERE clauses within the token-space of the query (='all_toks') for each question
                        Start of each list is 7, end of each list is 1, any token not present in the set union(SQL_TOK, q, col) is 0
                        E.g.[[7, 2, 27, 4, 34, 1], [7, 2, 14, 15, 16, 17, 4, 38, 1]]
        :param gt_cond: (list) the actual sets of conditions for each query [column_index, operator_index, condition]
                         e.g [[1, 0, '9:00PM'], [(some other condition if present)]]
        :param reinforce: (bool) True if RL is to be done

        :return cond_score: (Variable) shape = [batch_size, max(gt_tok_len), max_x_len]
                        Returns the score (logits) for all tokens among token classes.
                        cond_score(x,y,z) gives score for xth example's yth token for all the z classes.
                        scores beyond y=x_len[x] (last token for that specific example) don't mean anything and are given value -100
        """
        max_x_len = max(x_len) # Max length of x in the batch
        B = len(x_len) # batch size
        mask = torch.ByteTensor(np.array(np.arange(max_x_len) >= np.array(x_len)[:,None], dtype=np.int))

        h_enc, hidden = run_lstm(self.cond_lstm, x_emb_var, x_len)
        # concat forward and backward parts of each of the states so that
        # shape of state is again (num_layers, batch_size, hidden_size)
        decoder_hidden = tuple(torch.cat((hid[:2], hid[2:]),dim=2)
                for hid in hidden)

        #Teacher force training
        if gt_where is not None: #Condition contained WHERE token
            gt_tok_seq, gt_tok_len = self.gen_gt_batch(gt_where, gen_inp=True)
            g_s, _ = run_lstm(self.cond_decoder,
                    gt_tok_seq, gt_tok_len, decoder_hidden)

            h_enc_expand = h_enc.unsqueeze(1) #encoder output
            g_s_expand = g_s.unsqueeze(2) #decoder output
            cond_score = self.cond_out( self.cond_out_h(h_enc_expand) +
                    self.cond_out_g(g_s_expand) ).squeeze() #Equation 2 in paper

            # for each training example...
            #is training example padded on the outside?
            # if yes, set the score for padded entries to -100

            for idx, num in enumerate(x_len):
                if num < max_x_len:
                    cond_score[idx, :, num:] = -100
        else:
            h_enc_expand = h_enc.unsqueeze(1)
            scores = []
            choices = []
            done_set = set()
            mask = torch.ByteTensor(np.array(np.arange(max_x_len) >= np.array(x_len)[:,None], dtype=np.int))

            t = 0

            cur_inp = Variable(torch.zeros(B, 1, self.max_tok_num))
            cur_inp.data[:,0,7] = 1  #Set the <BEG> token
            cur_h = decoder_hidden
            while len(done_set) < B and t < 50:
                g_s, cur_h = self.cond_decoder(cur_inp, cur_h)
                g_s_expand = g_s.unsqueeze(2)

                cur_cond_score = self.cond_out(self.cond_out_h(h_enc_expand) +
                        self.cond_out_g(g_s_expand)).squeeze() #shape = (batch_size, max_x_len)

                cur_cond_score[mask] = -100
                scores.append(cur_cond_score)

                if reinforce:
                    ans_tok_var = self.softmax(cur_cond_score).multinomial() #Convert to a prob distribution for each row and then sample one value per training example
                    choices.append(ans_tok_var)
                else:
                    _, ans_tok_var = cur_cond_score.max(1).unsqueeze(1) #get highest score value and corresponding index for each row

                # Get the next sets of actions
                cur_inp = Variable(torch.zeros(B, self.max_tok_num).scatter_(1, ans_tok_var.data, 1)) # one hot
                cur_inp = cur_inp.unsqueeze(1)

                #Find the <END> (=1) token and add to done_set if found
                [done_set.add(a.numpy()[0]) for a in torch.nonzero(ans_tok_var.data == 1)]
                t += 1

            cond_score = torch.stack(scores, 1)

        if reinforce:
        	return cond_score, choices # choices are the prob action values
            # utilized during reinforce backward - In policy Gradient method
        else:
            return cond_score


import os
from collections import OrderedDict

from conans.errors import ConanException
from conans.util.conan_v2_mode import conan_v2_behavior

DEFAULT_INCLUDE = "include"
DEFAULT_LIB = "lib"
DEFAULT_BIN = "bin"
DEFAULT_RES = "res"
DEFAULT_SHARE = "share"
DEFAULT_BUILD = ""
DEFAULT_FRAMEWORK = "Frameworks"

class DefaultOrderedDict(OrderedDict):

    def __init__(self, factory):
        self.factory = factory
        super(DefaultOrderedDict, self).__init__()

    def __getitem__(self, key):
        if key not in self.keys():
            super(DefaultOrderedDict, self).__setitem__(key, self.factory())
        return super(DefaultOrderedDict, self).__getitem__(key)

class _CppInfo(object):
    """ Object that stores all the necessary information to build in C/C++.
    It is intended to be system independent, translation to
    specific systems will be produced from this info
    """
    def __init__(self):
        self.name = None
        self.names = {}
        self.system_libs = []  # Ordered list of system libraries
        self.includedirs = []  # Ordered list of include paths
        self.srcdirs = []  # Ordered list of source paths
        self.libdirs = []  # Directories to find libraries
        self.resdirs = []  # Directories to find resources, data, etc
        self.bindirs = []  # Directories to find executables and shared libs
        self.builddirs = []
        self.frameworks = []  # Macos .framework
        self.frameworkdirs = []
        self.rootpaths = []
        self.libs = []  # The libs to link against
        self.defines = []  # preprocessor definitions
        self.cflags = []  # pure C flags
        self.cxxflags = []  # C++ compilation flags
        self.sharedlinkflags = []  # linker flags
        self.exelinkflags = []  # linker flags
        self.build_modules = []
        self.rootpath = ""
        self.sysroot = ""
        self._build_modules_paths = None
        self._include_paths = None
        self._lib_paths = None
        self._bin_paths = None
        self._build_paths = None
        self._res_paths = None
        self._src_paths = None
        self._framework_paths = None
        self.version = None  # Version of the conan package
        self.description = None  # Description of the conan package
        # When package is editable, filter_empty=False, so empty dirs are maintained
        self.filter_empty = True

    def _filter_paths(self, paths):
        abs_paths = [os.path.join(self.rootpath, p)
                     if not os.path.isabs(p) else p for p in paths]
        if self.filter_empty:
            return [p for p in abs_paths if os.path.isdir(p)]
        else:
            return abs_paths

    @property
    def build_modules_paths(self):
        if self._build_modules_paths is None:
            self._build_modules_paths = [os.path.join(self.rootpath, p) if not os.path.isabs(p)
                                         else p for p in self.build_modules]
        return self._build_modules_paths

    @property
    def include_paths(self):
        if self._include_paths is None:
            self._include_paths = self._filter_paths(self.includedirs)
        return self._include_paths

    @property
    def lib_paths(self):
        if self._lib_paths is None:
            self._lib_paths = self._filter_paths(self.libdirs)
        return self._lib_paths

    @property
    def src_paths(self):
        if self._src_paths is None:
            self._src_paths = self._filter_paths(self.srcdirs)
        return self._src_paths

    @property
    def bin_paths(self):
        if self._bin_paths is None:
            self._bin_paths = self._filter_paths(self.bindirs)
        return self._bin_paths

    @property
    def build_paths(self):
        if self._build_paths is None:
            self._build_paths = self._filter_paths(self.builddirs)
        return self._build_paths

    @property
    def res_paths(self):
        if self._res_paths is None:
            self._res_paths = self._filter_paths(self.resdirs)
        return self._res_paths

    @property
    def framework_paths(self):
        if self._framework_paths is None:
            self._framework_paths = self._filter_paths(self.frameworkdirs)
        return self._framework_paths

    def get_name(self, generator):
        return self.names.get(generator, self.name)

    # Compatibility for 'cppflags' (old style property to allow decoration)
    def get_cppflags(self):
        conan_v2_behavior("'cpp_info.cppflags' is deprecated, use 'cxxflags' instead")
        return self.cxxflags

    def set_cppflags(self, value):
        conan_v2_behavior("'cpp_info.cppflags' is deprecated, use 'cxxflags' instead")
        self.cxxflags = value

    cppflags = property(get_cppflags, set_cppflags)

class Component(_CppInfo):

    def __init__(self, rootpath):
        super(Component, self).__init__()
        self.rootpath = rootpath
        self.includedirs.append(DEFAULT_INCLUDE)
        self.libdirs.append(DEFAULT_LIB)
        self.bindirs.append(DEFAULT_BIN)
        self.resdirs.append(DEFAULT_RES)
        self.builddirs.append(DEFAULT_BUILD)
        self.frameworkdirs.append(DEFAULT_FRAMEWORK)

class CppInfo(_CppInfo):
    """ Build Information declared to be used by the CONSUMERS of a
    conans. That means that consumers must use this flags and configs i order
    to build properly.
    Defined in user CONANFILE, directories are relative at user definition time
    """
    def __init__(self, root_folder):
        super(CppInfo, self).__init__()
        self.rootpath = root_folder  # the full path of the package in which the conans is found
        self.includedirs.append(DEFAULT_INCLUDE)
        self.libdirs.append(DEFAULT_LIB)
        self.bindirs.append(DEFAULT_BIN)
        self.resdirs.append(DEFAULT_RES)
        self.builddirs.append(DEFAULT_BUILD)
        self.frameworkdirs.append(DEFAULT_FRAMEWORK)
        self.components = DefaultOrderedDict(lambda: Component(self.rootpath))
        # public_deps is needed to accumulate list of deps for cmake targets
        self.public_deps = []
        self.configs = {}

    def __getattr__(self, config):
        def _get_cpp_info():
            result = _CppInfo()
            result.rootpath = self.rootpath
            result.sysroot = self.sysroot
            result.includedirs.append(DEFAULT_INCLUDE)
            result.libdirs.append(DEFAULT_LIB)
            result.bindirs.append(DEFAULT_BIN)
            result.resdirs.append(DEFAULT_RES)
            result.builddirs.append(DEFAULT_BUILD)
            result.frameworkdirs.append(DEFAULT_FRAMEWORK)
            return result

        return self.configs.setdefault(config, _get_cpp_info())

    def _raise_if_mixing_components(self):
        if (self.includedirs != [DEFAULT_INCLUDE] or
                self.libdirs != [DEFAULT_LIB] or
                self.bindirs != [DEFAULT_BIN] or
                self.resdirs != [DEFAULT_RES] or
                self.frameworkdirs != [DEFAULT_FRAMEWORK] or
                self.libs or
                self.system_libs or
                self.frameworks or
                self.defines or
                self.cflags or
                self.cxxflags or
                self.sharedlinkflags or
                self.exelinkflags or
                self.build_modules) and self.components:
            raise ConanException("self.cpp_info.components cannot be used with self.cpp_info "
                                 "global values at the same time")
        if self.configs and self.components:
            raise ConanException("self.cpp_info.components cannot be used with self.cpp_info configs"
                                 " (release/debug/...) at the same time")

class _BaseDepsCppInfo(_CppInfo):
    def __init__(self):
        super(_BaseDepsCppInfo, self).__init__()

    def update(self, dep_cpp_info):

        def merge_lists(seq1, seq2):
            return [s for s in seq1 if s not in seq2] + seq2

        self.system_libs = merge_lists(self.system_libs, dep_cpp_info.system_libs)
        self.includedirs = merge_lists(self.includedirs, dep_cpp_info.include_paths)
        self.srcdirs = merge_lists(self.srcdirs, dep_cpp_info.src_paths)
        self.libdirs = merge_lists(self.libdirs, dep_cpp_info.lib_paths)
        self.bindirs = merge_lists(self.bindirs, dep_cpp_info.bin_paths)
        self.resdirs = merge_lists(self.resdirs, dep_cpp_info.res_paths)
        self.builddirs = merge_lists(self.builddirs, dep_cpp_info.build_paths)
        self.frameworkdirs = merge_lists(self.frameworkdirs, dep_cpp_info.framework_paths)
        self.libs = merge_lists(self.libs, dep_cpp_info.libs)
        self.frameworks = merge_lists(self.frameworks, dep_cpp_info.frameworks)
        self.build_modules = merge_lists(self.build_modules, dep_cpp_info.build_modules_paths)
        self.rootpaths.append(dep_cpp_info.rootpath)

        # Note these are in reverse order
        self.defines = merge_lists(dep_cpp_info.defines, self.defines)
        self.cxxflags = merge_lists(dep_cpp_info.cxxflags, self.cxxflags)
        self.cflags = merge_lists(dep_cpp_info.cflags, self.cflags)
        self.sharedlinkflags = merge_lists(dep_cpp_info.sharedlinkflags, self.sharedlinkflags)
        self.exelinkflags = merge_lists(dep_cpp_info.exelinkflags, self.exelinkflags)

        if not self.sysroot:
            self.sysroot = dep_cpp_info.sysroot

    @property
    def build_modules_paths(self):
        return self.build_modules

    @property
    def include_paths(self):
        return self.includedirs

    @property
    def lib_paths(self):
        return self.libdirs

    @property
    def src_paths(self):
        return self.srcdirs

    @property
    def bin_paths(self):
        return self.bindirs

    @property
    def build_paths(self):
        return self.builddirs

    @property
    def res_paths(self):
        return self.resdirs

    @property
    def framework_paths(self):
        return self.frameworkdirs

class DepCppInfo(object):

    def __init__(self, cpp_info):
        self._cpp_info = cpp_info
        self._libs = None
        self._system_libs = None
        self._frameworks = None
        self._defines = None
        self._cxxflags = None
        self._cflags = None
        self._sharedlinkflags = None
        self._exelinkflags = None

        self._include_paths = None
        self._lib_paths = None
        self._bin_paths = None
        self._build_paths = None
        self._res_paths = None
        self._src_paths = None
        self._framework_paths = None
        self._build_module_paths = None

    def __getattr__(self, item):
        try:
            attr = self._cpp_info.__getattribute__(item)
        except AttributeError:  # item is not defined, get config (CppInfo)
            attr = self._cpp_info.__getattr__(item)
        return attr

    @staticmethod
    def _merge_lists(seq1, seq2):
        return seq1 + [s for s in seq2 if s not in seq1]

    def _aggregated_values(self, item):
        values = getattr(self, "_%s" % item)
        if values is not None:
            return values
        values = getattr(self._cpp_info, item)
        if self._cpp_info.components:
            for component in self._cpp_info.components.values():
                values = self._merge_lists(values, getattr(component, item))
        setattr(self, "_%s" % item, values)
        return values

    def _aggregated_paths(self, item):
        paths = getattr(self, "_%s_paths" % item)
        if paths is not None:
            return paths
        paths = getattr(self._cpp_info, "%s_paths" % item)
        if self._cpp_info.components:
            for component in self._cpp_info.components.values():
                paths = self._merge_lists(paths, getattr(component, "%s_paths" % item))
        setattr(self, "_%s_paths" % item, paths)
        return paths

    @property
    def build_modules_paths(self):
        return self._aggregated_paths("build_modules")

    @property
    def include_paths(self):
        return self._aggregated_paths("include")

    @property
    def lib_paths(self):
        return self._aggregated_paths("lib")

    @property
    def src_paths(self):
        return self._aggregated_paths("src")

    @property
    def bin_paths(self):
        return self._aggregated_paths("bin")

    @property
    def build_paths(self):
        return self._aggregated_paths("build")

    @property
    def res_paths(self):
        return self._aggregated_paths("res")

    @property
    def framework_paths(self):
        return self._aggregated_paths("framework")

    @property
    def libs(self):
        return self._aggregated_values("libs")

    @property
    def system_libs(self):
        return self._aggregated_values("system_libs")

    @property
    def frameworks(self):
        return self._aggregated_values("frameworks")

    @property
    def defines(self):
        return self._aggregated_values("defines")

    @property
    def cxxflags(self):
        return self._aggregated_values("cxxflags")

    @property
    def cflags(self):
        return self._aggregated_values("cflags")

    @property
    def sharedlinkflags(self):
        return self._aggregated_values("sharedlinkflags")

    @property
    def exelinkflags(self):
        return self._aggregated_values("exelinkflags")

class DepsCppInfo(_BaseDepsCppInfo):
    """ Build Information necessary to build a given conans. It contains the
    flags, directories and options if its dependencies. The conans CONANFILE
    should use these flags to pass them to the underlaying build system (Cmake, make),
    so deps info is managed
    """

    def __init__(self):
        super(DepsCppInfo, self).__init__()
        self._dependencies = OrderedDict()
        self.configs = {}

    def __getattr__(self, config):
        return self.configs.setdefault(config, _BaseDepsCppInfo())

    @property
    def dependencies(self):
        return self._dependencies.items()

    @property
    def deps(self):
        return self._dependencies.keys()

    def __getitem__(self, item):
        return self._dependencies[item]

    def update(self, cpp_info, pkg_name):
        assert isinstance(cpp_info, (CppInfo, DepCppInfo))
        self._dependencies[pkg_name] = cpp_info
        super(DepsCppInfo, self).update(cpp_info)
        for config, cpp_info in cpp_info.configs.items():
            self.configs.setdefault(config, _BaseDepsCppInfo()).update(cpp_info)


import numpy as np
import tensorflow as tf

def random_attractors(pts, n_attractors=6):
    pts = pts[np.all(~np.isnan(pts), axis=1)]
    mean = np.mean(pts, axis=0)
    avg_radius = np.mean(np.linalg.norm(pts - mean, axis=1))
    radius = avg_radius / 2
    n_pts = pts.shape[0]

    indices = np.random.choice(n_pts, n_attractors)
    base = pts[indices]
    return base + np.random.uniform(low=-radius, high=+radius, size=(n_attractors, pts.shape[1]))

def tf_random_attractors(pts, n_attractors):
    # batch_size, point, features
    assert len(pts.shape) == 3
    with tf.name_scope('random_attractors'):
        mean = tf.reduce_mean(pts, axis=-1, keepdims=True)
        avg_radius = tf.reduce_mean(tf.linalg.norm(pts - mean, axis=-1), axis=-1)
        # [batch]
        radius = avg_radius / 2

        shape = tf.shape(pts)
        batch_size = shape[0]
        n_pts = shape[1]

        base_indices = tf.random_uniform(
            shape=[batch_size, n_attractors], minval=0, maxval=n_pts,
            dtype=tf.int32
        )
        batch_indices = tf.range(batch_size, dtype=tf.int32)
        batch_indices = tf.expand_dims(batch_indices, -1)
        batch_indices = tf.tile(batch_indices, (1, n_attractors))

        # [batch, n_attractors, 2]
        base = tf.gather_nd(pts, tf.stack([batch_indices, base_indices], axis=-1))
        rad_expand = tf.expand_dims(tf.expand_dims(radius, axis=-1), axis=-1)
        return base + tf.random_uniform(
            minval=-rad_expand, maxval=rad_expand,
            shape=tf.shape(base)
        )

def attract_pts(pts, attractors, beta=8, gamma=1):
    n_pts = pts.shape[0]
    n_attractors = attractors.shape[0]

    # [n_pts, n_attractors]
    distances = np.linalg.norm(
        np.expand_dims(pts, 1) - np.expand_dims(attractors, 0),
        axis=-1
    )

    # Normalize all distances to a mean of 1, to make sure the noising
    # function doesn't vary with the scale of the input
    distances /= np.nanmean(distances)

    # [n_pts, n_attractors]
    weights = np.exp(- distances * beta)

    # The neutral weight should be 10 times the average sum of weights
    # from all attractors.
    neutral = gamma * np.nanmean(np.sum(weights, 1))

    # [n_pts, 1 + n_attractors]
    weights = np.concatenate((weights, neutral * np.ones((n_pts, 1))), axis=1)
    weight_sums = np.sum(weights, axis=1, keepdims=True)

    # [n_pts, 1 + n_attractors, 1]
    weights = np.expand_dims(weights, axis=-1)

    # [n_pts, 1 + n_attractors, dim]
    candidates = np.concatenate(
        (np.tile(np.expand_dims(attractors, axis=0),
                 (n_pts, 1, 1)),
         np.expand_dims(pts, axis=1)
         ), axis=1
    )

    return np.sum(weights * candidates, axis=1) / weight_sums

def tf_attract_pts(pts, attractors, beta=8, gamma=1):
    n_pts = tf.shape(pts)[1]
    n_attractors = tf.shape(pts)[1]

    # [batch, n_pts, n_attractors]
    distances = tf.linalg.norm(
        tf.expand_dims(pts, -2) - tf.expand_dims(attractors, -3),
        axis=-1
    )

    # Normalize all distances to a mean of 1, to make sure the noising
    # function doesn't vary with the scale of the input
    distances /= tf.reduce_mean(
        tf.reduce_mean(distances, axis=-1, keepdims=True),
        axis=-2, keepdims=True
    )

    # [batch, n_pts, n_attractors]
    weights = tf.exp(- distances ** 3 * beta)

    # The neutral weight should be 10 times the average sum of weights
    # from all attractors.
    # [batch]
    neutral = gamma * tf.reduce_mean(tf.reduce_sum(weights, axis=-1), axis=-1)

    # [batch, n_pts, 1 + n_attractors]
    weights = tf.concat(
        (weights,
         tf.tile(
             tf.expand_dims(tf.expand_dims(neutral, axis=-1), axis=-1),
             (1, n_pts, 1)
         )
         ),
        axis=2
    )
    # [batch, n_pts, 1]
    weight_sums = tf.reduce_sum(weights, axis=-1, keepdims=True)

    # [batch, n_pts, 1 + n_attractors, 1]
    weights = tf.expand_dims(weights, axis=-1)

    # [batch, n_pts, 1 + n_attractors, dim]
    candidates = tf.concat(
        (tf.tile(tf.expand_dims(attractors, axis=1),
                 (1, n_pts, 1, 1)),
         tf.expand_dims(pts, axis=2)
         ), axis=2
    )

    return tf.reduce_sum(weights * candidates, axis=2) / weight_sums

def random_attract_pts(pts, n_attractors=6, **kwargs):
    attractors = random_attractors(pts, n_attractors=n_attractors)
    return attract_pts(pts, attractors, **kwargs)

def tf_random_attract_pts(pts, n_attractors):
    attractors = tf_random_attractors(pts, n_attractors)
    return tf_attract_pts(pts, attractors)


from __future__ import print_function, absolute_import
import logging
import re
import json
import requests
import uuid
import time
import os
import socket
import argparse
import uuid
import datetime
import apache_beam as beam
from apache_beam.io import ReadFromText
from apache_beam.io import WriteToText
from apache_beam.io.filesystems import FileSystems
from apache_beam.metrics import Metrics
from apache_beam.metrics.metric import MetricsFilter
from apache_beam import pvalue
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions

now = datetime.datetime.now()

TABLE_SCHEMA = (
	'id:STRING,'
	'fecha:STRING,'
	'accion:STRING,'
	'idLupe:STRING,'
	'idAgente:STRING,'
	'correo:STRING,'
	'telMarcado:STRING,'
	'comentarioTipificacion:STRING,'
	'codigoTipificacion:STRING'
	)

class formatearData(beam.DoFn):

	def process(self, element):
		arrayCSV = element.split(';')

		tupla= {'id': arrayCSV[0],
				'fecha': arrayCSV[1],
				'accion': arrayCSV[2],
				'idLupe': arrayCSV[3],
				'idAgente': arrayCSV[4],
				'correo': arrayCSV[5],
				'telMarcado': arrayCSV[6],
				'comentarioTipificacion': arrayCSV[7],
				'codigoTipificacion': arrayCSV[8]
				}

		return [tupla]

def run(output):

	gcs_path = "gs://ct-rappi" #Definicion de la raiz del bucket
	gcs_project = "contento-bi"

	mi_runer = ("DirectRunner", "DataflowRunner")[socket.gethostname()=="contentobi"]
	pipeline =  beam.Pipeline(runner=mi_runer, argv=[
        "--project", gcs_project,
        "--staging_location", ("%s/dataflow_files/staging_location" % gcs_path),
        "--temp_location", ("%s/dataflow_files/temp" % gcs_path),
        "--output", ("%s/dataflow_files/output" % gcs_path),
        "--setup_file", "./setup.py",
        "--max_num_workers", "10",
		"--subnetwork", "https://www.googleapis.com/compute/v1/projects/contento-bi/regions/us-central1/subnetworks/contento-subnet1"
	])

	lines = pipeline | 'Lectura de Archivo' >> ReadFromText(output)
	transformed = (lines | 'Formatear Data' >> beam.ParDo(formatearData()))
	transformed | 'Escritura a BigQuery Rappi' >> beam.io.WriteToBigQuery(
		gcs_project + ":Rappi.flujo_react",
		schema=TABLE_SCHEMA,
		create_disposition=beam.io.BigQueryDisposition.CREATE_IF_NEEDED,
		write_disposition=beam.io.BigQueryDisposition.WRITE_APPEND
		)

	jobObject = pipeline.run()
	# jobID = jobObject.job_id()

	return ("El proceso de cargue a bigquery fue ejecutado con exito")


from osc_tui import main
from osc_tui import selectableGrid

class Grid(selectableGrid.SelectableGrid):
    def __init__(self, screen,  *args, **keywords):
        super().__init__(screen, *args, **keywords)
        self.col_titles = ["Id", "AccepterNet", "State"]

    def refresh_call(self, name_filter=None):
        return main.GATEWAY.ReadNetPeerings(form=self.form)['NetPeerings']

    def refresh(self):
        groups = main.do_search(self.data.copy(), ['NetPeeringId'], state_msg=True, accepted_net=True)

        values = list()
        for g in groups:
            AccepterNet = g["AccepterNet"]
            AccepterNetId = AccepterNet["NetId"] if AccepterNet else "Not Accepted"
            values.append([g['NetPeeringId'], AccepterNetId, g['State']['Message']])
        self.values = values


import pandas as pd
import numpy as np
from typing import Union, List, Optional, Iterable
import h5py

#######################
## Loading metadata  ##
#######################

def _load_samples_metadata(model):
    samples_metadata = pd.DataFrame(
        [
            [cell, group]
            for group, cell_list in model.samples.items()
            for cell in cell_list
        ],
        columns=["sample", "group"],
    )
    if "samples_metadata" in model.model:
        if len(list(model.model["samples_metadata"][model.groups[0]].keys())) > 0:
            _samples_metadata = pd.concat(
                [
                    pd.concat(
                        [
                            pd.Series(model.model["samples_metadata"][g][k])
                            for k in model.model["samples_metadata"][g].keys()
                        ],
                        axis=1,
                    )
                    for g in model.groups
                ],
                axis=0,
            )
            _samples_metadata.columns = list(
                model.model["samples_metadata"][model.groups[0]].keys()
            )

            if "group" in _samples_metadata.columns:
                del _samples_metadata["group"]
            if "sample" in _samples_metadata.columns:
                del _samples_metadata["sample"]

            samples_metadata = pd.concat(
                [
                    samples_metadata.reset_index(drop=True),
                    _samples_metadata.reset_index(drop=True),
                ],
                axis=1,
            )

            # Decode objects as UTF-8 strings
            for column in samples_metadata.columns:
                if samples_metadata[column].dtype == "object":
                    try:
                        samples_metadata[column] = [
                            i.decode() for i in samples_metadata[column].values
                        ]
                    except (UnicodeDecodeError, AttributeError):
                        pass

    samples_metadata = samples_metadata.set_index("sample")
    return samples_metadata

def _load_features_metadata(model):
    features_metadata = pd.DataFrame(
        [
            [feature, view]
            for view, feature_list in model.features.items()
            for feature in feature_list
        ],
        columns=["feature", "view"],
    )
    if "features_metadata" in model.model:
        if len(list(model.model["features_metadata"][model.views[0]].keys())) > 0:
            features_metadata_dict = {
                m: pd.concat(
                    [
                        pd.Series(model.model["features_metadata"][m][k])
                        for k in model.model["features_metadata"][m].keys()
                    ],
                    axis=1,
                )
                for m in model.views
            }

            for m in features_metadata_dict.keys():
                features_metadata_dict[m].columns = list(
                    model.model["features_metadata"][m].keys()
                )

            _features_metadata = pd.concat(features_metadata_dict, axis=0)

            if "view" in _features_metadata.columns:
                del _features_metadata["view"]
            if "feature" in _features_metadata.columns:
                del _features_metadata["feature"]

            features_metadata = pd.concat(
                [
                    features_metadata.reset_index(drop=True),
                    _features_metadata.reset_index(drop=True),
                ],
                axis=1,
            )

            # Decode objects as UTF-8 strings
            for column in features_metadata.columns:
                if features_metadata[column].dtype == "object":
                    try:
                        features_metadata[column] = [
                            i.decode() for i in features_metadata[column].values
                        ]
                    except (UnicodeDecodeError, AttributeError):
                        pass

    features_metadata = features_metadata.set_index("feature")
    return features_metadata

def _load_covariates(model):
    # Covariates names
    cov_names = None
    if "covariates" in model.model:
        cov_root = model.model["covariates"]
        if (type(cov_root) == h5py.Group) and ("covariates" in cov_root):
            cov_root = cov_root["covariates"]
        cov_names = np.array(cov_root)
        try:
            cov_names = [i.decode() for i in cov_names]
        except (UnicodeDecodeError, AttributeError):
            pass
    else:
        if "cov_samples" in model.model:
            cov_len = model.model["cov_samples"][model.groups[0]].shape[-1]
            cov_names = [f"Covariates{i+1}" for i in range(cov_len)]
        else:
            return None, None

    # Covariates values
    samples_covariates = pd.DataFrame(
        [
            [cell, group]
            for group, cell_list in model.samples.items()
            for cell in cell_list
        ],
        columns=["sample", "group"],
    )

    for attr in ["cov_samples", "cov_samples_transformed"]:
        if attr in model.model:
            if len(model.model[attr].keys()) > 0:  # groups are not empty
                attr_covariates = np.concatenate(
                    [np.array(model.model[attr][g]) for g in model.groups]
                )
                attr_covariates = pd.DataFrame(attr_covariates)
                if attr == "cov_samples":
                    attr_covariates.columns = cov_names
                elif attr == "cov_samples_transformed":
                    attr_covariates.columns = [f"{n}_transformed" for n in cov_names]
                else:
                    attr_covariates.columns = [
                        f"{attr}{i+1}" for i in range(attr_covariates.shape[-1])
                    ]
                samples_covariates = pd.concat(
                    [samples_covariates, attr_covariates], axis=1
                )

    # Covariates are None when there is only sample name and group name
    if len(samples_covariates.columns) == 2:
        return None, None

    samples_covariates = samples_covariates.set_index("sample")

    return cov_names, samples_covariates

#######################
#### Reading HDF5 #####
#######################

def _read_simple(data):
    if isinstance(data, h5py.Group):
        return {k: np.array(v) for k, v in data.items()}
    elif isinstance(data, h5py.Dataset):
        return np.array(data)
    else:
        raise ValueError("Attempting to read neither Group nor Dataset from HDF5 file")

#######################
## Utility functions ##
#######################

### Factors

def factor_indices_to_factors(x):
    if x is None:
        return None
    elif isinstance(x, int):
        x = [x]
    return [f"Factor{i+1}" for i in x]

def maybe_factor_indices_to_factors(x):
    if x is None:
        return None
    elif isinstance(x, int):
        return f"Factor{x+1}"
    elif isinstance(x, str):
        return x
    return [f"Factor{i+1}" if isinstance(i, int) else i for i in x]

def _is_iter(x):
    return isinstance(x, Iterable) and not isinstance(x, str)

def _make_iterable(x):
    if not _is_iter(x):  # e.g. int
        x = [x]
    return x

### R2 statistics

def calculate_r2(Z, W, Y):
    a = np.nansum((Y - Z.T.dot(W)) ** 2.0)
    b = np.nansum(Y ** 2)
    r2 = (1.0 - a / b) * 100
    return r2

def padjust_fdr(xs):
    """
    Adjust p-values using the BH procedure
    """
    from scipy.stats import rankdata

    ranked_p_values = rankdata(xs)
    fdr = xs * len(xs) / ranked_p_values
    fdr[fdr > 1] = 1
    return fdr

def padjust_fdr_2d(mx):
    """
    Adjust p-values in a matrix using the BH procedure
    """
    from scipy.stats import rankdata

    ranked_p_values = rankdata(mx).reshape((-1, mx.shape[1]))
    fdr = mx * mx.shape[0] * mx.shape[1] / ranked_p_values
    fdr[fdr > 1] = 1
    return fdr