import copy
import math
import random
from typing import *

import torch
from torch import Tensor
from torch import nn
from torch.nn import functional as F
from transformers import modeling_bart as bart
from transformers.modeling_utils import BeamHypotheses, calc_banned_ngram_tokens, calc_banned_bad_words_ids, \
    top_k_top_p_filtering

def extract_backreferences(ids, num_embeddings, backpointer_idx):
    ids_mask = ids >= num_embeddings
    backreferences = ids.clone() - num_embeddings
    backreferences[~ids_mask] = 0
    backreferences += (~ids_mask).long() * torch.arange(
        ids.size(1),
        dtype=ids.dtype,
        device=ids.device)
    ids = ids.clone()
    ids[ids_mask] = backpointer_idx
    return ids, backreferences

class AMRBartEncoder(nn.Module):
    """
    Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer
    is a :class:`EncoderLayer`.

    Args:
        config: BartConfig
    """

    def __init__(self, config: bart.BartConfig, embed_tokens, backpointer_idx):
        super().__init__()

        self.backpointer_idx = backpointer_idx

        self.dropout = config.dropout
        self.layerdrop = config.encoder_layerdrop
        self.output_attentions = config.output_attentions
        self.output_hidden_states = config.output_hidden_states

        embed_dim = embed_tokens.embedding_dim
        self.embed_scale = math.sqrt(embed_dim) if config.scale_embedding else 1.0
        self.padding_idx = embed_tokens.padding_idx
        self.max_source_positions = config.max_position_embeddings

        self.embed_tokens = embed_tokens

        if config.static_position_embeddings:
            self.embed_positions = bart.SinusoidalPositionalEmbedding(
                config.max_position_embeddings, embed_dim, self.padding_idx
            )
        else:
            self.embed_positions = bart.LearnedPositionalEmbedding(
                config.max_position_embeddings, embed_dim, self.padding_idx, #config.extra_pos_embeddings,
            )

        self.layers = nn.ModuleList([bart.EncoderLayer(config) for _ in range(config.encoder_layers)])
        self.layernorm_embedding = bart.LayerNorm(embed_dim) if config.normalize_embedding else nn.Identity()
        # mbart has one extra layer_norm
        self.layer_norm = bart.LayerNorm(config.d_model) if config.normalize_before else None

    def forward(
        self, input_ids, embedded=None, attention_mask=None,
    ):
        """
        Args:
            input_ids (LongTensor): tokens in the source language of shape
                `(batch, src_len)`
            attention_mask (torch.LongTensor): indicating which indices are padding tokens.
        Returns:
            Tuple comprised of:
                - **x** (Tensor): the last encoder layer's output of
                  shape `(src_len, batch, embed_dim)`
                - **encoder_states** (List[Tensor]): all intermediate
                  hidden states of shape `(src_len, batch, embed_dim)`.
                  Only populated if *self.output_hidden_states:* is True.
                - **all_attentions** (List[Tensor]): Attention weights for each layer.
                During training might not be of length n_layers because of layer dropout.
        """
        # check attention mask and invert
        if attention_mask is not None:
            attention_mask = bart.invert_mask(attention_mask)

        input_ids, backreferences = extract_backreferences(
            input_ids, self.embed_tokens.num_embeddings, self.backpointer_idx)
        inputs_embeds = self.embed_tokens(input_ids) * self.embed_scale
        embed_pos = self.embed_positions(input_ids)
        x = inputs_embeds + embed_pos

        if embedded is not None:
            x += embedded

        x = self.layernorm_embedding(x)
        x = F.dropout(x, p=self.dropout, training=self.training)

        # B x T x C -> T x B x C
        x = x.transpose(0, 1)

        encoder_states, all_attentions = [], []
        for encoder_layer in self.layers:
            if self.output_hidden_states:
                encoder_states.append(x)
            # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description)
            dropout_probability = random.uniform(0, 1)
            if self.training and (dropout_probability < self.layerdrop):  # skip the layer
                attn = None
            else:
                x, attn = encoder_layer(x, attention_mask)

            if self.output_attentions:
                all_attentions.append(attn)

        if self.layer_norm:
            x = self.layer_norm(x)
        if self.output_hidden_states:
            encoder_states.append(x)

        # T x B x C -> B x T x C
        encoder_states = [hidden_state.transpose(0, 1) for hidden_state in encoder_states]
        x = x.transpose(0, 1)

        return x, encoder_states, all_attentions

class AMRBartDecoder(nn.Module):
    """
    Transformer decoder consisting of *config.decoder_layers* layers. Each layer
    is a :class:`DecoderLayer`.
    Args:
        config: BartConfig
        embed_tokens (torch.nn.Embedding): output embedding
    """

    def __init__(self, config: bart.BartConfig, embed_tokens: nn.Embedding, backpointer_idx, amr_mode=True):
        super().__init__()
        self.output_attentions = config.output_attentions
        self.output_hidden_states = config.output_hidden_states
        self.dropout = config.dropout
        self.layerdrop = config.decoder_layerdrop
        self.padding_idx = embed_tokens.padding_idx
        self.max_target_positions = config.max_position_embeddings
        self.embed_scale = math.sqrt(config.d_model) if config.scale_embedding else 1.0
        self.backpointer_idx = backpointer_idx

        embed_dim = embed_tokens.embedding_dim

        self.embed_tokens = embed_tokens
        if config.static_position_embeddings:
            self.embed_positions = bart.SinusoidalPositionalEmbedding(
                config.max_position_embeddings, embed_dim, self.padding_idx
            )
        else:
            self.embed_positions = bart.LearnedPositionalEmbedding(
                config.max_position_embeddings, embed_dim, self.padding_idx, #config.extra_pos_embeddings,
            )

        self.layers = nn.ModuleList(
            [bart.DecoderLayer(config) for _ in range(config.decoder_layers)]
        )  # type: List[DecoderLayer]
        self.layernorm_embedding = bart.LayerNorm(config.d_model) if config.normalize_embedding else nn.Identity()
        self.layer_norm = bart.LayerNorm(config.d_model) if config.add_final_layer_norm else None

        self.pointer_k = nn.Linear(config.d_model, config.d_model)
        # self.pointer_k.weight.data = self.layers[-1].self_attn.k_proj.weight.data.clone()

        self.pointer_q = nn.Linear(config.d_model, config.d_model)
        # self.pointer_q.weight.data = self.layers[-1].self_attn.q_proj.weight.data.clone()

        # self.pointer_k = nn.Sequential(
        #     nn.Linear(config.d_model, config.decoder_ffn_dim),
        #     nn.GELU(),
        #     nn.Linear(config.decoder_ffn_dim, config.d_model),
        # )
        # self.pointer_q = nn.Sequential(
        #     nn.Linear(config.d_model, config.decoder_ffn_dim),
        #     nn.GELU(),
        #     nn.Linear(config.decoder_ffn_dim, config.d_model),
        # )

        self.amr_mode = amr_mode

    def forward(
        self,
        input_ids,
        encoder_hidden_states,
        encoder_padding_mask,
        decoder_padding_mask,
        decoder_causal_mask,
        decoder_cached_states=None,
        use_cache=False,
        **unused
    ):
        """
        Includes several features from "Jointly Learning to Align and
        Translate with Transformer Models" (Garg et al., EMNLP 2019).

        Args:
            input_ids (LongTensor): previous decoder outputs of shape
                `(batch, tgt_len)`, for teacher forcing
            encoder_hidden_states: output from the encoder, used for
                encoder-side attention
            encoder_padding_mask: for ignoring pad tokens
            decoder_cached_states (dict or None): dictionary used for storing state during generation

        Returns:
            tuple:
                - the decoder's features of shape `(batch, tgt_len, embed_dim)`
                - hidden states
                - attentions
        """

        # check attention mask and invert
        if encoder_padding_mask is not None:
            encoder_padding_mask = bart.invert_mask(encoder_padding_mask)

        input_ids, backreferences = extract_backreferences(
            input_ids,
            self.embed_tokens.num_embeddings,
            self.backpointer_idx)
        # embed positions
        embed_pos = self.embed_positions(input_ids, use_cache=use_cache)
        positions = embed_pos

        # to do this during prediction the old positions should be removed
        if use_cache:
            input_ids = input_ids[:, -1:]
            positions = positions[:, -1:]  # happens after we embed them
            # assert input_ids.ne(self.padding_idx).any()

        x = self.embed_tokens(input_ids) * self.embed_scale
        x += positions
        x = self.layernorm_embedding(x)
        x = F.dropout(x, p=self.dropout, training=self.training)

        # Convert to Bart output format: (seq_len, BS, model_dim) -> (BS, seq_len, model_dim)
        x = x.transpose(0, 1)
        encoder_hidden_states = encoder_hidden_states.transpose(0, 1)

        # decoder layers
        all_hidden_states = ()
        all_self_attns = ()
        next_decoder_cache = []
        for idx, decoder_layer in enumerate(self.layers):
            # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description)
            if self.output_hidden_states:
                all_hidden_states += (x,)
            dropout_probability = random.uniform(0, 1)
            if self.training and (dropout_probability < self.layerdrop):
                continue

            layer_state = decoder_cached_states[idx] if decoder_cached_states is not None else None

            x, layer_self_attn, layer_past = decoder_layer(
                x,
                encoder_hidden_states,
                encoder_attn_mask=encoder_padding_mask,
                decoder_padding_mask=decoder_padding_mask,
                layer_state=layer_state,
                causal_mask=decoder_causal_mask,
            )

            if use_cache:
                next_decoder_cache.append(layer_past.copy())

            if self.layer_norm and (idx == len(self.layers) - 1):  # last layer of mbart
                x = self.layer_norm(x)
            if self.output_attentions:
                all_self_attns += (layer_self_attn,)

        # Convert to standard output format: (seq_len, BS, model_dim) -> (BS, seq_len, model_dim)
        all_hidden_states = [hidden_state.transpose(0, 1) for hidden_state in all_hidden_states]
        x = x.transpose(0, 1)
        encoder_hidden_states = encoder_hidden_states.transpose(0, 1)

        xq = self.pointer_q(x)
        xk = self.pointer_k(x)

        if decoder_cached_states is not None:
            if 'prev_key' in decoder_cached_states[-1].get('pointer', {}):
                last_state = decoder_cached_states[-1]['pointer']
                xk = torch.cat([last_state['prev_key'], xk], dim=1)

        next_state = {'pointer': {'prev_key': xk}}

        if use_cache:
            next_decoder_cache.append(next_state)

        if self.amr_mode:
            scores = torch.einsum('bqh,bkh->bqk', xq, xk)

            if decoder_cached_states:
                mask = torch.full_like(scores[0], float('-inf'))
                mask = mask.triu(diagonal=xk.size(1) - 1)
            else:
                mask = torch.full_like(scores[0], float('-inf'))
                mask = mask.triu()
            scores += mask.unsqueeze(0)
        else:
            scores = torch.full((xq.size(0), xq.size(1), xk.size(1)), float('-inf'), device=xq.device)

        if use_cache:
            next_cache = ((encoder_hidden_states, encoder_padding_mask), next_decoder_cache)
        else:
            next_cache = None
        return (x, scores), next_cache, all_hidden_states, list(all_self_attns)

class AMRBartModel(bart.PretrainedBartModel):
    def __init__(self, config: bart.BartConfig, backpointer_idx=None):
        super().__init__(config)
        self.output_attentions = True
        self.output_hidden_states = config.output_hidden_states

        self.padding_idx, vocab_size = config.pad_token_id, config.vocab_size
        self.shared = nn.Embedding(vocab_size, config.d_model, self.padding_idx)

        if backpointer_idx is not None:
            self.backpointer_idx = backpointer_idx
        else:
            self.backpointer_idx = self.shared.num_embeddings - 1

        self.encoder = AMRBartEncoder(config, self.shared, backpointer_idx=self.backpointer_idx)
        self.decoder = AMRBartDecoder(config, self.shared, backpointer_idx=self.backpointer_idx)

        self.init_weights()

    @property
    def sentence_mode(self):
        return self.decoder.amr_mode

    @sentence_mode.setter
    def sentence_mode(self, value):
        assert isinstance(value, bool)
        self.decoder.amr_mode = value

    def forward(
        self,
        input_ids,
        attention_mask=None,
        decoder_input_ids=None,
        encoder_outputs: Optional[Tuple] = None,
        decoder_attention_mask=None,
        decoder_cached_states=None,
        use_cache=False,
    ):

        # make masks if user doesn't supply
        if not use_cache:
            decoder_input_ids, decoder_padding_mask, causal_mask = bart._prepare_bart_decoder_inputs(
                self.config,
                input_ids,
                decoder_input_ids=decoder_input_ids,
                decoder_padding_mask=decoder_attention_mask,
                causal_mask_dtype=self.shared.weight.dtype,
            )
        else:
            decoder_padding_mask, causal_mask = None, None

        assert decoder_input_ids is not None
        if encoder_outputs is None:
            encoder_outputs = self.encoder(input_ids=input_ids, attention_mask=attention_mask)
        assert isinstance(encoder_outputs, tuple)
        # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
        decoder_outputs = self.decoder(
            decoder_input_ids,
            encoder_outputs[0],
            attention_mask,
            decoder_padding_mask,
            decoder_causal_mask=causal_mask,
            decoder_cached_states=decoder_cached_states,
            use_cache=use_cache,
        )
        # Attention and hidden_states will be [] or None if they aren't needed
        # decoder_outputs: Tuple = bart._filter_out_falsey_values(decoder_outputs)
        assert isinstance(decoder_outputs[0][0], torch.Tensor)
        assert isinstance(decoder_outputs[0][1], torch.Tensor)
        encoder_outputs: Tuple = bart._filter_out_falsey_values(encoder_outputs)
        return decoder_outputs + encoder_outputs

    def get_input_embeddings(self):
        return self.shared

    def set_input_embeddings(self, value):
        self.shared = value
        self.encoder.embed_tokens = self.shared
        self.decoder.embed_tokens = self.shared

    def get_output_embeddings(self):
        return bart._make_linear_from_emb(self.shared)  # make it on the fly

class AMRBartForConditionalGeneration(bart.PretrainedBartModel):
    base_model_prefix = "model"

    def __init__(self, config: bart.BartConfig, backpointer_idx=None):
        super().__init__(config)
        base_model = AMRBartModel(config, backpointer_idx)
        self.model = base_model
        self.pad_index = base_model.shared.padding_idx
        self.register_buffer("final_logits_bias", torch.zeros((1, self.model.shared.num_embeddings)))
        self.backpointer_idx = backpointer_idx
        self._rev = None

    def init_reverse_model(self):
        rev = AMRBartForConditionalGeneration(self.model.config, self.backpointer_idx)
        rev.model.shared = self.model.shared
        rev.model.encoder = self.model.encoder
        rev.model.decoder.embed_tokens = self.model.decoder.embed_tokens
        rev.model.decoder.embed_positions = self.model.decoder.embed_positions
        self.amr_mode = True
        rev.amr_mode = False
        self._rev = rev

    @property
    def rev(self):
        if self._rev is None:
            return self
        else:
            return self._rev

    @property
    def amr_mode(self):
        return self.model.decoder.amr_mode

    @amr_mode.setter
    def amr_mode(self, value):
        assert isinstance(value, bool)
        self.model.decoder.amr_mode = value

    def forward(
        self,
        input_ids,
        attention_mask=None,
        encoder_outputs=None,
        decoder_input_ids=None,
        decoder_attention_mask=None,
        decoder_cached_states=None,
        lm_labels=None,
        use_cache=False,
        **unused
    ):
        r"""
        lm_labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`, defaults to :obj:`None`):
            Labels for computing the masked language modeling loss.
            Indices should either be in ``[0, ..., config.vocab_size]`` or -100 (see ``input_ids`` docstring).
            Tokens with indices set to ``-100`` are ignored (masked), the loss is only computed for the tokens
            with labels
            in ``[0, ..., config.vocab_size]``.

    Returns:
        :obj:`tuple(torch.FloatTensor)` comprising various elements depending on the configuration (:class:`~transformers.RobertaConfig`) and inputs:
        masked_lm_loss (`optional`, returned when ``lm_labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``:
            Masked language modeling loss.
        prediction_scores (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, config.vocab_size)`)
            Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
        hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_hidden_states=True``):
            Tuple of :obj:`torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
            of shape :obj:`(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
        attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_attentions=True``):
            Tuple of :obj:`torch.FloatTensor` (one for each layer) of shape
            :obj:`(batch_size, num_heads, sequence_length, sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.

    Examples::

            # Mask filling only works for bart-large
            from transformers import BartTokenizer, BartForConditionalGeneration
            tokenizer = BartTokenizer.from_pretrained('bart-large')
            TXT = "My friends are <mask> but they eat too many carbs."
            model = BartForConditionalGeneration.from_pretrained('bart-large')
            input_ids = tokenizer.batch_encode_plus([TXT], return_tensors='pt')['input_ids']
            logits = model(input_ids)[0]
            masked_index = (input_ids[0] == tokenizer.mask_token_id).nonzero().item()
            probs = logits[0, masked_index].softmax(dim=0)
            values, predictions = probs.topk(5)
            tokenizer.decode(predictions).split()
            # ['good', 'great', 'all', 'really', 'very']
        """
        # outputs = self.model(
        #     input_ids,
        #     attention_mask=attention_mask,
        #     decoder_input_ids=decoder_input_ids,
        #     encoder_outputs=encoder_outputs,
        #     decoder_attention_mask=decoder_attention_mask,
        #     decoder_cached_states=decoder_cached_states,
        #     use_cache=use_cache,
        # )
        # lm_logits = F.linear(outputs[0][0], self.model.shared.weight, bias=self.final_logits_bias)
        # po_logits = outputs[0][1]
        # po_padding = torch.full_like(po_logits[:, :, 0:1], float('-inf'))
        # po_padding = po_padding.repeat(1, 1, 1024 - po_logits.size(-1))
        # po_logits = torch.cat([po_logits, po_padding], -1)
        # uni_logits = torch.cat([lm_logits, po_logits], -1)
        #
        # outputs = (uni_logits,) + outputs[1:]  # Add cache, hidden states and attention if they are here

        outputs = self.compute_logits(
            input_ids,
            attention_mask=attention_mask,
            decoder_input_ids=decoder_input_ids,
            encoder_outputs=encoder_outputs,
            decoder_attention_mask=decoder_attention_mask,
            decoder_cached_states=decoder_cached_states,
            use_cache=use_cache,
        )

        if lm_labels is not None:
            uni_logits = outputs[0]
            masked_lm_loss = F.nll_loss(
                uni_logits.log_softmax(-1).contiguous().view(-1, uni_logits.size(-1)),
                lm_labels.contiguous().view(-1),
                ignore_index=self.pad_index)
            outputs = (masked_lm_loss,) + outputs

        return outputs

    def compute_logits(
        self,
        input_ids,
        attention_mask=None,
        encoder_outputs=None,
        decoder_input_ids=None,
        decoder_attention_mask=None,
        decoder_cached_states=None,
        use_cache=False,
    ):
        outputs = self.model(
            input_ids,
            attention_mask=attention_mask,
            decoder_input_ids=decoder_input_ids,
            encoder_outputs=encoder_outputs,
            decoder_attention_mask=decoder_attention_mask,
            decoder_cached_states=decoder_cached_states,
            use_cache=use_cache,
        )

        lm_logits = F.linear(outputs[0][0], self.model.shared.weight, bias=self.final_logits_bias)
        po_logits = outputs[0][1]
        po_padding = torch.full_like(po_logits[:, :, 0:1], float('-inf'))
        po_padding = po_padding.repeat(1, 1, 1024 - po_logits.size(-1))
        po_logits = torch.cat([po_logits, po_padding], -1)
        uni_logits = torch.cat([lm_logits, po_logits], -1)
        outputs = (uni_logits,) + outputs[1:]  # Add cache, hidden states and attention if they are here
        return outputs

    @torch.no_grad()
    def generate(
            self,
            input_ids: Optional[torch.LongTensor] = None,
            max_length: Optional[int] = None,
            min_length: Optional[int] = None,
            do_sample: Optional[bool] = None,
            early_stopping: Optional[bool] = None,
            num_beams: Optional[int] = None,
            temperature: Optional[float] = None,
            top_k: Optional[int] = None,
            top_p: Optional[float] = None,
            repetition_penalty: Optional[float] = None,
            bad_words_ids: Optional[Iterable[int]] = None,
            bos_token_id: Optional[int] = None,
            pad_token_id: Optional[int] = None,
            eos_token_id: Optional[int] = None,
            length_penalty: Optional[float] = None,
            no_repeat_ngram_size: Optional[int] = None,
            num_return_sequences: Optional[int] = None,
            attention_mask: Optional[torch.LongTensor] = None,
            decoder_start_token_id: Optional[int] = None,
            use_cache: Optional[bool] = None,
            **model_specific_kwargs
    ) -> torch.LongTensor:
        r""" Generates sequences for models with a LM head. The method currently supports greedy decoding, beam-search decoding, sampling with temperature, sampling with top-k or nucleus sampling.

        Adapted in part from `Facebook's XLM beam search code`_.

        .. _`Facebook's XLM beam search code`:
           https://github.com/facebookresearch/XLM/blob/9e6f6814d17be4fe5b15f2e6c43eb2b2d76daeb4/src/model/transformer.py#L529

        Parameters:

            input_ids: (`optional`) `torch.LongTensor` of shape `(batch_size, sequence_length)`
                The sequence used as a prompt for the generation. If `None` the method initializes
                it as an empty `torch.LongTensor` of shape `(1,)`.

            max_length: (`optional`) int
                The max length of the sequence to be generated.  Between `min_length` and infinity. Default to 20.

            min_length: (`optional`) int
                The min length of the sequence to be generated.  Between 0 and infinity. Default to 0.

            do_sample: (`optional`) bool
                If set to `False` greedy decoding is used. Otherwise sampling is used. Defaults to `False` as defined in `configuration_utils.PretrainedConfig`.

            early_stopping: (`optional`) bool
                if set to `True` beam search is stopped when at least `num_beams` sentences finished per batch. Defaults to `False` as defined in `configuration_utils.PretrainedConfig`.

            num_beams: (`optional`) int
                Number of beams for beam search. Must be between 1 and infinity. 1 means no beam search. Default to 1.

            temperature: (`optional`) float
                The value used to module the next token probabilities. Must be strictly positive. Default to 1.0.

            top_k: (`optional`) int
                The number of highest probability vocabulary tokens to keep for top-k-filtering. Between 1 and infinity. Default to 50.

            top_p: (`optional`) float
                The cumulative probability of parameter highest probability vocabulary tokens to keep for nucleus sampling. Must be between 0 and 1. Default to 1.

            repetition_penalty: (`optional`) float
                The parameter for repetition penalty. Between 1.0 and infinity. 1.0 means no penalty. Default to 1.0.

            pad_token_id: (`optional`) int
                Padding token. Default to specicic model pad_token_id or None if it does not exist.

            bos_token_id: (`optional`) int
                BOS token. Defaults to `bos_token_id` as defined in the models config.

            eos_token_id: (`optional`) int
                EOS token. Defaults to `eos_token_id` as defined in the models config.

            length_penalty: (`optional`) float
                Exponential penalty to the length. Default to 1.

            no_repeat_ngram_size: (`optional`) int
                If set to int > 0, all ngrams of size `no_repeat_ngram_size` can only occur once.
            bad_words_ids: (`optional`) list of lists of int
                `bad_words_ids` contains tokens that are not allowed to be generated. In order to get the tokens of the words that should not appear in the generated text, use `tokenizer.encode(bad_word, add_prefix_space=True)`.

            num_return_sequences: (`optional`) int
                The number of independently computed returned sequences for each element in the batch. Default to 1.

            attention_mask (`optional`) obj: `torch.LongTensor` of same shape as `input_ids`
                Mask to avoid performing attention on padding token indices.
                Mask values selected in ``[0, 1]``:
                ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens.
                Defaults to `None`.

                `What are attention masks? <../glossary.html#attention-mask>`__

            decoder_start_token_id=None: (`optional`) int
                If an encoder-decoder model starts decoding with a different token than BOS.
                Defaults to `None` and is changed to `BOS` later.

            use_cache: (`optional`) bool
                If `use_cache` is True, past key values are used to speed up decoding if applicable to model. Defaults to `True`.

            model_specific_kwargs: (`optional`) dict
                Additional model specific kwargs will be forwarded to the `forward` function of the model.

        Return:

            output: `torch.LongTensor` of shape `(batch_size * num_return_sequences, sequence_length)`
                sequence_length is either equal to max_length or shorter if all batches finished early due to the `eos_token_id`

        Examples::

            tokenizer = AutoTokenizer.from_pretrained('distilgpt2')   # Initialize tokenizer
            model = AutoModelWithLMHead.from_pretrained('distilgpt2')    # Download model and configuration from S3 and cache.
            outputs = model.generate(max_length=40)  # do greedy decoding
            print('Generated: {}'.format(tokenizer.decode(outputs[0], skip_special_tokens=True)))

            tokenizer = AutoTokenizer.from_pretrained('openai-gpt')   # Initialize tokenizer
            model = AutoModelWithLMHead.from_pretrained('openai-gpt')    # Download model and configuration from S3 and cache.
            input_context = 'The dog'
            input_ids = tokenizer.encode(input_context, return_tensors='pt')  # encode input context
            outputs = model.generate(input_ids=input_ids, num_beams=5, num_return_sequences=3, temperature=1.5)  # generate 3 independent sequences using beam search decoding (5 beams) with sampling from initial context 'The dog'
            for i in range(3): #  3 output sequences were generated
                print('Generated {}: {}'.format(i, tokenizer.decode(outputs[i], skip_special_tokens=True)))

            tokenizer = AutoTokenizer.from_pretrained('distilgpt2')   # Initialize tokenizer
            model = AutoModelWithLMHead.from_pretrained('distilgpt2')    # Download model and configuration from S3 and cache.
            input_context = 'The dog'
            input_ids = tokenizer.encode(input_context, return_tensors='pt')  # encode input context
            outputs = model.generate(input_ids=input_ids, max_length=40, temperature=0.7, num_return_sequences=3)  # 3 generate sequences using by sampling
            for i in range(3): #  3 output sequences were generated
                print('Generated {}: {}'.format(i, tokenizer.decode(outputs[i], skip_special_tokens=True)))

            tokenizer = AutoTokenizer.from_pretrained('ctrl')   # Initialize tokenizer
            model = AutoModelWithLMHead.from_pretrained('ctrl')    # Download model and configuration from S3 and cache.
            input_context = 'Legal My neighbor is'  # "Legal" is one of the control codes for ctrl
            input_ids = tokenizer.encode(input_context, return_tensors='pt')  # encode input context
            outputs = model.generate(input_ids=input_ids, max_length=50, temperature=0.7, repetition_penalty=1.2)  # generate sequences
            print('Generated: {}'.format(tokenizer.decode(outputs[0], skip_special_tokens=True)))

            tokenizer = AutoTokenizer.from_pretrained('gpt2')   # Initialize tokenizer
            model = AutoModelWithLMHead.from_pretrained('gpt2')    # Download model and configuration from S3 and cache.
            input_context = 'My cute dog'  # "Legal" is one of the control codes for ctrl
            bad_words_ids = [tokenizer.encode(bad_word, add_prefix_space=True) for bad_word in ['idiot', 'stupid', 'shut up']]
            input_ids = tokenizer.encode(input_context, return_tensors='pt')  # encode input context
            outputs = model.generate(input_ids=input_ids, max_length=100, do_sample=True, bad_words_ids=bad_words_ids)  # generate sequences without allowing bad_words to be generated
        """

        # We cannot generate if the model does not have a LM head
        if self.get_output_embeddings() is None:
            raise AttributeError(
                "You tried to generate sequences with a model that does not have a LM Head."
                "Please use another model class (e.g. `OpenAIGPTLMHeadModel`, `XLNetLMHeadModel`, `GPT2LMHeadModel`, `CTRLLMHeadModel`, `T5WithLMHeadModel`, `TransfoXLLMHeadModel`, `XLMWithLMHeadModel`, `BartForConditionalGeneration` )"
            )

        max_length = max_length if max_length is not None else self.config.max_length
        min_length = min_length if min_length is not None else self.config.min_length
        do_sample = do_sample if do_sample is not None else self.config.do_sample
        early_stopping = early_stopping if early_stopping is not None else self.config.early_stopping
        use_cache = use_cache if use_cache is not None else self.config.use_cache
        num_beams = num_beams if num_beams is not None else self.config.num_beams
        temperature = temperature if temperature is not None else self.config.temperature
        top_k = top_k if top_k is not None else self.config.top_k
        top_p = top_p if top_p is not None else self.config.top_p
        repetition_penalty = repetition_penalty if repetition_penalty is not None else self.config.repetition_penalty
        bos_token_id = bos_token_id if bos_token_id is not None else self.config.bos_token_id
        pad_token_id = pad_token_id if pad_token_id is not None else self.config.pad_token_id
        eos_token_id = eos_token_id if eos_token_id is not None else self.config.eos_token_id
        length_penalty = length_penalty if length_penalty is not None else self.config.length_penalty
        no_repeat_ngram_size = (
            no_repeat_ngram_size if no_repeat_ngram_size is not None else self.config.no_repeat_ngram_size
        )
        bad_words_ids = bad_words_ids if bad_words_ids is not None else self.config.bad_words_ids
        num_return_sequences = (
            num_return_sequences if num_return_sequences is not None else self.config.num_return_sequences
        )
        decoder_start_token_id = (
            decoder_start_token_id if decoder_start_token_id is not None else self.config.decoder_start_token_id
        )

        if input_ids is not None:
            batch_size = input_ids.shape[0]  # overriden by the input batch_size
        else:
            batch_size = 1

        assert isinstance(max_length, int) and max_length > 0, "`max_length` should be a strictly positive integer."
        assert isinstance(min_length, int) and min_length >= 0, "`min_length` should be a positive integer."
        assert isinstance(do_sample, bool), "`do_sample` should be a boolean."
        assert isinstance(early_stopping, bool), "`early_stopping` should be a boolean."
        assert isinstance(use_cache, bool), "`use_cache` should be a boolean."
        assert isinstance(num_beams, int) and num_beams > 0, "`num_beams` should be a strictly positive integer."
        assert temperature > 0, "`temperature` should be strictly positive."
        assert isinstance(top_k, int) and top_k >= 0, "`top_k` should be a positive integer."
        assert 0 <= top_p <= 1, "`top_p` should be between 0 and 1."
        assert repetition_penalty >= 1.0, "`repetition_penalty` should be >= 1."
        assert input_ids is not None or (
                isinstance(bos_token_id, int) and bos_token_id >= 0
        ), "If input_ids is not defined, `bos_token_id` should be a positive integer."
        assert pad_token_id is None or (
                isinstance(pad_token_id, int) and (pad_token_id >= 0)
        ), "`pad_token_id` should be a positive integer."
        assert (eos_token_id is None) or (
                isinstance(eos_token_id, int) and (eos_token_id >= 0)
        ), "`eos_token_id` should be a positive integer."
        assert length_penalty > 0, "`length_penalty` should be strictly positive."
        assert (
                isinstance(no_repeat_ngram_size, int) and no_repeat_ngram_size >= 0
        ), "`no_repeat_ngram_size` should be a positive integer."
        assert (
                isinstance(num_return_sequences, int) and num_return_sequences > 0
        ), "`num_return_sequences` should be a strictly positive integer."
        assert (
                bad_words_ids is None or isinstance(bad_words_ids, list) and isinstance(bad_words_ids[0], list)
        ), "`bad_words_ids` is either `None` or a list of lists of tokens that should not be generated"

        if input_ids is None:
            assert isinstance(bos_token_id, int) and bos_token_id >= 0, (
                "you should either supply a context to complete as `input_ids` input "
                "or a `bos_token_id` (integer >= 0) as a first token to start the generation."
            )
            input_ids = torch.full(
                (batch_size, 1), bos_token_id, dtype=torch.long, device=next(self.parameters()).device,
            )
        else:
            assert input_ids.dim() == 2, "Input prompt should be of shape (batch_size, sequence length)."

        # not allow to duplicate outputs when greedy decoding
        if do_sample is False:
            if num_beams == 1:
                # no_beam_search greedy generation conditions
                assert (
                        num_return_sequences == 1
                ), "Greedy decoding will always produce the same output for num_beams == 1 and num_return_sequences > 1. Please set num_return_sequences = 1"

            else:
                # beam_search greedy generation conditions
                assert (
                        num_beams >= num_return_sequences
                ), "Greedy beam search decoding cannot return more sequences than it has beams. Please set num_beams >= num_return_sequences"

        # create attention mask if necessary
        # TODO (PVP): this should later be handled by the forward fn() in each model in the future see PR 3140
        if (attention_mask is None) and (pad_token_id is not None) and (pad_token_id in input_ids):
            attention_mask = input_ids.ne(pad_token_id).long()
        elif attention_mask is None:
            attention_mask = input_ids.new_ones(input_ids.shape)

        # set pad_token_id to eos_token_id if not set. Important that this is done after
        # attention_mask is created
        if pad_token_id is None and eos_token_id is not None:
            logger.warning(
                "Setting `pad_token_id` to {} (first `eos_token_id`) to generate sequence".format(eos_token_id)
            )
            pad_token_id = eos_token_id

        # current position and vocab size
        if hasattr(self.config, "vocab_size"):
            vocab_size = self.config.vocab_size
        elif (
                self.config.is_encoder_decoder
                and hasattr(self.config, "decoder")
                and hasattr(self.config.decoder, "vocab_size")
        ):
            vocab_size = self.config.decoder.vocab_size

        vocab_size += 1024

        # set effective batch size and effective batch multiplier according to do_sample
        if do_sample:
            effective_batch_size = batch_size * num_return_sequences
            effective_batch_mult = num_return_sequences
        else:
            effective_batch_size = batch_size
            effective_batch_mult = 1

        if self.config.is_encoder_decoder:
            if decoder_start_token_id is None:
                decoder_start_token_id = bos_token_id

            assert (
                    decoder_start_token_id is not None
            ), "decoder_start_token_id or bos_token_id has to be defined for encoder-decoder generation"
            assert hasattr(self, "get_encoder"), "{} should have a 'get_encoder' function defined".format(self)
            assert callable(self.get_encoder), "{} should be a method".format(self.get_encoder)

            # get encoder and store encoder outputs
            encoder = self.get_encoder()

            encoder_outputs: tuple = encoder(input_ids, attention_mask=attention_mask)

        # Expand input ids if num_beams > 1 or num_return_sequences > 1
        if num_return_sequences > 1 or num_beams > 1:
            input_ids_len = input_ids.shape[-1]
            input_ids = input_ids.unsqueeze(1).expand(batch_size, effective_batch_mult * num_beams, input_ids_len)
            attention_mask = attention_mask.unsqueeze(1).expand(
                batch_size, effective_batch_mult * num_beams, input_ids_len
            )

            input_ids = input_ids.contiguous().view(
                effective_batch_size * num_beams, input_ids_len
            )  # shape: (batch_size * num_return_sequences * num_beams, cur_len)
            attention_mask = attention_mask.contiguous().view(
                effective_batch_size * num_beams, input_ids_len
            )  # shape: (batch_size * num_return_sequences * num_beams, cur_len)

        if self.config.is_encoder_decoder:
            # create empty decoder_input_ids
            input_ids = torch.full(
                (effective_batch_size * num_beams, 1),
                decoder_start_token_id,
                dtype=torch.long,
                device=next(self.parameters()).device,
            )
            cur_len = 1

            assert (
                    batch_size == encoder_outputs[0].shape[0]
            ), f"expected encoder_outputs[0] to have 1st dimension bs={batch_size}, got {encoder_outputs[0].shape[0]} "

            # expand batch_idx to assign correct encoder output for expanded input_ids (due to num_beams > 1 and num_return_sequences > 1)
            expanded_batch_idxs = (
                torch.arange(batch_size)
                    .view(-1, 1)
                    .repeat(1, num_beams * effective_batch_mult)
                    .view(-1)
                    .to(input_ids.device)
            )
            # expand encoder_outputs
            encoder_outputs = (encoder_outputs[0].index_select(0, expanded_batch_idxs), *encoder_outputs[1:])

        else:
            encoder_outputs = None
            cur_len = input_ids.shape[-1]

        if num_beams > 1:
            output = self._generate_beam_search(
                input_ids,
                cur_len=cur_len,
                max_length=max_length,
                min_length=min_length,
                do_sample=do_sample,
                early_stopping=early_stopping,
                temperature=temperature,
                top_k=top_k,
                top_p=top_p,
                repetition_penalty=repetition_penalty,
                no_repeat_ngram_size=no_repeat_ngram_size,
                bad_words_ids=bad_words_ids,
                bos_token_id=bos_token_id,
                pad_token_id=pad_token_id,
                decoder_start_token_id=decoder_start_token_id,
                eos_token_id=eos_token_id,
                batch_size=effective_batch_size,
                num_return_sequences=num_return_sequences,
                length_penalty=length_penalty,
                num_beams=num_beams,
                vocab_size=vocab_size,
                encoder_outputs=encoder_outputs,
                attention_mask=attention_mask,
                use_cache=use_cache,
                model_specific_kwargs=model_specific_kwargs,
            )
        else:
            output = self._generate_no_beam_search(
                input_ids,
                cur_len=cur_len,
                max_length=max_length,
                min_length=min_length,
                do_sample=do_sample,
                temperature=temperature,
                top_k=top_k,
                top_p=top_p,
                repetition_penalty=repetition_penalty,
                no_repeat_ngram_size=no_repeat_ngram_size,
                bad_words_ids=bad_words_ids,
                bos_token_id=bos_token_id,
                pad_token_id=pad_token_id,
                decoder_start_token_id=decoder_start_token_id,
                eos_token_id=eos_token_id,
                batch_size=effective_batch_size,
                encoder_outputs=encoder_outputs,
                attention_mask=attention_mask,
                use_cache=use_cache,
                model_specific_kwargs=model_specific_kwargs,
            )

        return output

    def _generate_beam_search(
        self,
        input_ids,
        cur_len,
        max_length,
        min_length,
        do_sample,
        early_stopping,
        temperature,
        top_k,
        top_p,
        repetition_penalty,
        no_repeat_ngram_size,
        bad_words_ids,
        bos_token_id,
        pad_token_id,
        eos_token_id,
        decoder_start_token_id,
        batch_size,
        num_return_sequences,
        length_penalty,
        num_beams,
        vocab_size,
        encoder_outputs,
        attention_mask,
        use_cache,
        model_specific_kwargs,
    ):
        """ Generate sequences for each example with beam search.
        """

        # generated hypotheses
        generated_hyps = [
            BeamHypotheses(num_beams, max_length, length_penalty, early_stopping=early_stopping)
            for _ in range(batch_size)
        ]

        # scores for each sentence in the beam
        beam_scores = torch.zeros((batch_size, num_beams), dtype=torch.float, device=input_ids.device)

        # for greedy decoding it is made sure that only tokens of the first beam are considered to avoid sampling the exact same tokens three times
        if do_sample is False:
            beam_scores[:, 1:] = -1e9
        beam_scores = beam_scores.view(-1)  # shape (batch_size * num_beams,)

        # cache compute states
        past = encoder_outputs  # defined for encoder-decoder models, None for decoder-only models

        # done sentences
        done = [False for _ in range(batch_size)]

        while cur_len < max_length:
            model_inputs = self.prepare_inputs_for_generation(
                input_ids, past=past, attention_mask=attention_mask, use_cache=use_cache, **model_specific_kwargs
            )
            outputs = self(**model_inputs)  # (batch_size * num_beams, cur_len, vocab_size)
            next_token_logits = outputs[0][:, -1, :]  # (batch_size * num_beams, vocab_size)

            # if model has past, then set the past variable to speed up decoding
            if self._use_cache(outputs, use_cache):
                past = outputs[1]

            # repetition penalty (from CTRL paper https://arxiv.org/abs/1909.05858)
            if repetition_penalty != 1.0:
                self.enforce_repetition_penalty_(
                    next_token_logits, batch_size, num_beams, input_ids, repetition_penalty,
                )

            if temperature != 1.0:
                next_token_logits = next_token_logits / temperature

            if self.config.is_encoder_decoder and do_sample is False:
                # TODO (PVP) still a bit hacky here - there might be a better solution
                next_token_logits = self.prepare_logits_for_generation(
                    next_token_logits, cur_len=cur_len, max_length=max_length
                )

            scores = F.log_softmax(next_token_logits, dim=-1)  # (batch_size * num_beams, vocab_size)

            # set eos token prob to zero if min_length is not reached
            if eos_token_id is not None and cur_len < min_length:
                scores[:, eos_token_id] = -float("inf")

            if no_repeat_ngram_size > 0:
                # calculate a list of banned tokens to prevent repetitively generating the same ngrams
                num_batch_hypotheses = batch_size * num_beams
                # from fairseq: https://github.com/pytorch/fairseq/blob/a07cb6f40480928c9e0548b737aadd36ee66ac76/fairseq/sequence_generator.py#L345
                banned_batch_tokens = calc_banned_ngram_tokens(
                    input_ids, num_batch_hypotheses, no_repeat_ngram_size, cur_len
                )
                for i, banned_tokens in enumerate(banned_batch_tokens):
                    scores[i, banned_tokens] = -float("inf")

            if bad_words_ids is not None:
                # calculate a list of banned tokens according to bad words
                banned_tokens = calc_banned_bad_words_ids(input_ids, bad_words_ids)

                for i, banned_tokens in enumerate(banned_tokens):
                    scores[i, banned_tokens] = -float("inf")

            assert scores.shape == (batch_size * num_beams, vocab_size), "Shapes of scores: {} != {}".format(
                scores.shape, (batch_size * num_beams, vocab_size)
            )

            if do_sample:
                _scores = scores + beam_scores[:, None].expand_as(scores)  # (batch_size * num_beams, vocab_size)
                # Top-p/top-k filtering
                _scores = top_k_top_p_filtering(
                    _scores, top_k=top_k, top_p=top_p, min_tokens_to_keep=2
                )  # (batch_size * num_beams, vocab_size)
                # re-organize to group the beam together to sample from all beam_idxs
                _scores = _scores.contiguous().view(
                    batch_size, num_beams * vocab_size
                )  # (batch_size, num_beams * vocab_size)

                # Sample 2 next tokens for each beam (so we have some spare tokens and match output of greedy beam search)
                probs = F.softmax(_scores, dim=-1)
                next_tokens = torch.multinomial(probs, num_samples=2 * num_beams)  # (batch_size, num_beams * 2)
                # Compute next scores
                next_scores = torch.gather(_scores, -1, next_tokens)  # (batch_size, num_beams * 2)
                # sort the sampled vector to make sure that the first num_beams samples are the best
                next_scores, next_scores_indices = torch.sort(next_scores, descending=True, dim=1)
                next_tokens = torch.gather(next_tokens, -1, next_scores_indices)  # (batch_size, num_beams * 2)

            else:
                next_scores = scores + beam_scores[:, None].expand_as(scores)  # (batch_size * num_beams, vocab_size)

                # re-organize to group the beam together (we are keeping top hypothesis accross beams)
                next_scores = next_scores.view(
                    batch_size, num_beams * vocab_size
                )  # (batch_size, num_beams * vocab_size)

                next_scores, next_tokens = torch.topk(next_scores, 2 * num_beams, dim=1, largest=True, sorted=True)

            assert next_scores.size() == next_tokens.size() == (batch_size, 2 * num_beams)

            # next batch beam content
            next_batch_beam = []

            # for each sentence
            for batch_idx in range(batch_size):

                # if we are done with this sentence
                if done[batch_idx]:
                    assert (
                        len(generated_hyps[batch_idx]) >= num_beams
                    ), "Batch can only be done if at least {} beams have been generated".format(num_beams)
                    assert (
                        eos_token_id is not None and pad_token_id is not None
                    ), "generated beams >= num_beams -> eos_token_id and pad_token have to be defined"
                    next_batch_beam.extend([(0, pad_token_id, 0)] * num_beams)  # pad the batch
                    continue

                # next sentence beam content
                next_sent_beam = []

                # next tokens for this sentence
                for beam_token_rank, (beam_token_id, beam_token_score) in enumerate(
                    zip(next_tokens[batch_idx], next_scores[batch_idx])
                ):
                    # get beam and token IDs
                    beam_id = beam_token_id // vocab_size
                    token_id = beam_token_id % vocab_size

                    effective_beam_id = batch_idx * num_beams + beam_id
                    # add to generated hypotheses if end of sentence or last iteration
                    if (eos_token_id is not None) and (token_id.item() == eos_token_id):
                        # if beam_token does not belong to top num_beams tokens, it should not be added
                        is_beam_token_worse_than_top_num_beams = beam_token_rank >= num_beams
                        if is_beam_token_worse_than_top_num_beams:
                            continue
                        generated_hyps[batch_idx].add(
                            input_ids[effective_beam_id].clone(), beam_token_score.item(),
                        )
                    else:
                        # add next predicted token if it is not eos_token
                        next_sent_beam.append((beam_token_score, token_id, effective_beam_id))

                    # the beam for next step is full
                    if len(next_sent_beam) == num_beams:
                        break

                # Check if were done so that we can save a pad step if all(done)
                done[batch_idx] = done[batch_idx] or generated_hyps[batch_idx].is_done(
                    next_scores[batch_idx].max().item(), cur_len=cur_len
                )

                # update next beam content
                assert len(next_sent_beam) == num_beams, "Beam should always be full"
                next_batch_beam.extend(next_sent_beam)
                assert len(next_batch_beam) == num_beams * (batch_idx + 1)

            # stop when we are done with each sentence
            if all(done):
                break

            # sanity check / prepare next batch
            assert len(next_batch_beam) == batch_size * num_beams
            beam_scores = beam_scores.new([x[0] for x in next_batch_beam])
            beam_tokens = input_ids.new([x[1] for x in next_batch_beam])
            beam_idx = input_ids.new([x[2] for x in next_batch_beam])

            # re-order batch and update current length
            input_ids = input_ids[beam_idx, :]
            input_ids = torch.cat([input_ids, beam_tokens.unsqueeze(1)], dim=-1)
            cur_len = cur_len + 1

            # re-order internal states
            if past is not None:
                past = self._reorder_cache(past, beam_idx)

            # extend attention_mask for new generated input if only decoder
            if self.config.is_encoder_decoder is False:
                attention_mask = torch.cat(
                    [attention_mask, attention_mask.new_ones((attention_mask.shape[0], 1))], dim=-1
                )

        # finalize all open beam hypotheses and end to generated hypotheses
        for batch_idx in range(batch_size):
            if done[batch_idx]:
                continue

            # test that beam scores match previously calculated scores if not eos and batch_idx not done
            if eos_token_id is not None and all(
                (token_id % vocab_size).item() is not eos_token_id for token_id in next_tokens[batch_idx]
            ):
                assert torch.all(
                    next_scores[batch_idx, :num_beams] == beam_scores.view(batch_size, num_beams)[batch_idx]
                ), "If batch_idx is not done, final next scores: {} have to equal to accumulated beam_scores: {}".format(
                    next_scores[:, :num_beams][batch_idx], beam_scores.view(batch_size, num_beams)[batch_idx],
                )

            # need to add best num_beams hypotheses to generated hyps
            for beam_id in range(num_beams):
                effective_beam_id = batch_idx * num_beams + beam_id
                final_score = beam_scores[effective_beam_id].item()
                final_tokens = input_ids[effective_beam_id]
                generated_hyps[batch_idx].add(final_tokens, final_score)

        # depending on whether greedy generation is wanted or not define different output_batch_size and output_num_return_sequences_per_batch
        output_batch_size = batch_size if do_sample else batch_size * num_return_sequences
        output_num_return_sequences_per_batch = 1 if do_sample else num_return_sequences

        # select the best hypotheses
        sent_lengths = input_ids.new(output_batch_size)
        best = []

        # retrieve best hypotheses
        for i, hypotheses in enumerate(generated_hyps):
            sorted_hyps = sorted(hypotheses.beams, key=lambda x: x[0])
            for j in range(output_num_return_sequences_per_batch):
                effective_batch_idx = output_num_return_sequences_per_batch * i + j
                best_hyp = sorted_hyps.pop()[1]
                sent_lengths[effective_batch_idx] = len(best_hyp)
                best.append(best_hyp)

        # shorter batches are filled with pad_token
        if sent_lengths.min().item() != sent_lengths.max().item():
            assert pad_token_id is not None, "`Pad_token_id` has to be defined"
            sent_max_len = min(sent_lengths.max().item() + 1, max_length)
            decoded = input_ids.new(output_batch_size, sent_max_len).fill_(pad_token_id)

            # fill with hypothesis and eos_token_id if necessary
            for i, hypo in enumerate(best):
                decoded[i, : sent_lengths[i]] = hypo
                if sent_lengths[i] < max_length:
                    decoded[i, sent_lengths[i]] = eos_token_id
        else:
            # none of the hypotheses have an eos_token
            assert (len(hypo) == max_length for hypo in best)
            decoded = torch.stack(best).type(torch.long).to(next(self.parameters()).device)

        return decoded

    @staticmethod
    def _reorder_cache(past: Tuple, beam_idx: Tensor) -> Tuple[Tensor]:
        return tuple(layer_past.index_select(1, beam_idx) for layer_past in past)

    def resize_token_embeddings(self, new_num_tokens: int) -> nn.Embedding:
        old_num_tokens = self.model.shared.num_embeddings
        new_embeddings = super().resize_token_embeddings(new_num_tokens)
        self.model.shared = new_embeddings
        self._resize_final_logits_bias(new_num_tokens, old_num_tokens)
        return new_embeddings

    def _resize_final_logits_bias(self, new_num_tokens: int, old_num_tokens: int) -> None:
        if new_num_tokens <= old_num_tokens:
            new_bias = self.final_logits_bias[:, :new_num_tokens]
        else:
            extra_bias = torch.zeros((1, new_num_tokens - old_num_tokens), device=self.final_logits_bias.device)
            new_bias = torch.cat([self.final_logits_bias, extra_bias], dim=1)
        self.register_buffer("final_logits_bias", new_bias)

    def prepare_inputs_for_generation(self, decoder_input_ids, past, attention_mask, use_cache, **kwargs):
        assert past is not None, "past has to be defined for encoder_outputs"

        # first step, decoder_cached_states are empty
        if not past[1]:
            encoder_outputs, decoder_cached_states = past, None
        else:
            encoder_outputs, decoder_cached_states = past
        return {
            "input_ids": None,  # encoder_outputs is defined. input_ids not needed
            "encoder_outputs": encoder_outputs,
            "decoder_cached_states": decoder_cached_states,
            "decoder_input_ids": decoder_input_ids,
            "attention_mask": attention_mask,
            "use_cache": use_cache,  # change this to avoid caching (presumably for debugging)
        }

    def prepare_logits_for_generation(self, logits, cur_len, max_length):
        #if cur_len == 1:
        #    self._force_token_ids_generation(logits, self.config.bos_token_id)
        if cur_len == max_length - 1 and self.config.eos_token_id is not None:
            self._force_token_ids_generation(logits, self.config.eos_token_id)
        return logits

    def _force_token_ids_generation(self, scores, token_ids) -> None:
        """force one of token_ids to be generated by setting prob of all other tokens to 0"""
        if isinstance(token_ids, int):
            token_ids = [token_ids]
        all_but_token_ids_mask = torch.tensor(
            [x for x in range(self.config.vocab_size) if x not in token_ids],
            dtype=torch.long,
            device=next(self.parameters()).device,
        )
        assert len(scores.shape) == 2, "scores should be of rank 2 with shape: [batch_size, vocab_size]"
        scores[:, all_but_token_ids_mask] = -float("inf")

    @staticmethod
    def _reorder_cache(past, beam_idx):
        ((enc_out, enc_mask), decoder_cached_states) = past
        reordered_past = []
        for layer_past in decoder_cached_states:
            # get the correct batch idx from decoder layer's batch dim for cross and self-attn
            layer_past_new = {
                attn_key: bart._reorder_buffer(attn_cache, beam_idx) for attn_key, attn_cache in layer_past.items()
            }
            reordered_past.append(layer_past_new)

        new_enc_out = enc_out if enc_out is None else enc_out.index_select(0, beam_idx)
        new_enc_mask = enc_mask if enc_mask is None else enc_mask.index_select(0, beam_idx)

        past = ((new_enc_out, new_enc_mask), reordered_past)
        return past

    def get_encoder(self):
        return self.model.encoder

    def get_output_embeddings(self):
        return bart._make_linear_from_emb(self.model.shared)  # make it on the fly


import torch.nn as nn
import torch.nn.functional as F
from cnns.nnlib.pytorch_layers.conv_picker import Conv

def get_conv(args, in_channels, out_channels, kernel_size, stride=1,
             padding=0, bias=True):
    return Conv(kernel_sizes=[kernel_size], in_channels=in_channels,
                out_channels=[out_channels], strides=[stride],
                padding=[padding], args=args, is_bias=bias).get_conv()

class NetSynthetic(nn.Module):
    def __init__(self, args):
        super(NetSynthetic, self).__init__()
        self.args = args
        out_channels1 = 5
        self.conv1 = get_conv(args, in_channels=1, out_channels=out_channels1,
                              kernel_size=5, stride=1)
        self.out_channels2 = 10
        self.conv2 = get_conv(args, in_channels=out_channels1,
                              out_channels=self.out_channels2,
                              kernel_size=5, stride=1)

        self.fc1 = nn.Linear(4 * 4 * self.out_channels2, 500)
        self.fc2 = nn.Linear(500, 10)

    def forward(self, x):
        x = self.conv1(x)
        x = F.relu(x)
        x = F.max_pool2d(x, 2, 2)
        x = F.relu(self.conv2(x))
        x = F.max_pool2d(x, 2, 2)
        x = x.view(-1, 4 * 4 * self.out_channels2)
        x = F.relu(self.fc1(x))
        x = self.fc2(x)
        return F.log_softmax(x, dim=1)


"""Main script
"""

from application.process import Process
from application.app import App
from utils.data_reader import get_properties

properties = get_properties()

DEBUG = properties.get("DEBUG").data

def main():
    if DEBUG == "FALSE":
        proc = Process()
        proc.start()
    else:
        app = App()
        app.start()

if __name__ == "__main__":
    main()


import os
from io import open
from setuptools import setup

with open(os.path.join(os.path.dirname(__file__), 'README.rst'), encoding='utf-8') as readme:
    README = readme.read()

# allow setup.py to be run from any path
os.chdir(os.path.normpath(os.path.join(os.path.abspath(__file__), os.pardir)))

setup(
    name = 'django-wechat-pay',
    version = '0.0.6',
    packages = ['wechat_pay'],
    include_package_data = True,
    install_requires = ['django-wechat-base'],
    license = 'BSD License',
    description = 'django wechat pay, fixed view error',
    long_description = README,
    url = 'https://github.com/ChanMo/django-wechat-pay/',
    author = 'ChanMo',
    author_email = 'chen.orange@aliyun.com',
    classifiers = [
        'Environment :: Web Environment',
        'Framework :: Django',
        'Intended Audience :: Developers',
        'License :: OSI Approved :: BSD License', # example license
        'Operating System :: OS Independent',
        'Programming Language :: Python',
        # Replace these appropriately if you are stuck on Python 2.
        'Programming Language :: Python :: 3',
        'Programming Language :: Python :: 3.2',
        'Programming Language :: Python :: 3.3',
        'Topic :: Internet :: WWW/HTTP',
        'Topic :: Internet :: WWW/HTTP :: Dynamic Content',
    ],
)


"""
Authors: Asker Brejnrod & Arjun Sampath

This file contains useful methods for plotting binned ms2 spectra data

Notable libraries used are:
    - numpy: https://numpy.org/doc/stable/
    - pandas: https://pandas.pydata.org
    - seaborn: https://seaborn.pydata.org
    - matplotlib: https://matplotlib.org
    - nimfa: https://nimfa.biolab.si
"""
import numpy as np
import pandas as pd
import nimfa
import matplotlib
import matplotlib.pyplot as plt
import matplotlib.ticker as ticker
from matplotlib.backends.backend_pdf import PdfPages
import seaborn as sns

def softmax(x):
    """Compute softmax values for x

    Args:
        x: Array of values to find softmax() of

    Returns:
        Array of same dimensions as input, with the softmax of all values computed

    """
    return np.exp(x) / np.sum(np.exp(x), axis=0)

def close_windows(event):
    """Closes all Matplotlib windows

    Key listener function used to close all plt windows on escape

    Args:
        event: Keylistener event variable

    """
    if event.key == 'escape':
        plt.close('all')

def plot_ms2_components(binned_ms2data, num_components=10, output_file=None, headless=False):
    """ Plots binned ms2spectra data

    Takes binned ms2spectra and breaks it up into the specified number of components
    using a Non-Negative Matrix Factorization algorithm (NMF). It uses a softmax to
    normalize each component. It plots all the spectra intensities by component in
    a general graph that shows how each component relates to others by intensity

    Args:
        binned_ms2data: Binned matrix of ms2spectra
        num_components: Number of components to split the spectra into
        output_file: File to save the plot to
        headless: Whether to show the plot or not for cases of plotting on a server without a GUI

    Returns:
        Matplotlib Axes object that contains the plotted graph data

    """
    if headless:
        matplotlib.use('Agg') #for plotting w/out GUI on servers

    # Use NMF to split the data into components
    nmf_model = nimfa.Nmf(binned_ms2data, rank=num_components)
    model = nmf_model()

    # Get the H matrix which is components on the rows and spectra intensities on the columns
    H = model.fit.H
    H_norm = []
    for x in H:
            H_norm.append(softmax(x.toarray()[0])) # Get the softmax of each component

    H_norm = np.array(H_norm).T # Transpose for easier use with pandas DataFrame

    labels = []
    # Create a tuple of all the component labels
    for i in np.arange(1, num_components+1):
        labels.append('Component ' + str(i))

    # Create a new DF in the proper format with component labels
    df = pd.DataFrame(H_norm, columns=labels)

    # Create a strip plot of the spectra intensities grouped by the different components
    fig = plt.figure()
    ax = sns.stripplot(data=df, size=2.5, jitter=.05)

    # Make the x-tick labels look nice/readable
    ax.set_xticklabels(ax.get_xticklabels(), rotation=55, ha='right')

    # Set y-labels and window titles. X-label is included with the DF
    ax.set_ylabel(r"$Normalized\ m/z\ Intensity$")
    ax.set_title("Spectra Intensities by Component", fontweight="bold")
    fig.canvas.set_window_title("Spectra Intensities by Component")

    plt.tight_layout()

    # If expected, create the output file and save
    if output_file != None:
        pdf_file = PdfPages(output_file)
        pdf_file.savefig()
        print("Plot saved to " + output_file)
        pdf_file.close()

    #attaches keylistener to plt figure
    fig.canvas.mpl_connect('key_press_event', close_windows)

    plt.show()

    return ax

def plot_ms2_components_fine(binned_ms2data, bins, num_components=10, output_file=None, headless=False):
    """ Plots binned ms2spectra data

    Takes binned ms2spectra and breaks it up into the specified number of components
    using a Non-Negative Matrix Factorization algorithm (NMF). It uses a softmax to
    normalize each component. It plots each individual component on a separate graph
    with bins vs intensity shown as a bar graph

    Args:
        binned_ms2data: Binned matrix of ms2spectra
        bins: Bins that correlate to the binned input matrix
        num_components: Number of components to split the spectra into
        output_file: File to save the plot to
        headless: Whether to show the plot or not for cases of plotting on a server without a GUI

    Returns:
        Tuple of  multiple Matplotlib Axes objects that contains each component's plotted graph data

    """

    if headless:
        matplotlib.use('Agg') #for plotting w/out GUI on servers

    # Use NMF to split the data into components
    nmf_model = nimfa.Nmf(binned_ms2data, rank=num_components)
    model = nmf_model()

    # Get the matrix that splits bins into the different components
    # Transpose so components is the rows and bins are the columns
    W = model.fit.W.T
    W_norm = []
    for x in W:
            W_norm.append(softmax(x.toarray()[0])) # calculate the softmax of each component

    subplots = []

    # If an output file is specified then create a PdfPages object for it
    pdf_file = PdfPages("fine_" + output_file) if output_file != None else None

    index = 1 # index is for determining component number

    # loop through all components
    for comp in W_norm:

        # Create new figure and plot the data
        fig = plt.figure()
        ax = sns.barplot(x=bins, y=comp*100, color='steelblue') # *100 ensures that data is as a percentage

        # Sets the locations of the x tick marks to automatic
        ax.xaxis.set_major_locator(ticker.AutoLocator())
        ax.xaxis.set_minor_locator(ticker.AutoMinorLocator())

        # Labelling of axis, title, and window title
        ax.set_ylabel(r"$Normalized\ Intensity\,[\%]$")
        ax.set_xlabel(r"$Binned\ m/z$")
        ax.set_title("Component " + str(index), fontweight="bold")
        fig.canvas.set_window_title("Component " + str(index))

        # Y-axis limits, sets grid-lines parallel to x-axis and sets a tight layout
        ax.set_ylim(0,100)
        ax.grid(True, axis="y", color='black', linestyle=':', linewidth=0.1)
        plt.tight_layout()

        #attaches keylistener to plt figure
        fig.canvas.mpl_connect('key_press_event', close_windows)

        subplots.append(ax)

        # Saves the figure if expected
        if pdf_file != None and output_file != None:
            pdf_file.savefig()

        index += 1 # increment component number

    # Closes the PDF file
    if pdf_file != None and output_file != None:
        print("Plot saved to " + "file_" + output_file)
        pdf_file.close()

    plt.show()

    return subplots


from django.shortcuts import render, redirect
from django.http import HttpResponse
from django.forms import inlineformset_factory
from .models import *
from .form import OrderForm
from django.contrib.auth.forms import UserCreationForm
from django.contrib import messages

# Create your views here.

def home(request):
    orders = Order.objects.all()
    last_five_order = orders
    customers = Customer.objects.all()

    total_orders = orders.count()
    total_customers =customers.count()
    delivered = orders.filter(status = 'Delivered').count()
    pending = orders.filter(status='Pending').count()
    context ={
        'orders':orders,
        'customers':customers,
        'last_five_order':last_five_order,
        'total_orders':total_orders,
        'total_customers':total_customers,
        'delivered':delivered,
        'pending':pending
    }
    return render(request, 'crm_app/index.html', context)

def products(request):
    product = Product.objects.all()
    return render(request, 'crm_app/products.html', {'product':product})

def customer(request, pk):
    customer = Customer.objects.get(id=pk)
    orders = customer.order_set.all()
    total_order = orders.count()
    context = {
        'customer':customer,
        'orders':orders,
        'total_order':total_order,}
    return render(request, 'crm_app/customer.html', context)

def createOrder(request, pk):
    customer = Customer.objects.get(id=pk)
    form = OrderForm(initial={'customer':customer})

    if request.method == 'POST':
        form = OrderForm(request.POST)
        if form.is_valid():
            form.save()
            return redirect('/')

    context = {'form':form, 'customer': customer}
    return render(request, 'crm_app/create_order.html', context)

def updateOrder(request, pk):
    orders = Order.objects.get(id=pk)
    form = OrderForm(instance=orders)

    if request.method =='POST':
        form = OrderForm(request.POST, instance=orders)
        if form.is_valid():
            form.save()
            return redirect('/')

    context = {'orders':orders, 'form':form,}
    return render(request, 'crm_app/update_order.html', context)

def deleteOrder(request, pk):
    order = Order.objects.get(id=pk)
    if request.method == 'POST':
        order.delete()
        return redirect('/')

    context = {'order':order}
    return render(request, 'crm_app/delete_order.html', context)

def register(request):
    forms = UserCreationForm()
    if request.method == 'POST':
        f = UserCreationForm(request.POST)
        if f.is_valid():
            f.save()
            messages.success(request, 'Account created successfully')
            return redirect('home')

    context = {'forms': forms}
    return render(request, 'crm_app/register.html', context)

def login(request):

    context = {}
    return render(request, 'crm_app/login.html', context)


"""IngrafenApp URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/2.1/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path
from cotizacion import views
from . import settings
from django.contrib.staticfiles.urls import static
from django.contrib.staticfiles.urls import staticfiles_urlpatterns

urlpatterns = [
    path('admin/', admin.site.urls),
    path("",views.home,name="home"),
    path("solicitud/", views.solicitud_cot, name="solicitud"),
    path("registro/",views.creacion_usuario, name="creacion_usuario"),
    path("login/",views.user_login,name="login"),
    path("logout/", views.user_logout,name="logout"),
    path("solicitudes/",views.solicitudes_existentes, name="solicitudes_existentes"),
    path("cotizacion/",views.cotizaciones_completadas, name="cotizaciones_completadas"),
    path("crear_material/",views.creacion_material,name="crear_material"),
    path("crear_cliente/",views.creacion_cliente,name="crear_cliente"),
    path("crear_trabajo/",views.creacion_trabajo,name="crear_trabajo"),
    path("contactenos/",views.contactenos,name="contactenos"),
    path("compañia/", views.quienes_somos,name="quienes_somos"),
    path("servicios/",views.servicios,name="servicios"),
    path("reportes/",views.reportes,name="reportes"),
    path("solicitud_ot/",views.solicitud_ot,name="solicitud_ot"),
    path("ordenes_por_aperturar/",views.ordenes_por_aperturar,name="ordenes_por_aperturar"),
    path("ordenes_aperturadas/",views.ordenes_aperturadas,name="ordenes_aperturadas"),
    path("ordenes_en_proceso/",views.ordenes_en_proceso,name="ordenes_en_proceso"),
    path("ordenes_sin_fecha/",views.ordenes_sin_fecha,name="ordenes_sin_fecha"),
    path("error01/",views.error01,name="error01"),
    path("clientes_ot/",views.creacion_cliente_ot,name="creacion_cliente_ot"),
    path("bloqueo_clientes/",views.bloqueo_clientes,name="bloqueo_clientes"),
    path("confirmacion_material/",views.confirmacion_material,name="confirmacion_material"),
    path("cambio_contraseña/",views.change_password,name="cambio_contraseña"),
    path("eliminar_solicitud/",views.eliminar_solicitud,name="eliminar_solicitud"),
    path("solicitud_gigantografia",views.solicitud_gigantografia,name="solicitud_gigantografia"),
    path("crear_material_gig",views.creacion_material_gig,name="crear_material_gig"),
    path("crear_trabajo_gig",views.creacion_trabajo_gig,name="crear_trabajo_gig"),
    path("ordenes_por_aperturar_gig",views.ordenes_por_aperturar_gig,name="ordenes_por_aperturar_gig"),
    path("ordenes_en_proceso_gig/",views.ordenes_en_proceso_gig,name="ordenes_en_proceso_gig"),
    path("ordenes_sin_fecha_gig/",views.ordenes_sin_fecha_gig,name="ordenes_sin_fecha_gig"),
    path("ordenes_aperturadas_gig/",views.ordenes_aperturadas_gig,name="ordenes_aperturadas_gig"),
    path("bloqueo_usuario/",views.bloqueo_usuario,name="bloqueo_usuario"),
    path("cambio_vendedor/",views.cambio_vendedor,name="cambio_vendedor"),
    path("modificar/",views.modificar,name="modificar"),

]

urlpatterns += staticfiles_urlpatterns()
urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)


import sys
import pandas as pd
import numpy as np
from scipy.stats import chi2_contingency
from scipy.stats import mannwhitneyu

OUTPUT_TEMPLATE = (
    '"Did more/less users use the search feature?" p-value:  {more_users_p:.3g}\n'
    '"Did users search more/less?" p-value:  {more_searches_p:.3g} \n'
    '"Did more/less instructors use the search feature?" p-value:  {more_instr_p:.3g}\n'
    '"Did instructors search more/less?" p-value:  {more_instr_searches_p:.3g}'
)

def main():
    searchdata_file = sys.argv[1]
    data = pd.read_json(searchdata_file,orient='records',lines=True)
    even = data[data['uid']%2 == 0].reset_index().drop(['index'],axis = 1)
    odd = data[data['uid']%2 != 0].reset_index().drop(['index'], axis=1)

    even_not_search = even[even['search_count'] == 0]
    even_search = even[even['search_count'] !=0 ]
    odd_not_search = odd[odd['search_count'] == 0]
    odd_search = odd[odd['search_count'] != 0]

    even_not_search_count = len(even_not_search)
    even_search_count = len(even_search)
    odd_not_search_count = len(odd_not_search)
    odd_search_count = len(odd_search)

    obs =  np.array([[even_search_count,even_not_search_count],
            [odd_search_count,odd_not_search_count]])

    stat, more_users_p, dof, expected = chi2_contingency(obs)
    more_searches_p = mannwhitneyu(even['search_count'],odd['search_count']).pvalue

    instr_even = even[even['is_instructor'] == True].reset_index().drop(['index'],axis = 1)
    instr_odd = odd[odd['is_instructor'] == True].reset_index().drop(['index'],axis = 1)

    instr_even_not_search = instr_even[instr_even['search_count'] == 0]
    instr_even_search = instr_even[instr_even['search_count'] !=0 ]
    instr_odd_not_search = instr_odd[instr_odd['search_count'] == 0]
    instr_odd_search = instr_odd[instr_odd['search_count'] != 0]

    instr_even_not_search_count = len(instr_even_not_search)
    instr_even_search_count = len(instr_even_search)
    instr_odd_not_search_count = len(instr_odd_not_search)
    instr_odd_search_count = len(instr_odd_search)

    instr_obs =  np.array([[instr_even_search_count,instr_even_not_search_count],
            [instr_odd_search_count,instr_odd_not_search_count]])

    instr_stat, more_instr_p, instr_dof, instr_expected = chi2_contingency(instr_obs)
    more_instr_searches_p = mannwhitneyu(instr_even['search_count'],instr_odd['search_count']).pvalue

    print(OUTPUT_TEMPLATE.format(
        more_users_p=more_users_p,
        more_searches_p=more_searches_p,
        more_instr_p=more_instr_p,
        more_instr_searches_p=more_instr_searches_p,
    ))

if __name__ == '__main__':
    main()


import numpy as np
from numpy import linalg
from matplotlib import pyplot as plt
import Rosenbrock

def EnsembleSampler(X0, N, L, alpha):
    # Functions to evaluate the density, grad log density, and Hessian of the
    # minus log density.
    log_pi = Rosenbrock.log_pi

    MCMC_chain = np.zeros((N+1, L, len(X0[0,:])))
    MCMC_chain[0,:,:] = X0 # Initialize the L walkers.

    X = X0
    accept = 0
    for k in range(1, N+1):
        for i in range(L):
            # Update the l-th walker.

            # Randomly select an index j != i
            j =  np.mod(i + 1 + np.random.randint(L-1), L)

            # Inverse transform sampling for Z.
            sa = np.sqrt(alpha)
            Z = ( (sa - 1/sa) * np.random.rand() + 1/sa )**2

            # Get the proposal.
            y = X[j,:] + Z*(X[i,:] - X[j,:])

            d = len(X[0,:]) # Get the dimension.
            # Compute the log of the acceptance probability.
            log_pacc = (d-1)*np.log(Z) + log_pi(y) - log_pi(X[i,:])

            # Metropolize.
            if np.log(np.random.rand()) <= log_pacc:
                accept += 1
                MCMC_chain[k,i,:] = y
            else:
                MCMC_chain[k,i,:] = X[i,:]

        X = MCMC_chain[k,:,:]

    return [MCMC_chain, accept/(N*L)]

def plot_trajectory(X0, N, L, alpha):
    [chain, accept] = EnsembleSampler(X0, N, L, alpha)

    print("\nAcceptance Rate = {:0.1f}%.\n".format(100*accept))

    pi = Rosenbrock.pi
    x1 = np.linspace(-4, 4, 100)
    x2 = np.linspace(-2, 10, 100)

    X1, X2 = np.meshgrid(x1, x2, sparse=True)

    Y = pi([X1, X2])

    plt.figure(1)
    plt.rc('text', usetex=True)
    plt.rc('font', family='serif')
    plt.contourf(x1, x2, Y)
    for l in range(L):
        plt.plot(chain[N,l,0], chain[N,l,1], 'r x')
    plt.xlabel('$x_1$')
    plt.ylabel('$x_2$')
    plt.title('Affine Invariant Ensemble Trajectories')
    plt.show()

L = 50
X0 = 0.5*np.random.randn(2*L)
X0 = X0.reshape((L, 2))
N = 100
alpha = 1.2
plot_trajectory(X0, N, L, alpha)


import numpy as np
import matplotlib.pyplot as plt
from numpy.ma import log

a = .6
s = np.random.logseries(a, 10000)
count, bins, ignored = plt.hist(s)

def logseries(k, p):
    return -p ** k / (k * log(1 - p))

plt.plot(bins, logseries(bins, a) * count.max() /
         logseries(bins, a).max(), 'r')
plt.show()


import torch
import torch.nn as nn
import math

class LSTM(nn.Module):
    def __init__(
        self,
        input_size,
        hidden_size,
    ):
        super().__init__()
        self.input_size = input_size
        self.hidden_size = hidden_size

        # forget layer
        self.W_f = nn.Parameter(torch.Tensor(self.input_size + self.hidden_size, self.hidden_size))
        self.b_f = nn.Parameter(torch.Tensor(self.hidden_size))

        # input layer
        self.W_i = nn.Parameter(torch.Tensor(self.input_size + self.hidden_size, self.hidden_size))
        self.b_i = nn.Parameter(torch.Tensor(self.hidden_size))
        self.W_Ctilde = nn.Parameter(torch.Tensor(self.input_size + self.hidden_size, self.hidden_size))
        self.b_Ctilde = nn.Parameter(torch.Tensor(self.hidden_size))

        # output layer
        self.W_o = nn.Parameter(torch.Tensor(self.input_size + self.hidden_size, self.hidden_size))
        self.b_o = nn.Parameter(torch.Tensor(self.hidden_size))

        self.init_weights()

    def forward(self, x, hidden_states):
        # list with the outputs
        h = []

        # initialize the hidden_states if a empty array is received
        if hidden_states is None:
            h_t1 = torch.zeros(x.size()[1], self.hidden_size, device='cuda')
            C_t1 = torch.zeros(x.size()[1], self.hidden_size, device='cuda')
        else:
            h_t1 = hidden_states[0]
            C_t1 = hidden_states[1]

        for t in range(x.size()[0]):
            # the input
            x_t = x[t, :, :]

            # concatenate x_t with h_{t-1}
            x_t_h_t1 = torch.cat((h_t1, x_t), dim=1)

            # forget layer
            f_t = torch.sigmoid(torch.matmul(x_t_h_t1, self.W_f) + self.b_f)

            # input layer
            i_t = torch.sigmoid(torch.matmul(x_t_h_t1, self.W_i) + self.b_i)
            Ctilde_t = torch.tanh(torch.matmul(x_t_h_t1, self.W_Ctilde) + self.b_Ctilde)

            # update cell state
            C_t = f_t * C_t1 + i_t * Ctilde_t

            # output layer
            o_t = torch.sigmoid(torch.matmul(x_t_h_t1, self.W_o) + self.b_o)
            h_t = o_t * torch.tanh(C_t)

            # append the current output to the list of outputs
            h.append(h_t.unsqueeze(0))

            # the actual cell state should go in the previous cell state for the next iteration
            h_t1 = h_t
            C_t1 = C_t

        return torch.cat(h), [h_t, C_t]

    def init_weights(self):
        for parameter in self.parameters():
            nn.init.uniform_(parameter, -0.05, 0.05)


import itertools

n = int(input())

nails = []
for i in range(n):
    x, y = map(int, input().split())
    nails.append((x, y))

max_area = 0

for corners in itertools.combinations(nails, 4):
    x_values = [c[0] for c in corners]
    y_values = [c[1] for c in corners]
    if len(set(x_values)) == 2 and len(set(y_values)) == 2:
        x1 = min(x_values)
        y1 = min(y_values)
        x2 = max(x_values)
        y2 = max(y_values)
        area = (x2 - x1) * (y2 - y1)
        if area > max_area:
            max_area = area

print(max_area)


from numpy.core.numeric import ones
import pandas as pd
import sys
import numpy as np
import matplotlib.pyplot as plt
import math

filename = sys.argv[1]

sigma = float(sys.argv[2])

D = pd.read_csv(filename)
D.pop('date')
D.pop('rv2')
#print(D)

np.set_printoptions(precision=3, suppress=False, threshold=5)

# Convert to numpy datatype
D = D.to_numpy()
row = np.size(D, 0)
col = np.size(D, 1)

def KERNEL_DISCRIMINANT(rm_first, sigma):
    rm_row = np.size(rm_first, 0)
    rm_col = np.size(rm_first, 1)
    # kernel matrix
    K = []
    for i in range(rm_row):
        K_row = []
        for j in range(rm_row):
            diff = rm_first[i,:] - rm_first[j,:]
            kernel = math.e ** (-np.dot(diff, diff) / (2 * sigma))
            K_row.append(kernel)
        K.append(K_row)

    K = np.array(K)
    appliances_col = D[range(0,1000),:][:,0]
    #print(appliances_col[0])
    c1 = []
    c2 = []
    for i in range(np.size(appliances_col, 0)):
        if appliances_col[i] <= 50:
            c1.append(i)
        else:
            c2.append(i)

    # compute class kernel matrix
    K_1 = K[:,c1]
    K_1_row, K_1_col = np.size(K_1, 0), np.size(K_1, 1)
    K_2 = K[:,c2]
    K_2_row, K_2_col = np.size(K_2, 0), np.size(K_2, 1)

    # compute class means
    m_1 = np.matmul(K_1, np.ones((K_1_col, 1))) / K_1_col
    m_2 = np.matmul(K_2, np.ones((K_2_col, 1))) / K_2_col

    # between-class scatter matrix
    M = np.outer((m_1 - m_2), np.transpose(m_1 - m_2))

    #print(np.size(M,0), np.size(M,1))

    # class scatter matrices
    N_1 = np.matmul(np.matmul(K_1, np.identity(K_1_col)-np.ones((K_1_col, K_1_col))/K_1_col), np.transpose(K_1))
    N_2 = np.matmul(np.matmul(K_2, np.identity(K_2_col)-np.ones((K_2_col, K_2_col))/K_2_col), np.transpose(K_2))

    # within-class scatter matrix
    N = N_1 + N_2
    #print(np.size(N,0), np.size(N,1))

    term = np.matmul(np.linalg.pinv(N), M)
    e_values, e_vectors = np.linalg.eig(term)

    a = e_vectors[:, [0]]
    lamb = e_values[0]
    a = np.real(a)
    term = np.matmul(np.matmul(np.transpose(a), K), a)
    a = a / math.sqrt(term)

    return K, K_1, K_2, lamb, a, appliances_col

'''
he matrix used to compute kernel
remove the first column since it
will determine the class
'''
rm_first = D[:, range(1,27)]
rm_first = rm_first[range(0,1000),:]
#print(rm_first)

K, K_1, K_2, lamb, a, appliances_col = KERNEL_DISCRIMINANT(rm_first, sigma)
# print(K)
print("a:", a)
# print(np.size(a, 0))
proj = np.matmul(K, a)
tmp = np.reshape(proj, (1000))
c1 = []
c2 = []
for i in range(1000):
    if appliances_col[i] <= 50:
        c1.append(proj[i])
    else:
        c2.append(proj[i])

plt.plot(c1, np.array(len(c1)*[0]), 'bo')
plt.plot(c2, np.array(len(c2)*[0]), 'rx')
plt.show()
# sigma = 0.001
# while(sigma <= 100):

#     sigma *= 10


#!/usr/bin/python3
from bs4 import BeautifulSoup
import requests
from time import sleep
from datetime import date,datetime
import lxml
import re

def user_ans(in_name, default, example):
    print(f"\nE.g.: {example}")
    in_name = in_name.replace('_',' ').title()
    ans = False
    while ans is False:
        user_input = input( f"{in_name}".center(21,'░') + ': ' )
        if user_input == '':
            user_input = default
            ans = True
        else:
            try:
                user_input = type(default)(user_input)
                ans = True
            except:
                print(f'{in_name}={user_input} is not valid. Try again')
                ans = False
    return user_input

def user_ans_dict(defaults_dict):
    query_dict = {}
    for key, value in defaults_dict.items():
        query_dict[key] = user_ans(key, value[0], value[1])
    server_query_dict = {    #rules for all answers to fit the query syntax requirements (differs from site to site)
    'title':        re.sub(r'\s','+', query_dict['title']),  ## name must be 'the+godfather'  \s => +
    'title_type':   re.sub(r'\s','_', query_dict['title_type']).lower(),
    'release_date': re.sub(r'\s','', query_dict['release_date']).split(','),
    'user_rating':  re.sub(r'\s','', query_dict['user_rating']).split(','),
    'genres':       re.sub(r'\s','', query_dict['genres']).lower(),
    'certificates': re.sub(r'\s','', query_dict['certificates']).upper().split(','),
    'countries':    re.sub(r'\s','', query_dict['countries']).lower(),
    'languages':    re.sub(r'\s','', query_dict['languages']).lower(),
    'adult':        re.sub(r'\s','', query_dict['adult']).lower(),
    'sort':         re.sub(r'\s','', query_dict['sort']).lower()
    }
    try:
        for i in range(len(server_query_dict['certificates'])):
            server_query_dict['certificates'][i] = 'US%3A' + server_query_dict['certificates'][i]
        server_query_dict['certificates'] = ','.join(server_query_dict['certificates'])
        server_query_dict['release_date'] = f"{date(int(server_query_dict['release_date'][0]),1,1)},{date(int(server_query_dict['release_date'][1]),12,31)}"
        server_query_dict['user_rating'] = f"{round(float(server_query_dict['user_rating'][0]),1)},{round(float(server_query_dict['user_rating'][1]),1)}"
    except:
        for key, value in query_dict.items():
            if value == '':
                server_query_dict[key] = defaults_dict[key][0]
            else:
                pass
    print('11 - Corrected all answers...')
    return server_query_dict

def imdb_search(query_dic,items,per_page):
    "takes query as input - outputs list of URLs for multiple result pages for this query"
    url_list = []
    query_str = ''
    for key in query_dic:
        if query_dic[key] != '':
            query_str += key + '=' + str( query_dic[key] ) + '&'
        else:
            pass
    url = 'https://www.imdb.com/search/title/?' + query_str

    if items >= per_page > 0:
        start = [ i for i in range(1,items+1,per_page) ]
        count = [per_page for i in start]
        if items%per_page == 0:
            pass
        else:
            count[-1] = items%per_page
    elif 0 < items < per_page:
        start = [1]
        count = [items]
    else:
        print('00 - Error: Number of Items and PageCount must be positive integers.')
        raise ValueError
    for i, start in enumerate(start):
        url_list.append( url + f'count={count[i]}&start={start}&ref_=adv_nxt')
    #del query_dic, query_str, url, start, count
    return url_list

def page_title(soup):
    title = soup.find('title')
    return title.text

def download_html(url, timeout):
    """Takes a list of URLs, file name, and downloads html files of them.
    Custom headers are enabled by default"""
    payload = {
        'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.162 Safari/537.36',
        'Accept':'text/html',
        'Accept-Charset':'utf-8',
        'Accept-Language':'en-US'}
    try:
        print('11 - Connected: Grabbing data...')
        req = requests.get(url, timeout=timeout, headers=payload) # custom headers go here: requsts.get( headers=head )
        print(f'11 - Took {req.elapsed} to load')
        #req.encoding = 'ISO-8859-1' ## utf-8 is better, non-latin charachters mis-translated with  ISO-8859-1
        soup = BeautifulSoup(req.text, 'lxml')
        print('11 - Parsing page...')
    except (requests.exceptions.ConnectTimeout, requests.exceptions.ReadTimeout):
        soup = 0
        print('00 - Loading Timed out....')
    return soup

defaults_dict = {                         ## Every dict value contains: key : [default, example]
    'title':        ['','The Dark Knight'],  ## name must be 'the+godfather'  \s => +
    'title_type':   ['','feature, TV series, Documentary,..'],  ##tv_movie,tv_series,tv_episode,tv_special,tv_miniseries,documentary,video_game,short,video,tv_short
    'release_date': ['','(range) - 1995, 2020 or 2000 , 2000'],
    'user_rating':  ['','(range) - 6.5 , 10'],
    'genres':       ['','horror, crime, comedy,..'],
    'certificates': ['','G, PG, PG-13, R, TV-14, TV-MA'], ## must add US%3A before every rating -- HTML encoding: means US:
    'countries':    ['','us,uk,fr,..'],
    'languages':    ['','en,fr,es,de,..'],
    'adult':        ['','include - skip=exclude'],
    'sort':         ['','boxoffice_gross_us,desc - skip=popularity'] ## moviemeter,desc (least popular)
    }

main_params = {
    'items':100,
    'per_page':50,
    'file_name':'crime',
    'wait_time': 10,
    'wait_counter':1,
    'timeout':10.0
    }

query_dic = user_ans_dict(defaults_dict)

for key, value in main_params.items():
    main_params[key] = user_ans(key, value, value)

urls = imdb_search(query_dic,main_params['items'], main_params['per_page'])