# -*- coding: utf-8 -*-
"""
@author: bai
"""
from data_prepare import pad_sequences, next_batch
import tensorflow as tf
from tensorflow.contrib.rnn import LSTMCell
from tensorflow.contrib.crf import crf_log_likelihood
from tensorflow.contrib.crf import viterbi_decode
from evaluation import conlleval
from utils import get_logger
import os
import sys

class Encoder(object):
    def __init__(self, args, embeddings, config):
        self.hidden_dim = args.hidden_dim
        self.embeddings = embeddings
        self.trainable = args.trainable
        self.dropout = args.dropout
        self.batch_size = args.batch_size
        self.outputs = None
        self.state = None
        self.config = config
        self.encoder_fw_cell = None
        self.encoder_bw_cell = None
        self.word_ids = tf.placeholder(tf.int32, shape=[None, None], name="word_ids")
        self.sequence_lengths = tf.placeholder(tf.int32, shape=[None], name="sequence_lengths")
        self.dropout_pl = tf.placeholder(dtype=tf.float32, shape=[], name="dropout")
    def lookup_layer(self):
        with tf.variable_scope("words"):
            _word_embeddings = tf.Variable(self.embeddings,
                                           dtype=tf.float32,
                                           trainable=self.trainable,
                                           name="_word_embeddings")
            word_embeddings = tf.nn.embedding_lookup(params=_word_embeddings,
                                                     ids=self.word_ids,
                                                     name="word_embeddings")
        self.word_embeddings =  tf.nn.dropout(word_embeddings, self.dropout_pl)
    def _encoder(self,inputs):
        with tf.variable_scope("encoder"):
            self.encoder_fw_cell = LSTMCell(self.hidden_dim)
            self.encoder_bw_cell = LSTMCell(self.hidden_dim)
            outputs, state = tf.nn.bidirectional_dynamic_rnn(
                cell_fw=self.encoder_fw_cell,
                cell_bw=self.encoder_fw_cell,
                inputs=inputs,
                sequence_length=self.sequence_lengths,
                dtype=tf.float32)
            outputs = tf.concat(outputs, axis=-1)
            state = (tf.reduce_mean((state[0][0], state[1][0]), axis=0), tf.reduce_mean((state[0][1], state[1][1]), axis=0))
            outputs = tf.nn.dropout(outputs, self.dropout_pl)
        return outputs,state
    def build(self):
        self.lookup_layer()
        self.outputs,self.state = self._encoder(inputs=self.word_embeddings)

class Decoder(object):
    def __init__(self,encoder,args, tag2label, vocab, paths, config):
        self.batch_size = args.batch_size
        self.epoch_num = args.epoch
        self.hidden_dim = args.hidden_dim
        self.decode_method = args.decode_method
        self.trainable = args.trainable
        self.dropout = args.dropout
        self.embedding_dim=args.embedding_dim
        self.lr = args.lr
        self.clip_grad = args.clip
        self.tag2label = tag2label
        self.num_tags = len(tag2label)
        self.vocab = vocab
        self.shuffle = args.shuffle
        self.model_path = paths['save_path']
        self.result_path = paths['result_path']
        self.logger = get_logger(paths['log_path'])
        self.config = config
        self.encoder=encoder
        self.labels = tf.placeholder(tf.int32, shape=[None, None], name="labels")
        self.lr_pl = tf.placeholder(dtype=tf.float32, shape=[], name="lr")
    def get_prop(self,output):
        with tf.variable_scope("predict"):
            W = tf.get_variable(name="W",
                                shape=[2 * self.hidden_dim, self.num_tags],
                                initializer=tf.contrib.layers.xavier_initializer(),
                                dtype=tf.float32)

            b = tf.get_variable(name="b",
                                shape=[self.num_tags],
                                initializer=tf.zeros_initializer(),
                                dtype=tf.float32)
            s = tf.shape(output)
            output = tf.reshape(output, [-1, 2*self.hidden_dim])
            pred = tf.matmul(output, W) + b
            self.logits = tf.reshape(pred, [-1, s[1], self.num_tags])
    def _loss(self):
        if self.decode_method==1:
            losses = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.logits,
                                                                    labels=self.labels)
            mask = tf.sequence_mask(self.encoder.sequence_lengths)
            losses = tf.boolean_mask(losses, mask)
            self.loss = tf.reduce_mean(losses)
        if self.decode_method==0:
            log_likelihood, self.transition_params = crf_log_likelihood(inputs=self.logits,
                                                                   tag_indices=self.labels,
                                                                   sequence_lengths=self.encoder.sequence_lengths)
            self.loss = -tf.reduce_mean(log_likelihood)
    def _predict(self):
        if self.decode_method==1:
            self.labels_softmax_ = tf.argmax(self.logits, axis=-1)
            self.labels_softmax_ = tf.cast(self.labels_softmax_, tf.int32)

    def _optimize(self):
        with tf.variable_scope("Optimizing"):
            self.global_step = tf.Variable(0, name="global_step", trainable=False)
            optim = tf.train.AdamOptimizer(learning_rate=self.lr_pl)
            grads_and_vars = optim.compute_gradients(self.loss)
            grads_and_vars_clip = [[tf.clip_by_value(g, -self.clip_grad, self.clip_grad), v] for g, v in grads_and_vars]
            self.train_op = optim.apply_gradients(grads_and_vars_clip, global_step=self.global_step)
    def init(self):
        self.init_op = tf.global_variables_initializer()
    def train(self, train, dev):
        saver = tf.train.Saver(tf.global_variables())
        with tf.Session(config=self.config) as sess:
            sess.run(self.init_op)
            for epoch in range(self.epoch_num):
                self.train_one_epoch(sess, train, dev, self.tag2label, epoch, saver)

    def test(self, test):
        saver = tf.train.Saver()
        with tf.Session(config=self.config) as sess:
            print('---------- testing -------')
            saver.restore(sess, self.model_path)
            label_list, seq_len_list = self.dev_one_epoch(sess, test)
            self.evaluate(label_list, seq_len_list, test)

    def demo_one(self, sess, sent):
        label_list = []
        for seqs, labels in next_batch(sent, self.batch_size, self.vocab, self.tag2label, shuffle=False):
            label_list_, _ = self.predict_one_batch(sess, seqs)
            label_list.extend(label_list_)
        label2tag = {}
        for tag, label in self.tag2label.items():
            label2tag[label] = tag if label != 0 else label
        tag = [label2tag[label] for label in label_list[0]]
        return tag

    def train_one_epoch(self, sess, train, dev, tag2label, epoch, saver):
        num_batches = (int)(len(train)/self.batch_size)
        batches = next_batch(train, self.batch_size, self.vocab, self.tag2label, shuffle=self.shuffle)
        for step, (seqs, labels) in enumerate(batches):
            sys.stdout.write(' processing: {} batch / {} batches.'.format(step + 1, num_batches) + '\r')
            feed_dict, _ = self.update(seqs, labels, self.lr, self.dropout)
            _, loss_train = sess.run([self.train_op, self.loss],feed_dict=feed_dict)
            if step + 1 == num_batches:
                self.logger.info('+++++++++++++epoch {}, loss: {:.4}++++++++++'.format(epoch + 1,loss_train))
                saver.save(sess, self.model_path, global_step=epoch)
        self.logger.info('------------------validation----------------')
        label_list_dev, seq_len_list_dev = self.dev_one_epoch(sess, dev)
        self.evaluate(label_list_dev, seq_len_list_dev, dev, epoch)

    def update(self, seqs, labels=None, lr=None, dropout=None):
        word_ids, seq_len_list = pad_sequences(seqs,pad_mark=0)

        feed_dict = {self.encoder.word_ids: word_ids,
                     self.encoder.sequence_lengths: seq_len_list,
                     }
        if labels is not None:
            labels_, _ = pad_sequences(labels, pad_mark=0)
            feed_dict[self.labels] = labels_
        if lr is not None:
            feed_dict[self.lr_pl] = lr
        if dropout is not None:
            feed_dict[self.encoder.dropout_pl] = dropout

        return feed_dict, seq_len_list

    def dev_one_epoch(self, sess, dev):
        label_list, seq_len_list = [], []
        for seqs, labels in next_batch(dev, self.batch_size, self.vocab, self.tag2label, shuffle=False):
            label_list_, seq_len_list_ = self.predict_one_batch(sess, seqs)
            label_list.extend(label_list_)
            seq_len_list.extend(seq_len_list_)
        return label_list, seq_len_list

    def predict_one_batch(self, sess, seqs):
        feed_dict, seq_len_list = self.update(seqs, dropout=1.0)
        if self.decode_method==1:
            label_list = sess.run(self.labels_softmax_, feed_dict=feed_dict)
            return label_list, seq_len_list
        if self.decode_method==0:
            logits, transition_params = sess.run([self.logits, self.transition_params],
                                                 feed_dict=feed_dict)
            label_list = []
            for logit, seq_len in zip(logits, seq_len_list):
                viterbi_seq, _ = viterbi_decode(logit[:seq_len], transition_params)
                label_list.append(viterbi_seq)
            return label_list, seq_len_list

    def evaluate(self, label_list, seq_len_list, data, epoch=None):
        label2tag = {}
        for tag, label in self.tag2label.items():
            label2tag[label] = tag if label != 0 else label

        model_predict = []
        for label_, (sent, tag) in zip(label_list, data):
            tag_ = [label2tag[label__] for label__ in label_]
            sent_res = []
            for i in range(len(sent)):
                sent_res.append([sent[i], tag[i], tag_[i]])
            model_predict.append(sent_res)
        epoch_num = str(epoch+1) if epoch != None else 'test'
        label_path = os.path.join(self.result_path, 'label_' + epoch_num)
        metric_path = os.path.join(self.result_path, 'result_metric_' + epoch_num)
        for _ in conlleval(model_predict, label_path, metric_path):
            self.logger.info(_)

class softmaxOrCRFDecoder(Decoder):
    def build(self):
        self.get_prop(output=self.encoder.outputs)
        self._predict()
        self._loss()
        self._optimize()
        self.init()

class LSTMDecoder(Decoder):
    pass


#!/usr/bin/env python
import io
import os
import sys
import unittest

import glfw
import pyngl

def setUpModule():
    if not glfw.init():
        print("error with glfw init")
        sys.exit()
    glfw.window_hint(glfw.CONTEXT_VERSION_MAJOR, 4)
    glfw.window_hint(glfw.CONTEXT_VERSION_MINOR, 1)
    glfw.window_hint(glfw.OPENGL_FORWARD_COMPAT, True)
    glfw.window_hint(glfw.OPENGL_PROFILE, glfw.OPENGL_CORE_PROFILE)
    window = glfw.create_window(640, 480, "", None, None)
    if not window:
        glfw.terminate()

    # Make the window's context current
    glfw.make_context_current(window)
    pyngl.NGLInit.initialize()

def tearDownModule() -> None:
    glfw.terminate()

class TestVAOPrimitives(unittest.TestCase):
    @classmethod
    def setUpClass(self):
        self.meshes = [
            pyngl.teapot,
            pyngl.octahedron,
            pyngl.dodecahedron,
            pyngl.icosahedron,
            pyngl.tetrahedron,
            pyngl.football,
            pyngl.cube,
            pyngl.troll,
            pyngl.bunny,
            pyngl.dragon,
            pyngl.buddah,
        ]

    def testGetCreatedIDs(self):
        for m in self.meshes:
            self.assertTrue(pyngl.VAOPrimitives.getVAOFromName(m) != 0)

    def testCheckCreated(self):
        for m in self.meshes:
            vao = pyngl.VAOPrimitives.getVAOFromName(m)
            vao.bind()
            self.assertTrue(vao.getBufferID(0) > 0)
            vao.unbind()

    def testCheckCreateSphere(self):
        pyngl.VAOPrimitives.createSphere("sphere", 1.0, 200)
        self.assertTrue(pyngl.VAOPrimitives.getVAOFromName("sphere") != None)

    def testCheckCreateLineGrid(self):
        pyngl.VAOPrimitives.createLineGrid("grid", 100.0, 100.0, 1000)
        self.assertTrue(pyngl.VAOPrimitives.getVAOFromName("grid") != None)

    def testCreateTrianglePlane(self):
        pyngl.VAOPrimitives.createTrianglePlane(
            "plane", 100.0, 100.0, 1000, 1000, pyngl.Vec3.up()
        )
        self.assertTrue(pyngl.VAOPrimitives.getVAOFromName("plane") != None)

    def testCreateCylinder(self):
        pyngl.VAOPrimitives.createCylinder("cylinder", 1.0, 2.0, 100, 100)
        self.assertTrue(pyngl.VAOPrimitives.getVAOFromName("cylinder") != None)

    def testCreateTorus(self):
        pyngl.VAOPrimitives.createTorus("torus", 1.0, 2.0, 100, 100, True)
        self.assertTrue(pyngl.VAOPrimitives.getVAOFromName("torus") != None)

    def testCreateCone(self):
        pyngl.VAOPrimitives.createCone("cone", 1.0, 2.0, 100, 100)
        self.assertTrue(pyngl.VAOPrimitives.getVAOFromName("cone") != None)

    def testCreateCapsule(self):
        pyngl.VAOPrimitives.createCapsule("capsule", 1.0, 2.0, 100)
        self.assertTrue(pyngl.VAOPrimitives.getVAOFromName("capsule") != None)

    # def testClear(self) :
    #  prim = pyngl.VAOPrimitives.instance()
    #  prim.clear()
    #  for m in self.meshes :
    #    self.assertFalse(prim.getVAOFromName(m) != None)

if __name__ == "__main__":
    unittest.main()


#!/usr/bin/env python
"""
Module containing useful general functions that don't belong to another
module.
"""

import math
from typing import Any, Dict, List, Optional, Tuple, TypeVar, Union, cast

import numba
import numpy as np
from scipy.special import erfc

IntOrIntArray = TypeVar("IntOrIntArray", np.ndarray, int)
NumberOrArrayUnion = Union[np.ndarray, float]

def gmd(U: np.ndarray,
        S: np.ndarray,
        V_H: np.ndarray,
        tol: float = 0.0) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
    """
    Perform the Geometric Mean Decomposition of a matrix A, whose SVD is
    given by `[U, S, V_H] = np.linalg.svd(A)`.

    The Geometric Mean Decomposition (GMD) is described in paper "Joint
    Transceiver Design for MIMO Communications Using Geometric Mean
    Decomposition."

    Parameters
    ----------
    U : np.ndarray
        First matrix obtained from the SVD decomposition of the original
        matrix you want to decompose.
    S : np.ndarray
        Second matrix obtained from the SVD decomposition of the original
        matrix you want to decompose.
    V_H : np.ndarray
       Third matrix obtained from the SVD decomposition of the original
       matrix you want to decompose.
    tol : float
        The tolerance.

    Returns
    -------
    (np.ndarray,np.ndarray,np.ndarray)
        The three matrices `Q`, `R` and `P` such that `A = QRP^H`, `R` is
        an upper triangular matrix and `Q` and `P` are unitary matrices.
    """
    # Note: The code here was adapted from the MATLAB code provided by the
    # original GMD authors in
    # http://www.sal.ufl.edu/yjiang/papers/gmd.m

    # \(\mtA = \mtU \mtS \mtV^H\)
    # \(\mtR = \mtU_r \mtS \mtV_r^H\)
    # \(A = \mtU \mtU_r^H S \mtV_R \mtV\)
    m = U.shape[0]
    n = V_H.shape[0]

    # Initialize R, P and Q
    R = np.zeros([m, n])
    P = V_H.conj().T.copy()
    Q = U.copy()

    # 'd' is a vector with the singular values
    d = np.copy(S)  # We copy here to avoid changing 'S'

    # l = min(m, n)
    # noinspection PyTypeChecker
    p = np.sum(S >= tol).item()  # Number of singular values >= tol

    # If there is no singular value greater than the tolerance, then we
    # throw an exception
    if p < 1:  # pragma: no cover
        raise RuntimeError(
            "This is no singular value greater than the tolerance")

    # If we only have one singular value, that will be our diagonal
    # element
    if p < 2:
        R[0, 0] = d[0]

    z = np.zeros([p - 1])  # Vector
    large = 1  # index of the largest diagonal element
    small = p - 1  # index of the smallest diagonal element
    perm = np.r_[0:p]  # perm (i) = location in d of i-th largest entry
    invperm = np.r_[0:p]  # maps diagonal entries to perm

    # Geometric Mean of the 'p' largest singular values
    sigma_bar = np.prod(S[0:p])**(1. / p)

    for k in range(p - 1):
        flag = 0

        # xxxxx If flag is changed to 1 here we will not rotate xxxxxxx
        if d[k] >= sigma_bar:
            i = perm[small]
            small -= 1
            if d[i] >= sigma_bar:
                flag = 1
        else:
            i = perm[large]
            large += 1
            if d[i] <= sigma_bar:
                flag = 1
        # xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx

        k1 = k + 1
        if i != k1:  # Apply permutation Pi of paper
            t = d[k1]  # Interchange d[i] and d[k1]
            d[k1] = d[i]
            d[i] = t

            j = invperm[k1]  # Update perm arrays
            perm[j] = i
            invperm[i] = j

            # Interchange columns i and k+1 of the Q and P matrices
            I = np.array([k1, i])
            J = np.array([i, k1])
            Q[:, I] = Q[:, J]
            P[:, I] = P[:, J]

        # Deltas
        delta1 = d[k]
        delta2 = d[k1]
        sq_delta1 = delta1**2
        sq_delta2 = delta2**2
        if flag:
            c = 1.0
            s = 0.0
        else:
            c = math.sqrt((sigma_bar**2 - sq_delta2) / (sq_delta1 - sq_delta2))
            s = math.sqrt(1 - c**2)

        d[k1] = delta1 * delta2 / sigma_bar  # = y in paper
        z[k] = s * c * (sq_delta2 - sq_delta1) / sigma_bar  # = x in paper
        R[k, k] = sigma_bar

        if k > 0:
            R[0:k, k] = z[0:k] * c  # new column of R
            z[0:k] = -z[0:k] * s  # new column of Z

        # First Givens Rotation matrix
        G1 = np.array([[c, -s], [s, c]])

        J = np.array([k, k1])
        P[:, J] = P[:, J].dot(G1)  # apply G1 to P

        # Second Givens Rotation Matrix
        G2 = (1. / sigma_bar) * np.array([[c * delta1, -s * delta2],
                                          [s * delta2, c * delta1]])

        Q[:, J] = Q[:, J].dot(G2)  # apply G2 to Q

    R[p - 1, p - 1] = sigma_bar
    R[0:p - 1, p - 1] = z

    return Q, R, P

def peig(A: np.ndarray, n: int) -> Tuple[np.ndarray, np.ndarray]:
    """
    Returns a matrix whose columns are the `n` dominant eigenvectors of
    `A` (eigenvectors corresponding to the `n` dominant eigenvalues).

    Parameters
    ----------
    A : np.ndarray
        A symmetric matrix (bi-dimensional numpy array).
    n : int
        Number of desired dominant eigenvectors.

    Returns
    -------
    np.ndarray, np.ndarray
        A list with two elements where the first element is a 2D numpy
        array with the desired eigenvectors, while the second element is a
        1D numpy array with the corresponding eigenvalues.

    Notes
    -----
    `A` must be a symmetric matrix so that its eigenvalues are real and
    positive.

    Raises
    ------
    ValueError
        If `n` is greater than the number of columns of `A`.

    Examples
    --------
    >>> A = np.random.randn(3,3) + 1j*np.random.randn(3,3)
    >>> V, D = peig(A, 1)

    """
    (_, ncols) = A.shape
    if n > ncols:  # A is symmetric -> we could get either nrows or ncols
        raise ValueError("`n` must be lower then the number of columns "
                         "in `A`")

    [D, V] = np.linalg.eig(A)
    indexes = np.argsort(D.real)
    indexes = indexes[::-1]
    V = V[:, indexes[0:n]]
    D = D[indexes[0:n]]
    return V, D

def leig(A: np.ndarray, n: int) -> Tuple[np.ndarray, np.ndarray]:
    """
    Returns a matrix whose columns are the `n` least significant
    eigenvectors of `A` (eigenvectors corresponding to the `n` dominant
    eigenvalues).

    Parameters
    ----------
    A : np.ndarray
        A symmetric matrix (bi-dimensional numpy array)
    n : int
        Number of desired least significant eigenvectors.

    Returns
    -------
    np.ndarray,np.ndarray
        A list with two elements where the first element is a 2D numpy
        array with the desired eigenvectors, while the second element is a
        1D numpy array with the corresponding eigenvalues.

    Notes
    -----
    `A` must be a symmetric matrix so that its eigenvalues are real and
    positive.

    Raises
    ------
    ValueError
        If `n` is greater than the number of columns of `A`.

    Examples
    --------
    >>> A = np.random.randn(3,3) + 1j*np.random.randn(3,3)
    >>> V, D = peig(A, 1)

    """
    (_, ncols) = A.shape
    if n > ncols:  # A is symmetric -> we could get either nrows or ncols
        raise ValueError("`n` must be lower then the number of columns "
                         "in `A`")

    [D, V] = np.linalg.eig(A)
    indexes = np.argsort(D.real)
    V = V[:, indexes[0:n]]
    D = D[indexes[0:n]]
    return V, D

def pretty_time(time_in_seconds: float) -> str:
    """
    Return the time in a more friendly way.

    Parameters
    ----------
    time_in_seconds : float
        Time in seconds.

    Returns
    -------
    time_string : str
        Pretty time representation as a string.

    Examples
    --------
    >>> pretty_time(30)
    '30.00s'
    >>> pretty_time(76)
    '1m:16s'
    >>> pretty_time(4343)
    '1h:12m:23s'
    """
    seconds = time_in_seconds
    minutes = int(seconds) // 60
    seconds = int(round(seconds % 60))

    hours = minutes // 60
    minutes %= 60

    if hours > 0:
        return "%sh:%02dm:%02ds" % (hours, minutes, seconds)

    if minutes > 0:
        return "%sm:%02ds" % (minutes, seconds)

    return "%.2fs" % time_in_seconds

def xor(a: int, b: int) -> int:
    """
    Calculates the xor operation between a and b.

    In python this is performed with a^b. However, sage changed the "^"
    operator. This xor function was created so that it can be used in
    either sage or in regular python.

    Parameters
    ----------
    a : int
        First number.
    b : int
        Second number.

    Returns
    -------
    int
        The result of the `xor` operation between `a` and `b`.

    Examples
    --------
    >>> xor(3,7)
    4
    >>> xor(15,6)
    9
    """
    return a.__xor__(b)

def randn_c(*args: int) -> np.ndarray:
    """
    Generates a random circularly complex gaussian matrix.

    Parameters
    ----------
    *args : any
        Variable number of arguments (int values) specifying the
        dimensions of the returned array. This is directly passed to the
        numpy.random.randn function.

    Returns
    -------
    result : np.ndarray
        A random N-dimensional numpy array (complex dtype) where the `N` is
        equal to the number of parameters passed to `randn_c`.

    Examples
    --------
    >>> a = randn_c(4,3)
    >>> a.shape
    (4, 3)
    >>> a.dtype
    dtype('complex128')

    """
    # noinspection PyArgumentList
    return (1.0 / math.sqrt(2.0)) * (np.random.randn(*args) +
                                     (1j * np.random.randn(*args)))

def randn_c_RS(RS: np.random.RandomState,
               *args: int) -> np.ndarray:  # pragma: no cover
    """
    Generates a random circularly complex gaussian matrix.

    This is essentially the same as the the randn_c function. The only
    difference is that the randn_c function uses the global RandomState
    object in numpy, while randn_c_RS use the provided RandomState
    object. This allow us greater control.

    Parameters
    ----------
    RS : np.random.RandomState
        The RandomState object used to generate the random values.
    *args : any
        Variable number of arguments specifying the dimensions of the
        returned array. This is directly passed to the
        numpy.random.randn function.

    Returns
    -------
    result : np.ndarray
        A random N-dimensional numpy array (complex dtype) where the
        `N` is equal to the number of parameters passed to `randn_c`.

    """
    if RS is None:
        # noinspection PyArgumentList
        return randn_c(*args)

    # noinspection PyArgumentList
    return (1.0 / math.sqrt(2.0)) * (RS.randn(*args) + (1j * RS.randn(*args)))

def level2bits(n: int) -> int:
    """
    Calculates the number of bits needed to represent n different
    values.

    Parameters
    ----------
    n : int
        Number of different levels.

    Returns
    -------
    num_bits : int
        Number of bits required to represent `n` levels.

    Examples
    --------
    >>> list(map(level2bits,range(1,20)))
    [1, 1, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5]
    """
    if n < 1:
        raise ValueError("level2bits: n must be greater then one")
    return int2bits(n - 1)

def int2bits(n: int) -> int:
    """
    Calculates the number of bits needed to represent an integer n.

    Parameters
    ----------
    n : int
        The integer number.

    Returns
    -------
    num_bits : int
        The number of bits required to represent the number `n`.

    Examples
    --------
    >>> list(map(int2bits, range(0,19)))
    [1, 1, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5]
    """
    if n < 0:
        raise ValueError("int2bits: n must be greater then zero")

    if n == 0:
        return 1

    bits = 0
    while n:
        n >>= 1
        bits += 1
    return bits

@numba.vectorize
def count_bits(n: IntOrIntArray) -> IntOrIntArray:  # pragma: no cover
    """
    Count the number of bits that are set in `n`.

    Parameters
    ----------
    n : int | np.ndarray
        An integer number or a numpy array of integer numbers.

    Returns
    -------
    Number of bits that are equal to 1 in the bit representation of the
    number `n`.

    Examples
    --------
    >>> a = np.array([3, 0, 2])
    >>> print(count_bits(a))
    [2 0 1]

    """
    count = 0
    while n > 0:
        if n & 1 == 1:
            count += 1
        n >>= 1
    return count

# # Note: This method works only for an integer `n` and returns an
# # integer. However, we are writing the documentation as if it were a numpy
# # ufunc because we will create the count_bits ufunc with it using
# # numpy.vectorize and count_bits will inherit the documentation.
# def _count_bits_single_element(n: IntOrIntArray
#                                ) -> IntOrIntArray:  # pragma: no cover
#     """
#     Count the number of bits that are set in `n`.

#     Parameters
#     ----------
#     n : int | np.ndarray
#         An integer number or a numpy array of integer numbers.

#     Returns
#     -------
#     Number of bits that are equal to 1 in the bit representation of the
#     number `n`.

#     Examples
#     --------
#     >>> a = np.array([3, 0, 2])
#     >>> print(count_bits(a))
#     [2 0 1]

#     """
#     count = 0
#     while n > 0:
#         if n & 1 == 1:
#             count += 1
#         n >>= 1
#     return count

# # Make count_bits an ufunc
# count_bits = np.vectorize(_count_bits_single_element)

# # count_bits = np.frompyfunc(_count_bits_single_element, 1, 1,
# #                            doc=_count_bits_single_element.__doc__)

def count_bit_errors(first: IntOrIntArray,
                     second: IntOrIntArray,
                     axis: Optional[Any] = None) -> IntOrIntArray:
    """
    Compare `first` and `second` and count the number of equivalent bit
    errors.

    The two arguments are assumed to have the index of transmitted and
    decoded symbols. The count_bit_errors function will compare each
    element in `first` with the corresponding element in `second`,
    determine how many bits changed and then return the total number of
    changes bits. For instance, if we compare the numbers 3 and 0, we see
    that 2 bits have changed, since 3 corresponds to '11', while 0
    corresponds to '00'.

    Parameters
    ----------
    first : int | np.ndarray
        The decoded symbols.
    second : int | np.ndarray
        The transmitted symbols.
    axis : int, optional
        Since first and second can be numpy arrays, when axis is not
        provided (that is, it is None) then the total number of bit errors
        of all the elements of the 'difference array' is returned. If axis
        is provided, then an array of bit errors is returned where the
        number of bit errors summed along the provided axis is returned.

    Returns
    -------
    bit_errors : int | np.ndarray
        The total number of bit errors.

    Examples
    --------
    >>> first = np.array([[2, 3, 3, 0], [1, 3, 1, 2]])
    >>> second = np.array([[0, 3, 2, 0], [2, 0, 1, 2]])
    >>> # The number of changed bits in each element is equal to
    >>> # array([1, 0, 1, 0, 2, 2, 0])
    >>> count_bit_errors(first, second)
    6
    >>> count_bit_errors(first, second, 0)
    array([3, 2, 1, 0])
    >>> count_bit_errors(first, second, 1)
    array([2, 4])
    """
    different_bits = xor(first, second)
    return np.sum(count_bits(different_bits), axis)  # type: ignore

def qfunc(x: float) -> float:
    """
    Calculates the 'q' function of x.

    Parameters
    ----------
    x : float
        The value to apply the Q function.

    Returns
    -------
    result : float
        Qfunc(x)

    Examples
    --------
    >>> qfunc(0.0)
    0.5
    >>> round(qfunc(1.0), 9)
    0.158655254
    >>> round(qfunc(3.0), 9)
    0.001349898
    """
    return cast(float, 0.5 * erfc(x / math.sqrt(2)))

def least_right_singular_vectors(
        A: np.ndarray, n: int) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
    """
    Return the three matrices. The first one is formed by the `n` least
    significative right singular vectors of `A`, the second one is
    formed by the remaining right singular vectors of `A` and the third
    one has the singular values of the singular vectors of the second
    matrix (the most significative ones).

    Parameters
    ----------
    A : np.ndarray
        A 2D numpy array.
    n : int
        An integer between 0 and the number of columns of `A`.

    Returns
    -------
    np.ndarray, np.ndarray, np.ndarray
        The tree matrices V0, V1 and S.

        The matrix V0 has the right singular vectors corresponding to
        the `n` least significant singular values.

        The matrix V1 has the remaining right singular vectors.

        The matrix S has the singular values corresponding to the
        remaining singular vectors `V1`.

    Notes
    -----
    Because of the sort operation, if you call
    least_right_singular_vectors(A, ncols_of_A) you will get all the right
    singular vectors of A with the column order reversed.

    Examples
    --------
    >>> A = np.array([1,2,3,6,5,4,2,2,1])
    >>> A.shape = (3,3)
    >>> (min_Vs, remaining_Vs, S) = least_right_singular_vectors(A,1)
    >>> min_Vs
    array([[-0.4474985 ],
           [ 0.81116484],
           [-0.3765059 ]])
    >>> remaining_Vs
    array([[-0.62341491, -0.64116998],
           [ 0.01889071, -0.5845124 ],
           [ 0.78166296, -0.49723869]])
    >>> S
    array([1.88354706, 9.81370681])
    """
    # Note that numpy.linalg.svd returns the hermitian of V
    [_, S, V_H] = np.linalg.svd(A, full_matrices=True)

    V = V_H.conjugate().transpose()

    # Index in crescent order of the singular values

    # Since the SVD gives the values in descending order, we just need to
    # reverse the order instead of performing a full sort
    sort_indexes = list(reversed(range(0, V.shape[0])))
    # sort_indexes = S.argsort()

    # The `n` columns corresponding to the least significative singular
    # values
    V0 = V[:, sort_indexes[0:n]]
    V1 = V[:, sort_indexes[n:]]

    return V0, V1, S[sort_indexes[n:]]

# New versions of numpy already have this method
# https://gist.github.com/1511969/222e3316048bce5763b1004331af898088ffcd9e
# def ravel_multi_index(indexes, shape):
#     """
#     Get the linear index corresponding to `indexes`.

#     The linear index is calculated in 'C' order. That is, it "travels"
#     the array faster in the fist dimension than in the last (row order
#     in bi-dimensional arrays).

#     Arguments
#     - `indexes`: A list with the indexes of each dimension in the array.
#     - `shape`: Shape of the array

#     Ex:
#     For shape=[3,3] we get the matrix
#     array([[0, 1, 2],
#            [3, 4, 5],
#            [6, 7, 8]])
#     Therefore (the indexes start at zero),
#     >>> ravel_multi_index([0,2],[3,3])
#     2

#     Similarly
#     >>> ravel_multi_index([3,1],[4,3])
#     10
#     """
#     #c order only
#     base_c = np.arange(np.prod(shape)).reshape(*shape)
#     return base_c[tuple(indexes)]

def calc_unorm_autocorr(x: np.ndarray) -> np.ndarray:
    """
    Calculates the unormalized auto-correlation of an array x starting
    from lag 0.

    Parameters
    ----------
    x : np.ndarray
        A 1D numpy array.

    Returns
    -------
    result : np.ndarray
        The unormalized auto-correlation of `x`.

    Examples
    --------
    >>> x = np.array([4, 2, 1, 3, 7, 3, 8])
    >>> calc_unorm_autocorr(x)
    array([152,  79,  82,  53,  42,  28,  32])

    """
    # R = np.convolve(x, x[::-1], 'full')
    R = np.correlate(x, x, 'full')

    # Return the auto-correlation for indexes greater then or equal to 0
    return R[R.size // 2:]

def calc_autocorr(x: np.ndarray) -> np.ndarray:
    """
    Calculates the (normalized) auto-correlation of an array x starting
    from lag 0.

    Parameters
    ----------
    x : np.ndarray
        A 1D numpy array.

    Returns
    -------
    result : np.ndarray
        The normalized auto-correlation of `x`.

    Examples
    --------
    >>> x = np.array([4, 2, 1, 3, 7, 3, 8])
    >>> calc_autocorr(x)
    array([ 1.   , -0.025,  0.15 , -0.175, -0.25 , -0.2  ,  0.   ])
    """
    x2 = x - np.mean(x)
    variance = float(np.var(x2))  # Biased variance of x2

    # We divide by x2.size because the calculated variance is the biased
    # version. If it was the unbiased we would have to divide by x2.size-1
    # instead.
    return calc_unorm_autocorr(x2) / (x2.size * variance)

# noinspection PyPep8
def update_inv_sum_diag(invA: np.ndarray, diagonal: np.ndarray) -> np.ndarray:
    """
    Calculates the inverse of a matrix :math:`(A + D)`, where :math:`D` is a diagonal
    matrix, given the inverse of :math`A` and the diagonal of :math:`D`.

    This calculation is performed using the Sherman-Morrison formula, given
    my

    .. math::
       (A+uv^T)^{-1} = A^{-1} - {A^{-1}uv^T A^{-1} \\over 1 + v^T A^{-1}u},

    where :math:`u` and :math:`v` are vectors.

    Parameters
    ----------
    invA : np.ndarray
        A 2D numpy array.
    diagonal : np.ndarray
        A 1D numpy array with the elements in the diagonal of :math:`D`.

    Returns
    -------
    new_inv : np.ndarray
        The inverse of :math:`A+D`.
    """

    #$$(A+uv^T)^{-1} = A^{-1} - {A^{-1}uv^T A^{-1} \over 1 + v^T A^{-1}u}$$

    # This function updates the inverse as the equation above when the
    # vectors "u" and "v" are equal and correspond to a column of the
    # identity matrix multiplied by a constant (only one element is
    # different of zero).
    # pylint: disable=C0111
    def calc_update_term(inv_matrix: np.ndarray, p_index: int,
                         p_indexed_element: float,
                         p_diagonal_element: float) -> np.ndarray:
        term1 = (p_diagonal_element *
                 np.outer(inv_matrix[:, p_index], inv_matrix[p_index, :]))
        return term1 / (1 + p_diagonal_element * p_indexed_element)

    new_inv = invA.copy()
    for index, diagonal_element in zip(range(diagonal.size), diagonal):
        indexed_element = new_inv[index, index]
        new_inv -= calc_update_term(new_inv, index, indexed_element,
                                    diagonal_element)

    return new_inv

def calc_confidence_interval(mean: float,
                             std: float,
                             n: int,
                             P: float = 95.0) -> Tuple[float, float]:
    """
    Calculate the confidence interval that contains the true mean (of a
    normal random variable) with a certain probability `P`, given the
    measured `mean`, standard deviation `std` for number of samples `n`.

    Only a few values are allowed for the probability `P`, which are: 50%,
    60%, 70%, 80%, 90%, 95%, 98%, 99%, 99.5%, 99.8% and 99.9%.

    Parameters
    ----------
    mean : float
        The measured mean value.
    std : float
        The measured standard deviation.
    n : int
        The number of samples used to measure the mean and standard
        deviation.
    P : float
        The desired confidence (probability in %) that true value is inside
        the calculated interval.

    Returns
    -------
    float, float
        A list with two float elements, the interval minimum and maximum
        values.

    Notes
    -----
    This function assumes that the estimated random variable is a normal
    variable.
    """
    # Dictionary that maps a desired "confidence" to the corresponding
    # critical value. See https://en.wikipedia.org/wiki/Student%27s_
    # t-distribution
    table_of_values = {
        50: 0.674,
        60: 0.842,
        70: 1.036,
        80: 1.282,
        90: 1.645,
        95: 1.960,
        98: 2.326,
        99: 2.576,
        99.5: 2.807,
        99.8: 3.090,
        99.9: 3.291
    }

    # Critical value used in the calculation of the confidence interval
    C = table_of_values[P]

    norm_std = std / np.sqrt(n)
    min_value = mean - (C * norm_std)
    max_value = mean + (C * norm_std)

    return min_value, max_value

def get_principal_component_matrix(A: np.ndarray,
                                   num_components: int) -> np.ndarray:
    """
    Returns a matrix without the "principal components" of `A`.

    This function returns a new matrix formed by the most significative
    components of `A`.

    Parameters
    ----------
    A : np.ndarray
        The original matrix, which is a 2D numpy matrix.
    num_components : int
        Number of components to be kept.

    Returns
    -------
    out : np.ndarray
        The new matrix with the dead dimensions removed.

    Notes
    -----
    You might want to normalize the returned matrix `out` after calling
    get_principal_component_matrix to have the same norm as the original
    matrix `A`.
    """
    # Note 'S' as returned by np.linalg.svd contains the singular values in
    # descending order. Therefore, we only need to keep the first
    # 'num_components' singular values (and vectors).
    [U, S, V_H] = np.linalg.svd(A)
    num_rows = U.shape[0]
    num_cols = V_H.shape[1]
    newS = np.zeros(num_rows, dtype=A.dtype)
    newS[:num_components] = S[:num_components]
    newS = np.diag(newS)[:, :num_cols]

    out = np.dot(U, np.dot(newS, V_H[:, :num_components]))

    return out

def get_range_representation(array: np.ndarray,
                             filename_mode: bool = False) -> Optional[str]:
    """
    Get the "range representation" of a numpy array consisting of a
    arithmetic progression. If no valid range representation exists,
    return None.

    Suppose you have the array
    n = [5, 10, 15, 20, 25, 30, 35, 40]
    This array is an arithmetic progression with step equal to 5 and can be
    represented as "5:5:40", which is exactly what get_range_representation
    will return for such array.

    Parameters
    ----------
    array : np.ndarray
        The array to be represented as a range expression.
    filename_mode : bool, optional
        If True, the returned representation will be more suitable to be
        used as part of a file-name. That is instead of "5:5:40" the string
        "5_(5)_40" would be returned.

    Returns
    -------
    expr : str
        A string expression representing `array`.
    """
    # Special case-> If len(array) < 4 we simply return the array
    if array.size < 4:
        return ','.join(array.astype(str))

    step = array[1] - array[0]

    # Change step from numpy.int64 or numpy.float64 to a regular int or
    # float. In the case of float, we round to 12 decimal digits. All of
    # this is only important in Python3.
    if step.dtype == int:
        step = int(step)
    elif step.dtype == float:
        step = round(float(step), 12)

    # noinspection PyTypeChecker
    if np.allclose(array[1:] - step, array[0:-1]):
        # array is an arithmetic progression
        if filename_mode is True:
            return "{0}_({1})_{2}".format(array[0], step, array[-1])
        return "{0}:{1}:{2}".format(array[0], step, array[-1])

    # array is not an arithmetic progression
    return None

def get_mixed_range_representation(array: np.ndarray,
                                   filename_mode: bool = False) -> str:
    """
    Get the "range representation" of a numpy array. This is similar to
    get_range_representation, but it no pure range representation is
    possible it will try to represent as least part of the array as range
    representations.

    Suppose you have the array
    n = [1, 2, 3, 5, 10, 15, 20, 25, 30, 35, 40, 100]

    Except for the 3 initial and the final elements, this array is an
    arithmetic progression with step equal to 5. Lets keep the 3 initial
    and the final values and represent the other values as a range
    representation.

    Parameters
    ----------
    array : np.ndarray
        The array to be represented as a range expression.
    filename_mode : bool, optional
        If True, the returned representation will be more suitable to be
        used as part of a file-name. That is instead of "5:5:40" the string
        "5_(5)_40" would be returned.

    Returns
    -------
    expr : str
        A string expression representing `array`.
    """
    if len(array) < 2:
        return '{0}'.format(array[0])

    diff = array[1:] - array[0:-1]
    diff = np.hstack([diff[0], diff])
    start = 0

    current_value = diff[0]

    output_expressions = []

    while start < len(diff):
        i = -1  # Just a start value in case the range below is empty
        for i in range(start, len(diff)):
            # if diff[i] != current_value:
            if not np.allclose(diff[i], current_value):
                end = i
                # Interval including the start, but not including the end
                output_expressions.append([start, end])
                start = end
                current_value = diff[end]
                # Break the for loop. The else statements will not run.
                break
        else:
            # If the for loop terminated normally, this code will run
            end = i
            output_expressions.append([start, end + 1])
            start = end + 1

    # xxxxx Process the results from the previous while loop. xxxxxxxxxxxxx
    # The first pair will never be changed
    for i, pair in enumerate(output_expressions[1:]):
        if pair[1] - pair[0] > 3:
            # This is a range expression

            # Get the step of this range expression
            step = array[pair[0] + 1] - array[pair[0]]

            # Get the first element in the range
            first_element = array[pair[0]]

            # Get the previous element (the element in array before this
            # range)
            previous_element = array[output_expressions[i][1] - 1]

            # If the difference of the first element in the range to the
            # previous element in the range is equal to the step of the
            # range, that means that this previous element should be in
            # this pair, and not in the previous one.
            if np.allclose(first_element - previous_element, step):
                output_expressions[i][1] -= 1
                output_expressions[i + 1][0] -= 1

    out = []
    for pair in output_expressions:
        value = get_range_representation(array[pair[0]:pair[1]], filename_mode)
        assert (value is not None)
        if value != '':
            out.append(value)

    return ','.join(out)

# noinspection PyPep8
def replace_dict_values(name: str,
                        dictionary: Dict[str, str],
                        filename_mode: bool = False) -> str:
    """
    Perform the replacements in `name` with the value of dictionary[name].

    See the usage example below:

    >>> name = "results_snr_{snr}_param_a_{param_a}"
    >>> replacements = {'snr': np.array([0,5,10,15,20]), 'param_a': 'something'}
    >>> replace_dict_values(name, replacements)
    'results_snr_[0:5:20]_param_a_something'

    Note that some small changes are performed in the dictionary prior to the
    replacement. More specifically, modifications such as changing a numpy
    array to a more compact representation (when possible). This is done by
    converting the numpy arrays with the get_mixed_range_representation
    function.

    If the string is going to be used as a filename, pass True to
    `filename_mode` as in the example below

    >>> replace_dict_values(name, replacements, True)
    'results_snr_[0_(5)_20]_param_a_something'

    Parameters
    ----------
    name : str
        The name fo be formatted.
    dictionary : dict
        The dictionary with the values to be replaced in `name`.
    filename_mode : bool, optional
        Extra parameter passed to the get_mixed_range_representation
        function. If True, the returned representation will be more
        suitable to be used as part of a file-name. That is instead of
        "5:5:40" the string "5_(5)_40" would be used.

    Returns
    -------
    new_name : str
        The value of `name` after the replacements in `dictionary`.

    Examples
    --------
    >>> name = "something {value1} - {value2} something else {value3}"
    >>> dictionary = {'value1':'bla bla', \
                      'value2':np.array([5, 10, 15, 20, 25, 30]), \
                      'value3': 76}
    >>> replace_dict_values(name, dictionary, True)
    'something bla bla - [5_(5)_30] something else 76'
    """
    new_dict = {}
    for n, v in dictionary.items():
        if isinstance(v, np.ndarray):
            v = "[{0}]".format(get_mixed_range_representation(
                v, filename_mode))
        new_dict[n] = v

    return name.format(**new_dict)

# Function taken from
# http://stackoverflow.com/questions/10480806/compare-dictionaries-ignoring-specific-keys
def equal_dicts(a: Dict[Any, Any], b: Dict[Any, Any],
                ignore_keys: List[Any]) -> bool:
    """
    Test if two dictionaries are equal ignoring certain keys.

    Parameters
    ----------
    a : dict
        The first dictionary
    b : dict
        The second dictionary
    ignore_keys : list
        A list or tuple with the keys to be ignored.
    """
    ka = set(a).difference(ignore_keys)
    kb = set(b).difference(ignore_keys)
    return ka == kb and all(a[k] == b[k] for k in ka)

def calc_decorrelation_matrix(cov_matrix: np.ndarray) -> np.ndarray:
    """
    Calculates the decorrelation matrix that can be applied to a data vector
    whose covariance matrix is ``cov_matrix`` so that the new vector covariance
    matrix is a diagonal matrix.

    Parameters
    ----------
    cov_matrix : np.ndarray
        The covariance matrix of the original data that will be
        decorrelated. This must be a symmetric and positive semi-definite
        matrix

    Returns
    -------
    np.ndarray
        The decorrelation matrix :math:`\\mtW_D`. If the original data is a vector
        :math:`\\vtX` it can be decorrelated with :math:`\\mtW_D^T \\vtX`.

    See also
    --------
    calc_whitening_matrix
    """
    _, V = np.linalg.eig(cov_matrix)
    return V

# noinspection PyPep8
def calc_whitening_matrix(cov_matrix: np.ndarray) -> np.ndarray:
    """
    Calculates the whitening matrix that can be applied to a data vector
    whose covariance matrix is ``cov_matrix`` so that the new vector
    covariance matrix is an identity matrix

    Parameters
    ----------
    cov_matrix : np.ndarray
        The covariance matrix of the original data that will be
        decorrelated. This must be a symmetric and positive semi-definite
        matrix

    Returns
    -------
    whitening_matrix : np.ndarray
        The whitening matrix :math:`\\mtW_W`. If the original data is a
        vector :math:`$\\vtX$` it can be whitened with
        :math:`\\mtW_W^H \\vtX`.

    Notes
    -----
    The returned `whitening_matrix` matrix will make the covariance of the
    filtered data an identity matrix. If you only need the the covariance
    matrix of the filtered data to be a diagonal matrix (not necessarily an
    identity matrix) what you want to calculate is the "decorrelation
    matrix". See the :func:`calc_decorrelation_matrix` function for that.

    See also
    --------
    calc_decorrelation_matrix
    """
    L, V = np.linalg.eig(cov_matrix)
    W = np.dot(V, np.diag(1. / (L**0.5)))
    return W

def calc_shannon_sum_capacity(sinrs: NumberOrArrayUnion) -> float:
    """
    Calculate the sum of the Shannon capacity of the values in `sinrs`

    Parameters
    ----------
    sinrs : float | np.ndarray
        SINR values (in linear scale).

    Returns
    -------
    sum_capacity : float
        Sum capacity.

    Examples
    --------
    >>> calc_shannon_sum_capacity(11.4)
    3.6322682154995127
    >>> calc_shannon_sum_capacity(20.3)
    4.412781525338476
    >>> sinrs_linear = np.array([11.4, 20.3])
    >>> print(calc_shannon_sum_capacity(sinrs_linear))
    8.045049740837989
    """
    sum_capacity = np.sum(np.log2(1 + sinrs))

    return cast(float, sum_capacity)

# xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
# xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
# xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx

# # xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
# # xxxxx Load Cython reimplementation of functions here xxxxxxxxxxxxxxxxxxxx
# # xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
# try:
#     # If the misc_c.so extension was compiled then any method defined there
#     # will replace the corresponding method defined here.
#     # pylint: disable=E0611,F0401
#     from ..c_extensions.misc_c import *  # type: ignore
#     USING_CYTHON = True
# except ImportError:  # pragma: no cover
#     import warnings
#     USING_CYTHON = False
#     warnings.warn(
#         "util.misc.count_bits will be slow, since cythonized version was not used"
#     )
# # xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx

if __name__ == '__main__':  # pragma: nocover
    import doctest

    doctest.testmod()


from pydantic import BaseModel
from enum import Enum
from typing import Optional

class RealStateParameter(BaseModel):
    text: Optional[str] = ''
    min_price: Optional[float] = 0
    max_price: Optional[float] = 0
    city: Optional[str] = ''
    rooms: Optional[int] = 0

class RealStateAdId(BaseModel):
    id: Optional[str]=""
    url: Optional[str]=""
    website: Optional[str]=""
class RealPortals(str,Enum):
    Seloger = "Seloger"
    Leboncoin = "Leboncoin"
    Bienci = "Bienci"
    Paruvendu = "Paruvendu"
    Pap = "Pap"
    Logicimmo = "Logicimmo"

class VendorType(str, Enum):
    individuals = "individuals"
    professionals = "professionals"

class RealStateType(str, Enum):
    sale = "sale"
    rental = "rental"
    latest = "Updated/Latest Ads"
    deletedCheck = "deletedCheck"

class PropertyType(str, Enum):
    house = "house"
    apartment = "apartment"
    ground = "ground"
    garage_parking = ""


from readData import readData
from model import Ciudad
from model import Ciudades
from time import time
from translate import translate

matriz_ciudades = readData('data/ciudades_data_prueba.xlsx')
matriz_gasolina = readData('data/gasolina_data_prueba.xlsx')

identificadores = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
diccionario_identificadores = {i:f'C{idx}' for idx,i in enumerate(identificadores)}

diccionario_ciudades = {}

matriz_total = [ [matriz_ciudades[idxf][idxc] + matriz_gasolina[idxf][idxc] for idxc,celda in enumerate(matriz_ciudades)] for idxf, fila in enumerate(matriz_ciudades)]

for idx,lista_costo in enumerate(matriz_total):
    fila_ciudad = { identificadores[idxf]: element for idxf, element in enumerate(lista_costo)}
    ciudad = Ciudad(identificadores[idx],fila_ciudad)
    diccionario_ciudades[identificadores[idx]] = ciudad

ciudades = Ciudades(diccionario_ciudades)

comienzo = time()
costos_de_caminos = { ciudades.camino_voraz(idx): ciudades.calcular_costo(ciudades.camino_voraz(idx)) for idx in identificadores[:20] }
camino_mas_corto = min(costos_de_caminos, key=costos_de_caminos.get)
costo = costos_de_caminos[camino_mas_corto]
final = time()
print('Tiempo: ', (final - comienzo))
print('Camino más corto: ', camino_mas_corto)
print('Traducción de camino:' , translate(camino_mas_corto))
print('Costo: ', costo)


from tfidf import TfidfModel

from summarize import summarize
from lib.db import load_docs_for_training, load_reviews_and_split_to_sentences

docs = load_docs_for_training()
tfidf = TfidfModel()
model, dictionary = tfidf.generate(docs)

target_id = 0

sentences_unfiltered = load_reviews_and_split_to_sentences(target_id)

summary_sentences = summarize(
    sentences_unfiltered, model, dictionary, max_characters=70, user_mmr=True, sent_limit=50
)
for sentence in summary_sentences:
    print(sentence.strip())


#%%
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import anthro.viz
colors = anthro.viz.plotting_style()

# Load the data.
data = pd.read_csv('../../../data/agriculture/FAOSTAT_agriculture_landuse/processed/FAOSTAT_global_crop_pasture_land_use.csv')

# Set up the figure canvas.
fig, ax = plt.subplots(1, 1, figsize=(3, 2.25))
ax.xaxis.set_tick_params(labelsize=6)
ax.yaxis.set_tick_params(labelsize=6)
ax.set_xlabel('year', fontsize=6)
ax.set_ylabel('land area [million km$^2$]', fontsize=6)
ax.set_ylim([10, 60])
ax.set_xlim([1960, 2018])

crops = data[data['usage_type']=='crops']
pasture = data[data['usage_type']=='pasture']
total = data.groupby(['year']).sum().reset_index()

ax.plot(crops['year'], crops['area_km2'].values / 10**6, '-o', color='#eac264',
        ms=2, label='crops', lw=1)
ax.plot(pasture['year'], pasture['area_km2'].values / 10**6, '-o',
        color=colors['dark_brown'], ms=2, label='pasture', lw=1)
ax.plot(total['year'], total['area_km2'].values / 10**6, '-o',
        color=colors['dark_green'], ms=2, label='total', lw=1)
ax.legend(fontsize=6, ncol=3)
plt.tight_layout()
plt.savefig('../../../figures/terra_number/agricultural_land.svg')


from setuptools import setup, find_packages

packages = find_packages(
        where='.',
        include=['kkdetection*']
)

with open("README.md", "r") as fh:
    long_description = fh.read()

setup(
    name='kkdetection',
    version='1.0.5',
    description='my object detection library.',
    long_description=long_description,
    long_description_content_type="text/markdown",
    url="https://github.com/kazukingh01/kkdetection",
    author='kazuking',
    author_email='kazukingh01@gmail.com',
    license='Public License',
    packages=packages,
    classifiers=[
        "Programming Language :: Python :: 3",
        "License :: Private License",
        "Operating System :: OS Independent",
    ],
    install_requires=[
        'detectron2 @ git+https://github.com/facebookresearch/detectron2.git@c9cf7c91c454bb875705af1b7f01e2a15da0d18f',
        'kkannotation @ git+https://github.com/kazukingh01/kkannotation.git@da938d026a8ad6edf1e556efa010067dc488fc50',
        'kkimgaug @ git+https://github.com/kazukingh01/kkimgaug.git@d4a715b4ff25988ce4b17324f9843c04dc99fd1a',
        'paddledet @ git+https://github.com/PaddlePaddle/PaddleDetection.git@eaf2dbe091d79f4824329ea885503e5f08fbb7ec',
        'Cython>=0.29.24',
        'cython-bbox>=0.1.3',
        'lap>=0.4.0',
    ],
    python_requires='>=3.8'
)


import logging

logger = logging.getLogger(__name__)

def info(msg):
    logger.info(msg)

def error(msg):
    logger.error(msg)

def warning(msg):
    logger.warning(msg)

def setup_custom_logger(lfname, stream_to_console=True):
    fHandler = logging.FileHandler(lfname)
    logger.setLevel(logging.DEBUG)
    file_formatter = logging.Formatter('%(asctime)s - %(levelname)s: %(message)s')
    fHandler.setFormatter(file_formatter)
    logger.addHandler(fHandler)
    if stream_to_console:
        ch = logging.StreamHandler()
        ch.setFormatter(ColoredLogFormatter('%(asctime)s - %(message)s'))
        ch.setLevel(logging.DEBUG)
        logger.addHandler(ch)

class ColoredLogFormatter(logging.Formatter):
    def __init__(self, fmt, datefmt=None, style='%'):
        # Logging defines
        self.__GREEN = "\033[92m"
        self.__RED = '\033[91m'
        self.__YELLOW = '\033[33m'
        self.__ENDC = '\033[0m'
        super().__init__(fmt, datefmt, style)

    def formatMessage(self,record):
        if record.levelname=='INFO':
            record.message = self.__GREEN + record.message + self.__ENDC
        elif record.levelname == 'WARNING':
            record.message = self.__YELLOW + record.message + self.__ENDC
        elif record.levelname == 'ERROR':
            record.message = self.__RED + record.message + self.__ENDC
        return super().formatMessage(record)


############################################################
# -*- coding: utf-8 -*-
#
#       #   #  #   #   #    #
#      ##  ##  #  ##  #    #
#     # # # #  # # # #    #  #
#    #  ##  #  ##  ##    ######
#   #   #   #  #   #       #
#
# Python-based Tool for interaction with the 10micron mounts
# GUI with PyQT5 for python
#
# written in python3, (c) 2019-2023 by mworion
# Licence APL2.0
#
###########################################################
# standard libraries
import pytest
import threading
from unittest import mock

# external packages
from PyQt5.QtGui import QPixmap
from skyfield.api import wgs84
import numpy as np

# local import
from tests.unit_tests.unitTestAddOns.baseTestApp import App
from gui.utilities.toolsQtWidget import MWidget
from gui.widgets.main_ui import Ui_MainWindow
from gui.mainWmixin.tabAlmanac import Almanac

@pytest.fixture(autouse=True, scope='module')
def function(qapp):

    class Mixin(MWidget, Almanac):
        def __init__(self):
            super().__init__()
            self.app = App()
            self.threadPool = self.app.threadPool
            self.ui = Ui_MainWindow()
            self.ui.setupUi(self)
            Almanac.__init__(self)

    window = Mixin()
    yield window
    window.threadPool.waitForDone(1000)

def test_initConfig_1(function):
    with mock.patch.object(function,
                           'showMoonPhase'):
        with mock.patch.object(function,
                               'showTwilightDataPlot'):
            with mock.patch.object(function,
                                   'showTwilightDataList'):
                with mock.patch.object(function,
                                       'listTwilightData'):
                    suc = function.initConfig()
                    assert suc

def test_storeConfig_1(function):
    function.thread = None
    suc = function.storeConfig()
    assert suc

def test_storeConfig_2(function):
    function.thread = threading.Thread()
    with mock.patch.object(threading.Thread,
                           'join'):
        suc = function.storeConfig()
        assert suc

def test_setColors(function):
    suc = function.setColors()
    assert suc

def test_colorChangeAlmanac(function):
    with mock.patch.object(function,
                           'showTwilightDataPlot'):
        suc = function.colorChangeAlmanac()
        assert suc

def test_plotTwilightData_1(function):
    function.closing = False
    ts = function.app.mount.obsSite.ts
    tsNow = ts.now()
    t = ts.tt_jd([tsNow.tt, tsNow.tt])
    e = np.array([1, 1])
    result = (ts, t, e)
    suc = function.plotTwilightData(result)
    assert suc

def test_plotTwilightData_2(function):
    function.closing = True
    ts = function.app.mount.obsSite.ts
    tsNow = ts.now()
    t = ts.tt_jd([tsNow.tt, tsNow.tt])
    e = np.array([1, 1])
    result = (ts, t, e)
    suc = function.plotTwilightData(result)
    assert suc

def test_calcTwilightData_1(function):
    ts = function.app.mount.obsSite.ts
    location = wgs84.latlon(latitude_degrees=0,
                            longitude_degrees=0,
                            elevation_m=0)
    val = function.calcTwilightData(ts, location, tWinL=0, tWinH=0)
    assert val

def test_searchTwilightWorker_1(function):
    location = wgs84.latlon(latitude_degrees=0,
                            longitude_degrees=0,
                            elevation_m=0)
    ts = function.app.mount.obsSite.ts
    tsNow = ts.now()
    t = ts.tt_jd([tsNow.tt, tsNow.tt])
    e = np.array([1, 1])
    with mock.patch.object(function,
                           'calcTwilightData',
                           return_value=(t, e)):
        suc = function.workerCalcTwilightDataPlot(ts, location, timeWindow=0)
        assert suc

def test_searchTwilightPlot_1(function):
    function.app.mount.obsSite.location = None
    with mock.patch.object(threading.Thread,
                           'start'):
        suc = function.showTwilightDataPlot()
        assert not suc

def test_searchTwilightPLot_2(function):
    function.app.mount.obsSite.location = wgs84.latlon(latitude_degrees=0,
                                                       longitude_degrees=0,
                                                       elevation_m=0)
    with mock.patch.object(function.threadPool,
                           'start'):
        suc = function.showTwilightDataPlot()
        assert suc

def test_searchTwilightList_1(function):
    function.app.mount.obsSite.location = None
    with mock.patch.object(threading.Thread,
                           'start'):
        suc = function.showTwilightDataList()
        assert not suc

def test_searchTwilightList_2(function):
    function.app.mount.obsSite.location = wgs84.latlon(latitude_degrees=0,
                                                       longitude_degrees=0,
                                                       elevation_m=0)
    with mock.patch.object(function,
                           'listTwilightData'):
        suc = function.showTwilightDataList()
        assert suc

def test_displayTwilightData_1(function):
    tsNow = function.app.mount.obsSite.ts.now()
    t = [tsNow, tsNow]
    e = [1, 1]
    suc = function.listTwilightData(t, e)
    assert suc

def test_calcMoonPhase_1(function):
    val = function.calcMoonPhase()
    assert len(val) == 8

def test_generateMoonMask_1(function):
    q = QPixmap(64, 64)
    function.generateMoonMask(q, 45)

def test_generateMoonMask_2(function):
    q = QPixmap(64, 64)
    function.generateMoonMask(q, 135)

def test_generateMoonMask_3(function):
    q = QPixmap(64, 64)
    function.generateMoonMask(q, 225)

def test_generateMoonMask_4(function):
    q = QPixmap(64, 64)
    function.generateMoonMask(q, 315)

def test_updateMoonPhase_1(function):
    ts = function.app.mount.obsSite.ts
    tsNow = ts.now()
    t = ts.tt_jd([tsNow.tt, tsNow.tt])
    val = (20, 45, .20, 0, t, [0, 1], t, [0, 1])
    with mock.patch.object(function,
                           'calcMoonPhase',
                           return_value=val):
        suc = function.showMoonPhase()
        assert suc

def test_updateMoonPhase_2(function):
    ts = function.app.mount.obsSite.ts
    tsNow = ts.now()
    t = ts.tt_jd([tsNow.tt, tsNow.tt])
    val = (20, 45, .20, 0, t, [0, 1], t, [0, 1])
    with mock.patch.object(function,
                           'calcMoonPhase',
                           return_value=val):
        suc = function.showMoonPhase()
        assert suc


import torch
from torch.autograd import Variable
from nltk import CFG, Nonterminal

grammar = """S -> S '+' T
S -> S '*' T
S -> S '/' T
S -> T
T -> '(' S ')'
T -> 'sin(' S ')'
T -> 'exp(' S ')'
T -> 'x'
T -> '1'
T -> '2'
T -> '3'
Nothing -> None"""

GCFG = CFG.fromstring(grammar)

S, T = Nonterminal('S'), Nonterminal('T')

def get_mask(nonterminal, grammar, as_variable=False):
    if isinstance(nonterminal, Nonterminal):
        mask = [rule.lhs() == nonterminal for rule in grammar.productions()]
        mask = Variable(torch.FloatTensor(mask)) if as_variable else mask
        return mask
    else:
        raise ValueError('Input must be instance of nltk.Nonterminal')

if __name__ == '__main__':
    # Usage:
    GCFG = nltk.CFG.fromstring(grammar)

    print(get_mask(T))


#！/usr/bin/env Python3
# -*- coding:UTF-8 -*-

import urllib.request
import string, ssl, sys
from bs4 import BeautifulSoup

class DownloadNovel():
    def __init__ (self, server):
        self.server = server
        # 存放章节名
        self.chapterNames = []
        # 存放章节链接
        self.chapterUrls = []
        # 存放章节数
        self.nums = 0
    def get_download_chapter_url (self):
        # 用于爬取https
        context = ssl._create_unverified_context()
        response = urllib.request.urlopen(self.server, context = context)
        text = str(response.read(), encoding = 'utf-8')
        bf = BeautifulSoup(text, 'html.parser')
        div = bf.find_all('div', class_ = 'volume')
        for i in range(len(div)):
            li = div[i].find_all('li')
            for j in range(len(li)):
                a = li[j].find_all('a')
                html = BeautifulSoup(str(a), 'lxml')
                # 章节url
                self.chapterUrls.append('https:' + str(html.a.get('href')))
                # 章节名
                self.chapterNames.append(html.a.string)
    def get_contents (self, target):
        context = ssl._create_unverified_context()
        response = urllib.request.urlopen(target, context = context)
        text = str(response.read(), encoding = 'utf-8')
        bf = BeautifulSoup(text, 'html.parser')
        txts = bf.find_all('div', class_='read-content j_readContent')
        txts = txts[0].text.replace('\xa0'*8, '\n\n')
        return txts
    def write (self, name, path, txt):
        write_flag = True
        with open(path, 'a', encoding = 'utf-8') as f:
            f.write(name + '\n')
            f.writelines(txt)
            f.write('\n\n')

if __name__ == '__main__':
    dl = DownloadNovel('https://book.qidian.com/info/1012053141#Catalog')
    dl.get_download_chapter_url()
    print('开始下载')
    for i in range(len(dl.chapterNames)):
        dl.write(dl.chapterNames[i], '无限火力大暴走.txt', dl.get_contents(dl.chapterUrls[i]))
        sys.stdout.write("  已下载:%.3f%%" %  float(i/len(dl.chapterNames)) + '\r')
        sys.stdout.flush()
    print('下载完成')


import json
import bs4 as bs
import requests

veggie = ["https://www.allrecipes.com/recipes/87/everyday-cooking/vegetarian/"]
health = ["https://www.allrecipes.com/recipes/84/healthy-recipes/","https://www.allrecipes.com/recipes/739/healthy-recipes/diabetic/"]
          #"https://www.allrecipes.com/recipes/742/healthy-recipes/low-carb/", "https://www.allrecipes.com/recipes/22485/healthy-recipes/heart-healthy-recipes/",
          #"https://www.allrecipes.com/recipes/1232/healthy-recipes/low-calorie/","https://www.allrecipes.com/recipes/1231/healthy-recipes/low-fat/",
          #"https://www.allrecipes.com/recipes/22607/healthy-recipes/weight-loss/"]
unheathly = []
# To add - Custom cuisine mappings

def write_urls(urls,output_file):

    for url in urls:
        response = requests.get(url)

        # Extracting the source code of the page.
        data = response.text

        # Passing the source code to BeautifulSoup to create a BeautifulSoup object for it.
        soup = bs.BeautifulSoup(data, 'lxml')

        # Extracting all the <a> tags into a list.
        tags = soup.find_all('a')

        links = []

        # Extracting URLs from the attribute href in the <a> tags.
        for tag in tags:
            x = tag.get('href')
            if not x: x = "none"

            if "/recipe/" in x and x not in links and x != "none":
                links.append(x)

        # write links to output file
        try:
            with open(output_file, 'r') as outfile:
                prev = json.load(outfile)
            prev.extend(links)
            links = prev

        except:
            pass

        with open(output_file, 'w') as outfile:
            json.dump(links, outfile)

# Standard run call

if __name__ == "__main__":
    # write_urls(veggie, "json_created_files/veggie_links.json")
    write_urls(health, "json_created_files/health_links.json")