# -*- coding: utf-8 -*-
"""
Created on Thu Aug  6 15:38:25 2020

@author: Carlos Coronel

Some auxiliary functions.

All the graph metrics employed were taken from the Brain Connectivity Toolbox for Python [2]:
https://github.com/aestrivex/bctpy [2]

[1] Lancaster, G., Iatsenko, D., Pidde, A., Ticcinelli, V., & Stefanovska, A.
(2018). Surrogate data for hypothesis testing of physical systems.
Physics Reports, 748, 1-60.

[2] Rubinov, M., & Sporns, O. (2010). Complex network measures of brain connectivity:
uses and interpretations. Neuroimage, 52(3), 1059-1069.

"""

import numpy as np
from scipy import stats
from statsmodels.sandbox.stats.multicomp import multipletests

def get_uptri(x):
    """
    Gets the vectorized upper triangle of a matrix x.

    Parameters
    ----------
    x : numpy array.
        connectivity matrix.
    Returns
    -------
    vector : numpy array.
             upper triangle of x in vector form.
    """
    nnodes = x.shape[0]
    npairs = (nnodes**2 - nnodes) // 2
    vector = np.zeros(npairs)

    idx = 0
    for row in range(0, nnodes - 1):
        for col in range(row + 1, nnodes):
            vector[idx] = x[row, col]
            idx = idx + 1

    return(vector)

def matrix_recon(x):
    """
    Reconstructs a connectivity matrix from its upper triangle.

    Parameters
    ----------
    x : numpy array.
        upper triangle of the original matrix.
    Returns
    -------
    matrix : numpy array.
             original connectivity matrix.
    """
    npairs = len(x)
    nnodes = int((1 + np.sqrt(1 + 8 * npairs)) // 2)

    matrix = np.zeros((nnodes, nnodes))
    idx = 0
    for row in range(0, nnodes - 1):
        for col in range(row + 1, nnodes):
            matrix[row, col] = x[idx]
            idx = idx + 1
    matrix = matrix + matrix.T

    return(matrix)

def probabilistic_thresholding(y, surr_number = 50, alpha = 0.05):
    """
    Threshold for neglecting spurious connectivity values in functional fata.
    It uses a phase randomization to destroy the pairwise correlations, while
    preserving the spectral properties of the original signals. Type I error
    (multiple comparisons) corrected by the Benjamini-Hochberg procedure.

    Parameters
    ----------
    y : txN numpy array.
        time series.
        t -> time.
        N -> nodes.
    surr_number : integer.
                  number of surrogates.
    alpha : float (between 0 and 1).
            critical p_value for statistical inference. By default is 0.05.
    Returns
    -------
    FC_adjusted : NxN numpy array.
                  thresholded functional connectivity matrix.
    p_matrix : NxN numpy array.
               p_values for each connectivity pair.
    """

    y = y - np.mean(y, axis = 0)
    yf = np.fft.fft(y, axis = 0)

    nnodes = y.shape[1]
    npairs = (nnodes**2 - nnodes) // 2
    matrix_surr = np.zeros((npairs, surr_number))

    FC_real = np.corrcoef(y.T)
    FC_real = FC_real[np.triu_indices(n = nnodes, k = 1)]

    for i in range(surr_number):
        np.random.seed(i + 1)
        random_vector = np.random.uniform(0, 2 * np.pi, ((yf.shape[0] // 2), yf.shape[1]))
        random_vector = np.row_stack((random_vector, random_vector[::-1,:]))
        yfR = yf * np.exp(1j * random_vector)
        surrogate = np.fft.ifft(yfR, axis = 0)
        surrogate = surrogate.real
        FC_surr = np.corrcoef(surrogate.T)
        matrix_surr[:,i] = FC_surr[np.triu_indices(n = nnodes, k = 1)]

    p_vector = np.zeros(npairs)
    for i in range(npairs):
        mu, sigma = stats.norm.fit(matrix_surr[i,:])
        p_vector[i] = 1 - stats.norm.cdf(FC_real[i], mu, sigma)

    reject, p_adjust, alphacSidak, alphacBonf = multipletests(p_vector, alpha = alpha, method='fdr_bh')

    FC_adjusted = FC_real * ((p_adjust < alpha) * 1)
    FC_adjusted = matrix_recon(FC_adjusted)
    p_matrix = matrix_recon(p_adjust)

    return([FC_adjusted, p_matrix])

#Kuramoto order parameter
def simple_order_parameter(data):
    """
    Calculates the global phase synchronization over time as the Kuramoto order
    parameter.

    Parameters
    ----------
    data : phixN numpy array.
           phi -> phases.
           N -> nodes.
    Returns
    -------
    PhaseSynch : 1xphi numpy array.
                 Kuramoto order parameter.
    """
    phases = data
    PhaseSynch = np.abs(np.mean(np.exp(1j * phases), axis = 1))
    return(PhaseSynch)

###################################
###FUNCTIONS FOR GRAPH ANALYSIS###
###################################
###################################
class BCTParamError(RuntimeError):
    pass

def cuberoot(x):
    """
    Correctly handle the cube root for negative weights, instead of uselessly
    crashing as in python or returning the wrong root as in matlab
    """
    return(np.sign(x) * np.abs(x)**(1 / 3))

def invert(W, copy = True):
    """
    Inverts elementwise the weights in an input connection matrix.
    In other words, change the from the matrix of internode strengths to the
    matrix of internode distances.
    If copy is not set, this function will *modify W in place.*
    Parameters
    ----------
    W : :obj:`numpy.ndarray`
        weighted connectivity matrix
    copy : bool
        if True, returns a copy of the matrix. Otherwise, modifies the matrix
        in place. Default value=True.
    Returns
    -------
    W : :obj:`numpy.ndarray`
        inverted connectivity matrix
    """
    if copy:
        W = W.copy()
    E = np.where(W)
    W[E] = 1. / W[E]
    return(W)

def dummyvar(cis, return_sparse=False):
    """
    This is an efficient implementation of matlab's "dummyvar" command
    using sparse matrices.
    input: partitions, NxM array-like containing M partitions of N nodes
        into <=N distinct communities
    output: dummyvar, an NxR matrix containing R column variables (indicator
        variables) with N entries, where R is the total number of communities
        summed across each of the M partitions.
        i.e.
        r = sum((max(len(unique(partitions[i]))) for i in range(m)))
    """
    # num_rows is not affected by partition indexes
    n = np.size(cis, axis=0)
    m = np.size(cis, axis=1)
    r = np.sum((np.max(len(np.unique(cis[:, i])))) for i in range(m))
    nnz = np.prod(cis.shape)

    ix = np.argsort(cis, axis=0)
    # s_cis=np.sort(cis,axis=0)
    # FIXME use the sorted indices to sort by row efficiently
    s_cis = cis[ix][:, range(m), range(m)]

    mask = np.hstack((((True,),) * m, (s_cis[:-1, :] != s_cis[1:, :]).T))
    indptr, = np.where(mask.flat)
    indptr = np.append(indptr, nnz)

    import scipy.sparse as sp
    dv = sp.csc_matrix((np.repeat((1,), nnz), ix.T.flat, indptr), shape=(n, r))
    return dv.toarray()

def agreement(ci, buffsz=1000):
    """
    Takes as input a set of vertex partitions CI of
    dimensions [vertex x partition]. Each column in CI contains the
    assignments of each vertex to a class/community/module. This function
    aggregates the partitions in CI into a square [vertex x vertex]
    agreement matrix D, whose elements indicate the number of times any two
    vertices were assigned to the same class.
    In the case that the number of nodes and partitions in CI is large
    (greater than ~1000 nodes or greater than ~1000 partitions), the script
    can be made faster by computing D in pieces. The optional input BUFFSZ
    determines the size of each piece. Trial and error has found that
    BUFFSZ ~ 150 works well.
    Parameters
    ----------
    ci : NxM :obj:`numpy.ndarray`
        set of M (possibly degenerate) partitions of N nodes
    buffsz : int | None
        sets buffer size. If not specified, defaults to 1000
    Returns
    -------
    D : NxN :obj:`numpy.ndarray`
        agreement matrix
    """
    ci = np.array(ci)
    n_nodes, n_partitions = ci.shape

    if n_partitions <= buffsz:  # Case 1: Use all partitions at once
        ind = dummyvar(ci)
        D = np.dot(ind, ind.T)
    else:  # Case 2: Add together results from subsets of partitions
        a = np.arange(0, n_partitions, buffsz)
        b = np.arange(buffsz, n_partitions, buffsz)
        if len(a) != len(b):
            b = np.append(b, n_partitions)
        D = np.zeros((n_nodes, n_nodes))
        for i, j in zip(a, b):
            y = ci[:, i:j]
            ind = dummyvar(y)
            D += np.dot(ind, ind.T)

    np.fill_diagonal(D, 0)
    return D

def consensus_und(D, tau, reps = 1000):
    """
    This algorithm seeks a consensus partition of the
    agreement matrix D. The algorithm used here is almost identical to the
    one introduced in Lancichinetti & Fortunato (2012): The agreement
    matrix D is thresholded at a level TAU to remove an weak elements. The
    resulting matrix is then partitions REPS number of times using the
    Louvain algorithm (in principle, any clustering algorithm that can
    handle weighted matrixes is a suitable alternative to the Louvain
    algorithm and can be substituted in its place). This clustering
    produces a set of partitions from which a new agreement is built. If
    the partitions have not converged to a single representative partition,
    the above process repeats itself, starting with the newly built
    agreement matrix.
    NOTE: In this implementation, the elements of the agreement matrix must
    be converted into probabilities.
    NOTE: This implementation is slightly different from the original
    algorithm proposed by Lanchichinetti & Fortunato. In its original
    version, if the thresholding produces singleton communities, those
    nodes are reconnected to the network. Here, we leave any singleton
    communities disconnected.
    Parameters
    ----------
    D : NxN :obj:`numpy.ndarray`
        agreement matrix with entries between 0 and 1 denoting the probability
        of finding node i in the same cluster as node j
    tau : float
        threshold which controls the resolution of the reclustering
    reps : int
        number of times the clustering algorithm is reapplied. default value
        is 1000.
    Returns
    -------
    ciu : Nx1 :obj:`numpy.ndarray`
        consensus partition
    """
    def unique_partitions(cis):
        # relabels the partitions to recognize different numbers on same
        # topology

        n, r = np.shape(cis)  # ci represents one vector for each rep
        ci_tmp = np.zeros(n)

        for i in range(r):
            for j, u in enumerate(sorted(
                    np.unique(cis[:, i], return_index=True)[1])):
                ci_tmp[np.where(cis[:, i] == cis[u, i])] = j
            cis[:, i] = ci_tmp
            # so far no partitions have been deleted from ci

        # now squash any of the partitions that are completely identical
        # do not delete them from ci which needs to stay same size, so make
        # copy
        ciu = []
        cis = cis.copy()
        c = np.arange(r)
        # count=0
        while (c != 0).sum() > 0:
            ciu.append(cis[:, 0])
            dup = np.where(np.sum(np.abs(cis.T - cis[:, 0]), axis=1) == 0)
            cis = np.delete(cis, dup, axis=1)
            c = np.delete(c, dup)
        return np.transpose(ciu)

    n = len(D)
    flag = True
    while flag:
        flag = False
        dt = D * (D >= tau)
        np.fill_diagonal(dt, 0)

        if np.size(np.where(dt == 0)) == 0:
            ciu = np.arange(1, n + 1)
        else:
            cis = np.zeros((n, reps))
            for i in np.arange(reps):
                cis[:, i], _ = modularity_louvain_und_sign(dt)
            ciu = unique_partitions(cis)
            nu = np.size(ciu, axis=1)
            if nu > 1:
                flag = True
                D = agreement(cis) / reps

    return np.squeeze(ciu + 1)

def transitivity_wu(W):
    """
    Transitivity is the ratio of 'triangles to triplets' in the network.
    (A classical version of the clustering coefficient).
    Parameters
    ----------
    W : NxN :obj:`numpy.ndarray`
        weighted undirected connection matrix
    Returns
    -------
    T : int
        transitivity scalar
    """
    K = np.sum(np.logical_not(W == 0), axis=1)
    ws = cuberoot(W)
    cyc3 = np.diag(np.dot(ws, np.dot(ws, ws)))
    return np.sum(cyc3, axis=0) / np.sum(K * (K - 1), axis=0)

def modularity_louvain_und_sign(W, gamma=1, qtype='sta', seed=None):
    """
    The optimal community structure is a subdivision of the network into
    nonoverlapping groups of nodes in a way that maximizes the number of
    within-group edges, and minimizes the number of between-group edges.
    The modularity is a statistic that quantifies the degree to which the
    network may be subdivided into such clearly delineated groups.
    The Louvain algorithm is a fast and accurate community detection
    algorithm (at the time of writing).
    Use this function as opposed to modularity_louvain_und() only if the
    network contains a mix of positive and negative weights.  If the network
    contains all positive weights, the output will be equivalent to that of
    modularity_louvain_und().
    Parameters
    ----------
    W : NxN :obj:`numpy.ndarray`
        undirected weighted/binary connection matrix with positive and
        negative weights
    qtype : str
        modularity type. Can be 'sta' (default), 'pos', 'smp', 'gja', 'neg'.
        See Rubinov and Sporns (2011) for a description.
    gamma : float
        resolution parameter. default value=1. Values 0 <= gamma < 1 detect
        larger modules while gamma > 1 detects smaller modules.
    seed : int | None
        random seed. default value=None. if None, seeds from /dev/urandom.
    Returns
    -------
    ci : Nx1 :obj:`numpy.ndarray`
        refined community affiliation vector
    Q : float
        optimized modularity metric
    Notes
    -----
    Ci and Q may vary from run to run, due to heuristics in the
    algorithm. Consequently, it may be worth to compare multiple runs.
    """
    np.random.seed(seed)

    n = len(W)  # number of nodes

    W0 = W * (W > 0)  # positive weights matrix
    W1 = -W * (W < 0)  # negative weights matrix
    s0 = np.sum(W0)  # weight of positive links
    s1 = np.sum(W1)  # weight of negative links

    if qtype == 'smp':
        d0 = 1 / s0
        d1 = 1 / s1  # dQ=dQ0/s0-sQ1/s1
    elif qtype == 'gja':
        d0 = 1 / (s0 + s1)
        d1 = d0  # dQ=(dQ0-dQ1)/(s0+s1)
    elif qtype == 'sta':
        d0 = 1 / s0
        d1 = 1 / (s0 + s1)  # dQ=dQ0/s0-dQ1/(s0+s1)
    elif qtype == 'pos':
        d0 = 1 / s0
        d1 = 0  # dQ=dQ0/s0
    elif qtype == 'neg':
        d0 = 0
        d1 = 1 / s1  # dQ=-dQ1/s1
    else:
        raise KeyError('modularity type unknown')

    if not s0:  # adjust for absent positive weights
        s0 = 1
        d0 = 0
    if not s1:  # adjust for absent negative weights
        s1 = 1
        d1 = 0

    h = 1  # hierarchy index
    nh = n  # number of nodes in hierarchy
    ci = [None, np.arange(n) + 1]  # hierarchical module assignments
    q = [-1, 0]  # hierarchical modularity values
    while q[h] - q[h - 1] > 1e-10:
        if h > 300:
            raise BCTParamError('Modularity Infinite Loop Style A.  Please '
                                'contact the developer with this error.')
        kn0 = np.sum(W0, axis=0)  # positive node degree
        kn1 = np.sum(W1, axis=0)  # negative node degree
        km0 = kn0.copy()  # positive module degree
        km1 = kn1.copy()  # negative module degree
        knm0 = W0.copy()  # positive node-to-module degree
        knm1 = W1.copy()  # negative node-to-module degree

        m = np.arange(nh) + 1  # initial module assignments
        flag = True  # flag for within hierarchy search
        it = 0
        while flag:
            it += 1
            if it > 1000:
                raise BCTParamError('Infinite Loop was detected and stopped. '
                                    'This was probably caused by passing in a '
                                    'directed matrix.')
            flag = False
            # loop over nodes in random order
            for u in np.random.permutation(nh):
                ma = m[u] - 1
                # positive dQ
                dQ0 = ((knm0[u, :] + W0[u, u] - knm0[u, ma]) -
                       gamma * kn0[u] * (km0 + kn0[u] - km0[ma]) / s0)
                # negative dQ
                dQ1 = ((knm1[u, :] + W1[u, u] - knm1[u, ma]) -
                       gamma * kn1[u] * (km1 + kn1[u] - km1[ma]) / s1)

                dQ = d0 * dQ0 - d1 * dQ1  # rescaled changes in modularity
                dQ[ma] = 0  # no changes for same module

                max_dQ = np.max(dQ)  # maximal increase in modularity
                if max_dQ > 1e-10:  # if maximal increase is positive
                    flag = True
                    mb = np.argmax(dQ)

                    # change positive node-to-module degrees
                    knm0[:, mb] += W0[:, u]
                    knm0[:, ma] -= W0[:, u]
                    # change negative node-to-module degrees
                    knm1[:, mb] += W1[:, u]
                    knm1[:, ma] -= W1[:, u]
                    km0[mb] += kn0[u]  # change positive module degrees
                    km0[ma] -= kn0[u]
                    km1[mb] += kn1[u]  # change negative module degrees
                    km1[ma] -= kn1[u]

                    m[u] = mb + 1  # reassign module

        h += 1
        ci.append(np.zeros((n,)))
        _, m = np.unique(m, return_inverse=True)
        m += 1

        for u in range(nh):  # loop through initial module assignments
            ci[h][np.where(ci[h - 1] == u + 1)] = m[u]  # assign new modules

        nh = np.max(m)  # number of new nodes
        wn0 = np.zeros((nh, nh))  # new positive weights matrix
        wn1 = np.zeros((nh, nh))

        for u in range(nh):
            for v in range(u, nh):
                wn0[u, v] = np.sum(W0[np.ix_(m == u + 1, m == v + 1)])
                wn1[u, v] = np.sum(W1[np.ix_(m == u + 1, m == v + 1)])
                wn0[v, u] = wn0[u, v]
                wn1[v, u] = wn1[u, v]

        W0 = wn0
        W1 = wn1

        q.append(0)
        # compute modularity
        q0 = np.trace(W0) - np.sum(np.dot(W0, W0)) / s0
        q1 = np.trace(W1) - np.sum(np.dot(W1, W1)) / s1
        q[h] = d0 * q0 - d1 * q1

    _, ci_ret = np.unique(ci[-1], return_inverse=True)
    ci_ret += 1

    return ci_ret, q[-1]

def efficiency_wei(Gw, local = False):
    """
    The global efficiency is the average of inverse shortest path length,
    and is inversely related to the characteristic path length.
    The local efficiency is the global efficiency computed on the
    neighborhood of the node, and is related to the clustering coefficient.
    Parameters
    ----------
    Gw : NxN :obj:`numpy.ndarray`
        undirected weighted connection matrix
        (all weights in W must be between 0 and 1)
    local : bool
        If True, computes local efficiency instead of global efficiency.
        Default value = False.
    Returns
    -------
    Eglob : float
        global efficiency, only if local=False
    Eloc : Nx1 :obj:`numpy.ndarray`
        local efficiency, only if local=True
    Notes
    -----
       The  efficiency is computed using an auxiliary connection-length
    matrix L, defined as L_ij = 1/W_ij for all nonzero L_ij; This has an
    intuitive interpretation, as higher connection weights intuitively
    correspond to shorter lengths.
       The weighted local efficiency broadly parallels the weighted
    clustering coefficient of Onnela et al. (2005) and distinguishes the
    influence of different paths based on connection weights of the
    corresponding neighbors to the node in question. In other words, a path
    between two neighbors with strong connections to the node in question
    contributes more to the local efficiency than a path between two weakly
    connected neighbors. Note that this weighted variant of the local
    efficiency is hence not a strict generalization of the binary variant.
    Algorithm:  Dijkstra's algorithm
    """
    def distance_inv_wei(G):
        n = len(G)
        D = np.zeros((n, n))  # distance matrix
        D[np.logical_not(np.eye(n))] = np.inf

        for u in range(n):
            # distance permanence (true is temporary)
            S = np.ones((n,), dtype=bool)
            G1 = G.copy()
            V = [u]
            while True:
                S[V] = 0  # distance u->V is now permanent
                G1[:, V] = 0  # no in-edges as already shortest
                for v in V:
                    W, = np.where(G1[v, :])  # neighbors of smallest nodes
                    td = np.array(
                        [D[u, W].flatten(), (D[u, v] + G1[v, W]).flatten()])
                    D[u, W] = np.min(td, axis=0)

                if D[u, S].size == 0:  # all nodes reached
                    break
                minD = np.min(D[u, S])
                if np.isinf(minD):  # some nodes cannot be reached
                    break
                V, = np.where(D[u, :] == minD)

        np.fill_diagonal(D, 1)
        D = 1 / D
        np.fill_diagonal(D, 0)
        return D

    n = len(Gw)
    Gl = invert(Gw, copy=True)  # connection length matrix
    A = np.array((Gw != 0), dtype=int)
    if local:
        E = np.zeros((n,))  # local efficiency
        for u in range(n):
            # find pairs of neighbors
            V, = np.where(np.logical_or(Gw[u, :], Gw[:, u].T))
            # symmetrized vector of weights
            sw = cuberoot(Gw[u, V]) + cuberoot(Gw[V, u].T)
            # inverse distance matrix
            e = distance_inv_wei(Gl[np.ix_(V, V)])
            # symmetrized inverse distance matrix
            se = cuberoot(e) + cuberoot(e.T)

            numer = np.sum(np.outer(sw.T, sw) * se) / 2
            if numer != 0:
                # symmetrized adjacency vector
                sa = A[u, V] + A[V, u].T
                denom = np.sum(sa)**2 - np.sum(sa * sa)
                # print numer,denom
                E[u] = numer / denom  # local efficiency

    else:
        e = distance_inv_wei(Gl)
        E = np.sum(e) / (n * n - n)
    return(E)

def charpath(D, include_diagonal=False, include_infinite=True):
    """
    The characteristic path length is the average shortest path length in
    the network. The global efficiency is the average inverse shortest path
    length in the network.
    Parameters
    ----------
    D : NxN :obj:`numpy.ndarray`
        distance matrix
    include_diagonal : bool
        If True, include the weights on the diagonal. Default value is False.
    include_infinite : bool
        If True, include infinite distances in calculation
    Returns
    -------
    lambda : float
        characteristic path length
    efficiency : float
        global efficiency
    ecc : Nx1 :obj:`numpy.ndarray`
        eccentricity at each vertex
    radius : float
        radius of graph
    diameter : float
        diameter of graph
    Notes
    -----
    The input distance matrix may be obtained with any of the distance
    functions, e.g. distance_bin, distance_wei.
    Characteristic path length is calculated as the global mean of
    the distance matrix D, excludings any 'Infs' but including distances on
    the main diagonal.
    """
    D = D.copy()

    if not include_diagonal:
        np.fill_diagonal(D, np.nan)

    if not include_infinite:
        D[np.isinf(D)] = np.nan

    Dv = D[np.logical_not(np.isnan(D))].ravel()

    # mean of finite entries of D[G]
    lambda_ = np.mean(Dv)

    # efficiency: mean of inverse entries of D[G]
    efficiency = np.mean(1 / Dv)

    # eccentricity for each vertex (ignore inf)
    ecc = np.array(np.ma.masked_where(np.isnan(D), D).max(axis=1))

    # radius of graph
    radius = np.min(ecc)  # but what about zeros?

    # diameter of graph
    diameter = np.max(ecc)

    return lambda_, efficiency, ecc, radius, diameter

def community_louvain(W, gamma= 1, ci=None, B='modularity', seed=None):
    """
    The optimal community structure is a subdivision of the network into
    nonoverlapping groups of nodes which maximizes the number of within-group
    edges and minimizes the number of between-group edges.
    This function is a fast an accurate multi-iterative generalization of the
    louvain community detection algorithm. This function subsumes and improves
    upon modularity_[louvain,finetune]_[und,dir]() and additionally allows to
    optimize other objective functions (includes built-in Potts Model i
    Hamiltonian, allows for custom objective-function matrices).
    Parameters
    ----------
    W : NxN np.array
        directed/undirected weighted/binary adjacency matrix
    gamma : float
        resolution parameter. default value=1. Values 0 <= gamma < 1 detect
        larger modules while gamma > 1 detects smaller modules.
        ignored if an objective function matrix is specified.
    ci : Nx1 np.arraylike
        initial community affiliation vector. default value=None
    B : str | NxN np.arraylike
        string describing objective function type, or provides a custom
        NxN objective-function matrix. builtin values
            'modularity' uses Q-metric as objective function
            'potts' uses Potts model Hamiltonian.
            'negative_sym' symmetric treatment of negative weights
            'negative_asym' asymmetric treatment of negative weights
    seed : int | None
        random seed. default value=None. if None, seeds from /dev/urandom.
    Returns
    -------
    ci : Nx1 np.array
        final community structure
    q : float
        optimized q-statistic (modularity only)
    """
    np.random.seed(seed)

    n = len(W)
    s = np.sum(W)

    if np.min(W) < -1e-10:
        raise BCTParamError('adjmat must not contain negative weights')

    if ci is None:
        ci = np.arange(n) + 1
    else:
        if len(ci) != n:
            raise BCTParamError('initial ci vector size must equal N')
        _, ci = np.unique(ci, return_inverse=True)
        ci += 1
    Mb = ci.copy()

    if B in ('negative_sym', 'negative_asym'):
        W0 = W * (W > 0)
        s0 = np.sum(W0)
        B0 = W0 - gamma * np.outer(np.sum(W0, axis=1), np.sum(W, axis=0)) / s0

        W1 = W * (W < 0)
        s1 = np.sum(W1)
        if s1:
            B1 = (W1 - gamma * np.outer(np.sum(W1, axis=1), np.sum(W1, axis=0))
                  / s1)
        else:
            B1 = 0

    elif np.min(W) < -1e-10:
        raise BCTParamError('Input connection matrix contains negative '
                            'weights but objective function dealing with '
                            'negative weights was not selected')

    if B == 'potts' and np.any(np.logical_not(np.logical_or(W == 0, W == 1))):
        raise BCTParamError('Potts hamiltonian requires binary input matrix')

    if B == 'modularity':
        B = W - gamma * np.outer(np.sum(W, axis=1), np.sum(W, axis=0)) / s
    elif B == 'potts':
        B = W - gamma * np.logical_not(W)
    elif B == 'negative_sym':
        B = B0 / (s0 + s1) - B1 / (s0 + s1)
    elif B == 'negative_asym':
        B = B0 / s0 - B1 / (s0 + s1)
    else:
        try:
            B = np.array(B)
        except BCTParamError:
            print('unknown objective function type')

        if B.shape != W.shape:
            raise BCTParamError('objective function matrix does not match '
                                'size of adjacency matrix')
        if not np.allclose(B, B.T):
            print('Warning: objective function matrix not symmetric, '
                  'symmetrizing')
            B = (B + B.T) / 2

    Hnm = np.zeros((n, n))
    for m in range(1, n + 1):
        Hnm[:, m - 1] = np.sum(B[:, ci == m], axis=1)  # node to module degree
    H = np.sum(Hnm, axis=1)  # node degree
    Hm = np.sum(Hnm, axis=0)  # module degree

    q0 = -np.inf
    # compute modularity
    q = np.sum(B[np.tile(ci, (n, 1)) == np.tile(ci, (n, 1)).T]) / s

    first_iteration = True

    while q - q0 > 1e-10:
        it = 0
        flag = True
        while flag:
            it += 1
            if it > 1000:
                raise BCTParamError('Modularity infinite loop style G. '
                                    'Please contact the developer.')
            flag = False
            for u in np.random.permutation(n):
                ma = Mb[u] - 1
                dQ = Hnm[u, :] - Hnm[u, ma] + B[u, u]  # algorithm condition
                dQ[ma] = 0

                max_dq = np.max(dQ)
                if max_dq > 1e-10:
                    flag = True
                    mb = np.argmax(dQ)

                    Hnm[:, mb] += B[:, u]
                    Hnm[:, ma] -= B[:, u]  # change node-to-module strengths

                    Hm[mb] += H[u]
                    Hm[ma] -= H[u]  # change module strengths

                    Mb[u] = mb + 1

        _, Mb = np.unique(Mb, return_inverse=True)
        Mb += 1

        M0 = ci.copy()
        if first_iteration:
            ci = Mb.copy()
            first_iteration = False
        else:
            for u in range(1, n + 1):
                ci[M0 == u] = Mb[u - 1]  # assign new modules

        n = np.max(Mb)
        b1 = np.zeros((n, n))
        for i in range(1, n + 1):
            for j in range(i, n + 1):
                # pool weights of nodes in same module
                bm = np.sum(B[np.ix_(Mb == i, Mb == j)])
                b1[i - 1, j - 1] = bm
                b1[j - 1, i - 1] = bm
        B = b1.copy()

        Mb = np.arange(1, n + 1)
        Hnm = B.copy()
        H = np.sum(B, axis=0)
        Hm = H.copy()

        q0 = q
        q = np.trace(B) / s  # compute modularity

    return ci, q

def get_agreement_matrix(W, reps = 100, tau = 0.5, gamma = 1, ci = None, B = 'modularity', seeds = None):
    nnodes = W.shape[0]
    n_p = reps #number of partitions
    D_matrix = np.zeros((nnodes,nnodes)) #Agreement matrix

    if seeds == None:
        seed_vector = np.arange(0, n_p, 1)
    elif len(seed_vector) == n_p:
        seed_vector = seeds
    else:
        print('seeds must be an array of length equal to reps, or None')

    for rand in range(0,n_p):
        partition = community_louvain(W, gamma, ci, B, seed_vector[rand])[0]
        for row in range(0,nnodes):
            for col in range(0,nnodes):
                D_matrix[row,col] += (partition[row] == partition[col]) * 1
    D_matrix /= n_p
    D_matrix[D_matrix < tau] = 0
    np.fill_diagonal(D_matrix, 0)

    return(D_matrix)

def distance_wei(G):
    """
    The distance matrix contains lengths of shortest paths between all
    pairs of nodes. An entry (u,v) represents the length of shortest path
    from node u to node v. The average shortest path length is the
    characteristic path length of the network.
    Parameters
    ----------
    G : NxN :obj:`numpy.ndarray`
        Directed/undirected connection-length matrix.
        NB L is not the adjacency matrix. See below.
    Returns
    -------
    D : NxN :obj:`numpy.ndarray`
        distance (shortest weighted path) matrix
    B : NxN :obj:`numpy.ndarray`
        matrix of number of edges in shortest weighted path
    Notes
    -----
       The input matrix must be a connection-length matrix, typically
    obtained via a mapping from weight to length. For instance, in a
    weighted correlation network higher correlations are more naturally
    interpreted as shorter distances and the input matrix should
    consequently be some inverse of the connectivity matrix.
       The number of edges in shortest weighted paths may in general
    exceed the number of edges in shortest binary paths (i.e. shortest
    paths computed on the binarized connectivity matrix), because shortest
    weighted paths have the minimal weighted distance, but not necessarily
    the minimal number of edges.
       Lengths between disconnected nodes are set to Inf.
       Lengths on the main diagonal are set to 0.
    Algorithm: Dijkstra's algorithm.
    """
    n = len(G)
    D = np.zeros((n, n))  # distance matrix
    D[np.logical_not(np.eye(n))] = np.inf
    B = np.zeros((n, n))  # number of edges matrix

    for u in range(n):
        # distance permanence (true is temporary)
        S = np.ones((n,), dtype=bool)
        G1 = G.copy()
        V = [u]
        while True:
            S[V] = 0  # distance u->V is now permanent
            G1[:, V] = 0  # no in-edges as already shortest
            for v in V:
                W, = np.where(G1[v, :])  # neighbors of shortest nodes

                td = np.array(
                    [D[u, W].flatten(), (D[u, v] + G1[v, W]).flatten()])
                d = np.min(td, axis=0)
                wi = np.argmin(td, axis=0)

                D[u, W] = d  # smallest of old/new path lengths
                ind = W[np.where(wi == 1)]  # indices of lengthened paths
                # increment nr_edges for lengthened paths
                B[u, ind] = B[u, v] + 1

            if D[u, S].size == 0:  # all nodes reached
                break
            minD = np.min(D[u, S])
            if np.isinf(minD):  # some nodes cannot be reached
                break

            V, = np.where(D[u, :] == minD)

    return D, B

def participation_coef(W, ci, degree = 'undirected'):
    """
    Participation coefficient is a measure of diversity of intermodular
    connections of individual nodes.
    Parameters
    ----------
    W : NxN :obj:`numpy.ndarray`
        binary/weighted directed/undirected connection matrix
    ci : Nx1 :obj:`numpy.ndarray`
        community affiliation vector
    degree : {'undirected', 'in', 'out'}, optional
        Flag to describe nature of graph. 'undirected': For undirected graphs,
        'in': Uses the in-degree, 'out': Uses the out-degree
    Returns
    -------
    P : Nx1 :obj:`numpy.ndarray`
        participation coefficient
    """
    if degree == 'in':
        W = W.T

    _, ci = np.unique(ci, return_inverse=True)
    ci += 1

    n = len(W)  # number of vertices
    Ko = np.sum(W, axis=1)  # (out) degree
    Gc = np.dot((W != 0), np.diag(ci))  # neighbor community affiliation
    Kc2 = np.zeros((n,))  # community-specific neighbors

    for i in range(1, int(np.max(ci)) + 1):
        Kc2 += np.square(np.sum(W * (Gc == i), axis=1))

    P = np.ones((n,)) - Kc2 / np.square(Ko)
    # P=0 if for nodes with no (out) neighbors
    P[np.where(np.logical_not(Ko))] = 0

    return(P)

def modularity_und(A, gamma=1, kci=None):
    """
    The optimal community structure is a subdivision of the network into
    nonoverlapping groups of nodes in a way that maximizes the number of
    within-group edges, and minimizes the number of between-group edges.
    The modularity is a statistic that quantifies the degree to which the
    network may be subdivided into such clearly delineated groups.
    Parameters
    ----------
    A : NxN :obj:`numpy.ndarray`
        undirected weighted/binary connection matrix
    gamma : float
        resolution parameter. default value=1. Values 0 <= gamma < 1 detect
        larger modules while gamma > 1 detects smaller modules.
    kci : Nx1 :obj:`numpy.ndarray` | None
        starting community structure. If specified, calculates the Q-metric
        on the community structure giving, without doing any optimzation.
        Otherwise, if not specified, uses a spectral modularity maximization
        algorithm.
    Returns
    -------
    ci : Nx1 :obj:`numpy.ndarray`
        optimized community structure
    Q : float
        maximized modularity metric
    Notes
    -----
    This algorithm is deterministic. The matlab function bearing this
    name incorrectly disclaims that the outcome depends on heuristics
    involving a random seed. The louvain method does depend on a random seed,
    but this function uses a deterministic modularity maximization algorithm.
    """
    from scipy import linalg
    n = len(A)  # number of vertices
    k = np.sum(A, axis=0)  # degree
    m = np.sum(k)  # number of edges (each undirected edge
    # is counted twice)
    B = A - gamma * np.outer(k, k) / m  # initial modularity matrix

    init_mod = np.arange(n)  # initial one big module
    modules = []  # output modules list

    def recur(module):
        n = len(module)
        modmat = B[module][:, module]
        modmat -= np.diag(np.sum(modmat, axis=0))

        vals, vecs = linalg.eigh(modmat)  # biggest eigendecomposition
        rlvals = np.real(vals)
        max_eigvec = np.squeeze(vecs[:, np.where(rlvals == np.max(rlvals))])
        if max_eigvec.ndim > 1:  # if multiple max eigenvalues, pick one
            max_eigvec = max_eigvec[:, 0]
        # initial module assignments
        mod_asgn = np.squeeze((max_eigvec >= 0) * 2 - 1)
        q = np.dot(mod_asgn, np.dot(modmat, mod_asgn))  # modularity change

        if q > 0:  # change in modularity was positive
            qmax = q
            np.fill_diagonal(modmat, 0)
            it = np.ma.masked_array(np.ones((n,)), False)
            mod_asgn_iter = mod_asgn.copy()
            while np.any(it):  # do some iterative fine tuning
                # this line is linear algebra voodoo
                q_iter = qmax - 4 * mod_asgn_iter * \
                    (np.dot(modmat, mod_asgn_iter))
                qmax = np.max(q_iter * it)
                imax = np.argmax(q_iter * it)
                # imax, = np.where(q_iter == qmax)
                # if len(imax) > 1:
                #     imax = imax[0]
                # does switching increase modularity?
                mod_asgn_iter[imax] *= -1
                it[imax] = np.ma.masked
                if qmax > q:
                    q = qmax
                    mod_asgn = mod_asgn_iter
            if np.abs(np.sum(mod_asgn)) == n:  # iteration yielded null module
                modules.append(np.array(module).tolist())
                return
            else:
                mod1 = module[np.where(mod_asgn == 1)]
                mod2 = module[np.where(mod_asgn == -1)]

                recur(mod1)
                recur(mod2)
        else:  # change in modularity was negative or 0
            modules.append(np.array(module).tolist())

    # adjustment to one-based indexing occurs in ls2ci
    if kci is None:
        recur(init_mod)
        ci = ls2ci(modules)
    else:
        ci = kci
    s = np.tile(ci, (n, 1))
    q = np.sum(np.logical_not(s - s.T) * B / m)
    return ci, q

def ls2ci(ls, zeroindexed=False):
    """
    Convert from a 2D python list of modules to a community index vector.
    The list is a pure python list, not requiring numpy.
    Parameters
    ----------
    ls : listof(list)
        pure python list with lowest value zero-indexed
        (regardless of value of zeroindexed parameter)
    zeroindexed : bool
        If True, ci uses zero-indexing (lowest value is 0). Defaults to False.
    Returns
    -------
    ci : Nx1 :obj:`numpy.ndarray`
        community index vector
    """
    if ls is None or np.size(ls) == 0:
        return ()  # list is empty
    nr_indices = sum(map(len, ls))
    ci = np.zeros((nr_indices,), dtype=int)
    z = int(not zeroindexed)
    for i, x in enumerate(ls):
        for j, y in enumerate(ls[i]):
            ci[ls[i][j]] = i + z
    return ci


import requests

city = input("City???: ")
api_url = 'http://api.openweathermap.org/data/2.5/weather'
params = {
    'q': city,
    'appid': 'API key',  # API key must be here from https://openweathermap.org
    'units': 'metric',
    'lang': 'ru'
}

res = requests.get(api_url, params=params)
# print(res.status_code)
# print(res.headers["Content-Type"])
# print(res.json())
data = res.json()
temp = int(data["main"]["temp"])
print(f"Current temperature in {city} is {temp} C\N{DEGREE SIGN}")


from django import forms
from django.contrib.auth.forms import UserCreationForm
from contester.models import *

# Create your forms here.

class NewUserForm(UserCreationForm):
    email = forms.EmailField(required=True)

    class Meta:
        model = Account
        fields = ("username", "email", "password1", "password2")

    def save(self, commit=True):
        user = super(NewUserForm, self).save(commit=False)
        user.email = self.cleaned_data['email']
        if commit:
            user.save()
        return user

class DocumentForm(forms.Form):
    docfile = forms.FileField(
        label='Select a file',
        help_text='max. 1 megabytes'
    )

class SuggestingTaskForm(forms.Form):
    CHOICES = (('Easy', 'Easy'), ('Medium', 'Medium'), ('Hard', 'Hard'),)
    title = forms.CharField(max_length=100)
    difficulty = forms.ChoiceField(choices=CHOICES)
    source = forms.CharField(max_length=100)
    isOriginal = forms.CheckboxInput()
    description = forms.TextInput()
    solution = forms.TextInput()

class CreateTaskForm(forms.Form):
    title = forms.CharField(max_length=100)
    description = forms.TextInput()
    input_description = forms.TextInput()
    output_description = forms.TextInput()
    CHOICES = (('Easy', 'Easy'), ('Medium', 'Medium'), ('Hard', 'Hard'),)
    complexity = forms.ChoiceField(choices=CHOICES)
    topic = forms.CharField(max_length=100)
    points = forms.NumberInput()


import copy
import logging
import os
from typing import Dict, Tuple, Union

import numpy as np
import torch
import torchmetrics

from ludwig.combiners.combiners import get_combiner_class
from ludwig.constants import MODEL_ECD, TYPE
from ludwig.globals import MODEL_WEIGHTS_FILE_NAME
from ludwig.models.base import BaseModel
from ludwig.schema.utils import load_config_with_kwargs
from ludwig.utils import output_feature_utils
from ludwig.utils.data_utils import clear_data_cache
from ludwig.utils.torch_utils import get_torch_device

logger = logging.getLogger(__name__)

class ECD(BaseModel):
    @staticmethod
    def type() -> str:
        return MODEL_ECD

    def __init__(
        self,
        input_features,
        combiner,
        output_features,
        random_seed=None,
        **_kwargs,
    ):
        self._input_features_def = copy.deepcopy(input_features)
        self._combiner_def = copy.deepcopy(combiner)
        self._output_features_def = copy.deepcopy(output_features)

        self._random_seed = random_seed

        super().__init__(random_seed=self._random_seed)

        # ================ Inputs ================
        try:
            self.input_features.update(self.build_inputs(self._input_features_def))
        except KeyError as e:
            raise KeyError(
                f"An input feature has a name that conflicts with a class attribute of torch's ModuleDict: {e}"
            )

        # ================ Combiner ================
        logger.debug(f"Combiner {self._combiner_def[TYPE]}")
        combiner_class = get_combiner_class(self._combiner_def[TYPE])
        config, kwargs = load_config_with_kwargs(
            combiner_class.get_schema_cls(),
            self._combiner_def,
        )
        self.combiner = combiner_class(input_features=self.input_features, config=config, **kwargs)

        # ================ Outputs ================
        self.output_features.update(self.build_outputs(self._output_features_def, self.combiner))

        # ================ Combined loss metric ================
        self.eval_loss_metric = torchmetrics.MeanMetric()
        self.eval_additional_losses_metrics = torchmetrics.MeanMetric()

        # After constructing all layers, clear the cache to free up memory
        clear_data_cache()

    def encode(
        self,
        inputs: Union[
            Dict[str, torch.Tensor], Dict[str, np.ndarray], Tuple[Dict[str, torch.Tensor], Dict[str, torch.Tensor]]
        ],
    ):
        # Convert inputs to tensors.
        for input_feature_name, input_values in inputs.items():
            if not isinstance(input_values, torch.Tensor):
                inputs[input_feature_name] = torch.from_numpy(input_values)
            else:
                inputs[input_feature_name] = input_values

        encoder_outputs = {}
        for input_feature_name, input_values in inputs.items():
            encoder = self.input_features[input_feature_name]
            encoder_output = encoder(input_values)
            encoder_outputs[input_feature_name] = encoder_output

        return encoder_outputs

    def combine(self, encoder_outputs):
        return self.combiner(encoder_outputs)

    def decode(self, combiner_outputs, targets, mask):
        # Invoke output features.
        output_logits = {}
        output_last_hidden = {}
        for output_feature_name, output_feature in self.output_features.items():
            # Use the presence or absence of targets to signal training or prediction.
            target = targets[output_feature_name] if targets is not None else None
            decoder_outputs = output_feature(combiner_outputs, output_last_hidden, mask=mask, target=target)

            # Add decoder outputs to overall output dictionary.
            for decoder_output_name, tensor in decoder_outputs.items():
                output_feature_utils.set_output_feature_tensor(
                    output_logits, output_feature_name, decoder_output_name, tensor
                )

            # Save the hidden state of the output feature (for feature dependencies).
            output_last_hidden[output_feature_name] = decoder_outputs["last_hidden"]
        return output_logits

    def forward(
        self,
        inputs: Union[
            Dict[str, torch.Tensor], Dict[str, np.ndarray], Tuple[Dict[str, torch.Tensor], Dict[str, torch.Tensor]]
        ],
        mask=None,
    ) -> Dict[str, torch.Tensor]:
        """Forward pass of the model.

        Args:
            inputs: Inputs to the model. Can be a dictionary of input names to
                input tensors or a tuple of (inputs, targets) where inputs is
                a dictionary of input names to input tensors and targets is a
                dictionary of target names to target tensors.
            mask: A mask for the inputs.

        Returns:
            A dictionary of output {feature name}::{tensor_name} -> output tensor.
        """

        if isinstance(inputs, tuple):
            inputs, targets = inputs
            # Convert targets to tensors.
            for target_feature_name, target_value in targets.items():
                if not isinstance(target_value, torch.Tensor):
                    targets[target_feature_name] = torch.from_numpy(target_value)
                else:
                    targets[target_feature_name] = target_value
        else:
            targets = None

        assert list(inputs.keys()) == self.input_features.keys()

        encoder_outputs = self.encode(inputs)
        combiner_outputs = self.combine(encoder_outputs)
        return self.decode(combiner_outputs, targets, mask)

    def save(self, save_path):
        """Saves the model to the given path."""
        weights_save_path = os.path.join(save_path, MODEL_WEIGHTS_FILE_NAME)
        torch.save(self.state_dict(), weights_save_path)

    def load(self, save_path):
        """Loads the model from the given path."""
        weights_save_path = os.path.join(save_path, MODEL_WEIGHTS_FILE_NAME)
        device = torch.device(get_torch_device())
        self.load_state_dict(torch.load(weights_save_path, map_location=device))

    def get_args(self):
        """Returns init arguments for constructing this model."""
        return (self._input_features_df, self._combiner_def, self._output_features_df, self._random_seed)


import cv2
import os
import numpy as np
from keras.models import load_model
from keras.preprocessing.image import array_to_img
import matplotlib.pyplot as plt
import csv

def batch_from_video(video):
	#all frames from the video extracted
	video_name = video.split('/')[1]
	video_number = video_name.split('.')[0]
	frame_array = np.empty((0,64,64,1))
	video_object = cv2.VideoCapture(video)
	success = 1
	while success:
		success, image = video_object.read()
		if image is not None:
			image = cv2.resize(image,(64,64))
			image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
			image = image.reshape((1,64,64,1))
			frame_array = np.append(frame_array, image,axis = 0)
	#dictionary for good frame checking
	gd_dictionary = {}
	csv_path = 'all-vids.csv'
	with open(csv_path) as csv_object:
		csv_file = csv.reader(csv_object,delimiter = ',')
		for row in csv_file:
			if row[0] not in gd_dictionary.keys():
				gd_dictionary[row[0]] = [i for i in range(int(row[1]),int(row[2])+1)]
			else:
				for i in range(int(row[1]),int(row[2])+1):
					gd_dictionary[row[0]].append(i)

	concatenated_array = np.empty((0,64,64,3))
	labels = np.empty((0,1))

	for index in range(1,(frame_array.shape[0]-2)):
		#concatenate three consecutive images. First and last image thrown away
		new_image = np.concatenate(
			(frame_array[index-1],frame_array[index],frame_array[index+1]),axis = 2)
		concatenated_array = np.append(concatenated_array,[new_image],axis = 0)

		#get label
		if index in gd_dictionary[video_number]:
			labels = np.append(labels,np.ones((1,1)),axis = 0)
		else:
			labels = np.append(labels,np.zeros((1,1)),axis = 0)

	return concatenated_array

batch = batch_from_video('videos/43.mp4')
for i in range(batch.shape[0]):
	image = array_to_img(batch[i])
	plt.imsave('%d.png'%i,image)


#! /usr/bin/env python2
# -*- coding: utf-8 -*-
import ROOT
ROOT.gSystem.Load("libTreeObjects.so")
import argparse
from multiplot import *
from treeFunctions import *
import Styles
import ratios
Styles.tdrStyle()
import os

import ConfigParser
axisConf = ConfigParser.SafeConfigParser()
axisConf.read("axis.cfg")

if __name__ == "__main__":
	arguments = argparse.ArgumentParser( description="Simple EWK" )
	arguments.add_argument( "--plot", default="photon.pt" )
	arguments.add_argument( "--input", default="slimEWK_V01.12_tree.root" )
	arguments.add_argument( "--savePrefix", default="new" )
	opts = arguments.parse_args()

	ROOT.gROOT.SetBatch()

	slimFileName = opts.input.replace( os.path.basename(opts.input), "slim"+os.path.basename(opts.input))

	genE = Dataset( opts.input, "susyTree", "Max$(abs(genElectron.eta)) < 1.4442 && @genElectron.size() > 0", "e_{gen}", 1 )
	recE = Dataset( slimFileName, "photonElectronTree", "photon.genInformation==1", "e", 2 )
	gamma = Dataset( slimFileName, "photonTree", "photon.genInformation==1", "#gamma", 3 )

	multihisto = Multihisto()

	label, unit = readAxisConf( opts.plot, axisConf)

	histForRatio = {}

	for dataset in [genE, recE, gamma]:
		if dataset.label == "e_{gen}":
			try:
				plot = opts.plot.replace("photon","genElectron")
			except:
				plot = opts.plot
		else:
			plot = opts.plot

		hist = createHistoFromTree( dataset.tree, plot, "weight*(%s)"%(dataset.additionalCut), firstBin=10, lastBin=200 )
		hist.SetTitle(";%s%s;Entries"%(label,unit))
		hist.SetLineColor( dataset.color )
		hist.SetLineWidth(2)
		multihisto.addHisto( hist, dataset.label )
		histForRatio[dataset.label] = hist

	can = ROOT.TCanvas()

	hPad = ROOT.TPad("hPad", "Histogram", 0, 0, 1, 1)
	#hPad = ROOT.TPad("hPad", "Histogram", 0, 0.2, 1, 1)
	hPad.cd()
	multihisto.Draw("hist")

	ratioPad = ROOT.TPad("ratioPad", "Ratio", 0, 0, 1, 0.2)
	ratioPad.cd()
	ratioPad.SetLogy(0)
	ratioGraph = ratios.RatioGraph(histForRatio["e"], histForRatio["e_{gen}"])
	ratioGraph.draw(ROOT.gPad, yMin=0.5, yMax=1.5, adaptiveBinning=False, errors="x")
	ratioGraph.hAxis.SetYTitle( "e/e_{gen}")
	ratioGraph.graph.Draw("same p")

	can.cd()
	hPad.Draw()
	#ratioPad.Draw()
	can.SaveAs("plots/%s_%s_%s.pdf"%(opts.input[0:5],opts.plot.replace(".",""),opts.savePrefix))
	del can


import pandas as pd
import json
import plotly
from flask import render_template, request, g
from flask_login import current_user
from flask_babel import _, lazy_gettext as _l
from ... import main
from .... import db
from ....models import Category, Diary, OptionalCategory

@main.route('/plot_option', methods=['GET'])
def plot_option():
    args_category = int(request.args.get('optional_category'))
    optional_category_name = ''
    for optional_category in g.optional_category_list:
        if optional_category[0] == args_category:
            optional_category_name = optional_category[1]
            break
    query = Diary.query.filter_by(
        user=current_user._get_current_object(),
        category=Category.OPTION,
        optional_category=args_category)
    df = pd.read_sql(query.statement, db.engine)

    df['date'] = pd.to_datetime(df['date'])
    df = df.set_index(df['date'])
    df = df.drop(columns=['date', 'id', 'user_id']).resample(rule='MS').count()

    graphs = [
        dict(
            data=[
                dict(
                    x=df.index,
                    # x=ts,
                    y=df.note,
                    type='bar',
                )
            ],
            layout=dict(
                title=_('The number of diaries by month'),
                xaxis=dict(
                    dtick='M1',
                ),
                yaxis=dict(
                    dtick='5',
                    title=_('The number of diaries'),
                ),
            ),
        )
    ]

    # Add "ids" to each of the graphs to pass up to the client
    # for templating
    ids = ['graph-{}'.format(i) for i, _ in enumerate(graphs)]

    # Convert the figures to JSON
    # PlotlyJSONEncoder appropriately converts pandas, datetime, etc
    # objects to their JSON equivalents
    graphJSON = json.dumps(graphs, cls=plotly.utils.PlotlyJSONEncoder)

    return render_template('plot/plot_option.html',
                           ids=ids,
                           graphJSON=graphJSON,
                           optional_category_name=optional_category_name)


from email.mime.text import MIMEText
from email.mime.image import MIMEImage
import smtplib
import mimetypes
from email.mime.base import MIMEBase
from email import encoders
from email.mime.multipart import MIMEMultipart

def send_mail(to_list, subject, body, attachments=None, error=False, filename_list=None,
              cc=None, reply_to=None):
    if not isinstance(to_list,list):
        to_list = [to_list]

    if cc:
        if not isinstance(cc, list):
            cc = [cc]

    msg = MIMEMultipart()

    to_list_string = ''
    for email in to_list:
        to_list_string += ', '
        to_list_string +=email

    cc_string = ''
    if cc:
        for email in cc:
            cc_string += ', '
            cc_string += email
    to_list_string = to_list_string.lstrip(', ')

    msg['Subject'] = subject
    msg['to'] = to_list_string
    msg['cc'] = cc_string
    text = MIMEText('%s' % body, 'html' )
    text['Content-Type'] = 'text/html; charset=UTF8'

    if reply_to:
        msg.add_header('reply_to', reply_to)
    msg.attach(text)

    if attachments:
        attach_files(msg, attachments, filename_list)

    s = smtplib.SMTP('smtp.gmail.com', 587)
    s.connect("smtp.gmail.com", 587)

    s.ehlo()
    s.starttls()
    s.ehlo()

    s.login("tkh2111.automation@gmail.com", "tmp1234!")

    if cc:
        for email in cc:
            to_list.append(email)
    s.sendmail("tkh2111.automation@gmail.com", to_list, msg.as_string())

    s.quit()


#Name: Chris Demundo
#Umich ID: cdemundo

# these should be the only imports you need
import sys
import sqlite3

# write your code here
# usage should be
#  python3 part2.py customers
#  python3 part2.py employees
#  python3 part2.py orders cust=<customer id>
#  python3 part2.py orders emp=<employee last name>

db_name = './Northwind_small.sqlite'

def main():

	correct_use = "Correct usage: part2.py customers OR part2py employees OR part2.py orders cust=CustID OR part2.py orders emp=EmpID"

	conn = MakeConnection()
	c = conn.cursor()

	if len(sys.argv) < 2:
		print(correct_use)
	else:
		if(sys.argv[1] == 'customers'):
			rows = GetCustomers(c)
			print("ID       Customer Name")
			for row in rows:
				print("{ID}     {CustName}".format(ID=row[0], CustName=row[1]))
		elif(sys.argv[1] == 'employees'):
			rows = GetEmployees(c)
			print("ID    Employee Name")
			for row in rows:
				print('{ID}     {FirstName} {LastName}'.format(ID=row[0], FirstName = row[1], LastName = row[2]))
		elif(sys.argv[1] == 'orders'):
			if len(sys.argv) > 2:
				if "cust" in sys.argv[2]:
					rows = GetOrderDatesByCust(c, sys.argv[2].split("=")[1])
					print("Order dates")
					for row in rows:
						print(row[0])
				elif "emp" in sys.argv[2]:
					rows = GetOrderDatesByEmp(c, sys.argv[2].split("=")[1])
					print("Order Dates")
					for row in rows:
						print(row[0])
				else:
					print(correct_use)
			else:
				print(correct_use)
		else:
			print(correct_use)

def MakeConnection():
	try:
		conn = sqlite3.connect(db_name)
		return conn
	except:
		print("DB Connection not working")

	return none

def GetCustomers(c):
	c.execute("SELECT ID, CompanyName FROM Customer")
	return c.fetchall()

def GetEmployees(c):
	c.execute("SELECT ID, FirstName, Lastname FROM Employee")
	return c.fetchall()

def GetOrderDatesByCust(c, customerID):
	c.execute("SELECT OrderDate FROM `Order` WHERE CustomerID = '{cid}'".format(cid=customerID))
	return c.fetchall()

def GetOrderDatesByEmp(c, empLastName):
	select_statement = "SELECT OrderDate FROM `Order` INNER JOIN Employee on Employee.Id = `Order`.EmployeeId WHERE Employee.LastName = '{eLN}'".format(eLN=empLastName)

	c.execute(select_statement)
	return c.fetchall()

if __name__ == "__main__":
	main()


import matplotlib.pyplot as plt

'''
y1 = [1,2,3,4,5]
y2 = [2,2,2,2,2]
y3 = [3,1,1,4,5]
x = [13,27,30,45,50]
def origin():
    plt.figure()
    plt.title('Accuracy per senteces')
    plt.xlabel('number of sentences seen /100')
    plt.plot(x, y1, 'r--', x, y2, 'b-', x, y3, 'g:')
    plt.legend(['y1', 'y2', 'eee'])
    plt.show()

Y = [y1, y2, y3]
X = [x, x, x]
name = ['a', 'b', 'c']
plt.figure()
plt.title('Accuracy per senteces')
plt.xlabel('number of sentences seen /100')
style = ['r--', 'b-.', 'g:']
for a, b, d in zip(X, Y, style):
    plt.plot(a, b, d)
plt.legend([n for n in name])
plt.show()
'''
path = 'new model out.txt'
with open(path, 'r') as f:
    loss, accu = [], []
    for line in f:
        line = line.split()
        loss.append(int(float(line[1])))
        accu.append(float(line[2]))

plt.figure(1)
plt.title('loss')
plt.xlabel('Iteration/5000')
plt.ylabel('loss')
plt.plot(loss)
plt.show()

plt.figure(2)
plt.title('Accuracy')
plt.xlabel('Iteration/5000')
plt.ylabel('Accuracy')
plt.plot(accu)
plt.show()


import numpy as np
import matplotlib.pyplot as plt
import os
from numpy import genfromtxt

################################
################################ data
################################

os.chdir(os.path.realpath(''))

################################
################################ Parameter definitions
################################

################################ within-host dynamics
R0 = 7.69
k = 5
delta = 0.595
c = 10.
p = 112000
mu = 0.001
B = mu*p/delta
T = 4000

beta = c*delta*R0/((mu*p-delta*R0)*T)

################################ drug efficacy
eps1 = 0.5
#eps2 = 0.25
eps3 = 0.75
V0 = np.arange(1,1000,1)

################################
################################ Import Data
################################

################################ neutral
my_data = genfromtxt('surv_HN_eps_0_sc_0_model_1.txt', delimiter=',')

data1 = []
eps_data1 = []
for i in range(len(my_data)):
    data1.append(float(my_data[i][1]))
    eps_data1.append(float(my_data[i][0]))

################################ Reducing burst size, eps = 0.2
#my_data = genfromtxt('surv_HN_eps_025_sc_0_model_1.txt', delimiter=',')

#data2 = []
#eps_data2 = []
#for i in range(len(my_data)):
#    data2.append(float(my_data[i][1]))
#    eps_data2.append(float(my_data[i][0]))

################################ Reducing infectivity, eps = 0.2
#my_data = genfromtxt('surv_HN_eps_025_sc_1_model_1.txt', delimiter=',')

#data3 = []
#eps_data3 = []
#for i in range(len(my_data)):
#    data3.append(float(my_data[i][1]))
#    eps_data3.append(float(my_data[i][0]))

################################ Reducing burst size, eps = 0.5
my_data = genfromtxt('surv_HN_eps_05_sc_0_model_1.txt', delimiter=',')

data4 = []
eps_data4 = []
for i in range(len(my_data)):
    data4.append(float(my_data[i][1]))
    eps_data4.append(float(my_data[i][0]))

################################ Reducing infectivity, eps = 0.5
my_data = genfromtxt('surv_HN_eps_05_sc_1_model_1.txt', delimiter=',')

data5 = []
eps_data5 = []
for i in range(len(my_data)):
    data5.append(float(my_data[i][1]))
    eps_data5.append(float(my_data[i][0]))

################################ Reducing burst size, eps = 0.8
my_data = genfromtxt('surv_HN_eps_075_sc_0_model_1.txt', delimiter=',')

data6 = []
eps_data6 = []
for i in range(len(my_data)):
    data6.append(float(my_data[i][1]))
    eps_data6.append(float(my_data[i][0]))

################################ Reducing infectivity, eps = 0.8
my_data = genfromtxt('surv_HN_eps_075_sc_1_model_1.txt', delimiter=',')

data7 = []
eps_data7 = []
for i in range(len(my_data)):
    data7.append(float(my_data[i][1]))
    eps_data7.append(float(my_data[i][0]))

################################
################################ Surv prob of virus
################################

################################ Reducing burst size
surv1 = []
#surv12 = []
surv13 = []
survneu = []

for i in range(len(V0)):
    p2 = (1-eps1)*p*mu
#    p21 = (1-eps2)*p
    p22 = (1-eps3)*p*mu

    surv1.append(max(0,1-(c/(c+beta*T)+delta/p2)**V0[i]))
#    surv12.append(max(0,1-(c/(c+beta*T)+delta/p21)**V0[i]))
    surv13.append(max(0,1-(c/(c+beta*T)+delta/p22)**V0[i]))
    survneu.append(max(0,1-(c/(c+beta*T)+delta/(mu*p))**V0[i]))

################################ Reducing infectivity
surv2 = []
#surv22 = []
surv23 = []

for i in range(len(V0)):
    beta2 = (1-eps1)*beta
#    beta21 = (1-eps2)*beta
    beta22 = (1-eps3)*beta

    surv2.append(max(0,1-(c/(c+beta2*T)+delta/(mu*p))**V0[i]))
#    surv22.append(max(0,1-(c/(c+beta21*T)+delta/p)**V0[i]))
    surv23.append(max(0,1-(c/(c+beta22*T)+delta/(mu*p))**V0[i]))

################################
################################ Plot surv prob
################################
plt.semilogx(V0,np.array(surv1)/np.array(survneu),linewidth=3,color='C0',linestyle='solid')
#plt.plot(eps_data2,np.array(data2)/np.array(data1),'o',color='C0',markersize=10)
plt.plot(V0,np.array(surv2)/np.array(survneu),linewidth=3,color='C1',linestyle='solid')
#plt.plot(eps_data3,np.array(data3)/np.array(data1),'o',color='C1',markersize=10)
#plt.plot(V0,np.array(surv12)/np.array(survneu),linewidth=3,color='C0',linestyle='solid')
plt.plot(eps_data4,np.array(data4)/np.array(data1),'o',color='C0',markersize=10)
#plt.plot(V0,np.array(surv22)/np.array(survneu),linewidth=3,color='C1',linestyle='solid')
plt.plot(eps_data5,np.array(data5)/np.array(data1),'o',color='C1',markersize=10)
plt.plot(V0,np.array(surv13)/np.array(survneu),linewidth=3,color='C0',linestyle='dashed')
plt.plot(eps_data6,np.array(data6)/np.array(data1),'o',color='C0',markersize=10)
plt.plot(V0,np.array(surv23)/np.array(survneu),linewidth=3,color='C1',linestyle='dashed')
plt.plot(eps_data7,np.array(data7)/np.array(data1),'o',color='C1',markersize=10)
#plt.plot(V0,survneu,linewidth=2,color='black')
#plt.plot(eps_data1,data1,'o',color='black',markersize=10)

plt.ylim((-0.05,1.05))
plt.tick_params(axis='both', which='major', labelsize=15, width=1, length=10)
plt.tick_params(axis='both', which='minor', labelsize=10, width=1, length=5)
plt.show()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
"""A CLI (diff) for the ramses_rf library - log diff utility.

ramses_rf is used to parse/process Honeywell's RAMSES-II packets.
"""

import argparse
import os
import re
from collections import deque, namedtuple
from datetime import datetime as dt
from datetime import timedelta as td

DEBUG_ADDR = "0.0.0.0"
DEBUG_PORT = 5678

RSSI_REGEXP = re.compile(r"(-{3}|\d{3})")
PKT_LINE = namedtuple("Packet", ["dtm", "rssi", "packet", "line"])

DATETIME_FORMAT = "YYYY-MM-DD HH:MM:SS.ssssss"
DATETIME_LENGTH = len(DATETIME_FORMAT)

# how far to look ahead for a matching packet - higher values will slow down
DEFAULT_LOOKAHEAD_SECS = 1  # may need to increase this if logs from diff systems
DEFAULT_LOOKAHEAD_PKTS = 5  # <3 has increased potential for false positives

# packets to print before/after a stall
DEFAULT_PKTS_BEFORE = 2
DEFAULT_PKTS_AFTER = 2

# exclude a stall unless they exceed the following
STALL_LIMIT_SECS = 30  # 30 is a useful minimum, max should be sync_cycle time, ~180
STALL_LIMIT_PKTS = 10  # <7 has increased potential for false positives

if False:
    BOTH, LEFT, RITE = "===", "<<<", ">>>"
else:
    BOTH, LEFT, RITE = " = ", "<  ", "  >"

def _parse_args():
    def extant_file(value):
        """Confirm value is the name of a existant file."""
        if not os.path.exists(value):
            raise argparse.ArgumentTypeError(f"{value} does not exist")
        return value

    def natural_int(value):
        """Confirm value is a positive integer."""
        i_value = int(value)
        if i_value < 1:  # or value != i_value:
            raise argparse.ArgumentTypeError(f"{value} is not a positive integer")
        return i_value

    def pos_int(value):
        """Confirm value is a non-negative integer."""
        i_value = int(value)
        if i_value < 0:  # or value is not i_value:
            raise argparse.ArgumentTypeError(f"{value} is not a non-negative integer")
        return i_value

    def dt_td(value):
        """Confirm value is a positive td (in seconds)."""
        f_value = float(value)
        if f_value < 0:  # or value is not f_value:
            raise argparse.ArgumentTypeError(f"{value} is not a non-negative float")
        return td(seconds=f_value)

    parser = argparse.ArgumentParser()

    group = parser.add_argument_group(title="File names")
    group.add_argument(
        "hgi80_log", type=extant_file, help=f"({LEFT}) reference packet log"
    )
    group.add_argument(
        "evofw_log", type=extant_file, help=f"({RITE}) packet log to test"
    )

    group = parser.add_argument_group(title="Context control")
    group.add_argument(
        "-B",
        "--before",
        default=DEFAULT_PKTS_BEFORE,
        type=pos_int,
        help="matching lines to print before each block",
    )
    group.add_argument(
        "-A",
        "--after",
        default=DEFAULT_PKTS_AFTER,
        type=pos_int,
        help="matching lines to print after each block",
    )
    group.add_argument(
        "-s",
        "--seconds",
        default=td(seconds=DEFAULT_LOOKAHEAD_SECS),
        type=dt_td,
        help="minimum lookahead in seconds (float)",
    )
    group.add_argument(
        "-p",
        "--packets",
        default=DEFAULT_LOOKAHEAD_PKTS,
        type=natural_int,
        help="minimum lookahead in packets (int), recommended >=3",
    )

    group = parser.add_argument_group(title="Stall detection")
    group.add_argument(
        "-D",
        "--pause-duration",
        default=td(seconds=STALL_LIMIT_SECS),
        type=dt_td,
        help="minimum duration of pauses in seconds (float)",
    )
    group.add_argument(
        "-L",
        "--pause-length",
        default=STALL_LIMIT_PKTS,
        type=natural_int,
        help="minimum length of pauses in packets (int), recommended >=7",
    )

    group = parser.add_argument_group(title="Debug options")
    group.add_argument(
        "-z", "--debug_mode", action="count", default=0, help="1=N/A, 2=enable, 3=wait"
    )

    return parser.parse_args()

def main():
    """Main loop."""
    args = _parse_args()

    if args.debug_mode == 1:
        # print(f"Debugging not enabled, additional logging enabled.")
        pass

    elif args.debug_mode > 1:
        import ptvsd

        print(f"Debugging is enabled, listening on: {DEBUG_ADDR}:{DEBUG_PORT}.")
        ptvsd.enable_attach(address=(DEBUG_ADDR, DEBUG_PORT))

        if args.debug_mode > 2:
            print("Execution paused, waiting for debugger to attach...")
            ptvsd.wait_for_attach()
            print("Debugger is attached, continuing execution.")

    compare(args)

def compare(config) -> dict:

    MICROSECONDS = td(microseconds=1)
    buffer = {
        "packets": deque(),
        "run_length": 0,
        "making_block": False,
        "pause_len": 0,
        "making_pause": False,
    }

    def time_diff(pkt_1, pkt_2) -> float:
        td = (pkt_2.dtm - pkt_1.dtm) / MICROSECONDS
        return 999999 if td > 999999 else max(td, -999999)

    def pkts_match(pkt, window, idx) -> bool:
        matched = pkt.packet == window[idx].packet
        return matched if not matched else time_diff(pkt, window[idx]) < 999999

    def buffer_print(buffer, num_lines):
        for _ in range(num_lines):
            print(buffer["packets"].popleft())

    def parse_line(raw_line: str) -> namedtuple:
        """Parse a line from a packet log into a dtm (detetime), payload (str) tuple.

        Assumes lines have a datetime stamp, e.g.: 'YYYY-MM-DD HH:MM:SS.ssssss'
        """
        line = raw_line.strip()  # if line != raw_line ("" != "/r/n"): # then, at EOF?
        if not line:
            return PKT_LINE(None, None, None, line)

        dtm = dt.fromisoformat(line[:DATETIME_LENGTH])
        pkt = line[DATETIME_LENGTH + 1 :]

        if pkt[:1] in ["#", "*"]:  # or not RSSI_REGEXP.match(pkt[:3]):
            return PKT_LINE(dtm, None, pkt, line)  # a pure diagnostic line
        return PKT_LINE(dtm, pkt[:3], pkt[4:], line)

    def slide_pkt_window(fh, pkt_window, until=None, min_length=config.packets):
        """Populate the window with packet log lines until at EOF.

        Assumes no blank lines in the middle of the file (thus no empty elements).
        """
        assert until is not None or min_length is not None

        if len(pkt_window) == 0:
            pkt_window.append(parse_line(fh.readline()))
        while pkt_window[-1].dtm is not None and len(pkt_window) < min_length:
            pkt_window.append(parse_line(fh.readline()))
        if until is not None:
            while pkt_window[-1].dtm is not None and pkt_window[-1].dtm < until:
                pkt_window.append(parse_line(fh.readline()))
        if pkt_window[-1].dtm is None:
            pkt_window.pop()

    def buffer_append(buffer, summary, diff, pkt, pkt2=None):
        """Populate the buffer, maintain counters and print any lines as able.

        Also includes code to track pauses.
        """

        def pause_began(buffer, summary, pkt):
            # idx = min(config.before, len(buffer["packets"]) - 1)
            # print("*** BEGAN:", buffer["packets"][idx])
            summary["pause_began"] = pkt

        def pause_ended(buffer, summary, pkt=None):
            # print("*** ENDED:", buffer["packets"][0])
            # duration = (
            #     dt.fromisoformat(buffer["packets"][0][4 : 4 + DATETIME_LENGTH])
            #     - summary["pause_began"].dtm
            # )
            duration = summary["last_pkt_1"].dtm - summary["pause_began"].dtm
            if (
                duration > config.pause_duration
                or buffer["pause_len"] > config.pause_length
            ):
                summary["pauses"].append({summary["pause_began"]: duration})

        td = time_diff(pkt, pkt2) if diff == BOTH else 0
        summary["dt_pos" if td > 0 else "dt_neg"] += td

        buffer["packets"].append(f"{diff} {pkt.dtm} ({td:+7.0f}) {pkt.packet}")
        buffer["run_length"] += 1

        if diff == LEFT:
            if not buffer["making_pause"]:
                buffer.update({"pause_len": 0, "making_pause": True})
                pause_began(buffer, summary, pkt)
            buffer["pause_len"] += 1

        elif buffer["making_pause"]:
            buffer["making_pause"] = False
            if buffer["pause_len"] > 3:
                pause_ended(buffer, summary)

        if diff in (LEFT, RITE):  # can print/clear the entire buffer
            # if not buffer["making_block"]:  # beginning of a block
            buffer_print(buffer, len(buffer["packets"]))
            buffer.update({"run_length": 0, "making_block": True})
            return

        if buffer["making_block"]:
            if buffer["run_length"] > config.before + config.after:  # end of a block
                buffer_print(buffer, config.after)
                buffer.update({"run_length": config.before, "making_block": False})
                buffer["packets"].popleft()  # the first BOTH pkt between AFTER & BEFORE
                print()

        elif buffer["run_length"] > config.before:
            buffer.update({"run_length": config.before, "making_block": False})
            buffer["packets"].popleft()  # an unwanted BOTH pkt before BEFORE

    def buffer_flush(buffer):
        if buffer["making_block"]:
            # block_ended(buffer, summary)

            buffer_print(buffer, min(config.after, buffer["run_length"]))
            print()

    def print_summary(dt_pos, dt_neg, num_pkts, num_1, num_2, warning, pauses, *args):
        num_total = sum([num_pkts, num_1, num_2])
        print(f"Of the {num_total} valid packets:")
        print(
            " - average time delta of matched packets:",
            f"{(dt_pos - dt_neg) / num_pkts:0.0f} "
            f"(+{dt_pos / num_pkts:0.0f}, {dt_neg / num_pkts:0.0f}) nanosecs",
        )
        print(
            " - there were:",
            f"{num_1} ({LEFT.strip()}, {num_1 / num_total * 100:0.2f}%), "
            f"{num_2} ({RITE.strip()}, {num_2 / num_total * 100:0.2f}%) "
            "unmatched packets",
        )

        lst = [v.total_seconds() for d in pauses for k, v in d.items()]
        print(
            f"\r\nOf the {len(lst)} pauses >{config.pause_length} packets "
            f"or >{config.pause_duration.total_seconds():0.2f} secs duration:"
        )
        if len(lst):
            print(
                f" - average duration: {sum(lst) / len(lst):.0f} secs; "
                f"maximum: {max(lst):.0f} secs"
            )
        else:
            print(" - average duration: 0 secs; maximum: 0 secs")

        # print()
        # [print(k.line) for d in pauses for k, v in d.items()]

        if warning is True:
            print("\r\nWARNING: The reference packet log is not from a HGI80.")

    SUMMARY_KEYS = ["dt_pos", "dt_neg", "num_pkts", "num_1", "num_2", "warning"]
    summary = {k: 0 for k in SUMMARY_KEYS}
    summary.update({"pauses": [], "last_pkt_1": None})

    pkt_2_window = deque()

    with open(config.hgi80_log) as fh_1, open(config.evofw_log) as fh_2:
        for raw_line in fh_1:
            pkt_1 = parse_line(raw_line)
            if "*" in pkt_1.packet or "#" in pkt_1.packet:
                summary["warning"] = True  # file_1 is evofw3, and not hgi80!

            slide_pkt_window(fh_2, pkt_2_window, until=pkt_1.dtm + config.seconds)

            for idx, pkt_2 in enumerate(list(pkt_2_window)):
                matched = pkts_match(pkt_1, pkt_2_window, idx)
                if matched:
                    for _ in range(idx):  # all pkts before the match
                        buffer_append(buffer, summary, RITE, pkt_2_window[0])
                        if not pkt_2_window[0].packet[:1] == "#":
                            summary["num_2"] += 1
                        pkt_2_window.popleft()

                    buffer_append(buffer, summary, BOTH, pkt_1, pkt_2)
                    summary["num_pkts"] += 1
                    pkt_2_window.popleft()
                    break

            else:
                buffer_append(buffer, summary, LEFT, pkt_1)
                summary["num_1"] += 1
                summary["last_pkt_1"] = pkt_1

        buffer_flush(buffer)

    print_summary(*list(summary.values()))

if __name__ == "__main__":
    main()


import os
from natsort import natsorted

if not os.path.exists('inputs'):
    os.makedirs('inputs')

alignments = natsorted(os.listdir('../alignment/results'))
for i,f in enumerate(alignments):
    open('inputs/{}.in'.format(i), 'w').write('{} {}'.format(i, f))