import numbers
import os

import numpy as np
import pandas as pd

from concurrent.futures import ProcessPoolExecutor
from typing import Union, Optional
from typing_extensions import Literal

from .. import utils, config
from ..core import NeuronList, Dotprops, make_dotprops
from .base import Blaster

__all__ = ['nblast', 'nblast_smart', 'nblast_allbyall', 'sim_to_dist']

fp = os.path.dirname(__file__)
smat_path = os.path.join(fp, 'score_mats')

logger = config.logger

class ScoringFunction:
    """Class representing scoring function."""

    def __init__(self, smat):
        if isinstance(smat, type(None)):
            self.scoring_function = self.pass_through
        elif isinstance(smat, (pd.DataFrame, str)):
            self.parse_matrix(smat)
            self.scoring_function = self.score_lookup
        else:
            raise TypeError

    def __call__(self, dist, dot):
        return self.scoring_function(dist, dot)

    @property
    def max_dist(self):
        """Max distance considered by the scoring matrix.

        Returns ``None`` if pass through.
        """
        if self.scoring_function == self.pass_through:
            return None
        # The last bin is always `np.inf`, so we need the second last bin
        return self.dist_bins[-2]

    def pass_through(self, dist, dot):
        """Pass-through scores if no scoring matrix."""
        return dist * dot

    def score_lookup(self, dist, dot):
        return self.cells[
                          np.digitize(dist, self.dist_bins),
                          np.digitize(dot, self.dot_bins),
                         ]

    def parse_matrix(self, smat):
        """Parse matrix."""
        if isinstance(smat, str):
            smat = pd.read_csv(smat, index_col=0)

        if not isinstance(smat, pd.DataFrame):
            raise TypeError(f'Excepted filepath or DataFrame, got "{type(smat)}"')

        self.cells = smat.to_numpy()

        self.dist_thresholds = [self.parse_interval(s) for s in smat.index]
        # Make sure right bin is open
        self.dist_thresholds[-1] = np.inf
        self.dist_bins = np.array(self.dist_thresholds, float)

        self.dot_thresholds = [self.parse_interval(s) for s in smat.columns]
        # Make sure right bin is open
        self.dot_thresholds[-1] = np.inf
        self.dot_bins = np.array(self.dot_thresholds, float)

    def parse_interval(self, s):
        """Strip brackets and parse right interval.

        Example
        -------
        >>> parse_intervals("(0,0.1]")                          # doctest: +SKIP
        0.1
        """
        return float(s.strip("([])").split(",")[-1])

class NBlaster(Blaster):
    """Implements version 2 of the NBLAST algorithm.

    Please note that some properties are computed on initialization and
    changing parameters (e.g. ``use_alpha``) at a later stage will mess things
    up!

    Parameters
    ----------
    use_alpha :     bool
                    Whether or not to use alpha values for the scoring.
                    If True, the dotproduct of nearest neighbor vectors will
                    be scaled by ``sqrt(alpha1 * alpha2)``.
    normalized :    bool
                    If True, will normalize scores by the best possible score
                    (i.e. self-self) of the query neuron.
    smat :          str | pd.DataFrame
                    Score matrix. If 'auto' (default), will use scoring matrices
                    from FCWB. Same behaviour as in R's nat.nblast
                    implementation. If ``smat=None`` the scores will be
                    generated as the product of the distances and the dotproduct
                    of the vectors of nearest-neighbor pairs.
    limit_dist :    float | "auto" | None
                    Sets the max distance for the nearest neighbor search
                    (`distance_upper_bound`). Typically this should be the
                    highest distance considered by the scoring function. If
                    "auto", will extract that value from the scoring matrix.
    progress :      bool
                    If True, will show a progress bar.

    """

    def __init__(self, use_alpha=False, normalized=True, smat='auto',
                 limit_dist=None, progress=True):
        """Initialize class."""
        super().__init__(progress=progress)
        self.use_alpha = use_alpha
        self.normalized = normalized
        self.desc = "NBlasting"

        if smat == 'auto':
            if self.use_alpha:
                smat = pd.read_csv(f'{smat_path}/smat_alpha_fcwb.csv',
                                   index_col=0)
            else:
                smat = pd.read_csv(f'{smat_path}/smat_fcwb.csv',
                                   index_col=0)

        self.score_fn = ScoringFunction(smat)

        if limit_dist == 'auto':
            self.distance_upper_bound = self.score_fn.max_dist
        else:
            self.distance_upper_bound = limit_dist

    def append(self, dotprops):
        """Append dotprops."""
        if isinstance(dotprops, (NeuronList, list)):
            for n in dotprops:
                self.append(n)
            return

        if not isinstance(dotprops, Dotprops):
            raise ValueError(f'Expected Dotprops, got "{type(dotprops)}"')

        self.neurons.append(dotprops)
        self.ids.append(dotprops.id)
        # Calculate score for self hit
        self.self_hits.append(self.calc_self_hit(dotprops))

    def calc_self_hit(self, dotprops):
        """Non-normalized value for self hit."""
        if not self.use_alpha:
            return len(dotprops.points) * self.score_fn(0, 1.0)
        else:
            dists = np.repeat(0, len(dotprops.points))
            alpha = dotprops.alpha * dotprops.alpha
            dots = np.repeat(1, len(dotprops.points)) * np.sqrt(alpha)
            return self.score_fn(dists, dots).sum()

    def single_query_target(self, q_idx, t_idx, scores='forward'):
        """Query single target against single target."""
        # Take a short-cut if this is a self-self comparison
        if q_idx == t_idx:
            if self.normalized:
                return 1
            return self.self_hits[q_idx]

        # Run nearest-neighbor search for query against target
        data = self.neurons[q_idx].dist_dots(self.neurons[t_idx],
                                             alpha=self.use_alpha,
                                             distance_upper_bound=self.distance_upper_bound)
        if self.use_alpha:
            dists, dots, alpha = data
            dots *= np.sqrt(alpha)
        else:
            dists, dots = data

        scr = self.score_fn(dists, dots).sum()

        # Normalize against best hit
        if self.normalized:
            scr /= self.self_hits[q_idx]

        # For the mean score we also have to produce the reverse score
        if scores in ('mean', 'min', 'max'):
            reverse = self.single_query_target(t_idx, q_idx, scores='forward')
            if scores == 'mean':
                scr = (scr + reverse) / 2
            elif scores == 'min':
                scr = min(scr, reverse)
            elif scores == 'max':
                scr = max(scr, reverse)

        return scr

def nblast_smart(query: Union[Dotprops, NeuronList],
                 target: Optional[str] = None,
                 t: Union[int, float] = 90,
                 criterion: Union[Literal['percentile'],
                                  Literal['score'],
                                  Literal['N']] = 'percentile',
                 scores: Union[Literal['forward'],
                               Literal['mean'],
                               Literal['min'],
                               Literal['max']] = 'forward',
                 return_mask: bool = False,
                 normalized: bool = True,
                 use_alpha: bool = False,
                 smat: Optional[Union[str, pd.DataFrame]] = 'auto',
                 limit_dist: Optional[Union[Literal['auto'], int, float]] = 'auto',
                 n_cores: int = os.cpu_count() // 2,
                 progress: bool = True) -> pd.DataFrame:
    """Smart(er) NBLAST query against target neurons.

    In contrast to :func:`navis.nblast` this function will first run a
    "pre-NBLAST" in which only 10% of the query dotprops' points are used.
    Using those initial scores, we select for each query the highest scoring
    targets and run the full NBLAST only on those query-target pairs (see
    ``t`` and ``criterion`` for fine-tuning).

    Parameters
    ----------
    query :         Dotprops | NeuronList
                    Query neuron(s) to NBLAST against the targets. Neurons
                    should be in microns as NBLAST is optimized for that and
                    have similar sampling resolutions.
    target :        Dotprops | NeuronList, optional
                    Target neuron(s) to NBLAST against. Neurons should be in
                    microns as NBLAST is optimized for that and have
                    similar sampling resolutions. If not provided, will NBLAST
                    queries against themselves.
    t :             int | float
                    Determines for which pairs we will run a full NBLAST. See
                    ``criterion`` parameter for details.
    criterion :     "percentile" | "score" | "N"
                    Criterion for selecting query-target pairs for full NBLAST:

                      - "percentile" runs full NBLAST on the ``t``-th percentile
                      - "score" runs full NBLAST on all scores above ``t``
                      - "N" runs full NBLAST on top ``t`` targets

    return_mask :   bool
                    If True, will also return a boolean mask that shows which
                    scores are based on a full NBLAST and which ones only on
                    the pre-NBLAST.
    scores :        'forward' | 'mean' | 'min' | 'max'
                    Determines the final scores:

                      - 'forward' (default) returns query->target scores
                      - 'mean' returns the mean of query->target and
                        target->query scores
                      - 'min' returns the minium between query->target and
                        target->query scores
                      - 'max' returns the maximum between query->target and
                        target->query scores

    n_cores :       int, optional
                    Max number of cores to use for nblasting. Default is
                    ``os.cpu_count() // 2``. This should ideally be an even
                    number as that allows optimally splitting queries onto
                    individual processes.
    use_alpha :     bool, optional
                    Emphasizes neurons' straight parts (backbone) over parts
                    that have lots of branches.
    normalized :    bool, optional
                    Whether to return normalized NBLAST scores.
    smat :          str | pd.DataFrame
                    Score matrix. If 'auto' (default), will use scoring matrices
                    from FCWB. Same behaviour as in R's nat.nblast
                    implementation. If ``smat=None`` the scores will be
                    generated as the product of the distances and the dotproduct
                    of the vectors of nearest-neighbor pairs.
    limit_dist :    float | "auto" | None
                    Sets the max distance for the nearest neighbor search
                    (`distance_upper_bound`). Typically this should be the
                    highest distance considered by the scoring function. If
                    "auto", will extract that value from the scoring matrix.
    progress :      bool
                    Whether to show progress bars.

    Returns
    -------
    scores :        pandas.DataFrame
                    Matrix with NBLAST scores. Rows are query neurons, columns
                    are targets.
    mask :          np.ndarray
                    Only if ``return_mask=True``: a boolean mask with same shape
                    as ``scores`` that shows which scores are based on a full
                    NBLAST and which ones only on the pre-NBLAST.

    References
    ----------
    Costa M, Manton JD, Ostrovsky AD, Prohaska S, Jefferis GS. NBLAST: Rapid,
    Sensitive Comparison of Neuronal Structure and Construction of Neuron
    Family Databases. Neuron. 2016 Jul 20;91(2):293-311.
    doi: 10.1016/j.neuron.2016.06.012.

    Examples
    --------
    >>> import navis
    >>> nl = navis.example_neurons(n=5)
    >>> nl.units
    <Quantity([8 8 8 8 8], 'nanometer')>
    >>> # Convert to microns
    >>> nl_um = nl * (8 / 1000)
    >>> # Convert to dotprops
    >>> dps = navis.make_dotprops(nl_um)
    >>> # Run a NBLAST where only the top target from the pre-NBLAST is run
    >>> # through a full NBLAST
    >>> scores = navis.nblast_smart(dps[:3], dps[3:], t=1, criterion='N')

    See Also
    --------
    :func:`navis.nblast`
                The conventional full NBLAST.
    :func:`navis.nblast_allbyall`
                A more efficient way than ``nblast(query=x, target=x)``.
    :func:`navis.synblast`
                A synapse-based variant of NBLAST.

    """
    utils.eval_param(criterion, name='criterion',
                     allowed_values=("percentile", "score", "N"))

    if isinstance(target, type(None)):
        target = query

    try:
        t = int(t)
    except BaseException:
        raise TypeError(f'`t` must be (convertable to) integer - got "{type(t)}"')

    if criterion == 'percentile':
        if (t <= 0 or t >= 100):
            raise ValueError('Expected `t` to be integer between 0 and 100 for '
                             f'criterion "percentile", got {t}')
    elif criterion == 'N':
        if (t < 0 or t >= len(target)):
            raise ValueError('`t` must be between 0 and the total number of'
                             f'targets ({len(target)} for criterion "N", '
                             f'got {t}')

    # Make sure we're working on NeuronLists
    query_dps = NeuronList(query)
    target_dps = NeuronList(target)

    # Run NBLAST preflight checks
    nblast_preflight(query_dps, target_dps, n_cores, req_unique_ids=True)

    # Find an optimal partition that minimizes the number of neurons
    # we have to send to each process
    n_rows, n_cols = find_optimal_partition(n_cores, query_dps, target_dps)

    # Make simplified dotprops
    query_dps_simp = query_dps.downsample(10, inplace=False)
    target_dps_simp = target_dps.downsample(10, inplace=False)

    # First we NBLAST the highly simplified dotprops against another
    nblasters = []
    with config.tqdm(desc='Prep. pre-NBLAST',
                     total=n_rows * n_cols,
                     leave=False,
                     disable=not progress) as pbar:
        for qq in np.array_split(query_dps_simp, n_rows):
            for tt in np.array_split(target_dps_simp, n_cols):
                # Initialize NBlaster
                this = NBlaster(use_alpha=use_alpha,
                                normalized=normalized,
                                smat=smat,
                                limit_dist=limit_dist,
                                progress=progress)
                # Add queries and targets
                for n in qq:
                    this.append(n)
                for n in tt:
                    this.append(n)
                # Keep track of indices of queries and targets
                this.queries = np.arange(len(qq))
                this.targets = np.arange(len(tt)) + len(qq)
                this.pbar_position = len(nblasters)
                this.desc = 'Pre(NBLASTING)'

                nblasters.append(this)
                pbar.update()

    # If only one core, we don't need to break out the multiprocessing
    if n_cores == 1:
        scr = this.multi_query_target(this.queries,
                                      this.targets,
                                      scores=scores)
    else:
        with ProcessPoolExecutor(max_workers=len(nblasters)) as pool:
            # Each nblaster is passed to it's own process
            futures = [pool.submit(this.multi_query_target,
                                   q_idx=this.queries,
                                   t_idx=this.targets,
                                   scores=scores) for this in nblasters]

            results = [f.result() for f in futures]

        scr = pd.DataFrame(np.zeros((len(query_dps), len(target_dps))),
                           index=query_dps.id, columns=target_dps.id)

        for res in results:
            scr.loc[res.index, res.columns] = res.values

    # Now select targets of interest for each query
    if criterion == 'percentile':
        # Generate a mask for the scores we want to recalculate from full dotprops
        sel = np.percentile(scr, q=t, axis=1)
        mask = scr >= sel.reshape(-1, 1)
    elif criterion == 'score':
        # Generate a mask for the scores we want to recalculate from full dotprops
        sel = np.full(scr.shape[0], fill_value=t)
        mask = scr >= sel.reshape(-1, 1)
    else:
        # Sort such that the top hit is to the left
        srt = np.argsort(scr.values, axis=1)[:, ::-1]
        # Generate the mask
        mask = pd.DataFrame(np.zeros(scr.shape), dtype=bool,
                            columns=scr.columns, index=scr.index)
        _ = np.arange(mask.shape[0])
        for N in range(t):
            mask.values[_, srt[:, N]] = True

    # Now re-generate the NBLASTERs with the full dotprops
    nblasters = []
    with config.tqdm(desc='Preparing full NBLASTs',
                     total=n_rows * n_cols,
                     leave=False,
                     disable=not progress) as pbar:
        for q in np.array_split(np.arange(len(query_dps)), n_rows):
            for t in np.array_split(np.arange(len(target_dps)), n_cols):
                # Initialize NBlaster
                this = NBlaster(use_alpha=use_alpha,
                                normalized=normalized,
                                smat=smat,
                                limit_dist=limit_dist,
                                progress=progress)
                # Add queries and targets
                for n in query_dps[q]:
                    this.append(n)
                for n in target_dps[t]:
                    this.append(n)

                # Find the pairs to NBLAST in this part of the matrix
                submask = mask.loc[query_dps[q].id,
                                   target_dps[t].id]
                # `pairs` is an array of `[[query, target], [...]]` pairs
                this.pairs = np.vstack(np.where(submask)).T

                # Offset the query indices
                this.pairs[:, 1] += len(q)

                # Track this NBLASTER's mask relative to the original big one
                this.mask = np.zeros(mask.shape, dtype=bool)
                this.mask[q[0]:q[-1]+1, t[0]:t[-1]+1] = submask

                # Make sure position of progress bar checks out
                this.pbar_position = len(nblasters)

                nblasters.append(this)
                pbar.update()

    # If only one core, we don't need to break out the multiprocessing
    if n_cores == 1:
        scr[mask] = this.pair_query_target(this.pairs, scores=scores)
    else:
        with ProcessPoolExecutor(max_workers=len(nblasters)) as pool:
            # Each nblaster is passed to its own process
            futures = [pool.submit(this.pair_query_target,
                                   pairs=this.pairs,
                                   scores=scores) for this in nblasters]

            results = [f.result() for f in futures]

        for res, nbl in zip(results, nblasters):
            scr[nbl.mask] = res

    if return_mask:
        return scr, mask

    return scr

def nblast(query: Union[Dotprops, NeuronList],
           target: Optional[str] = None,
           scores: Union[Literal['forward'],
                         Literal['mean'],
                         Literal['min'],
                         Literal['max']] = 'forward',
           normalized: bool = True,
           use_alpha: bool = False,
           smat: Optional[Union[str, pd.DataFrame]] = 'auto',
           limit_dist: Optional[Union[Literal['auto'], int, float]] = None,
           n_cores: int = os.cpu_count() // 2,
           batch_size: Optional[int] = None,
           progress: bool = True) -> pd.DataFrame:
    """NBLAST query against target neurons.

    This implements the NBLAST algorithm from Costa et al. (2016) (see
    references) and mirror the implementation in R's ``nat.nblast``
    (https://github.com/natverse/nat.nblast).

    Parameters
    ----------
    query :         Dotprops | NeuronList
                    Query neuron(s) to NBLAST against the targets. Neurons
                    should be in microns as NBLAST is optimized for that and
                    have similar sampling resolutions.
    target :        Dotprops | NeuronList, optional
                    Target neuron(s) to NBLAST against. Neurons should be in
                    microns as NBLAST is optimized for that and have
                    similar sampling resolutions. If not provided, will NBLAST
                    queries against themselves.
    scores :        'forward' | 'mean' | 'min' | 'max'
                    Determines the final scores:

                      - 'forward' (default) returns query->target scores
                      - 'mean' returns the mean of query->target and
                        target->query scores
                      - 'min' returns the minium between query->target and
                        target->query scores
                      - 'max' returns the maximum between query->target and
                        target->query scores

    use_alpha :     bool, optional
                    Emphasizes neurons' straight parts (backbone) over parts
                    that have lots of branches.
    normalized :    bool, optional
                    Whether to return normalized NBLAST scores.
    smat :          str | pd.DataFrame
                    Score matrix. If 'auto' (default), will use scoring matrices
                    from FCWB. Same behaviour as in R's nat.nblast
                    implementation. If ``smat=None`` the scores will be
                    generated as the product of the distances and the dotproduct
                    of the vectors of nearest-neighbor pairs.
    limit_dist :    float | "auto" | None
                    Sets the max distance for the nearest neighbor search
                    (`distance_upper_bound`). Typically this should be the
                    highest distance considered by the scoring function. If
                    "auto", will extract that value from the scoring matrix.
    n_cores :       int, optional
                    Max number of cores to use for nblasting. Default is
                    ``os.cpu_count() - 2``. This should ideally be an even
                    number as that allows optimally splitting queries onto
                    individual processes.
    batch_size :    int, optional
                    Set the number of Dotprops per process. This can be useful
                    to reduce the memory footprint at the cost of longer run
                    times. By default (``None``), queries and targets will be
                    evenly distributed across n_cores. Ignored if ``n_cores=1``.
    progress :      bool
                    Whether to show progress bars.

    References
    ----------
    Costa M, Manton JD, Ostrovsky AD, Prohaska S, Jefferis GS. NBLAST: Rapid,
    Sensitive Comparison of Neuronal Structure and Construction of Neuron
    Family Databases. Neuron. 2016 Jul 20;91(2):293-311.
    doi: 10.1016/j.neuron.2016.06.012.

    Examples
    --------
    >>> import navis
    >>> nl = navis.example_neurons(n=5)
    >>> nl.units
    <Quantity([8 8 8 8 8], 'nanometer')>
    >>> # Convert to microns
    >>> nl_um = nl * (8 / 1000)
    >>> # Convert to dotprops
    >>> dps = navis.make_dotprops(nl_um)
    >>> # Run the nblast
    >>> scores = navis.nblast(dps[:3], dps[3:])

    See Also
    --------
    :func:`navis.nblast_allbyall`
                A more efficient way than ``nblast(query=x, target=x)``.
    :func:`navis.nblast_smart`
                A smart(er) NBLAST suited for very large NBLAST.
    :func:`navis.synblast`
                A synapse-based variant of NBLAST.

    """
    # Make sure we're working on NeuronLists
    query_dps = NeuronList(query)
    target_dps = NeuronList(target)

    # Run NBLAST preflight checks
    nblast_preflight(query_dps, target_dps, n_cores,
                     batch_size=batch_size, req_unique_ids=True)

    # Find an optimal partition that minimizes the number of neurons
    # we have to send to each process
    if not batch_size or n_cores == 1:
        n_rows, n_cols = find_optimal_partition(n_cores, query_dps, target_dps)
    else:
        n_rows, n_cols = find_batch_partition(batch_size, query_dps, target_dps)

    nblasters = []
    with config.tqdm(desc='Preparing',
                     total=n_rows * n_cols,
                     leave=False,
                     disable=not progress) as pbar:
        for q in np.array_split(query_dps, n_rows):
            for t in np.array_split(target_dps, n_cols):
                # Initialize NBlaster
                this = NBlaster(use_alpha=use_alpha,
                                normalized=normalized,
                                smat=smat,
                                limit_dist=limit_dist,
                                progress=progress)

                # Use better description if we process in batches
                if batch_size:
                    this.desc += f' (batch {len(nblasters) + 1}/{n_rows * n_cols})'

                # Add queries and targets
                for n in q:
                    this.append(n)
                for n in t:
                    this.append(n)
                # Keep track of indices of queries and targets
                this.queries = np.arange(len(q))
                this.targets = np.arange(len(t)) + len(q)
                this.pbar_position = len(nblasters) % n_cores

                nblasters.append(this)
                pbar.update(1)

    # If only one core, we don't need to break out the multiprocessing
    if n_cores == 1:
        return this.multi_query_target(this.queries,
                                       this.targets,
                                       scores=scores)

    with ProcessPoolExecutor(max_workers=n_cores) as pool:
        # Each nblaster is passed to its own process
        futures = [pool.submit(this.multi_query_target,
                               q_idx=this.queries,
                               t_idx=this.targets,
                               scores=scores) for this in nblasters]

        results = [f.result() for f in futures]

    scores = pd.DataFrame(np.zeros((len(query_dps), len(target_dps))),
                          index=query_dps.id, columns=target_dps.id)

    for res in results:
        scores.loc[res.index, res.columns] = res.values

    return scores

def nblast_allbyall(x: NeuronList,
                    normalized: bool = True,
                    use_alpha: bool = False,
                    smat: Optional[Union[str, pd.DataFrame]] = 'auto',
                    limit_dist: Optional[Union[Literal['auto'], int, float]] = None,
                    n_cores: int = os.cpu_count() // 2,
                    progress: bool = True) -> pd.DataFrame:
    """All-by-all NBLAST of inputs neurons.

    A slightly more efficient way of running ``nblast(query=x, target=x)``.

    Parameters
    ----------
    x :             Dotprops | NeuronList
                    Neuron(s) to NBLAST against each other. Neurons should
                    be in microns as NBLAST is optimized for that and have
                    similar sampling resolutions.
    n_cores :       int, optional
                    Max number of cores to use for nblasting. Default is
                    ``os.cpu_count() - 2``. This should ideally be an even
                    number as that allows optimally splitting queries onto
                    individual processes.
    use_alpha :     bool, optional
                    Emphasizes neurons' straight parts (backbone) over parts
                    that have lots of branches.
    normalized :    bool, optional
                    Whether to return normalized NBLAST scores.
    smat :          str | pd.DataFrame, optional
                    Score matrix. If 'auto' (default), will use scoring matrices
                    from FCWB. Same behaviour as in R's nat.nblast
                    implementation. If ``smat=None`` the scores will be
                    generated as the product of the distances and the dotproduct
                    of the vectors of nearest-neighbor pairs.
    limit_dist :    float | "auto" | None
                    Sets the max distance for the nearest neighbor search
                    (`distance_upper_bound`). Typically this should be the
                    highest distance considered by the scoring function. If
                    "auto", will extract that value from the scoring matrix.
    progress :      bool
                    Whether to show progress bars.

    Returns
    -------
    scores :        pandas.DataFrame
                    Matrix with NBLAST scores.

    References
    ----------
    Costa M, Manton JD, Ostrovsky AD, Prohaska S, Jefferis GS. NBLAST: Rapid,
    Sensitive Comparison of Neuronal Structure and Construction of Neuron
    Family Databases. Neuron. 2016 Jul 20;91(2):293-311.
    doi: 10.1016/j.neuron.2016.06.012.

    Examples
    --------
    >>> import navis
    >>> nl = navis.example_neurons(n=5)
    >>> nl.units
    <Quantity([8 8 8 8 8], 'nanometer')>
    >>> # Convert to microns
    >>> nl_um = nl * (8 / 1000)
    >>> # Make dotprops
    >>> dps = navis.make_dotprops(nl_um)
    >>> # Run the nblast
    >>> scores = navis.nblast_allbyall(dps)

    See Also
    --------
    :func:`navis.nblast`
                For generic query -> target nblasts.

    """
    # Make sure we're working on NeuronLists
    dps = NeuronList(x)

    # Run NBLAST preflight checks
    # Note that we are passing the same dotprops twice to avoid having to
    # change the function's signature. Should have little to no overhead.
    nblast_preflight(dps, dps, n_cores, req_unique_ids=True)

    # Find an optimal partition that minimizes the number of neurons
    # we have to send to each process
    n_rows, n_cols = find_optimal_partition(n_cores, dps, dps)

    nblasters = []
    with config.tqdm(desc='Preparing',
                     total=n_rows * n_cols,
                     leave=False,
                     disable=not progress) as pbar:
        for qix in np.array_split(np.arange(len(dps)), n_rows):
            for tix in np.array_split(np.arange(len(dps)), n_cols):
                # Initialize NBlaster
                this = NBlaster(use_alpha=use_alpha,
                                normalized=normalized,
                                smat=smat,
                                limit_dist=limit_dist,
                                progress=progress)

                # Make sure we don't add the same neuron twice
                # Map indices to neurons
                to_add = list(set(qix) | set(tix))

                ixmap = {}
                for i, ix in enumerate(to_add):
                    this.append(dps[ix])
                    ixmap[ix] = i

                # Keep track of indices of queries and targets
                this.queries = [ixmap[ix] for ix in qix]
                this.targets = [ixmap[ix] for ix in tix]
                this.pbar_position = len(nblasters)

                nblasters.append(this)
                pbar.update()

    # If only one core, we don't need to break out the multiprocessing
    if n_cores == 1:
        return this.all_by_all()

    with ProcessPoolExecutor(max_workers=len(nblasters)) as pool:
        # Each nblaster is passed to it's own process
        futures = [pool.submit(this.multi_query_target,
                               q_idx=this.queries,
                               t_idx=this.targets,
                               scores='forward') for this in nblasters]

        results = [f.result() for f in futures]

    scores = pd.DataFrame(np.zeros((len(dps), len(dps))),
                          index=dps.id, columns=dps.id)

    for res in results:
        scores.loc[res.index, res.columns] = res.values

    return scores

def find_batch_partition(N, q, t):
    """Find partition such that each batch contains at most `N` neurons."""
    import math
    # Number of rows (i.e. how many splits for queries)
    n_rows = max(math.floor(len(q) / (N / 2)), 1)

    # Given the number of queries per batch, how many targets can we have
    remaining = N - (len(q) / n_rows)

    n_cols = max(math.floor(len(t) / remaining), 1)

    return int(n_rows), int(n_cols)

def find_optimal_partition(n_cores, q, t):
    """Find an optimal partition for given NBLAST query."""
    # Find an optimal partition that minimizes the number of neurons
    # we have to send to each process
    neurons_per_query = []
    for n_rows in range(1, n_cores + 1):
        # Skip splits we can't make
        if n_cores % n_rows:
            continue
        if n_rows > len(q):
            continue

        n_cols = min(int(n_cores / n_rows), len(t))

        n_queries = len(q) / n_rows
        n_targets = len(t) / n_cols

        neurons_per_query.append([n_rows, n_cols, n_queries + n_targets])

    # Find the optimal partition
    neurons_per_query = np.array(neurons_per_query)
    n_rows, n_cols = neurons_per_query[np.argmin(neurons_per_query[:, 2]), :2]

    return int(n_rows), int(n_cols)

def force_dotprops(x, k, resample, progress=False):
    """Force data into Dotprops."""
    if isinstance(x, (NeuronList, list)):
        dp = [force_dotprops(n, k, resample) for n in config.tqdm(x,
                                                                  desc='Dotprops',
                                                                  disable=not progress,
                                                                  leave=False)]
        return NeuronList(dp)

    # Try converting non-Dotprops
    if not isinstance(x, Dotprops):
        return make_dotprops(x, k=k, resample=resample)

    # Return Dotprops
    return x

def check_microns(x):
    """Check if neuron data is in microns.

    Returns either [True, None (=unclear), False]
    """
    microns = [config.ureg.Unit('microns'),
               config.ureg.Unit('um'),
               config.ureg.Unit('micrometer'),
               config.ureg.Unit('dimensionless')]
    if not isinstance(x, NeuronList):
        x = NeuronList(x)

    # For very large NeuronLists converting the unit string to pint units is
    # the time consuming step. Here we will first reduce to unique units:
    unit_str = []
    for n in x:
        if isinstance(n._unit_str, str):
            unit_str.append(n._unit_str)
        else:
            unit_str += list(n._unit_str)
    unit_str = np.unique(unit_str)

    any_not_microns = False
    all_units = True
    for u in unit_str:
        # If not a unit (i.e. `None`)
        if not u:
            all_units = False
            continue

        # Convert to proper unit
        u = config.ureg(u).to_compact().units

        if u not in microns:
            any_not_microns = True

    if any_not_microns:
        return False
    elif all_units:
        return True
    return None

def align_dtypes(*x, downcast=False, inplace=True):
    """Align data types of dotprops.

    Parameters
    ----------
    *x :        Dotprops | NeuronLists thereof
    downcast :  bool
                If True, will downcast all points to the lowest precision
                dtype.
    inplace :   bool
                If True, will modify the original neuron objects. If False, will
                make a copy before changing dtypes.

    Returns
    -------
    *x
                Input data with aligned dtypes.

    """
    dtypes = get_dtypes(x)

    if len(dtypes) == 1:
        return x

    if not inplace:
        for i in range(x):
            x[i] = x[i].copy()

    if downcast:
        target = lowest_type(*x)
    else:
        target = np.result_type(*dtypes)

    for n in x:
        if isinstance(n, NeuronList):
            for i in range(len(n)):
                n[i].points = n[i].points.astype(target, copy=False)
        elif isinstance(n, Dotprops):
            n.points = n.points.astype(target, copy=False)
        else:
            raise TypeError(f'Unable to process "{type(n)}"')

    return x

def get_dtypes(*x):
    """Collect data types of dotprops points."""
    dtypes = set()
    for n in x:
        if isinstance(n, NeuronList):
            dtypes = dtypes | get_dtypes(n)
        elif isinstance(n, Dotprops):
            dtypes.add(n.points.dtype)
        else:
            raise TypeError(f'Unable to process "{type(n)}"')
    return dtypes

def lowest_type(*x):
    """Find the lowest data type."""
    dtypes = get_dtypes(x)

    if len(dtypes) == 1:
        return dtypes[0]

    lowest = dtypes[0]
    for dt in dtypes[1:]:
        lowest = demote_types(lowest, dt)

    return lowest

def demote_types(a, b):
    """Determine the lower of two dtypes."""
    if isinstance(a, np.ndarray):
        a = a.dtype
    if isinstance(b, np.ndarray):
        b = b.dtype

    # No change is same
    if a == b:
        return a

    # First, get the "higher" type
    higher = np.promote_types(a, b)
    # Now get the one that's not higher
    if a != higher:
        return a
    return b

def sim_to_dist(x):
    """Convert similarity scores to distances.

    Parameters
    ----------
    x :     (M, M) np.ndarray | pandas.DataFrame
            Similarity score matrix to invert.

    Returns
    -------
    distances

    """
    if not isinstance(x, (np.ndarray, pd.DataFrame)):
        raise TypeError(f'Expected numpy array or pandas DataFrame, got "{type(x)}"')

    if isinstance(x, pd.DataFrame):
        mx = x.values.max()
    else:
        mx = x.max()

    return (x - mx) * -1

def nblast_preflight(query, target, n_cores, batch_size=None,
                     req_unique_ids=False, req_dotprops=True):
    """Run preflight checks for NBLAST."""
    if req_dotprops:
        if query.types != (Dotprops, ):
            raise TypeError(f'`query` must be Dotprop(s), got "{query.types}". '
                            'Use `navis.make_dotprops` to convert neurons.')

        if target.types != (Dotprops, ):
            raise TypeError(f'`target` must be Dotprop(s), got "{target.types}". '
                            'Use `navis.make_dotprops` to convert neurons.')

    if req_unique_ids:
        # At the moment, neurons need to have a unique ID for things to work
        if query.is_degenerated:
            raise ValueError('Queries have non-unique IDs.')
        if target.is_degenerated:
            raise ValueError('Targets have non-unique IDs.')

    # Check if query or targets are in microns
    # Note this test can return `None` if it can't be determined
    if check_microns(query) is False:
        logger.warning('NBLAST is optimized for data in microns and it looks '
                       'like your queries are not in microns.')
    if check_microns(target) is False:
        logger.warning('NBLAST is optimized for data in microns and it looks '
                       'like your targets are not in microns.')

    if not isinstance(n_cores, numbers.Number) or n_cores < 1:
        raise TypeError('`n_cores` must be an integer > 0')

    n_cores = int(n_cores)
    if n_cores > 1 and n_cores % 2:
        logger.warning('NBLAST is most efficient if `n_cores` is an even number')
    elif n_cores < 1:
        raise ValueError('`n_cores` must not be smaller than 1')
    elif n_cores > os.cpu_count():
        logger.warning('`n_cores` should not larger than the number of '
                       'available cores')

    if batch_size is not None:
        if batch_size <= 0:
            raise ValueError('`batch_size` must be >= 1 or `None`.')


from flask import Flask, request, Response
from gevent import spawn
from gevent.queue import Queue

from sse import ServerSentEvent

from crossdomain import crossdomain

app = Flask(__name__)
subscriptions = []

pixels = [" " for i in range(9)]
current_player = "x"

@app.route("/debug")
def debug():
    return "Currently %d subscriptions" % len(subscriptions)

@app.route("/status")
def status():
    global pixels
    global current_player

    change = ""

    if request.args["change"] == "player":
        if current_player == "x":
            current_player = "o"
        else:
            current_player = "x"
        change = current_player
    else:
        current_player = "x"
        pixels = [" " for i in range(9)]
        change = "reset"

    def notify():
        msg = change
        for sub in subscriptions[:]:
            sub.put(msg)

    spawn(notify)

    return str(request.args["change"])

@app.route("/target")
def target():
    change = str(request.args["pixels"])

    for i in range(len(pixels)):
        if pixels[i] == " " and change[i] == "0":
            pixels[i] = current_player

    def notify():
        msg = "".join(pixels)
        for sub in subscriptions[:]:
            sub.put(msg)

    spawn(notify)

    return str(request.args["pixels"])

@app.route("/subscribe")
@crossdomain(origin='*')
def subscribe():
    def notify():
        msg = str(len(subscriptions))
        for sub in subscriptions[:]:
            sub.put(msg)

    def gen():
        q = Queue()
        subscriptions.append(q)
        spawn(notify)
        try:
            while True:
                result = q.get()
                ev = ServerSentEvent(str(result))
                yield ev.encode()
        except GeneratorExit: # Or maybe use flask signals
            subscriptions.remove(q)
            spawn(notify)

    return Response(gen(), mimetype="text/event-stream")


import zmq
import threading

# ZeroMQ Context

context = zmq.Context()

# Define the socket using the "Context"
sock_pub = context.socket(zmq.PUB)
sock_pub.bind("tcp://127.0.0.1:5678")

sock_rep = context.socket(zmq.REP)
sock_rep.bind("tcp://127.0.0.1:5679")

# Run a simple "Echo" server
def chat_window():
    while True:
        msg = sock_rep.recv_json()
        sock_rep.send_string('0')
        sock_pub.send_json(msg)

if __name__ == "__main__":
    # t = threading.Thread(target=chat_window())
    # t.daemon = True
    # t.start()
    chat_window()


from sys import argv
import glob
from collections import defaultdict

#print(argv)
#script, input_file, output_file = argv

def Extract_Write(input_file, outfile):
    no_sense = 10   # number of senses
    # no_fac = no_sense + 2
    with open(input_file) as f:
        count = defaultdict(int)
        for line in f:
            tokens = line.split()
            new_line = list()
            new_token = list()
            for i in range(no_sense):
                new_token.append('-')
            new_line.append("￨".join(new_token))
         #   new_line.append('-')
            for token in tokens:
                new_token = list()
                factor = token.split("|")
                #  new_token.append(factor[0])   # add word to new token
                if len(factor)<=4:
                    new_line.append(0)
                    count[0] += 1
                else:
                    add = 0
                    for i in range(10):
                        if factor[i+3] == '-' or factor[i+3] == '--':
                            new_line.append(i)  # add sense to new token
                            count[i] += 1
                            add = 1
                    if add == 0:
                        new_line.append(10)
                        count[10] += 1

                new_line.append("￨".join(new_token))
        print(count)
           # outfile.write(' '.join(new_line)+'\n')

    # open(output_file, 'w').writelines(new_lines)

def Get_line_num(input_file):
    lines = open(input_file).readlines()
    new_lines = list()
    line_num = len(lines)
    return line_num

#file directory
# "/home/stefan/Downloads/Babel3.7/corpus/buildBPE/tst.wplmb.source/*.source"

files = glob.glob("/home/stefan/Downloads/Babel3.7/corpus/buildBPE/original_mfre.test.wplmb.source/*wplmb.source")
# /raid/data/yanzhe/babelWE/dev.wplmb014bcom
files = glob.glob("/raid/data/yanzhe/lattice/corpus/corpus_10senseBb40/tst.wplmb014bcom/*.wplmb014bcom")

input_files = list()
output_files = list()
files = sorted(files)

for file1 in files:
    segs = file1.split("/")
    new_infile = list()
    new_outfile = list()
    new_infile.append(segs[-1])
    new_outfile.append(segs[-1])
    new_outfile.append('1')
    input_files.append(segs[-1])
    output_files.append("".join(new_outfile))
## for i in range(len(input_files)):
   ## Get_some_factors(input_files[i], output_files[i])

line_nums = list()

file_name_w = 'train.wplmb.multi.source_Bb37_2'
# /raid/data/yanzhe/babelWE/
file_name_w = '/raid/data/yanzhe/lattice/corpus/corpus_10senseBb40/tst.wplmb.multi.source_Bb40_1to10_count'

with open(file_name_w, 'w') as outfile:
    for fname in files:
        Extract_Write(fname, outfile)
  #      outfile.writelines(Get_some_factors(fname))
        line_nums.append(str(Get_line_num(fname)))
        print(line_nums[-1])
        #print(fname)
with open(file_name_w+'num_list', 'w') as outfile:
    outfile.writelines(' '.join(line_nums))


# coding=utf-8
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.nn.utils.rnn import pack_padded_sequence
from PIL import Image
from .base_model import BaseModel
from . import networks
import pdb

class SalModel(BaseModel):
    def __init__(self, opt, **kwargs):
        super(SalModel, self).__init__(opt)

        self.name = 'SalSal_' + opt.base
        self.ws = 0.05

        net = networks.DeepLab(pretrained=True, c_output=1, base=opt.base)
        net = torch.nn.parallel.DataParallel(net)
        self.net = net.cuda()
        self.criterion = nn.BCEWithLogitsLoss()
        self.optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, self.net.parameters()),
                                            lr=opt.lr)

    def save(self, label):
        self.save_network(self.net, self.name, label)

    def load(self, label):
        print('loading %s'%label)
        self.load_network(self.net, self.name, label)

    def set_input(self, data):
        self.input = data['img_sal'].cuda()
        self.targets = data['gt_sal'].cuda()

    def forward(self):
        # print("We are Forwarding !!")
        self.big_mask_logits = self.net.forward(self.input)

    def test(self, input, name, WW, HH):
        with torch.no_grad():
            big_mask_logits = self.net.forward(input.cuda())
            outputs = F.sigmoid(big_mask_logits.squeeze(1))
        outputs = outputs.detach().cpu().numpy() * 255
        for ii, msk in enumerate(outputs):
            msk = Image.fromarray(msk.astype(np.uint8))
            msk = msk.resize((WW[ii], HH[ii]))
            msk.save('{}/{}.png'.format(self.opt.results_dir, name[ii]), 'PNG')

    def backward(self):
        # Combined loss
        loss = self.criterion(self.big_mask_logits, self.targets) * (1-self.ws)
        gt_self = F.sigmoid(self.big_mask_logits).detach()
        gt_self[gt_self>0.5] = 1
        gt_self[gt_self<=0.5] = 0
        loss += self.criterion(self.big_mask_logits, gt_self) * self.ws
        loss.backward()
        self.loss['sal'] = loss.item()

    def optimize_parameters(self, it):
        self.forward()
        self.optimizer.zero_grad()
        self.backward()
        self.optimizer.step()

    def switch_to_train(self):
        self.net.train()

    def switch_to_eval(self):
        self.net.eval()


import tensorflow as tf
from config import Config
import tensorflow_addons as tfa
from pathlib import Path
import senet
from glob import glob

def cosine_distance(matrix1, matrix2):
    matrix1_matrix2 = np.dot(matrix1, matrix2.transpose())
    matrix1_norm = np.sqrt(np.multiply(matrix1, matrix1).sum(axis=1))
    matrix1_norm = matrix1_norm[:, np.newaxis]
    matrix2_norm = np.sqrt(np.multiply(matrix2, matrix2).sum(axis=1))
    matrix2_norm = matrix2_norm[:, np.newaxis]
    cosine_distance = np.divide(matrix1_matrix2, np.dot(matrix1_norm, matrix2_norm.transpose()))
    return cosine_distance

class BaseModel(tf.keras.layers.Layer):
    def __init__(self):
        super(BaseModel, self).__init__()

    def build(self, input_shape):
        self.feature_model = tf.keras.applications.mobilenet_v2.MobileNetV2(
            input_shape=(Config.width, Config.height, 3), include_top=False,weights="./h5/mobilenet_v2_weights_tf_dim_ordering_tf_kernels_1.0_128_no_top.h5")  # Downloading data from https://storage.googleapis.com/tensorflow/keras-applications/mobilenet_v2/mobilenet_v2_weights_tf_dim_ordering_tf_kernels_1.0_224_no_top.h5
        self.senet = senet.SENet((4, 4, 1280), 128)
        self.dense_feature = (tf.keras.layers.Dense(units=128, activation=None))

        super(BaseModel, self).build(input_shape)  # 一定要在最后调用它

    def call(self, inputs):
        image_inputs = inputs

        features =self.feature_model(image_inputs)
        #pooled_features = tf.keras.layers.GlobalAveragePooling2D()(features)   #这里是替代下面的senet的
        pooled_features = self.senet(features)
        pooled_features = tf.keras.layers.Dropout(0.5217)(pooled_features)
        dense_features = self.dense_feature(pooled_features)
        embeddings = tf.keras.layers.Lambda(lambda x: tf.math.l2_normalize(tf.cast(x, dtype='float32'), axis=1))(dense_features)

        return embeddings

if __name__ == "__main__":
    with tf.device("/CPU:0"):
        image = tf.keras.Input(shape=(Config.width,Config.height,3),dtype=tf.float32)
        embedding = BaseModel()(image)
        model = tf.keras.Model(image,embedding)
        model.load_weights("./model.h5",by_name=True)

        import matplotlib.image as mpimg
        import numpy as np
        import cv2

        image_path_1_list = list(Path("C:/face_DATA/1000/1/").glob('*.jpg'))
        image_path_2_list = list(Path("C:/face_DATA/1000/2/").glob('*.jpg'))

        image_1_list = []
        image_2_list = []
        for image_1,image_2 in zip(image_path_1_list,image_path_2_list):

            img_1 = mpimg.imread(image_1)
            img_1 = cv2.resize(img_1, (128, 128))
            image_1_list.append(img_1)

            img_2 = mpimg.imread(image_2)
            img_2 = cv2.resize(img_2, (128, 128))
            image_2_list.append(img_2)

        image_1_list = np.array(image_1_list)
        image_2_list = np.array(image_2_list)

        image_embedding_1 = np.array(model.predict(image_1_list))
        image_embedding_2 = np.array(model.predict(image_2_list))

        print(image_embedding_1.shape)
        print(image_embedding_2.shape)

        result = (cosine_distance(image_embedding_1,image_embedding_2))
        result = np.argmax(result, axis=-1)
        counter = 0
        for i,j in zip(result,range(900)):
            if i == j:
                counter += 1
        print("正确率为：",counter/900)


#!/usr/bin/python3

import csv
import konstruktikon_browser
from lxml import etree

fname = "konstruktikon2.xml"
st = ["SemType1", "SemSubType1", "SemType2", "SemSubType2"]
synt = ["Syntactic tag_Old version", "Clause", "Biclausal", "Matrix", "Praedic", "Parenth", "Adj", "Adv", "Conj", "Prep", "Part", "Verb pattern", "Light verb (tag dismissed)", "Nominal pattern"]
karp = "http://spraakbanken.gu.se/eng/research/infrastructure/karp/karp"

class FileUpdate(konstruktikon_browser.Browser):
    def __init__(self, konst_file, base):
        super().__init__(konst_file)
        r_entries = self.lex.xpath("//LexicalEntry")
        bv = list(base.keys())
        for entry in r_entries:
            name = entry.xpath("Sense")[0].attrib["id"].replace("konstruktikon-rus--", "").replace("_", " ")
            if name in base:
                le = VirtualLE(entry, base[name])
                print(name)
                try:
                    bv.remove(name)
                except ValueError:
                    pass
        for name in bv:
            attrs = base[name]
            attrs["name"] = name
            le = VirtualLE(add_attrs=attrs)
            self.lex.addnext(le.entry)

        with open(fname, "wb") as kkn:
            kkn.write(etree.tostring(self.konst_xml, pretty_print=True))
            kkn.close()

class VirtualLE:
    def __init__(self, entry=None, add_attrs=None):
        if add_attrs is None:
            add_attrs = {}
        if entry is not None:
            self.entry = entry
        else:
            self.entry = etree.Element("LexicalEntry")
        if entry is None:
            #self.entry.set("xml:lang", "rus")
            etree.SubElement(self.entry, "feat", attrib={
                "att": "lastmodified",
                "val": "2019-11-28T02:00:00"
            })
            etree.SubElement(self.entry, "feat", attrib={
                "att": "lastmodifiedBy",
                "val": "f.sizov@yandex.ru"
            })
            etree.SubElement(self.entry, "Lemma")
            sense = etree.SubElement(self.entry, "Sense", attrib={
                "id": "konstruktikon-rus--%s" % "_".join(add_attrs["name"].split())
            })
            defn = etree.SubElement(sense, "definition",
                                    namespaces="http://spraakbanken.gu.se/eng/research/infrastructure/karp/karp")
            karp_text = etree.SubElement(defn, etree.QName(karp, "text"), attrib={
                "n": "0"
            })
            karp_text.text = "-"

            etree.SubElement(sense, "feat", attrib={
                "att": "illustration",
                "val": "" if "illustration" not in add_attrs else add_attrs["illustration"]
            })
            etree.SubElement(sense, "feat", attrib={
                "att": "cefr",
                "val": "" if "cefr" not in add_attrs else add_attrs["cefr"]
            })
            etree.SubElement(sense, "feat", attrib={
                "att": "type",
                "val": "konstruktikon"
            })
            etree.SubElement(sense, "feat", attrib={
                "att": "cee",
                "val": "" if "cee" not in add_attrs else add_attrs["cee"]
            })
        else:
            sense = self.entry.xpath("Sense")[0]

        for cat in ["SemType1", "SemSubType1", "SemType2", "SemSubType2"]:
            etree.SubElement(sense, "feat", attrib={
                "att": cat,
                "val": add_attrs[cat]
            })

        etree.SubElement(sense, "feat", attrib={
            "att": "Syntax",
            "val": ",".join(add_attrs["Syntax"])
        })

with open('synt-n-sem.csv', newline='\n', encoding='utf-8') as synt_n_sem:
    w = csv.reader(synt_n_sem, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
    base = {}
    for k, row in enumerate(w):
        if k < 2:
            continue
        attribs = {
            "Syntax": []
        }
        for n, k in enumerate(range(8, 12)):
            attribs[st[n]] = row[k]
        for n, k in enumerate(range(15, 15 + 14)):
            if row[k] != "":
                attribs["Syntax"].append(row[k])
        base[row[3]] = attribs

print(base)
fu = FileUpdate(fname, base)

#print(fu)


from django.dispatch import receiver
from django.contrib.auth.signals import user_logged_in, user_logged_out

from .constants import ONLINE, OFFLINE

@receiver(user_logged_in)
def set_online_status(sender, user, request, **kwargs):
    """
    Set user's status to online, than user login
    """
    user.profile.status = ONLINE
    user.profile.save()

@receiver(user_logged_out)
def set_offline_status(sender, user, request, **kwargs):
    """
    Set user status to offline, than user logout
    """
    user.profile.status = OFFLINE
    user.profile.save()


"""
Contains urls mapped with CueSearch app views
"""
from django.urls import path
from . import views

urlpatterns = [
    # Global Dimension
    path("dimension/", views.DimensionView.as_view(), name="dimension"),
    # path("metrics/", views.MetricsView.as_view(), name="metrics"),
    path(
        "global-dimension/", views.GlobalDimensionView.as_view(), name="globalDimension"
    ),
    path(
        "global-dimension/create/",
        views.GlobalDimensionView.as_view(),
        name="globalDimensionCreate",
    ),
    path(
        "global-dimension/delete/<int:id>",
        views.GlobalDimensionView.as_view(),
        name="global-dimension-delete",
    ),
    path(
        "global-dimension/publish", views.pubGlobalDimension, name="pubGlobalDimension"
    ),
    path(
        "global-dimension/<int:id>", views.globalDimensionById, name="globalDimensionId"
    ),
    path(
        "global-dimension/update/<int:id>",
        views.updateGlobalDimension,
        name="updateGlobalDimension",
    ),
    # Search Card Templates
    path("cardTemplates/", views.getCardTemplates, name="getCardTemplate"),
    path("getSearchCards/", views.getSearchCards, name="getSearchCards"),
    path("getSearchCardData/", views.getSearchCardData, name="getSearchCardData"),
    path(
        "searchsuggestions/",
        views.getSearchSuggestionsView,
        name="getSearchSuggestionsView",
    ),
    path("runIndexing/", views.elasticSearchIndexingView, name="runIndexing"),
    # Custom Card Templates
    # path("createTemplates/", views.createCardTemplates, name="createCardTemplates"),
    path("templates/", views.getTemplates, name="getTemplates"),
    path("templates/create/", views.createCardTemplates, name="createCardTemplates"),
    path(
        "templates/update/<int:id>",
        views.updateCardTemplate,
        name="updateCardTemplate",
    ),
    # Card Templates
    path("templates/<int:id>", views.getTemplatesById, name="getTemplatesById"),
    path(
        "templates/delete/<int:id>",
        views.deleteCardTemplate,
        name="cardTemplateDelete",
    ),
    path("templates/publish/", views.pubCardTemplate, name="pubCardTemplates"),
    path("templates/verify/", views.verifyCardTemplate, name="verifyCardTemplates"),
]


#
#
#
#
import itertools

from euler_python.utils import eulerlib

def problem047():
    """

    The first two consecutive numbers to have two distinct prime factors are:

       14 = 2 × 7
       15 = 3 × 5

       The first three consecutive numbers to have three distinct prime factors
       are:

       644 = 2² × 7 × 23
       645 = 3 × 5 × 43
       646 = 2 × 17 × 19.

       Find the first four consecutive integers to have four distinct prime
       factors. What is the first of these numbers?
    """

    def cond(i):
        return all((count_distinct_prime_factors(i + j) == 4) for j in range(4))

    ans = next(filter(cond, itertools.count()))
    return ans

@eulerlib.Memoize
def count_distinct_prime_factors(n):
    count = 0
    while n > 1:
        count += 1
        for i in range(2, eulerlib.sqrt(n) + 1):
            if n % i == 0:
                while True:
                    n //= i
                    if n % i != 0:
                        break
                break
        else:
            break  # n is prime
    return count

if __name__ == "__main__":
    print(problem047())


"""A setuptools based setup module.
See:
https://packaging.python.org/en/latest/distributing.html
https://github.com/pypa/sampleproject
"""

# Always prefer setuptools over distutils
from setuptools import setup, find_packages
# To use a consistent encoding
from codecs import open
from os import path

here = path.abspath(path.dirname(__file__))

# Get the long description from the README file
with open(path.join(here, 'README.md'), encoding='utf-8') as f:
    long_description = f.read()

setup(
    name='noderoot',
    version='0.0.2',
    description='Noderoot is a tool for chrooting into root file systems stored on a different system.',
    long_description=long_description,
    url='https://github.com/gusbricker/noderoot',
    author='Chris Lapa',
    author_email='chris@lapa.com.au',
    license='GPLv2',
    # See https://pypi.python.org/pypi?%3Aaction=list_classifiers
    classifiers=[
        # How mature is this project? Common values are
        #   3 - Alpha
        #   4 - Beta
        #   5 - Production/Stable
        'Development Status :: 3 - Alpha',

        # Indicate who your project is intended for
        'Intended Audience :: Developers',
        'Topic :: Software Development :: Build Tools',

        # Pick your license as you wish (should match "license" above)
        'License :: OSI Approved :: MIT License',

        # Specify the Python versions you support here. In particular, ensure
        # that you indicate whether you support Python 2, Python 3 or both.
        'Programming Language :: Python :: 2.7',
    ],
    keywords='rootfs development linux',
    packages=['noderoot'],
    install_requires=['cffi', 'cryptography', 'paramiko'],
	download_url = 'https://github.com/gusbricker/noderoot/tarball/0.0.2',

    # To provide executable scripts, use entry points in preference to the
    # "scripts" keyword. Entry points provide cross-platform support and allow
    # pip to create the appropriate form of executable for the target platform.
    entry_points={
        'console_scripts': [
            'noderoot = noderoot.__main__:main',
        ],
    },
)


import json
import logging

from library.logger import set_logging
from library.config import Config
from library.aws.rds import RdsSnapshotsChecker
from library.aws.utility import Account
from library.ddb_issues import IssueStatus, RdsPublicSnapshotIssue
from library.ddb_issues import Operations as IssueOperations
from library.aws.utility import DDB, Sns

def lambda_handler(event, context):
    """ Lambda handler to evaluate public RDS snapshots """
    set_logging(level=logging.INFO)

    try:
        payload = json.loads(event["Records"][0]["Sns"]["Message"])
        account_id = payload['account_id']
        account_name = payload['account_name']
        # get the last region from the list to process
        region = payload['regions'].pop()
        # if request_id is present in payload then this lambda was called from the API
        request_id = payload.get('request_id', None)
    except Exception:
        logging.exception(f"Failed to parse event\n{event}")
        return

    try:
        config = Config()

        main_account = Account(region=config.aws.region)
        ddb_table = main_account.resource("dynamodb").Table(config.rdsSnapshot.ddb_table_name)

        account = Account(id=account_id,
                          name=account_name,
                          region=region,
                          role_name=config.aws.role_name_identification)
        if account.session is None:
            return

        logging.debug(f"Checking for public RDS snapshots in {account}")

        # existing open issues for account to check if resolved
        open_issues = IssueOperations.get_account_open_issues(ddb_table, account_id, RdsPublicSnapshotIssue)
        # make dictionary for fast search by id
        # and filter by current region
        open_issues = {issue.issue_id: issue for issue in open_issues if issue.issue_details.region == region}
        logging.debug(f"Public RDS snapshots in DDB:\n{open_issues.keys()}")

        checker = RdsSnapshotsChecker(account=account)
        if checker.check():
            for snapshot in checker.snapshots:
                logging.debug(f"Checking {snapshot.id} ({snapshot.db})")
                issue = RdsPublicSnapshotIssue(account_id, snapshot.id)
                issue.issue_details.db = snapshot.db
                issue.issue_details.name = snapshot.name
                issue.issue_details.region = snapshot.account.region
                issue.issue_details.engine = snapshot.engine
                issue.issue_details.tags = snapshot.tags
                if config.rdsSnapshot.in_whitelist(account_id, snapshot.id):
                    issue.status = IssueStatus.Whitelisted
                else:
                    issue.status = IssueStatus.Open
                logging.debug(f"Setting {snapshot.id} status {issue.status}")
                IssueOperations.update(ddb_table, issue)
                # remove issue id from issues_list_from_db (if exists)
                # as we already checked it
                open_issues.pop(snapshot.id, None)

            logging.debug(f"RDS public snapshots in DDB:\n{open_issues.keys()}")
            # all other unresolved issues in DDB are for removed/remediated RDS snapshots
            for issue in open_issues.values():
                IssueOperations.set_status_resolved(ddb_table, issue)
        if request_id:
            api_table = main_account.resource("dynamodb").Table(config.api.ddb_table_name)
            DDB.track_progress(api_table, request_id)
    except Exception:
        logging.exception(f"Failed to check public RDS snapshots in '{region}' for '{account_id} ({account_name})'")

    # push SNS messages until the list with regions to check is empty
    if len(payload['regions']) > 0:
        try:
            Sns.publish(payload["sns_arn"], payload)
        except Exception:
            logging.exception("Failed to chain public RDS snapshots checking")

    logging.debug(f"Checked public RDS snapshots in '{region}' for '{account_id} ({account_name})'")


from __future__ import annotations # for postponed evaluation
from multiprocessing import Event
from pyneurode.processor_node.Processor import ProcessError
from pyneurode.processor_node.Message import Message
from .BatchProcessor import BatchProcessor
import dearpygui.dearpygui as dpg
import logging
from queue import Empty
import numpy as np
from .Visualizer import Visualizer
from typing import List, Dict
import time

class GUIProcessor(BatchProcessor):
    """    A GUIProcessor is a processor that receive messages for plotting and displaying the GUI.

    .. note:: its process() function will be called frequently. Avoid doing time consuming processing there.
    """

    def __init__(self, internal_buffer_size=1000, frame_rate=60):
        """

        Args:
            internal_buffer_size (int, optional): size of the internal buffer to hold messages before they are displayed. Defaults to 1000.
        """
        super().__init__(interval=1/frame_rate, internal_buffer_size=internal_buffer_size)
        self.visualizers:Dict(List(Visualizer)) = {}

    def startup(self):
        super().startup()

        self.log(logging.DEBUG, 'GUI starting up')
        self.plot_data = (np.arange(500), np.zeros((500,)))
        self.data_count = 0

        # build the GUI
        dpg.create_context()
        dpg.configure_app(docking=True, docking_space=True, init_file='dpg_layout.ini')
        dpg.create_viewport()
        dpg.setup_dearpygui()

        self.make_GUI()
        dpg.show_viewport()

    def register_visualizer(self, visualizer:Visualizer, filters:List[str]):
        """Register a Visualizer object with the GUIProcessor. Only message of types specified in the
        filter list will be passed to the visualizer.

        Args:
            visualizer (Visualizer): visualizer to be creat the GUI and call during frame update
            filters (List[str]): names of message type to be passed to the visualizer.
        """
        #
        # each visualizer is associate with one or more message type definied in the filters

        for f in filters:
            if not f in self.visualizers:
                self.visualizers[f] = []
                self.visualizers[f].append(visualizer)
            else:
                self.visualizers[f].append(visualizer)

    def main_loop(self):

        while not self.shutdown_event.is_set():
            while dpg.is_dearpygui_running():
                dpg.render_dearpygui_frame()
                if self.end_time is None:
                    self.end_time  = time.monotonic() + self.interval

                # Wait till the future trigger point
                time2wait = self.end_time - time.monotonic()

                if time2wait>0:
                    time.sleep(time2wait)

                if time.monotonic() >= self.end_time:
                    self.end_time = time.monotonic() + self.interval #update future trigger time
                    data_list = self.ring_buffer.readAvailable()
                    data = self._process(data_list)
                    self.send(data)

            self.shutdown_event.set() #shutdown all the nodes

    def shutdown(self):
        self.log(logging.DEBUG, 'Destroying GUI context')
        dpg.destroy_context()

    def make_control_panel(self):
        # make the window for control panel
        with dpg.window(label='Control panel',width=-1, height=-1):
            dpg.add_button(label='Save layout', callback=lambda: dpg.save_init_file('dpg_layout.ini'))

    def make_GUI(self):
        # build the GUI
        self.make_control_panel()

        for msg_name, visualizer_list in self.visualizers.items():
            for v in visualizer_list:
                print('Initializing ', v.name)
                v.init_gui()

    def process(self, messages: List[Message] = None):
        # sort the messages according to their type
        #TODO use a separate thread for each visualizer
        msg_list = {}

        for msg in messages:
            if not msg.type in msg_list:
                msg_list[msg.type] = []
                msg_list[msg.type].append(msg)
            else:
                msg_list[msg.type].append(msg)

        # pass each type of messages to their respective visualizer
        for msg_type, msgs in msg_list.items():
            if  msg_type in self.visualizers:
                for v in self.visualizers[msg_type]:
                        v.update(msgs)


# -*- coding: utf8 -*-
import re
import requests
from bs4 import BeautifulSoup
from datetime import datetime

HTML_PARSER = 'html.parser'
ROOT_URL = 'http://www.books.com.tw/'
RANK_URL = 'http://www.books.com.tw/web/sys_saletopb/books/19?attribute=30'
RANK_BEGIN, RANK_LIMIT = 0, 100
RE = re.compile('已追蹤作者|  ')
LIST_EN = ['author', 'publisher', 'pulication_date',
           'language', 'price', 'discount', 'deadline', 'content']
LIST_ZHTW = ['作者', '出版社', '出版日期', '語言', '定價', '優惠價', '優惠期限', '內容大綱']
SAVE_CONTENT = True

def get_books_link():
    rank_req = requests.get(RANK_URL)
    if rank_req.status_code == requests.codes.ok:
        soup = BeautifulSoup(rank_req.content, HTML_PARSER)
        rankshop_link = []
        for link in soup.find_all('li', {'class': 'item'}):
            rankshop_link.append(link.a['href'])
        parse_book(rankshop_link[RANK_BEGIN:RANK_LIMIT])

def parse_book(rankshop_link):
    dt = datetime.today()
    filename = 'TopBook{y}{m:02d}.md'.format(y=dt.year, m=dt.month)
    f = open(filename, 'w', encoding='utf-8')

    if not SAVE_CONTENT:
        del LIST_EN[-1]
        del LIST_ZHTW[-1]

    for number, link in enumerate(rankshop_link, 1):
        req = requests.get(link)
        if req.status_code == requests.codes.ok:
            book_dict = {}
            soup = BeautifulSoup(req.content, HTML_PARSER)

            def SaveValues(eleTags, isTitleImg):
                if isTitleImg:
                    book_dict['title'] = eleTags.find('h1', {'itemprop': 'name'}).text
                    book_dict['image'] = eleTags.find('img', {'class': 'cover M201106_0_getTakelook_P00a400020052_image_wrap'})['src']
                else:
                    for tag in eleTags.find('ul').find_all('li'):
                        for item in LIST_ZHTW:
                            index = LIST_ZHTW.index(item)
                            text = tag.text
                            if re.search(item, text):
                                try:
                                    text = RE.sub('', text)
                                    text_begin = text.find(LIST_ZHTW[index])
                                    text_end = text.find('\n', text.find(LIST_ZHTW[index]))
                                    book_dict[LIST_EN[index]] = text[
                                        text_begin: text_end if text_end != -1 else None]
                                except:
                                    pass
                            elif LIST_EN[index] == 'content' and SAVE_CONTENT:
                                book_dict[LIST_EN[index]] = soup.find('div', {'class': 'bd'}).text.lstrip()
            # print (book_dict)
            SaveValues(soup.find('div', {'class': 'mod type02_p01_wrap clearfix'}), True)
            SaveValues(soup.find('div', {'class': 'type02_p003 clearfix'}), False)
            SaveValues(soup.find('div', {'class': 'cnt_prod002 clearfix'}), False)
            write_md(f, book_dict, number)

    f.close()
    print('\a')     # Notice Program End.

def write_md(f, book_dict, number):
    try:
        f.write('##No.{no} {title}\n'.format(no=number, title=book_dict['title']))
        f.write('![img]({img})\n\n'.format(img=book_dict['image']))
        for index, item in enumerate(LIST_EN):
            if item in book_dict:
                if LIST_EN[index] == 'content' and SAVE_CONTENT:
                    f.write('\t* {item}：\n\t\t'.format(item=LIST_ZHTW[index]))
                    f.write(book_dict[LIST_EN[index]].replace('\u3000\u3000', '\t\t'))
                else:
                    f.write('\t* {item}\n'.format(item=book_dict[LIST_EN[index]]))
            else:
                print ("No.{no} {title}'s {item} is None.".format(
                    no=number, title=book_dict['title'], item=item))
    except:
        pass

if __name__ == '__main__':
    get_books_link()


import sys
import subprocess
import requests
import json
import time

if len(sys.argv) >= 3:
	mention_name = sys.argv[1]
	hipchat_token = sys.argv[2]

	if len(sys.argv) == 4:
		data_usage_threshold = int(sys.argv[3])
	else:
		data_usage_threshold = 2000

	print(data_usage_threshold)

	def on_skype_call():

		nettopArguments = ["nettop", "-P", "-m", "udp", "-p", "Skype Helper", "-l", "2", "-k", "tx_win,tc_class,interface,state,tc_mgt,cc_algo,time,rx_dupe,rx_ooo,rtt_avg,rcvsize,re-tx,P,C,R,W,bytes_out", "-x", "-d"]

		result = subprocess.check_output(nettopArguments)

		lines = result.splitlines()

		if len(lines) >= 4:
			bytes_in_value = lines[3].split()[2]

			if int(bytes_in_value) > data_usage_threshold:
				call_status = "On Call"

			else:
				call_status = "Available"

			print(bytes_in_value)
		else:
			print(0)
			call_status = "Available"

		return call_status

	def update_hipchat_status(new_status, status_message, mention_name, token):  # dnd or chat

		user_url = "https://api.hipchat.com/v2/user/@" + mention_name

		get_result = requests.get(user_url, params={"auth_token": token})

		print(get_result.status_code)

		if get_result.status_code == 200:
			try:
				user_data = json.loads(get_result.content)

				user_data["presence"]["show"] = new_status

				user_data["presence"]["status"] = status_message

				put_result = requests.put(user_url, params={"auth_token": token}, json=user_data)

				print(put_result.content)

			except TypeError:
				print('Error setting hipchat status')

	last_skype_result = "Available"

	while True:

		time.sleep(5)

		skype_result = on_skype_call()

		if skype_result != last_skype_result:

			print(skype_result)

			if skype_result == "On Call":
				update_hipchat_status("dnd", skype_result, mention_name, hipchat_token)
			else:
				update_hipchat_status("chat", skype_result, mention_name, hipchat_token)

		last_skype_result = skype_result

else:
	print("usage: python skypemonitor.py [mention_name] [token] [data_usage_threshold (optional)]")