r"""Inference components such as estimators, training losses and MCMC samplers."""

import torch
import torch.nn as nn
import torch.nn.functional as F

from itertools import islice
from torch import Tensor, BoolTensor, Size
from typing import *

from .distributions import Distribution, DiagNormal, AffineTransform
from .nn import Affine, MLP
from .nn.flows import Buffer, FlowModule, MAF
from .utils import broadcast

class NRE(nn.Module):
    r"""Creates a neural ratio estimation (NRE) classifier network.

    The principle of neural ratio estimation is to train a classifier network
    :math:`d_\phi(\theta, x)` to discriminate between pairs :math:`(\theta, x)`
    equally sampled from the joint distribution :math:`p(\theta, x)` and the
    product of the marginals :math:`p(\theta)p(x)`. Formally, the optimization
    problem is

    .. math:: \arg\min_\phi
        \mathbb{E}_{p(\theta, x)} \big[ \ell(d_\phi(\theta, x)) \big] +
        \mathbb{E}_{p(\theta)p(x)} \big[ \ell(1 - d_\phi(\theta, x)) \big]

    where :math:`\ell(p) = - \log p` is the negative log-likelihood.
    For this task, the decision function modeling the Bayes optimal classifier is

    .. math:: d(\theta, x)
        = \frac{p(\theta, x)}{p(\theta, x) + p(\theta) p(x)}

    thereby defining the likelihood-to-evidence (LTE) ratio

    .. math:: r(\theta, x)
        = \frac{d(\theta, x)}{1 - d(\theta, x)}
        = \frac{p(\theta, x)}{p(\theta) p(x)}
        = \frac{p(x | \theta)}{p(x)}
        = \frac{p(\theta | x)}{p(\theta)} .

    To prevent numerical stability issues when :math:`d_\phi(\theta, x) \to 0`,
    the neural network returns the logit of the class prediction
    :math:`\text{logit}(d_\phi(\theta, x)) = \log r_\phi(\theta, x)`.

    References:
        Approximating Likelihood Ratios with Calibrated Discriminative Classifiers
        (Cranmer et al., 2015)
        https://arxiv.org/abs/1506.02169

        Likelihood-free MCMC with Amortized Approximate Ratio Estimators
        (Hermans et al., 2019)
        https://arxiv.org/abs/1903.04057

    Arguments:
        theta_dim: The dimensionality :math:`D` of the parameter space.
        x_dim: The dimensionality :math:`L` of the observation space.
        moments: The parameters moments :math:`\mu` and :math:`\sigma`. If provided,
            the moments are used to standardize the parameters.
        build: The network constructor (e.g. :class:`lampe.nn.ResMLP`).
        kwargs: Keyword arguments passed to the constructor.
    """

    def __init__(
        self,
        theta_dim: int,
        x_dim: int,
        moments: Tuple[Tensor, Tensor] = None,
        build: Callable[[int, int], nn.Module] = MLP,
        **kwargs,
    ):
        super().__init__()

        if moments is None:
            self.standardize = nn.Identity()
        else:
            mu, sigma = moments
            self.standardize = Affine(-mu / sigma, 1 / sigma)

        self.net = build(theta_dim + x_dim, 1, **kwargs)

    def forward(self, theta: Tensor, x: Tensor) -> Tensor:
        r"""
        Arguments:
            theta: The parameters :math:`\theta`, with shape :math:`(*, D)`.
            x: The observation :math:`x`, with shape :math:`(*, L)`.

        Returns:
            The log-ratio :math:`\log r_\phi(\theta, x)`, with shape :math:`(*,)`.
        """

        theta = self.standardize(theta)
        theta, x = broadcast(theta, x, ignore=1)

        return self.net(torch.cat((theta, x), dim=-1)).squeeze(-1)

class NRELoss(nn.Module):
    r"""Creates a module that calculates the loss :math:`l` of a NRE classifier
    :math:`d_\phi`. Given a batch of :math:`N` pairs :math:`\{ (\theta_i, x_i) \}`,
    the module returns

    .. math:: l = \frac{1}{N} \sum_{i = 1}^N
        \ell(d_\phi(\theta_i, x_i)) + \ell(1 - d_\phi(\theta_{i+1}, x_i))

    where :math:`\ell(p) = - \log p` is the negative log-likelihood.

    Arguments:
        estimator: A classifier network :math:`d_\phi(\theta, x)`.
    """

    def __init__(self, estimator: nn.Module):
        super().__init__()

        self.estimator = estimator

    def forward(self, theta: Tensor, x: Tensor) -> Tensor:
        r"""
        Arguments:
            theta: The parameters :math:`\theta`, with shape :math:`(N, D)`.
            x: The observation :math:`x`, with shape :math:`(N, L)`.

        Returns:
            The scalar loss :math:`l`.
        """

        theta_prime = torch.roll(theta, 1, dims=0)

        log_r, log_r_prime = self.estimator(
            torch.stack((theta, theta_prime)),
            x,
        )

        l1 = -F.logsigmoid(log_r).mean()
        l0 = -F.logsigmoid(-log_r_prime).mean()

        return l1 + l0

class BNRELoss(nn.Module):
    r"""Creates a module that calculates the loss :math:`l` of a balanced NRE (BNRE)
    classifier :math:`d_\phi`. Given a batch of :math:`N` pairs
    :math:`\{ (\theta_i, x_i) \}`, the module returns

    .. math::
        \begin{align}
            l & = \frac{1}{N} \sum_{i = 1}^N
            \ell(d_\phi(\theta_i, x_i)) + \ell(1 - d_\phi(\theta_{i+1}, x_i)) \\
              & + \gamma \left(1 - \frac{1}{N} \sum_{i = 1}^N
                d_\phi(\theta_i, x_i) + d_\phi(\theta_{i+1}, x_i)
            \right)^2
        \end{align}

    where :math:`\ell(p) = - \log p` is the negative log-likelihood.

    References:
        Towards Reliable Simulation-Based Inference with Balanced Neural Ratio Estimation
        (Delaunoy et al., 2022)

    Arguments:
        estimator: A classifier network :math:`d_\phi(\theta, x)`.
    """

    def __init__(self, estimator: nn.Module, gamma: float = 42.0):
        super().__init__()

        self.estimator = estimator
        self.gamma = gamma

    def forward(self, theta: Tensor, x: Tensor) -> Tensor:
        r"""
        Arguments:
            theta: The parameters :math:`\theta`, with shape :math:`(N, D)`.
            x: The observation :math:`x`, with shape :math:`(N, L)`.

        Returns:
            The scalar loss :math:`l`.
        """

        theta_prime = torch.roll(theta, 1, dims=0)

        log_r, log_r_prime = self.estimator(
            torch.stack((theta, theta_prime)),
            x,
        )

        l1 = -F.logsigmoid(log_r).mean()
        l0 = -F.logsigmoid(-log_r_prime).mean()
        lb = (1 - torch.sigmoid(log_r) + torch.sigmoid(log_r_prime)).mean().square()

        return l1 + l0 + self.gamma * lb

class AMNRE(NRE):
    r"""Creates an arbitrary marginal neural ratio estimation (AMNRE) classifier
    network.

    The principle of AMNRE is to introduce, as input to the classifier, a binary mask
    :math:`b \in \{0, 1\}^D` indicating a subset of parameters :math:`\theta_b =
    (\theta_i: b_i = 1)` of interest. Intuitively, this allows the classifier to
    distinguish subspaces and to learn a different ratio for each of them. Formally,
    the classifier network takes the form :math:`d_\phi(\theta_b, x, b)` and the
    optimization problem becomes

    .. math:: \arg\min_\phi
        \mathbb{E}_{p(\theta, x) P(b)} \big[ \ell(d_\phi(\theta_b, x, b)) \big] +
        \mathbb{E}_{p(\theta)p(x) P(b)} \big[ \ell(1 - d_\phi(\theta_b, x, b)) \big],

    where :math:`P(b)` is a binary mask distribution. In this context, the Bayes
    optimal classifier is

    .. math:: d(\theta_b, x, b)
        = \frac{p(\theta_b, x)}{p(\theta_b, x) + p(\theta_b) p(x)}
        = \frac{r(\theta_b, x)}{1 + r(\theta_b, x)} .

    Therefore, a classifier network trained for AMNRE gives access to an estimator
    :math:`\log r_\phi(\theta_b, x, b)` of all marginal LTE log-ratios
    :math:`\log r(\theta_b, x)`.

    References:
        Arbitrary Marginal Neural Ratio Estimation for Simulation-based Inference
        (Rozet et al., 2021)
        https://arxiv.org/abs/2110.00449

    Arguments:
        theta_dim: The dimensionality :math:`D` of the parameter space.
        x_dim: The dimensionality :math:`L` of the observation space.
        args: Positional arguments passed to :class:`NRE`.
        kwargs: Keyword arguments passed to :class:`NRE`.
    """

    def __init__(
        self,
        theta_dim: int,
        x_dim: int,
        *args,
        **kwargs,
    ):
        super().__init__(theta_dim * 2, x_dim, *args, **kwargs)

    def forward(self, theta: Tensor, x: Tensor, b: BoolTensor) -> Tensor:
        r"""
        Arguments:
            theta: The parameters :math:`\theta`, with shape :math:`(*, D)`, or
                a subset :math:`\theta_b`, with shape :math:`(*, |b|)`.
            x: The observation :math:`x`, with shape :math:`(*, L)`.
            b: A binary mask :math:`b`, with shape :math:`(*, D)`.

        Returns:
            The log-ratio :math:`\log r_\phi(\theta_b, x, b)`, with shape :math:`(*,)`.
        """

        if theta.shape[-1] < b.shape[-1]:
            theta, b = broadcast(theta, b, ignore=1)
            theta = theta.new_zeros(b.shape).masked_scatter(b, theta)

        theta = self.standardize(theta) * b
        theta, x, b = broadcast(theta, x, b * 2.0 - 1.0, ignore=1)

        return self.net(torch.cat((theta, x, b), dim=-1)).squeeze(-1)

class AMNRELoss(nn.Module):
    r"""Creates a module that calculates the loss :math:`l` of a AMNRE classifier
    :math:`d_\phi`. Given a batch of :math:`N` pairs :math:`\{ (\theta_i, x_i) \}`,
    the module returns

    .. math:: l = \frac{1}{N} \sum_{i = 1}^N
        \ell(d_\phi(\theta_i \odot b_i, x_i, b_i)) +
        \ell(1 - d_\phi(\theta_{i+1} \odot b_i, x_i, b_i))

    where the binary masks :math:`b_i` are sampled from a distribution :math:`P(b)`.

    Arguments:
        estimator: A classifier network :math:`d_\phi(\theta, x, b)`.
        mask_dist: A binary mask distribution :math:`P(b)`.
    """

    def __init__(
        self,
        estimator: nn.Module,
        mask_dist: Distribution,
    ):
        super().__init__()

        self.estimator = estimator
        self.mask_dist = mask_dist

    def forward(self, theta: Tensor, x: Tensor) -> Tensor:
        r"""
        Arguments:
            theta: The parameters :math:`\theta`, with shape :math:`(N, D)`.
            x: The observation :math:`x`, with shape :math:`(N, L)`.

        Returns:
            The scalar loss :math:`l`.
        """

        theta_prime = torch.roll(theta, 1, dims=0)

        b = self.mask_dist.sample(theta.shape[:-1])

        log_r, log_r_prime = self.estimator(
            torch.stack((theta, theta_prime)),
            x,
            b,
        )

        l1 = -F.logsigmoid(log_r).mean()
        l0 = -F.logsigmoid(-log_r_prime).mean()

        return l1 + l0

class NPE(nn.Module):
    r"""Creates a neural posterior estimation (NPE) normalizing flow.

    The principle of neural posterior estimation is to train a parametric conditional
    distribution :math:`p_\phi(\theta | x)` to approximate the posterior distribution
    :math:`p(\theta | x)`. The optimization problem is to minimize the Kullback-Leibler
    (KL) divergence between the two distributions or, equivalently,

    .. math:: \arg\min_\phi \mathbb{E}_{p(\theta, x)} \big[ -\log p_\phi(\theta | x) \big] .

    Normalizing flows are typically used for :math:`p_\phi(\theta | x)` as they are
    differentiable parametric distributions enabling gradient-based optimization
    techniques.

    References:
        https://wikipedia.org/wiki/Kullback-Leibler_divergence

    Arguments:
        theta_dim: The dimensionality :math:`D` of the parameter space.
        x_dim: The dimensionality :math:`L` of the observation space.
        moments: The parameters moments :math:`\mu` and :math:`\sigma`. If provided,
            the moments are used to standardize the parameters.
        build: The flow constructor (e.g. :class:`lampe.nn.flows.NSF`).
        kwargs: Keyword arguments passed to the constructor.
    """

    def __init__(
        self,
        theta_dim: int,
        x_dim: int,
        moments: Tuple[Tensor, Tensor] = None,
        build: Callable[[int, int], FlowModule] = MAF,
        **kwargs,
    ):
        super().__init__()

        self.flow = build(theta_dim, x_dim, **kwargs)

        if moments is not None:
            mu, sigma = moments
            self.flow.transforms.insert(0, Buffer(AffineTransform, -mu / sigma, 1 / sigma))

    def forward(self, theta: Tensor, x: Tensor) -> Tensor:
        r"""
        Arguments:
            theta: The parameters :math:`\theta`, with shape :math:`(*, D)`.
            x: The observation :math:`x`, with shape :math:`(*, L)`.

        Returns:
            The log-density :math:`\log p_\phi(\theta | x)`, with shape :math:`(*,)`.
        """

        theta, x = broadcast(theta, x, ignore=1)

        return self.flow(x).log_prob(theta)

    @torch.no_grad()
    def sample(self, x: Tensor, shape: Size = ()) -> Tensor:
        r"""
        Arguments:
            x: The observation :math:`x`, with shape :math:`(*, L)`.
            shape: The shape :math:`S` of the samples.

        Returns:
            The samples :math:`\theta \sim p_\phi(\theta | x)`,
            with shape :math:`S + (*, D)`.
        """

        return self.flow(x).sample(shape)

class NPELoss(nn.Module):
    r"""Creates a module that calculates the loss :math:`l` of a NPE normalizing flow
    :math:`p_\phi`. Given a batch of :math:`N` pairs :math:`\{ (\theta_i, x_i) \}`,
    the module returns

    .. math:: l = \frac{1}{N} \sum_{i = 1}^N -\log p_\phi(\theta_i | x_i) .

    Arguments:
        estimator: A normalizing flow :math:`p_\phi(\theta | x)`.
    """

    def __init__(self, estimator: nn.Module):
        super().__init__()

        self.estimator = estimator

    def forward(self, theta: Tensor, x: Tensor) -> Tensor:
        r"""
        Arguments:
            theta: The parameters :math:`\theta`, with shape :math:`(N, D)`.
            x: The observation :math:`x`, with shape :math:`(N, L)`.

        Returns:
            The scalar loss :math:`l`.
        """

        log_p = self.estimator(theta, x)

        return -log_p.mean()

class AMNPE(NPE):
    r"""Creates an arbitrary marginal neural posterior estimation (AMNPE)
    normalizing flow.

    TODO

    Arguments:
        theta_dim: The dimensionality :math:`D` of the parameter space.
        x_dim: The dimensionality :math:`L` of the observation space.
        args: Positional arguments passed to :class:`NPE`.
        kwargs: Keyword arguments passed to :class:`NPE`.
    """

    def __init__(
        self,
        theta_dim: int,
        x_dim: int,
        *args,
        **kwargs,
    ):
        super().__init__(theta_dim, x_dim + theta_dim, *args, **kwargs)

    def forward(self, theta: Tensor, x: Tensor, b: BoolTensor) -> Tensor:
        r"""
        Arguments:
            theta: The parameters :math:`\theta`, with shape :math:`(*, D)`.
            x: The observation :math:`x`, with shape :math:`(*, L)`.
            b: A binary mask :math:`b`, with shape :math:`(*, D)`.

        Returns:
            The log-density :math:`\log p_\phi(\theta | x, b)`, with shape :math:`(*,)`.
        """

        theta, x, b = broadcast(theta, x, b * 2.0 - 1.0, ignore=1)

        return self.flow(torch.cat((x, b), dim=-1)).log_prob(theta)

    @torch.no_grad()
    def sample(self, x: Tensor, b: BoolTensor, shape: Size = ()) -> Tensor:
        r"""
        Arguments:
            x: The observation :math:`x`, with shape :math:`(*, L)`.
            b: A binary mask :math:`b`, with shape :math:`(D,)`.
            shape: The shape :math:`S` of the samples.

        Returns:
            The samples :math:`\theta_b \sim p_\phi(\theta_b | x, b)`,
            with shape :math:`S + (*, D)`.
        """

        x, b_ = broadcast(x, b * 2.0 - 1.0, ignore=1)

        return self.flow(torch.cat((x, b_), dim=-1)).sample(shape)[..., b]

class AMNPELoss(nn.Module):
    r"""Creates a module that calculates the loss :math:`l` of an AMNPE normalizing flow
    :math:`p_\phi`. Given a batch of :math:`N` pairs :math:`\{ (\theta_i, x_i) \}`,
    the module returns

    .. math:: l = \frac{1}{N} \sum_{i = 1}^N
        -\log p_\phi(\theta_i \odot b_i + \theta_{i + 1} \odot (1 - b_i) | x_i, b_i)

    where the binary masks :math:`b_i` are sampled from a distribution :math:`P(b)`.

    Arguments:
        estimator: A normalizing flow :math:`p_\phi(\theta | x, b)`.
        mask_dist: A binary mask distribution :math:`P(b)`.
    """

    def __init__(
        self,
        estimator: nn.Module,
        mask_dist: Distribution,
    ):
        super().__init__()

        self.estimator = estimator
        self.mask_dist = mask_dist

    def forward(self, theta: Tensor, x: Tensor) -> Tensor:
        r"""
        Arguments:
            theta: The parameters :math:`\theta`, with shape :math:`(N, D)`.
            x: The observation :math:`x`, with shape :math:`(N, L)`.

        Returns:
            The scalar loss :math:`l`.
        """

        theta_prime = torch.roll(theta, 1, dims=0)

        b = self.mask_dist.sample(theta.shape[:-1])
        theta = torch.where(b, theta, theta_prime)

        log_prob = self.estimator(theta, x, b)

        return -log_prob.mean()

class MetropolisHastings(object):
    r"""Creates a batched Metropolis-Hastings sampler.

    Metropolis-Hastings is a Markov chain Monte Carlo (MCMC) sampling algorithm used to
    sample from intractable distributions :math:`p(x)` whose density is proportional to a
    tractable function :math:`f(x)`, with :math:`x \in \mathcal{X}`. The algorithm
    consists in repeating the following routine for :math:`t = 1` to :math:`T`, where
    :math:`x_0` is the initial sample and :math:`q(x' | x)` is a pre-defined transition
    distribution.

    1. sample :math:`x' \sim q(x' | x_{t-1})`
    2. :math:`\displaystyle \alpha \gets \frac{f(x')}{f(x_{t-1})} \frac{q(x_{t-1} | x')}{q(x' | x_{t-1})}`
    3. sample :math:`u \sim \mathcal{U}(0, 1)`
    4. :math:`x_t \gets \begin{cases} x' & \text{if } u \leq \alpha \\ x_{t-1} & \text{otherwise} \end{cases}`

    Asymptotically, i.e. when :math:`T \to \infty`, the distribution of samples
    :math:`x_t` is guaranteed to converge towards :math:`p(x)`. In this implementation,
    a Gaussian transition :math:`q(x' | x) = \mathcal{N}(x'; x, \Sigma)` is used, which
    can be modified by sub-classing :class:`MetropolisHastings`.

    Wikipedia:
        https://wikipedia.org/wiki/Metropolis-Hastings_algorithm

    Arguments:
        x_0: A batch of initial points :math:`x_0`, with shape :math:`(*, L)`.
        f: A function :math:`f(x)` proportional to a density function :math:`p(x)`.
        log_f: The logarithm :math:`\log f(x)` of a function proportional
            to :math:`p(x)`.
        sigma: The standard deviation of the Gaussian transition.
            Either a scalar or a vector.

    Example:
        >>> x_0 = torch.randn(128, 7)
        >>> log_f = lambda x: -(x**2).sum(dim=-1) / 2
        >>> sampler = MetropolisHastings(x_0, log_f=log_f, sigma=0.5)
        >>> samples = [x for x in sampler(256, burn=128, step=4)]
        >>> samples = torch.stack(samples)
        >>> samples.shape
        torch.Size([32, 128, 7])
    """

    def __init__(
        self,
        x_0: Tensor,
        f: Callable[[Tensor], Tensor] = None,
        log_f: Callable[[Tensor], Tensor] = None,
        sigma: Union[float, Tensor] = 1.0,
    ):
        super().__init__()

        self.x_0 = x_0

        assert f is not None or log_f is not None, \
            "Either 'f' or 'log_f' has to be provided."

        if f is None:
            self.log_f = log_f
        else:
            self.log_f = lambda x: f(x).log()

        self.sigma = sigma

    def q(self, x: Tensor) -> Distribution:
        return DiagNormal(x, torch.ones_like(x) * self.sigma)

    @property
    def symmetric(self) -> bool:
        return True

    def __iter__(self) -> Iterator[Tensor]:
        x = self.x_0

        # log f(x)
        log_f_x = self.log_f(x)

        while True:
            # y ~ q(y | x)
            y = self.q(x).sample()

            # log f(y)
            log_f_y = self.log_f(y)

            #     f(y)   q(x | y)
            # a = ---- * --------
            #     f(x)   q(y | x)
            log_a = log_f_y - log_f_x

            if not self.symmetric:
                log_a = log_a + self.q(y).log_prob(x) - self.q(x).log_prob(y)

            a = log_a.exp()

            # u in [0; 1]
            u = torch.rand(a.shape).to(a)

            # if u < a, x <- y
            # else x <- x
            mask = u < a

            x = torch.where(mask.unsqueeze(-1), y, x)
            log_f_x = torch.where(mask, log_f_y, log_f_x)

            yield x

    def __call__(self, stop: int, burn: int = 0, step: int = 1) -> Iterable[Tensor]:
        return islice(self, burn, stop, step)


import argparse

from auxilliaries.settings_reader import SettingsReader
from graph_indexing.graph_indexing_components.graph_name_handler import GraphNameHandler
from graph_indexing.graph_indexing_components.graph_printer import GraphPrinter
from graph_indexing.graph_indexing_components.graph_relation_counter import GraphRelationCounter
from graph_indexing.graph_indexing_components.graph_relation_filter import GraphRelationFilter
from graph_indexing.graph_indexing_components.graph_relation_indexer import GraphRelationIndexer
from graph_indexing.graph_indexing_components.graph_type_handler import GraphTypeHandler
from graph_indexing.graph_indexing_components.graph_vertex_indexer import GraphVertexIndexer
from graph_indexing.graph_iterators.graph_file_iterator import GraphFileIterator
from indexes.element_cache import ElementCache
from indexes.element_index import ElementIndex
import sys

"""
Future-proof graph indexing system
"""

parser = argparse.ArgumentParser(description='Indexes a graph')
parser.add_argument('--graph', type=str, help='The location of the graph')
parser.add_argument('--preprocessor_settings', type=str, help='Settings file for preprocessor')
args = parser.parse_args()

settings_reader = SettingsReader()
settings = settings_reader.read(args.preprocessor_settings)

name_cache = settings["cache_locations"]["name_cache"]
name_relation = settings["other"]["name_relation"]
relation_index_location = settings["cache_locations"]["relation_index"]
vertex_index_location = settings["cache_locations"]["vertex_index"]
vertex_type_cache = settings["cache_locations"]["vertex_type_cache"]

discarded_name_file = settings["filters"]["names"]
discarded_relation_file = settings["filters"]["relations"]

graph_processor = GraphPrinter()
relation_index = ElementIndex(relation_index_location)
graph_processor = GraphRelationIndexer(graph_processor, relation_index)
graph_processor = GraphRelationCounter(graph_processor)

vertex_index = ElementIndex(vertex_index_location)
graph_processor = GraphVertexIndexer(graph_processor, vertex_index)

name_cache = ElementCache(name_cache)
type_cache = ElementCache(vertex_type_cache)
graph_processor = GraphTypeHandler(graph_processor, type_cache, name_cache)

graph_processor = GraphRelationFilter(graph_processor, discarded_relation_file, name_relation)

graph_iterator = GraphFileIterator(args.graph)

non_events = []

print("Indexing graph", file=sys.stderr)
for i,graph in enumerate(graph_iterator.iterate()):
    if i % 10000000 == 0:
        print(str(i), file=sys.stderr)
    processed = graph_processor.process(graph)
    if processed is not None:
        print("\t".join(processed))


import scrapy

class GoStdlibSpider(scrapy.Spider):
    """Spider for scraping the Go standard libraries."""

    name = "go-stdlib-spider"

    def start_requests(self):
        """Start making requests."""
        for url, kind in [
            ("https://godoc.org/-/go", "core"),
            ("https://godoc.org/-/subrepo", "subrepo"),
        ]:
            request = scrapy.Request(url, callback=self._parse)
            request.meta["kind"] = kind
            yield request

    def _parse(self, response):
        """Parse the response for the index."""
        library_metadata = [response.meta["kind"]]
        for library_name in response.css("td > a::attr(href)").getall():
            yield {
                "library_name": library_name.strip("/"),
                "lang": "go",
                "library_metadata": library_metadata,
            }


import json as js, pandas, os, re
from matplotlib import pyplot as plt

dataset_completo_path = "C:/Users/Mark/Marco/Magistrale/Anno I/Secondo semestre/DS & ML/Progetto/Social-Mapper-Extended/social_mapper2/dataset/dataset_completo.json"
cf_path = "C:/Users/Mark/Marco/Magistrale/Anno I/Secondo semestre/DS & ML/Progetto/Social-Mapper-Extended/social_mapper2/dataset/cf.json"

f = open(dataset_completo_path, "r")
dataset = js.load(f)

dictionary = dict()

for account, info in dataset.items():
    if account == "dylanchristopher":
        continue

    locations = []
    max_location = None
    if len(info["info_locations"]) > 0:
        max_location = info["info_locations"][0]
        for loc in info["info_locations"]:
            if loc == max_location:
                continue
            if loc["occurrences"] > max_location["occurrences"]:
                max_location = loc
            elif loc["occurrences"] == max_location["occurrences"]:
                locations.append(loc)

    remove_obj = []
    for locs in locations:
        if locs["occurrences"] < max_location["occurrences"]:
            remove_obj.append(locs)

    for obj in remove_obj:
        locations.remove(obj)

    if max_location != None:
        locations.insert(0,max_location)

    if len(locations) > 0:
        dictionary[account] = {"name": info["info_name"], "locations": locations}

with open(cf_path, "w") as cf:
    js.dump(dictionary, cf)


import pygame
from bullet import Bullet
from settings import *

life = 100

class Player(pygame.sprite.Sprite):

    def __init__(self, pos):
        global life
        super().__init__()
        '''
        # size of player
        self.image = pygame.Surface((32, 64))
        # color of player
        self.image.fill("orange")
        '''
        self.image = pygame.image.load("graphics/player/ddcritbcharacter-1.png (4).png")
        self.rect = self.image.get_rect(topleft=pos)
        # x and y axis
        self.direction = pygame.math.Vector2(0, 0)
        self.speed = 8
        # shooting
        self.bullets = pygame.sprite.Group()
        self.firing = False

        self.life = life

        self.score = 0

        self.last_direct = 0

        self.shoot = pygame.mixer.Sound("sounds/zapsplat_foley_rubber_tube_swish_whoosh_through_air_002_96127.mp3")

        self.hit = pygame.mixer.Sound("sounds/zapsplat_multimedia_beep_digital_soft_click_delayed_ascending_87487.mp3")

    def horizontal_movement_collision(self, tiles):
        self.rect.x += self.direction.x * self.speed
        keys = pygame.key.get_pressed()
        for tile in tiles.sprites():
            if tile.rect.colliderect(self.rect):
                if self.direction.x < 0:
                    self.rect.left = tile.rect.right
                    if keys[pygame.K_LSHIFT]:
                        tile.kill()
                elif self.direction.x > 0:
                    self.rect.right = tile.rect.left
                    if keys[pygame.K_LSHIFT]:
                        tile.kill()

    def vertical_movement_collision(self, tiles):
        self.rect.y += self.direction.y * self.speed
        keys = pygame.key.get_pressed()
        for tile in tiles.sprites():
            if tile.rect.colliderect(self.rect):
                if self.direction.y < 0:
                    self.rect.top = tile.rect.bottom
                    if keys[pygame.K_LSHIFT]:
                        tile.kill()
                elif self.direction.y > 0:
                    self.rect.bottom = tile.rect.top
                    if keys[pygame.K_LSHIFT]:
                        tile.kill()

    def get_input(self, tiles):

        keys = pygame.key.get_pressed()
        # going left and right
        # player coordinate9s are (self.rect.x, self.rect.y)
        if keys[pygame.K_RIGHT]:
            self.direction.x = 1
            self.last_direct = 1
        elif keys[pygame.K_LEFT]:
            self.direction.x = -1
            self.last_direct = -1
        # going up and down
        elif keys[pygame.K_UP]:
            self.direction.y = -1
        elif keys[pygame.K_DOWN]:
            self.direction.y = 1
        # not moving
        else:
            self.direction.x = 0
            self.direction.y = 0
        # preventing multiple bullets from being shot at once
        if keys[pygame.K_SPACE] and not self.firing:
            self.fire()
            self.firing = True
        elif not keys[pygame.K_SPACE] and self.firing:
            self.firing = False

    def fire(self):
        # we need to send it x and y coordinates
        # center x means it fires from the center of the player
        bullet = Bullet((self.rect.centerx, self.rect.centery), self.last_direct)
        self.bullets.add(bullet)
        pygame.mixer.Sound.play(self.shoot)

    def draw_bullets(self, surface):
        self.bullets.draw(surface)

    def removelife(self):
        self.life -= 1

    def getlife(self):
        return self.life

    def editscore(self):
        # play sound effect when enemy is killed
        pygame.mixer.Sound.play(self.hit)
        if self.life < 50:
            self.score += 10
        else:
            self.score += 15

    def update(self, tiles, enemies):
        self.get_input(tiles)
        self.enemies = enemies
        collided2 = pygame.sprite.groupcollide(self.bullets, self.enemies, True, True)
        if len(collided2) > 0:
            self.editscore()
        self.horizontal_movement_collision(tiles)
        self.vertical_movement_collision(tiles)
        self.bullets.update(tiles)
        # stops player from leaving the screen
        if self.rect.left < 0:
            self.rect.left = 0
        if self.rect.right > screen_width:
            self.rect.right = screen_width
        if self.rect.top <= 0:
            self.rect.top = 0
        if self.rect.bottom >= screen_height:
            self.rect.bottom = screen_height


#!/usr/bin/env python
from urllib.parse import urlparse, parse_qs
import json
import os
import logging
import re

import bs4

import trackleaders_scraper.common as common

def parse_riders_from_race_page(race_page_text):
    race_page = bs4.BeautifulSoup(race_page_text, "html.parser")
    rider_links = (
        race_page
        .find(string=re.compile('All Riders.*'))
        .find_parent('h3')
        .next_sibling
        .find_all('a', title="Click for individual history")
    )
    return [
        {
            'url_fragment': parse_qs(urlparse(rider_link['href']).query)['name'][0],
            'name': rider_link.string,
        }
        for rider_link in rider_links
    ]

def main():
    args = common.get_base_argparser().parse_args()
    common.configure_logging(args)

    logging.info('Getting Riders')

    session = common.get_trackleaders_session()

    race_page_response = session.get('http://trackleaders.com/{}f.php'.format(args.race))
    race_page_response.raise_for_status()
    riders = parse_riders_from_race_page(race_page_response.text)

    race_path = common.get_race_path(args.race)
    if not os.path.exists(race_path):
        os.mkdir(race_path)

    with open(common.get_riders_path(race_path), 'w') as f:
        json.dump(riders, f, indent=2, sort_keys=True)


import os

from celery import Celery
from celery.schedules import crontab

# Set the default Django settings module for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'sendmails.settings')

app = Celery('sendmails')

app.config_from_object('django.conf:settings', namespace='CELERY')

# Load task modules from all registered Django apps.
app.autodiscover_tasks()

app.conf.beat_schedule = {
    'generate_weekly_triggers': {
        'task': 'app.tasks.send_scheduled_mails',
        'schedule': 1,
    }
}

@app.task(bind=True, ignore_result=True)
def debug_task(self):
    print(f'Request: {self.request!r}')


import torch
from torch.utils.data import Dataset

from gpt3.utils import add_special_tokens
from data_utils import *

tokenizer = add_special_tokens()

class GPT21024Dataset(Dataset):

    def __init__(self, records, max_len=2048):
        self.data = records
        self.max_len = max_len
        self.tokenizer = add_special_tokens()
        self.sep_token = self.tokenizer.encode(self.tokenizer.sep_token)
        self.pad_token = self.tokenizer.encode(self.tokenizer.pad_token)

    def __len__(self):
        return len(self.data)

    def _truncate(self, article, abstract, addit_symbols_count = 1):
        len_abstract = len(abstract)
        max_len_article = self.max_len - len_abstract - addit_symbols_count
        return article[:max_len_article], abstract

    def __getitem__(self, idx):
        sample = self.data[idx]
        text = self.pad_token * self.max_len
        article, abstract = self._truncate(sample['text'], sample['summary'])
        content = article + self.sep_token + abstract
        text[:len(content)] = content
        text = torch.tensor(text)
        sample = {'article': text, 'sum_idx': len(article)}
        return sample

if __name__ == "__main__":
    train_records = read_gazeta_records("gazeta_train.jsonl")
    val_records = read_gazeta_records("gazeta_val.jsonl")
    test_records = read_gazeta_records("gazeta_test.jsonl")

    train_records = tokenize_gazeta(train_records)
    val_records = tokenize_gazeta(val_records)
    test_records = tokenize_gazeta(test_records)


# Import the dependencies.
import numpy as np
import pandas as pd
import datetime as dt
import sqlalchemy
from sqlalchemy.ext.automap import automap_base
from sqlalchemy.orm import Session
from sqlalchemy import create_engine, func
from flask import Flask, jsonify
engine = create_engine("sqlite:///Resources/hawaii.sqlite")

#################################################
# Database Setup
#################################################

# reflect an existing database into a new model
Base = automap_base()

# reflect the tables
Base.prepare(autoload_with=engine)

# Save references to each table
Measurement = Base.classes.measurement
Station = Base.classes.station

# Create our session (link) from Python to the DB
session = Session(engine)

#################################################
# Flask Setup
#################################################
app = Flask(__name__)

#################################################
# Flask Routes
#################################################
@app.route("/")
def climate():
    return(f"/api/v1.0/percipitation
" f"/api/v1.0/stations
" f"/api/v1.0/tobs
"
          f"/api/v1.0/ start
" f"/api/v1.0/start/end
")
@app.route("/api/v1.0/percipitation")
def percipitation():
    session = Session(engine)
    last_date = dt.date(2017, 8, 23) - dt.timedelta(days=365)
    results = session.query(Measurement.date, Measurement.prcp).filter(Measurement.date >= last_date).all()
    ##() generally means calling a function, when calling function parameters go into parantheses
    ##example dt.function(2017, 8, 23)
    ## set is in parantheses, different from list because immutable, cannot change what's in it

    ##{} dictionary, dictionary has key value pairs unlike list
    ##example key is a name of a column and values are names within column

    ##[] a list, in python can be different types of objects, also used to set key in dict

    ##tuples shows with parantheses, a tuple is one variable with multiple objects inside like a list
    ##tuples stores collection of variables like a list
    ##different from list because immutable, cannot change what's in it
    ##also ordered
    ##key value pair because want value = percipitation and key = date
    ##i is value and x is percipitation
    ##calling on object and jsonifying object
    results_dict = {}
    for i, x in results:
        results_dict[i] = x
        session.close()
    return(jsonify(results_dict))
    ##object is a unit of code that computer understands, computer understands c
    ##jsonify converts into object computer understands, similar to translation tool

    ##/api/v1.0/stations
@app.route("/api/v1.0/stations")
def stations():
    session = Session(engine)
    results = session.query(Station.station).all()
    stations = list(np.ravel(results))
    session.close()
    return(jsonify(stations))

    ##/api/v1.0/tobs
@app.route("/api/v1.0/tobs")
def tobs():
    session = Session(engine)
    last_date = dt.date(2017, 8, 23) - dt.timedelta(days=365)
    results = session.query(Measurement.tobs).filter(Measurement.station=="USC00519281").filter(Measurement.date>=last_date).all()
    stations = list(np.ravel(results))
    session.close()
    return(jsonify(stations))

##runs the app(variable created substantiating flask)
##dynamic route means that user can choose
##dynamic route means based on user input

##m, d y allows user to input date and year, uppercase year is full year, lowercase is last two digits
@app.route ("/api/v1.0/<start>")
@app.route ("/api/v1.0/<start>/<end>")
def stats(start = None, end = None):
    session = Session(engine)
    if not end:
        start = dt.datetime.strptime(start, "%m-%d-%Y")
        results = session.query(func.min(Measurement.tobs),func.max(Measurement.tobs),func.avg(Measurement.tobs)).filter(Measurement.station == "USC00519281").filter(Measurement.date >= start).all()
        session.close()
        return(jsonify(list(np.ravel(results))))

    start = dt.datetime.strptime(start, "%m-%d-%Y")
    end = dt.datetime.strptime(end, "%m-%d-%Y")
    results = session.query(func.min(Measurement.tobs),func.max(Measurement.tobs),func.avg(Measurement.tobs)).filter(Measurement.station == "USC00519281").filter(Measurement.date >= start).filter(Measurement.date <= end).all()
    session.close()
    return(jsonify(list(np.ravel(results))))

if __name__ == "__main__":
    app.run(debug=True)


# coding: utf-8
import os
import sys
import logging
#logging.basicConfig()
from traceback import print_exc

__location__ = os.path.realpath(
    os.path.join(os.getcwd(), os.path.dirname(__file__)))

def getLogger(name):
    formatter = logging.Formatter(
        fmt='%(asctime)s %(filename)s:%(lineno)s: %(levelname)-8s %(message)s',
        datefmt='%Y-%m-%d %H:%M:%S')

    console_handler = logging.StreamHandler(stream=sys.stderr)
    console_handler.setFormatter(formatter)

    logger = logging.getLogger(name)
    logger.setLevel(logging.DEBUG) # default level

    logger.addHandler(console_handler)

    log_file_path = os.path.join(
            __location__,
            'logs/common-logs.txt')

    log_dir = os.path.dirname(log_file_path)
    if os.path.isdir(log_dir):
        file_handler = logging.FileHandler(log_file_path,mode='a')
        file_handler.setFormatter(formatter)
        logger.addHandler(file_handler)

    return logger

if __name__ == '__main__':
    logger = getLogger(__name__)
    logger.warn('Hello, World')


from django.urls import path
from . import views

urlpatterns = [

    path('healths/', views.HealthListView.as_view(), name='healths'),
    path('health/<int:pk>', views.HealthDetailView.as_view(), name='healths-detail'),

    path('fashions/', views.FashionListView.as_view(), name='fashions'),
    path('fashion/<int:pk>', views.FashionDetailView.as_view(), name='fashion-detail'),

    path('cooks/', views.CookListView.as_view(), name='cooks'),
    path('cook/<int:pk>', views.CookDetailView.as_view(), name='cook-detail'),

]


import glob
import os
import sys
import platform
import typer
import shutil

from .utils import runCommand, loadPedaUserConfig

loadPedaUserConfig()
app = typer.Typer()

@app.command(
    context_settings={"allow_extra_args": True, "ignore_unknown_options": True}
)
def genbuild(ctx: typer.Context, buildDir: str = './build'):
    runCommand(["cmake", ".", "-B", buildDir] + ctx.args)

@app.command()
def delbuild(buildDir: str = './build'):
    shutil.rmtree(buildDir)

@app.command(
    context_settings={"allow_extra_args": True, "ignore_unknown_options": True}
)
def build(ctx: typer.Context, buildDir: str = './build'):
    runCommand(["cmake", "--build", buildDir] + ctx.args)

@app.command()
def clean(buildDir: str = './build'):
    runCommand(["cmake", "--build", buildDir, "--target", "clean"])

@app.command()
def sln(buildDir: str = './build'):
    if platform.system() == 'Windows':
        # print("We are in windows")
        slns = glob.glob(buildDir + '/*.sln')
        # print(slns)
        if len(slns) > 0:
            #runCommand(["start", slns[0]])
            os.startfile(slns[0])
        else:
            print("No solution files found!", file=sys.stderr)

@app.command(
    context_settings={"allow_extra_args": True, "ignore_unknown_options": True}
)
def run(ctx: typer.Context, prog: str, buildDir: str = './build', buildConfig: str = 'Debug'):
    if prog == None:
        print("Program name missing", file=sys.stderr)
    else:
        # for extra_arg in ctx.args:
        #     print(f"Got extra arg: {extra_arg}")
        if platform.system() == 'Windows':
            winExePath = os.path.join(buildDir, buildConfig, prog + '.exe')
            # print(winExePath)
            runCommand([winExePath] + ctx.args)
        else:
            exePath = os.path.join(buildDir, prog)
            # print(exePath)
            runCommand([exePath] + ctx.args)


import inspect
import logging
from logging.handlers import TimedRotatingFileHandler

log_server = logging.getLogger('server')
log_server.setLevel(logging.DEBUG)
rotate_handler = TimedRotatingFileHandler("log\logs\server.log", when='m', interval=1, backupCount=5)
rotate_handler.suffix = '%Y%m%d'
formatter = logging.Formatter("%(asctime)s %(levelname)-5s %(module)s %(message)s")
rotate_handler.setFormatter(formatter)

log_server.addHandler(rotate_handler)

def log_server_deco(func):
    def wrapper(*args, **kwargs):
        log_server.info(f"Функция {func.__name__} вызвана из функции {inspect.currentframe().f_back.f_code.co_name}")
        return func(*args, **kwargs)

    return wrapper


import os
import sys
import time

import jsbsim
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.utils.data import DataLoader, Dataset
from tqdm import tqdm

torch.set_num_threads(8)

sys.path.append(str(jsbsim.get_default_root_dir()) + '/pFCM/')

from repLearning.repLearning import Representation
from src.environments.jsbsim.jsbsimEnv import DogfightEnv as Env

class DogfightDataset(Dataset):

    def __init__(
        self,
        status,  # of size [num_of_data, 9, 10, 50, 50]
        property,  # of size [num_of_data, num_of_property]
        label,  # of size [num_of data]
    ):
        self.status = status
        self.property = property
        self.label = label

    def __len__(self):
        return len(self.status)

    def __getitem__(self, index):
        data = {}
        data["status"] = self.status[index]
        data["property"] = self.property[index]
        data["label"] = self.label[index]
        return data

class IDSD():

    def __init__(
        self,
        modelPath='/data/wnn_data/bestModel/',
    ) -> None:
        from repLearning.cnn import Model
        self.model = Model(

        )
        self.modelPath = modelPath
        if not os.path.exists(self.modelPath):
            os.mkdir(self.modelPath)
        if os.listdir(self.modelPath) != []:
            try:
                self.model.load_state_dict(torch.load(self.modelPath + 'Epoch.pt'))
            except:
                print("Model Loading Error!")
                time.sleep(1)

    def episode(
        self,
        device,
        optimizer,
    ):
        env = Env()
        print("**********Nof: {}**********".format(env.getNof()))
        wins_record1 = []
        wins_record2 = []

        while True:

            terminate = env.step(playSpeed=0)
            if terminate != 0:
                break

            rep1 = Representation(env)
            rep2 = Representation(env)
            rl1 = rep1.getRepresentation('IDSD', self.model, device, 1) + torch.cat([torch.rand([1, 4]) / 2, torch.zeros([1, 1])], dim=1).to(device)
            rl2 = rep2.getRepresentation('IDSD', self.model, device, 2)
            if env.getNof() % 12 == 0:
                if wins_record1 == []:
                    wins_record1 = torch.cat([rl1, torch.ones([1, 1]).to(device)], dim=1).unsqueeze(0)
                    input1 = rep1.getStatus()
                    inputp_1 = torch.Tensor(rep1.getProperty(1)).unsqueeze(0)

                    wins_record2 = torch.cat([rl2, torch.ones([1, 1]).to(device)], dim=1).unsqueeze(0)
                    input2 = rep2.getStatus()
                    inputp_2 = torch.Tensor(rep2.getProperty(2)).unsqueeze(0)
                else:
                    wins_record1 = torch.cat([wins_record1, torch.cat([rl1, torch.ones(1, 1).to(device)], dim=1).unsqueeze(0)], dim=0)
                    input1 = torch.cat([input1, rep1.getStatus()])
                    inputp_1 = torch.cat([inputp_1, torch.Tensor(rep1.getProperty(1)).unsqueeze(0)])

                    wins_record2 = torch.cat([wins_record2, torch.cat([rl2, torch.ones(1, 1).to(device)], dim=1).unsqueeze(0)], dim=0)
                    input2 = torch.cat([input2, rep2.getStatus()])
                    inputp_2 = torch.cat([inputp_2, torch.Tensor(rep2.getProperty(2)).unsqueeze(0)])
            env.getFdm(1).sendAction(rl1.tolist())
            env.getFdm(2).sendAction(rl2.tolist())
        if terminate == 1:
            wins_data = wins_record1
            wins_input = input1
            wins_inputp = inputp_1
        elif terminate == 2:
            wins_data = wins_record2
            wins_input = input2
            wins_inputp = inputp_2
        elif terminate == -1:
            return
        else:
            raise Exception("Return code error!", terminate)

        fullDataset = DogfightDataset(wins_input, wins_inputp, wins_data)
        trainLoader = DataLoader(dataset=fullDataset, batch_size=1, shuffle=True)
        for _ in range(1):
            for batch in trainLoader:
                self.model.train()
                status = batch['status']
                property = batch['property']
                label = batch['label']
                status = status.to(device)
                property = property.to(device)
                label = label.to(device)
                pred = self.model(status, property)  # torch.Size([1, 7])
                pred = pred.to(device)

                # for i in range(0, 3):
                #     label[0, i] = min(max(label[0, i], 1), -1)

                # for i in range(3, 6):
                #     label[0, i] = min(max(label[0, i], 1), 0)

                # print(batch_pred.size())
                # pred = F.normalize(pred, dim=0)
                # label = F.normalize(pred, dim=0)
                print(pred.size())
                loss = 0
                label = label.squeeze().unsqueeze(dim=0)
                print(label)
                for i in range(5):
                    # loss = loss + np.log(np.abs(pred[0, i].item()) + .0001) * np.abs(label[0, i].item())
                    loss = loss + (label[0, i].item() - pred[0, i].item()) ** 2
                # loss = loss_function(pred, label)
                print("pred: {}\nlabel: {}".format(pred, label))
                print("loss: {}".format(loss))
                loss = torch.tensor(loss, requires_grad=True)
                self.model.zero_grad()
                loss.backward(retain_graph=True)
                optimizer.step()

    def train(
        self,
        epochs=20000,
        cuda='0',
        optimizer='SGD',
        lr=1e-2,
        momentum=0.9,
        weight_decay=1e-3
    ):
        device = torch.device("cuda:{}".format(cuda) if torch.cuda.is_available() else "cpu")
        self.model.to(device)
        if optimizer == 'SGD':
            optimizer = optim.SGD(self.model.parameters(), lr, momentum, weight_decay)
        elif optimizer == 'Adam':
            raise Exception("Optimizer Adam isn't supported yet :(")
        else:
            raise Exception("Optimizer {} doesn't exist.".format(optimizer))
        for _ in tqdm(range(epochs)):
            self.episode(device, optimizer)
            torch.save(self.model.state_dict(), self.modelPath + 'Epoch.pt')

if __name__ == '__main__':
    model = IDSD(

    )

    model.train(
        cuda='3',
    )


import glob
import pathlib
import pandas as pd
import xlsxwriter
import ghgscenario as gs
import lukeghg.crf.ghginventory as ghg

uid_file_dict = dict()

def uidmatrix_cell_count(uid_set:set):
    return len(uid_set)

def possible_missing_uid(scenuid_set,uidmatrix_set):
    return scenuid_set.difference(uidmatrix_set)

def scen_files_to_uid_set(files:str,uid_mapping_file:str):
    file_ls = glob.glob(files)
    uid_set = set()
    for fname in file_ls:
        p=pathlib.Path(fname)
        ls = ghg.ParseGHGInventoryFile(fname,uid_mapping_file)
        for x in ls:
            if len(x) == 0:
                print("Empty line")
            else:
                uid = x.pop(0)
                uid_set.add(uid)
                uid_file_dict[uid]=p.name
    return uid_set

def uid_matrix_to_uid_cells(uid_matrix_file:str):
    df = gs.read_uid_matrix_file(uid_matrix_file)
    df = df.loc[:,'FL-FL':]
    a = df.to_numpy()
    a = a.flatten()
    uid_set = set(a)
    return uid_set

def uid_matrix_to_uid_set(uid_matrix_file:str):
    df = gs.read_uid_matrix_file(uid_matrix_file)
    #Start column is FL-FL
    df = df.loc[:,'FL-FL':]
    #This will be array of arrays
    a = df.to_numpy()
    #This will be an array
    a = a.flatten()
    #The set data type
    uid_set = set(a)
    #Filter out empty cells
    uid_set = {x for x in a if pd.notna(x)}
    return uid_set

def create_uid_df(uidmatrix_count,uidmatrix_set,scenuid_set,missing_uid_set):
    ls1 = sorted(list(uidmatrix_set))
    ls2 = sorted(list(missing_uid_set))
    ls3 = sorted(list(scenuid_set))
    fname_ls2 = [uid_file_dict[uid] for uid in ls2]
    fname_ls3 = [uid_file_dict[uid] for uid in ls3]
    ls_table = [ls1,ls2,fname_ls2,ls3,fname_ls3]
    df = pd.DataFrame(ls_table)
    df = df.transpose()
    df.columns=["In UIDMatrix: "+str(len(ls1)),"Missing from UIDMatrix: "+str(len(ls2)),"File",
                "In Scenario files: "+str(len(ls3)),"File"]
    df.index.name="All UIDMatrix entries: "+str(uidmatrix_count)
    return df

def create_missing_scen_uid_excel(uid_matrix_file:str,scen_files:str,uid_mapping_file:str,excel_file:str):
    """Create excel file containing missing uid from the inventory
    uid_matrix_file: template uid matrix file
    scen_files: scenario inventory files (wild card search)
    uid_mapping_file: the uid mapping file
    excel_file: output file for missing uid
    """
    uidmatrix_count = uidmatrix_cell_count(uid_matrix_to_uid_cells(uid_matrix_file))
    uidmatrix_set = uid_matrix_to_uid_set(uid_matrix_file)
    scenuid_set = scen_files_to_uid_set(scen_files,uid_mapping_file)
    missing_uid_set =  possible_missing_uid(scenuid_set,uidmatrix_set)
    df = create_uid_df(uidmatrix_count,uidmatrix_set,scenuid_set,missing_uid_set)
    writer = pd.ExcelWriter(excel_file,engine='xlsxwriter')
    df.to_excel(writer,sheet_name='UIDMatrix')
    writer.close()


import librosa
import tensorflow as tf
import numpy as np
from copy import deepcopy
from tensorflow.keras.layers import Dense, Activation, Dropout, Conv1D, MaxPooling1D, BatchNormalization
from tensorflow.contrib.rnn import GRUCell, RNNCell
from util.hparams import *

def pre_net(input_data, training):
    x = Dense(256)(input_data)
    x = Activation('relu')(x)
    x = Dropout(0.5)(x, training=training)
    x = Dense(128)(x)
    x = Activation('relu')(x)
    x = Dropout(0.5)(x, training=training)
    return x

def CBHG(input_data, sequence_length, K, conv_dim):
    x = tf.concat([
        Activation('relu')(BatchNormalization()(
            Conv1D(128, kernel_size=k, strides=1, padding='same')(input_data))) for k in range(1, K+1)], axis=-1)

    x = MaxPooling1D(pool_size=2, strides=1, padding='same')(x)
    x = Conv1D(conv_dim[0], kernel_size=3, strides=1, padding='same')(x)
    x = BatchNormalization()(x)
    x = Activation('relu')(x)
    x = Conv1D(conv_dim[1], kernel_size=3, strides=1, padding='same')(x)
    x = BatchNormalization()(x)
    highway_input = input_data + x

    if K == 8:
        highway_input = Dense(128)(highway_input)

    for _ in range(4):
        H = Dense(128)(highway_input)
        H = Activation('relu')(H)
        T = Dense(128, bias_initializer=tf.constant_initializer(-1.0))(highway_input)
        T = Activation('sigmoid')(T)
        highway_input = H * T + highway_input * (1.0 - T)

    x, _ = tf.nn.bidirectional_dynamic_rnn(
        GRUCell(128),
        GRUCell(128),
        highway_input,
        sequence_length=sequence_length,
        dtype=tf.float32)
    x = tf.concat(x, axis=2)

    return x

class ConcatWrapper(RNNCell):
    def __init__(self, cell):
        super(ConcatWrapper, self).__init__()
        self.cell = cell

    @property
    def state_size(self):
        return self.cell.state_size

    @property
    def output_size(self):
        return self.cell.output_size + self.cell.state_size.attention

    def zero_state(self, batch_size, dtype):
        return self.cell.zero_state(batch_size, dtype)

    def call(self, inputs, state):
        output, res_state = self.cell(inputs, state)
        return tf.concat([output, res_state.attention], axis=-1), res_state

def griffin_lim(spectrogram):
    spec = deepcopy(spectrogram)
    for i in range(50):
        est_wav = librosa.istft(spec, hop_length=hop_length, win_length=win_length)
        est_stft = librosa.stft(est_wav, n_fft=n_fft, hop_length=hop_length, win_length=win_length)
        phase = est_stft / np.maximum(1e-8, np.abs(est_stft))
        spec = spectrogram * phase
    wav = librosa.istft(spec, hop_length=hop_length, win_length=win_length)
    return np.real(wav)


from django.db import models
from django.conf import settings
from django.db import models, transaction
from django.core.mail import send_mail
import secrets
import string
from translation.metadata import TRANSLATION_LANGUAGE_CHOICES
from django.contrib.postgres.fields import ArrayField

TRANSCRIPT_TYPE = (
    ("ORIGINAL_SOURCE", "Original Source"),
    ("MACHINE_GENERATED", "Machine Generated"),
    ("MANUALLY_CREATED", "Manually Created"),
    ("MANUALLY_UPLOADED", "Manually Uploaded"),
)

TRANSLATION_TYPE_CHOICES = (
    ("MACHINE_GENERATED", "Machine Generated"),
    ("MANUALLY_CREATED", "Manually Created"),
    ("ORIGINAL_SOURCE", "Original Source"),
)

VOICEOVER_TYPE_CHOICES = (
    ("MACHINE_GENERATED", "Machine Generated"),
    ("MANUALLY_CREATED", "Manually Created"),
)

TASK_TYPE = (
    ("TRANSCRIPTION_EDIT", "Transcription Edit"),
    ("TRANSCRIPTION_REVIEW", "Transcription Review"),
    ("TRANSLATION_EDIT", "Translation Edit"),
    ("TRANSLATION_REVIEW", "Translation Review"),
    ("VOICEOVER_EDIT", "VoiceOver Edit"),
)

class Organization(models.Model):
    """
    Model for organizations
    """

    title = models.CharField(
        verbose_name="organization_title", max_length=512, null=False, unique=True
    )

    email_domain_name = models.CharField(
        verbose_name="organization_email_domain", max_length=512, null=True
    )

    is_active = models.BooleanField(
        verbose_name="organization_is_active",
        default=True,
        help_text=("Designates whether an organization is active or not."),
    )

    created_by = models.ForeignKey(
        settings.AUTH_USER_MODEL,
        on_delete=models.SET_NULL,
        null=True,
        related_name="organization_created",
        verbose_name="created_by",
    )

    organization_owner = models.OneToOneField(
        settings.AUTH_USER_MODEL,
        on_delete=models.SET_NULL,
        null=True,
        related_name="organization_owned",
        verbose_name="organization_owner",
    )

    default_transcript_editor = models.ForeignKey(
        settings.AUTH_USER_MODEL,
        verbose_name="transcript editor",
        related_name="transcript_editor",
        on_delete=models.SET_NULL,
        default=None,
        null=True,
        blank=True,
    )

    default_transcript_reviewer = models.ForeignKey(
        settings.AUTH_USER_MODEL,
        verbose_name="transcript reviewer",
        related_name="transcript_reviewer",
        on_delete=models.SET_NULL,
        default=None,
        null=True,
        blank=True,
    )

    default_translation_editor = models.ForeignKey(
        settings.AUTH_USER_MODEL,
        verbose_name="translation editor",
        related_name="translation_editor",
        on_delete=models.SET_NULL,
        default=None,
        null=True,
        blank=True,
    )

    default_translation_reviewer = models.ForeignKey(
        settings.AUTH_USER_MODEL,
        verbose_name="translation reviewer",
        related_name="translation_reviewer",
        on_delete=models.SET_NULL,
        default=None,
        null=True,
        blank=True,
    )

    default_transcript_type = models.CharField(
        choices=TRANSCRIPT_TYPE,
        max_length=35,
        default=None,
        verbose_name="default transcript type",
        null=True,
        blank=True,
    )

    default_translation_type = models.CharField(
        choices=TRANSLATION_TYPE_CHOICES,
        max_length=35,
        verbose_name="Default Translation Type",
        default=None,
        null=True,
        blank=True,
    )

    created_at = models.DateTimeField(verbose_name="created_at", auto_now_add=True)
    updated_at = models.DateTimeField(verbose_name="updated_at", auto_now=True)

    default_voiceover_type = models.CharField(
        choices=VOICEOVER_TYPE_CHOICES,
        max_length=35,
        default=None,
        verbose_name="Project Default VoiceOver Type",
        null=True,
        blank=True,
    )
    default_task_types = ArrayField(
        models.CharField(
            choices=TASK_TYPE,
            blank=True,
            default=None,
            null=True,
            max_length=50,
        ),
        blank=True,
        default=None,
        null=True,
        verbose_name="Organization Default Task Types",
    )

    default_target_languages = ArrayField(
        models.CharField(
            choices=TRANSLATION_LANGUAGE_CHOICES,
            blank=True,
            default=None,
            null=True,
            max_length=50,
        ),
        blank=True,
        default=None,
        null=True,
        verbose_name="Organization Default Target Languages",
    )
    description = models.TextField(
        max_length=1000, null=True, blank=True, help_text=("Organization Description")
    )
    enable_upload = models.BooleanField(
        verbose_name="enable_upload",
        default=False,
        help_text=("Indicates whether CSV upload is enable or not."),
    )

    def __str__(self):
        return self.title + ", id=" + str(self.pk)

class Invite(models.Model):
    """
    Invites to invite users to organizations.
    """

    user = models.OneToOneField(
        settings.AUTH_USER_MODEL,
        related_name="invite_users",
        on_delete=models.CASCADE,
        null=True,
    )

    organization = models.ForeignKey(
        Organization,
        on_delete=models.CASCADE,
        null=True,
        related_name="invite_oganization",
        verbose_name="organization",
    )

    invite_code = models.CharField(
        verbose_name="invite_code", max_length=256, null=True, unique=True
    )

    def __str__(self):
        return str(self.user.email)

    @classmethod
    def create_invite(cls, organization=None, users=None):
        with transaction.atomic():
            for user in users:
                try:
                    invite = Invite.objects.get(user=user)
                except:
                    invite = Invite.objects.create(organization=organization, user=user)
                    invite.invite_code = cls.generate_invite_code()
                    invite.save()
                if organization is not None:
                    organization_name = organization.title
                else:
                    organization_name = "be the Org Owner."
                send_mail(
                    "Invitation to join Organization",
                    f"Hello! You are invited to {organization_name}. Your Invite link is: https://chitralekha.ai4bharat.org/#/invite/{invite.invite_code}",
                    settings.DEFAULT_FROM_EMAIL,
                    [user.email],
                )

    # def has_permission(self, user):
    #     if self.organization.created_by.pk == user.pk or user.is_superuser:
    #         return True
    #     return False

    @classmethod
    def generate_invite_code(cls):
        return "".join(
            secrets.choice(string.ascii_uppercase + string.digits) for i in range(10)
        )


import pathlib
from typing import List

import kclvm.config
import kclvm.internal.util as util
import kclvm.internal.gpyrpc.gpyrpc_pb2 as pb2

KCL_MOD_PATH_ENV = "${KCL_MOD}"

def load_settings_files(
    work_dir: str, files: List[str]
) -> pb2.LoadSettingsFiles_Result:
    """Load KCL CLI config from the setting files.

    Parameter
    ---------
    work_dir : str
        The kcl run work directory.
    files:
        The setting YAML files.

    Returns
    -------
    result: LoadSettingsFiles_Result
        The merged kcl singleton config.
    """
    from kclvm.compiler.vfs import GetPkgRoot

    if not files:
        return pb2.LoadSettingsFiles_Result(
            kcl_cli_configs=pb2.CliConfig(), kcl_options=[]
        )
    key_value_pairs = [
        pb2.KeyValuePair(key=k, value=v)
        for k, v in util.merge_option_same_keys(
            kclvm.config.KCLCLISettingAction().deal(files)[0]
        ).items()
    ]
    if work_dir or kclvm.config.current_path:
        files = [
            str(
                pathlib.Path(work_dir)
                .joinpath(
                    str(x).replace(
                        KCL_MOD_PATH_ENV,
                        GetPkgRoot(work_dir or kclvm.config.current_path or files[0])
                        or "",
                    )
                )
                .resolve()
            )
            for x in kclvm.config.input_file
        ]
    return pb2.LoadSettingsFiles_Result(
        kcl_cli_configs=pb2.CliConfig(
            files=files,
            output=kclvm.config.output,
            overrides=kclvm.config.overrides,
            path_selector=kclvm.config.path_selector,
            strict_range_check=kclvm.config.strict_range_check,
            disable_none=kclvm.config.disable_none,
            verbose=kclvm.config.verbose,
            debug=kclvm.config.debug,
        ),
        kcl_options=key_value_pairs,
    )


import datetime

from flask import Blueprint
from core import Utils
from exts import db, APIResponse
from models import Interview, Person

listing_resources = Blueprint('listing_resources', __name__)

@listing_resources.route('/interview/<interview_id>', methods=['GET'])
def get_interview(interview_id):
    interview = db.session.query(Interview).filter_by(interview_id=interview_id).one()
    return APIResponse.respond(Utils.extract_details(interview))

@listing_resources.route('/interviews', methods=['GET'])
def list_interviews():
    now = datetime.datetime.now()
    interviews = db.session.query(Interview).filter(Interview.start >= now).order_by(Interview.start).all()
    interview_list = []
    for interview in interviews:
        print(interview.get_all())
        interview_list.append(Utils.extract_details(interview))
    return APIResponse.respond(interview_list)

@listing_resources.route('/persons', methods=['GET'])
def list_person():
    persons = db.session.query(Person).all()
    persons_list = []
    for person in persons:
        persons_list.append({'email': person.email, 'name': person.name})
    return APIResponse.respond(persons_list)


# -*- coding: utf-8 -*-
import os
import telebot
import time
import random
import threading
from emoji import emojize
from telebot import types
from pymongo import MongoClient
import game_classes
import lobbys
import cards
from tools import medit
import traceback
games=lobbys.games

from game_classes import codetoclass, findallenemy, findnearenemy

token = os.environ['TELEGRAM_TOKEN']
bot = telebot.TeleBot(token)

#client=MongoClient(os.environ['database'])
#db=client.unknown
#users=db.users

@bot.message_handler(commands=['creategame'])
def creategame(m):
    if m.chat.id not in games:
        if m.chat.id!=m.from_user.id:
            game=game_classes.Game(m)
            games.update({game.id:game})
            kb=types.InlineKeyboardMarkup()
            kb.add(types.InlineKeyboardButton(text='Присоединиться',callback_data='join'))
            msg=bot.send_message(m.chat.id,'Набор участников для экспедиции открыт! Жмите "Присоединиться" для вступления в игру.',reply_markup=kb)
            game=games[m.chat.id]
            game.message=msg
            t=threading.Timer(30,game.cancelgame)
            t.start()
            game.canceltimer=t
        else:
            bot.send_message(m.chat.id, 'В эту игру нельзя играть в личке! Добавьте бота в какой-нибудь чат.')

@bot.message_handler(commands=['del'])
def delett(m):
    try:
        del games[m.chat.id]
        bot.send_message(m.chat.id, 'Игра была удалена!')
    except:
        pass

@bot.message_handler(commands=['startgame'])
def startgame(m):
    try:
        game=games[m.chat.id]
    except:
        game=None
    if game!=None:
        game.canceltimer=None
        game.startgame()

@bot.callback_query_handler(func=lambda call:True)
def inline(call):
    if call.data=='join':
        try:
            game=games[call.message.chat.id]
        except:
            game=None
        if game!=None:
            if game.started==False:
                if call.from_user.id not in game.playerlist:
                    game.createplayer(call.from_user)
                    bot.send_message(call.message.chat.id,call.from_user.first_name+' присоединился!')
                    game.m_update()
        else:
            bot.send_message(call.message.chat.id, call.from_user.first_name+', в этом чате нет запущенной игры! Сначала начните её '+
                             'командой /creategame.')
    else:
      try:
          kb=types.InlineKeyboardMarkup()
          game=games[int(call.data.split(' ')[1])]
          for ids in game.playerlist:
              if game.playerlist[ids].id==call.from_user.id:
                  user=game.playerlist[ids]
          chat=game
          if 'playcard' in call.data:
              for ids in user.cards:
                  print(ids)
                  print(ids.name)
                  kb.add(types.InlineKeyboardButton(text=ids.name, callback_data='info '+str(chat.id)+' '+ids.code))
              medit('Выберите карту:', call.message.chat.id, call.message.message_id, reply_markup=kb)

          if 'info' in call.data:
              x=call.data.split(' ')[2]
              text='none'
              card=None
              for ids in user.cards:
                  if ids.code==x:
                      card=ids
              print('Карта: ')
              print(card)
              if card!=None:
                  text=card.info
                  if card.type!='unknown' and card.type!='infection':
                      kb.add(types.InlineKeyboardButton(text='⚡Использовать карту', callback_data='usecard '+str(chat.id)+' '+card.code))
                  kb.add(types.InlineKeyboardButton(text='↩Назад', callback_data='mainmenu '+str(chat.id)))
                  medit(text, call.message.chat.id, call.message.message_id, reply_markup=kb, parse_mode='markdown')

          if 'trade' in call.data:
              x=call.data.split(' ')[2]
              card=None
              for ids in user.cards:
                  if ids.code==x:
                      card=ids
              if card!=None:
                  user.fortrade=card
                  medit('Выбрано для обмена: "'+card.name+'".',call.message.chat.id, call.message.message_id)
              else:
                  bot.answer_callback_query(call.id, 'У вас этого нет!')

          if 'usecard' in call.data:
              x=call.data.split(' ')[2]
              card=None
              for ids in user.cards:
                  if ids.code==x:
                      card=ids
              try:
                  trgt=call.data.split(' ')[3]
                  for ids in chat.playerlist:
                      if chat.playerlist[ids].id==int(trgt):
                          target=chat.playerlist[ids]
              except:
                  target=None
              if card!=None:
                  print(card)
                  if card.type=='action' or card.type=='barrier':
                      if user.active:
                          if card.targetable:
                              if target==None:
                                  if card.targetall:
                                      enemies=findallenemy(user, game)
                                  else:
                                      enemies=findnearenemy(user, game)
                                  if card.target_self:
                                      enemies.append(user)
                                  for ids in enemies:
                                      kb.add(types.InlineKeyboardButton(text=ids.name, callback_data='usecard '+str(chat.id)+' '+x+' '+str(ids.id)))
                                  kb.add(types.InlineKeyboardButton(text='Назад', callback_data='mainmenu'))
                                  medit('Выберите цель для карты "'+card.name+'":', call.message.chat.id, call.message.message_id, reply_markup=kb)
                              else:
                                  if card.cancancelled!=[]:
                                      t=threading.Timer(10, card.use, args=[user, target, chat])
                                      t.start()
                                      target.defmenu(card)
                                  else:
                                      card.use(user, target, chat)
                                  medit('Выбрано: "'+card.name+'".', call.message.chat.id, call.message.message_id)
                          else:
                              try:
                                  enm=call.data.split(' ')[3]
                              except:
                                  enm=None
                              enemy=None
                              if enm!=None:
                                  for ids in chat.playerlist:
                                      if chat.playerlist[ids].id==int(enm):
                                          enemy=chat.playerlist[ids]
                              else:
                                  enemy=user
                              if card.cancancelled!=[]:
                                  t=threading.Timer(10, card.use, args=[user, enemy, chat])
                                  t.start()
                                  enemy.defmenu(card)
                              else:
                                  card.use(user, enemy, chat)
                              medit('Выбрано: "'+card.name+'".', call.message.chat.id, call.message.message_id)
                      else:
                          bot.answer_callback_query(call.id, 'Сейчас не ваш ход!')

                  elif card.type=='defence':
                      if user.active==False and user.attacked:
                          pass
                      else:
                          bot.answer_callback_query(call.id, 'Эту карту можно сыграть только в ответ на сыгранную на вас карту!')

      except Exception as e:
          print('Ошибка:\n', traceback.format_exc())
          bot.send_message(441399484, traceback.format_exc())

print('7777')
bot.polling(none_stop=True,timeout=600)


# Graph Valid Tree

# Description
# Given n nodes labeled from 0 to n - 1 and a list of undirected edges
# (each edge is a pair of nodes), write a function to check whether these
# edges make up a valid tree.

# You can assume that no duplicate edges will appear in edges. Since all
# edges are undirected, [0, 1] is the same as [1, 0] and thus will not
# appear together in edges.

# Example

# Example 1:

# Input: n = 5 edges = [[0, 1], [0, 2], [0, 3], [1, 4]]
# Output: true.

# Example 2:

# Input: n = 5 edges = [[0, 1], [1, 2], [2, 3], [1, 3], [1, 4]]
# Output: false.

from collections import defaultdict, deque

class Solution_BFS:
    """
    @param n: An integer
    @param edges: a list of undirected edges
    @return: true if it's a valid tree, or false
    """
    def __init__(self):
        self.neighbors = defaultdict(list)

    def validTree(self, n, edges):
        # write your code here
        if n - 1 != len(edges):
            return False

        self._build_neighbors(edges)
        queue = deque([0])
        seen = set([0])

        while queue:
            node = queue.popleft()
            for neighbor in self.neighbors[node]:
                if neighbor not in seen:
                    queue.append(neighbor)
                    seen.add(neighbor)

        return len(seen) == n

    def _build_neighbors(self, edges):
        for start, end in edges:
            self.neighbors[start].append(end)
            self.neighbors[end].append(start)

class UnionFind():
    def __init__(self, n):
        self.father = {}
        for i in range(n):
            self.father[i] = i
        self.count = n

    def find(self, other):
        path = []
        while other != self.father[other]:
            path.append(other)
            other = self.father[other]

        for p in path:
            self.father[p] = other

        return other

    def union(self, a, b):
        father_a = self.find(a)
        father_b = self.find(b)
        if father_a != father_b:
            self.father[father_a] = father_b
            self.count -= 1

class Solution_UnionFind:
    """
    @param n: An integer
    @param edges: a list of undirected edges
    @return: true if it's a valid tree, or false
    """
    def validTree(self, n, edges):
        # write your code here
        if n - 1 != len(edges):
            return False

        uf = UnionFind(n)
        for start, end in edges:
            uf.union(start, end)

        return uf.count == 1


import time
import datetime
from timeConvert import dayInSeconds
from futureTides import HighTideKeeper, LowTideKeeper
import json

dayTimeTimeStamp = 1602907205
capeCharlesTimeStamp = 1592057307 #Low Tide morning of 6/13/2020 in Cape Charles Harbor
jamesTimeStamp = capeCharlesTimeStamp + 27480 #Jmes River Locks 458 minutes behind cape charles
piankatankTimeStamp = capeCharlesTimeStamp + 3660 # Cherry points tide is 61 minutes behind cape charles
yorkTimeStamp = capeCharlesTimeStamp - 240  #York River Spit is 4 minutes behind the cape charles

def mergeDict(hDict, lDict):
    return {**hDict, **lDict}

#jDict = {'a': '44', 'b' : '77', 'c': '23'}

if __name__ == ('__main__'):
    with open('jTide.json', 'w') as f1:
        json.dump(mergeDict(HighTideKeeper(jamesTimeStamp), LowTideKeeper(jamesTimeStamp)), f1, indent=2)
    with open('pTide.json', 'w') as f2:
        json.dump(mergeDict(HighTideKeeper(piankatankTimeStamp), LowTideKeeper(piankatankTimeStamp)), f2, indent=2)
    with open('yTide.json', 'w') as f3:
        json.dump(mergeDict(HighTideKeeper(yorkTimeStamp), LowTideKeeper(yorkTimeStamp)), f3, indent=2)


import PySimpleGUI as sg
import os
from PIL import Image as I
import datetime
import time
from COCO import Coco, Image, Anno
import utilities

def worm(dir_path: str, category: str, coco):
    """
    recursive flow that adds image objects to the coco object
    :param dir_path: the dir of the folder with all the data
    :param category: the name of the category until this run
    :param coco: the coco object
    :return: coco object
    """
    # get all folders in the dir_path
    folders = [f for f in os.listdir(dir_path) if os.path.isdir(os.path.join(dir_path, f))]

    if folders:
        for f in folders:
            coco = worm(dir_path=os.path.join(dir_path, f), category=f'{category}_{f}' if category else f, coco=coco)
    # get all files in the dir_path
    files = [f for f in os.listdir(dir_path) if os.path.isfile(os.path.join(dir_path, f))]
    if not files:  # if there are no images...
        return coco
    im_exist = False  # lets check if there is an im in folder
    for i in files:
        ext = i.split('.')[-1].lower()  # ending of file
        valid_types = ["bmp", "jpg", 'jpeg', 'png']
        if ext in valid_types:
            im_exist = True
            break
    if not im_exist:
        return coco
    # create a new category
    elif not category:
        sg.popup_error('There is an image file which is not in a folder!', background_color='#181c21')
        return False  # abort process
    coco.categories.append(
        {
            'supercategory': 'None',
            'id': len(coco.categories) + 1,
            'name': category,
            'status': 'Pass'
                   }
    )
    for f in files:
        ext = f.split('.')[-1].lower()  # ending of file
        valid_types = ["bmp", "jpg", 'jpeg', 'png']
        if ext not in valid_types:
            # raise TypeError("Invalid type of file")
            pass
        else:
            full_path = os.path.join(dir_path, f)
            im = I.open(full_path)
            width, height = im.size
            # date and time the image was created
            created_time = os.path.getctime(full_path)
            date_captured = time.ctime(created_time)
            date_obj = time.strptime(date_captured)
            date_captured = time.strftime("%d/%m/%Y , %H:%M", date_obj)
            # create Image object
            im_o = Image(
                id=len(coco.output_images) + 1,
                path=full_path,  # full path
                name=f,
                width=width,
                height=height,
                license=0,
                date_captured=date_captured
            )
            # load annotations, in this case we dont care about seg, area, bbox, iscrowd
            im_o.add_anno(
                segmentation=[[width / 2 - 10, height / 2 - 10,
                               width / 2 - 10, height / 2 + 10,
                               width / 2 + 10, height / 2 + 10,
                               width / 2 + 10, height / 2 - 10]],
                area=100,
                bbox=[width / 2 - 10, height / 2 - 10, width / 2 + 10, height / 2 + 10],
                iscrowd=0,
                id=len(coco.output_images) + 1,
                image_id=len(coco.output_images) + 1,
                category_id=len(coco.categories),
                          )
            coco.output_images.append(im_o)
    return coco

def GUI():
    """
    build the window object
    :return: Window object
    """
    # setting up theme for gui window
    sg.LOOK_AND_FEEL_TABLE['MyCreatedTheme'] = {'BACKGROUND': '#181c21',
                                                'TEXT': '#FFFFFF',
                                                'INPUT': '#181c21',
                                                'TEXT_INPUT': '#FFFFFF',
                                                'SCROLL': '# 99CC99',
                                                'BUTTON': ('#FFFFFF', '#181c21'),
                                                'PROGRESS': ('# 99CC99', '#FFFFFF'),
                                                'BORDER': 1, 'SLIDER_DEPTH': 0,
                                                'PROGRESS_DEPTH': 0, }
    sg.theme('MyCreatedTheme')  # this is implementing the custom them that was build earlier
    # UI setup
    font: tuple = ("Arial", 20)
    frame_width: int = 1000

    frame = [[sg.Text('Main folder', font=font, background_color='#181c21'),
              sg.Input(key='-IMPORT DIR-', font=font, size=(32, 1)),
              sg.FolderBrowse('Browse',
              font=font,  # read only 'json' files
              button_color='#181c21')],
             [sg.Text('New folder name', font=font, background_color='#181c21'),
              sg.Input(key='-COCO NAME-', font=font, size=(35, 1))],
             [sg.Text('Folder to export to', font=font, background_color='#181c21'),
              sg.Input(key='-EXPORT DIR-', font=font, size=(29, 1)),
              sg.FolderBrowse('Browse',
                              font=font,  # read only 'json' files
                              button_color='#181c21')
              ],
             [sg.Push(), sg.Button('Export to COCO', font=font, key='-run-', button_color='#181c21')]
             ]

    layout = [
        [sg.Frame('Select folder with classified folders',
                  frame,
                  font=font,
                  background_color='#181c21',
                  size=(frame_width, 270))
         ]
    ]

    # GUI object
    window = sg.Window("Bright Machines Classification tool",
                       layout=layout,
                       icon='bm_logo.ico',
                       finalize=True,
                       return_keyboard_events=True,
                       grab_anywhere=True,
                       background_color='#181c21')

    return window

def run_gui(window: sg.Window):
    """
    will iterate until a run command is made
    :param window: sg object
    :return:
    """
    while True:
        event, values = window.read(timeout=100)
        if event == sg.WIN_CLOSED:
            break
        elif event == '-run-':
            if values['-IMPORT DIR-'] == '' or values['-EXPORT DIR-'] == '':
                sg.popup_error('Please enter a folder direction', background_color='#181c21')
            elif values['-COCO NAME-'] == '':
                sg.popup_error('Please enter wanted coco file name', background_color='#181c21')
            else:
                dir_name, base_name = values['-EXPORT DIR-'], values['-COCO NAME-']
                full_dir: str = os.path.join(dir_name, base_name)
                run_coco_build(folder=values['-IMPORT DIR-'], dir_for_coco=full_dir)

def copy_images(ims: list, destination: str):
    ims_d: list = [im.path for im in ims]  # list of full dirs of all images
    ok: bool = utilities.move_files_to_folder_sg(list_of_files=ims_d, destination_folder=destination)
    if ok:
        sg.popup('Images where copied to a single folder', background_color='#181c21',
                 auto_close=True, auto_close_duration=3)
    return

def run_coco_build(folder: str, dir_for_coco: str):
    """
    this will be the backbone for building and exporting the coco file
    :param folder: the dir of the folder with all the data
    :param dir_for_coco: the full dir of the location and name for the new coco json file
    :return:
    """
    # create coco object
    new_coco = Coco(coco_dir=dir_for_coco)
    time_now = datetime.datetime.now()
    new_coco.coco_info = {
        "year": time_now.year,
        "version": "1.0",
        "description": "BMA",
        "contributor": "",
        "url": "",
        "date_created": "%s/%s/%s , %s:%s" % (
            time_now.day, time_now.month, time_now.year, time_now.hour, time_now.minute)
    }
    new_coco.licenses = [{
        "id": 0,
        "name": "Unknown License",
        "url": ""
    }]
    coco = worm(dir_path=folder, category='', coco=new_coco)
    if not isinstance(coco, bool):  # the process went OK
        # create a folder
        # checking if the directory demo_folder
        # exist or not.
        while os.path.exists(dir_for_coco):
            dir_for_coco += '_new'
        os.makedirs(dir_for_coco)
        # save coco
        coco.export_coco('/' + os.path.basename(dir_for_coco))
        sg.popup('Export is done, the file was created', background_color='#181c21',
                 auto_close=True, auto_close_duration=3)
        # copy all images to the same folder
        os.makedirs(os.path.join(dir_for_coco, 'images'))
        copy_images(ims=coco.output_images, destination=os.path.join(dir_for_coco, 'images'))

def main():
    window = GUI()
    run_gui(window)

if __name__ == "__main__":
    main()


import json
import time
import sys
import requests

def try_get_json(url, timeout=20):
    t = time.time()
    try:
        with Timer("requests.get(%s)" % url):
            response = requests.get(url, timeout=timeout)
    except requests.exceptions.Timeout:
        log("GET %s timed out after %s." % (url, time.time()-t))
        raise
    except requests.exceptions.MissingSchema:
        log("%s is not a valid URL" % url)
        raise
    except requests.exceptions.ConnectionError as e:
        log("GET %s failed: %s" % (url, e))
        raise
    except:
        log("Unexpected error from %s : %s" % (url, sys.exc_info()[0]))
        raise

    if response.status_code == 200:
        with Timer("get text"):
            t = response.content
        log("len(t): %d" % len(t))
        with Timer("json.loads(%s)" % url):
            return json.loads(t)
    else:
        log("GET %s failed - Non 200 HTTP Error" % url)
        return False

def log(message):
    ts = time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime())
    print('%s %s' % (ts, message))

class Timer:
    def __init__(self, name):
        self.name = name

    def __enter__(self):
        log("%s starting" % (self.name))
        self.time = time.time()

    def __exit__(self, type, value, traceback):
        return log("%s took %ss" % (self.name, time.time() - self.time))


from flask import url_for, redirect, request
from flask_admin import BaseView, expose

from app.controllers import interceptors_controller
from app.models.models.http_method import HTTPMethod
from app.utils.utils import toast, call

class View(BaseView):
    def is_visible(self):
        return False

    @expose('/')
    def index(self):
        return self.render('admin/interceptors/interceptors.html')

    @expose('/<mock_id>/<response_id>/interceptors/back')
    def interceptor_back(self, mock_id, response_id):
        return redirect(url_for('mocks.mock_response', mock_id=mock_id, response_id=response_id))

    @expose('/<mock_id>/<response_id>/interceptors/<interceptor_id>')
    def interceptor(self, mock_id, response_id, interceptor_id):
        interceptor = interceptors_controller.interceptor(mock_id, response_id, interceptor_id)
        interceptor_configuration_example = interceptors_controller.interceptor_configuration_example(interceptor.type.value)
        interceptor_is_configurable = interceptors_controller.interceptor_is_configurable(interceptor.type.value)
        return self.render('admin/interceptors/interceptors.html', mock_id=mock_id, response_id=response_id,
                           interceptor=interceptor, interceptor_configuration_example=interceptor_configuration_example,
                           interceptor_is_configurable=interceptor_is_configurable)

    # interceptors
    @expose('/<mock_id>/<response_id>/interceptors/<interceptor_id>/update', methods=[HTTPMethod.POST.value])
    def interceptor_update(self, mock_id, response_id, interceptor_id):
        name = request.form.get('interceptors_definition_form_name')
        call(
            lambda: interceptors_controller.interceptor_update(mock_id, response_id, interceptor_id, name),
            lambda: toast('Interceptors has been updated', category='success')
        )
        return redirect(url_for('interceptors.interceptor', mock_id=mock_id, response_id=response_id,
                                interceptor_id=interceptor_id))

    @expose('/<mock_id>/<response_id>/interceptors/<interceptor_id>/enable', methods=[HTTPMethod.POST.value])
    def interceptor_enable(self, mock_id, response_id, interceptor_id):
        call(
            lambda: interceptors_controller.interceptor_enable(mock_id, response_id, interceptor_id),
            lambda: toast('Interceptor has been enabled', category='success')
        )
        return url_for('interceptors.interceptor', mock_id=mock_id, response_id=response_id,
                       interceptor_id=interceptor_id)

    @expose('/<mock_id>/<response_id>/interceptors/<interceptor_id>/disable', methods=[HTTPMethod.POST.value])
    def interceptor_disable(self, mock_id, response_id, interceptor_id):
        call(
            lambda: interceptors_controller.interceptor_disable(mock_id, response_id, interceptor_id),
            lambda: toast('Interceptor has been disabled', category='success')
        )
        return url_for('interceptors.interceptor', mock_id=mock_id, response_id=response_id,
                       interceptor_id=interceptor_id)

    @expose('/<mock_id>/<response_id>/interceptors/<interceptor_id>/configuration', methods=[HTTPMethod.POST.value])
    def interceptor_update_configuration(self, mock_id, response_id, interceptor_id):
        configuration = request.form.get("configuration")
        call(
            lambda: interceptors_controller.interceptor_update_configuration(mock_id, response_id, interceptor_id,
                                                                             configuration),
            lambda: toast('Interceptor has been updated', category='success'))
        return url_for('interceptors.interceptor', mock_id=mock_id, response_id=response_id,
                       interceptor_id=interceptor_id)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Feb 10 14:15:37 2022

@author: sunlin

@实现声音震动的单通道模型准确率计算和混淆矩阵生成
"""
from cProfile import label
from utils.confusionMatrixGenerator import confusionMatrixGenerator,plot_confusion_matrix
from utils.accCalculator import accCalculateFrame

from sklearn.metrics import classification_report
import torch
import numpy as np

def classificationPerform(model_aco,model_seis,frame_data_aco, frame_data_seis):
    predict_aco=model_aco(torch.tensor(frame_data_aco, dtype=torch.float).reshape(1,-1)).data.numpy()
    predict_seis=model_seis(torch.tensor(frame_data_seis, dtype=torch.float).reshape(1,-1)).data.numpy()
    return predict_aco.reshape(-1), predict_seis.reshape(-1)

def resultClassifier(predict_aco,predict_seis,predict_aco_mfcc,predict_seis_medium,predict_aco_wavelet,
                    predict_seis_wavelet,label_per):
    cm_aco,acc_aco = confusionMatrixGenerator(predict_aco, label_per)
    cm_seis,acc_seis  = confusionMatrixGenerator(predict_seis, label_per)
    cm_aco_mfcc,acc_aco_mfcc  = confusionMatrixGenerator(predict_aco_mfcc, label_per)
    cm_seis_medium,acc_seis_medium  = confusionMatrixGenerator(predict_seis_medium, label_per)
    cm_aco_wavelet,acc_aco_wavelet  = confusionMatrixGenerator(predict_aco_wavelet, label_per)
    cm_seis_wavelet,acc_seis_wavelet  = confusionMatrixGenerator(predict_seis_wavelet, label_per)
    #图片1、2，分类器的混淆矩阵
    plot_confusion_matrix(cm_aco,1,'confusion matrix of acoustic classifier')
    plot_confusion_matrix(cm_seis,2,'confusion matrix of seismic classifier')
    plot_confusion_matrix(cm_aco_mfcc,3,'confusion matrix of acoustic classifier with MFCC')
    plot_confusion_matrix(cm_seis_medium,4,'confusion matrix of seismic classifier with medium scale')
    plot_confusion_matrix(cm_aco_wavelet,5,'confusion matrix of acoustic classifier with wavelet')
    plot_confusion_matrix(cm_seis_wavelet,6,'confusion matrix of seismic classifier with wavelet')
    #输出准确率
    return [acc_aco,acc_seis,acc_aco_mfcc,acc_seis_medium,acc_aco_wavelet,acc_seis_wavelet ]

def resultPerform(n,info,label_per,predict,target_names):
    #label_per = np.argmax(np.array(label_per),axis=1)
    #label_per = [np.argmax(i) for i in label_per]
    predict = [np.argmax(i) for i in predict]
    #predict = np.argmax(np.array(predict),axis=1)
    #y = (predict == predict.max(axis=1,keepdims=1)).astype(int)
    #print(predict,label_per)
    #print(label_per,predict)
    print(classification_report(label_per,predict,target_names = target_names))
    cm_aco,acc_aco = confusionMatrixGenerator(predict, label_per)
    plot_confusion_matrix(cm_aco,n,'confuse_matrix of '+info)

def classificationProject(label_per,
                            predict_aco,predict_aco_mfcc,predict_aco_wavelet,
                            predict_seis,predict_seis_medium,predict_seis_wavelet,
                            target_names):
    # frame classifier performance
    print(" aco AlexNet classifier performance")
    resultPerform(1,'acoustic classifier',label_per=label_per, predict=predict_aco, target_names=target_names)

    print(" seis AlexNet classifier performance")
    resultPerform(2,'seismic classifier',label_per=label_per,predict=predict_seis,target_names=target_names)

    print(" aco_mfcc classifier performance")
    resultPerform(3,'mfcc classifier',label_per=label_per,predict=predict_aco_mfcc,target_names=target_names)

    print(" aco_wavelet classifier performance")
    resultPerform(4,'ACO wavelet classifier',label_per=label_per,predict=predict_aco_wavelet,target_names=target_names)

    print(" seis_wavelet classifier performance")
    resultPerform(5,'SEIS wavelet classifier',label_per=label_per,predict=predict_seis_wavelet,target_names=target_names)

    print(" medium classifier performance")
    resultPerform(6,'SEIS medium scale classifier',label_per=label_per,predict=predict_seis_medium,target_names=target_names)

    # frame fusion performance

    #LTCFN performance

if __name__ == "__main__":

    y_true = [0,1,2,2,2]
    y_pred = [0,0,2,2,1]
    target_names = ['class 0', 'class 1', 'class 2']
    print(classification_report(y_true,y_pred,target_names = target_names))


import numpy as np
from const import PitchExtractorType
from voice_changer.DiffusionSVC.pitchExtractor.PitchExtractor import PitchExtractor
from voice_changer.RVC.deviceManager.DeviceManager import DeviceManager
import onnxruntime

class RMVPOnnxEPitchExtractor(PitchExtractor):

    def __init__(self, file: str, gpu: int):
        super().__init__()
        self.file = file
        self.pitchExtractorType: PitchExtractorType = "rmvpe_onnx"
        self.f0_min = 50
        self.f0_max = 1100
        self.f0_mel_min = 1127 * np.log(1 + self.f0_min / 700)
        self.f0_mel_max = 1127 * np.log(1 + self.f0_max / 700)

        (
            onnxProviders,
            onnxProviderOptions,
        ) = DeviceManager.get_instance().getOnnxExecutionProvider(gpu)
        self.onnxProviders = onnxProviders
        self.onnxProviderOptions = onnxProviderOptions

        so = onnxruntime.SessionOptions()
        so.log_severity_level = 3
        self.onnx_session = onnxruntime.InferenceSession(self.file, sess_options=so, providers=onnxProviders, provider_options=onnxProviderOptions)

    def extract(self, audio, pitchf, f0_up_key, sr, window, silence_front=0):
        try:
            # データ変換
            if isinstance(audio, np.ndarray) is False:
                audio = audio = audio.cpu().numpy()

            if isinstance(pitchf, np.ndarray) is False:
                pitchf = pitchf.cpu().numpy().astype(np.float32)

            if audio.ndim != 1:
                raise RuntimeError(f"Exeption in {self.__class__.__name__} audio.ndim is not 1 (size :{audio.ndim}, {audio.shape})")
            if pitchf.ndim != 1:
                raise RuntimeError(f"Exeption in {self.__class__.__name__} pitchf.ndim is not 1 (size :{pitchf.ndim}, {pitchf.shape})")

            # 処理
            silenceFrontFrame = silence_front * sr
            startWindow = int(silenceFrontFrame / window)  # 小数点以下切り捨て
            slienceFrontFrameOffset = startWindow * window
            targetFrameLength = len(audio) - slienceFrontFrameOffset
            minimumFrames = 0.01 * sr
            targetFrameLength = max(minimumFrames, targetFrameLength)
            audio = audio[-targetFrameLength:]
            audio = np.expand_dims(audio, axis=0)

            output = self.onnx_session.run(
                ["f0", "uv"],
                {
                    "waveform": audio.astype(np.float32),
                    "threshold": np.array([0.3]).astype(np.float32),
                },
            )

            f0 = output[0].squeeze()

            f0 *= pow(2, f0_up_key / 12)
            pitchf[-f0.shape[0]:] = f0[: pitchf.shape[0]]

            f0_mel = 1127.0 * np.log(1.0 + pitchf / 700.0)
            f0_mel[f0_mel > 0] = (f0_mel[f0_mel > 0] - self.f0_mel_min) * 254 / (self.f0_mel_max - self.f0_mel_min) + 1
            f0_mel[f0_mel <= 1] = 1
            f0_mel[f0_mel > 255] = 255
            f0_coarse = np.rint(f0_mel).astype(int)

        except Exception as e:
            raise RuntimeError(f"Exeption in {self.__class__.__name__}", e)

        return f0_coarse, pitchf