import os
import h5py
import numpy as np
from copy import deepcopy
from scipy.stats import chi2
from openquake.hazardlib.gsim.base import CoeffsTable
from openquake.hazardlib.gsim.gmpe_table import GMPETable, hdf_arrays_to_dict
from openquake.hazardlib.imt import PGV
from openquake.hazardlib import const

# Common interpolation function
def ITPL(mag, tu, tl, ml, f):
    return tl + (tu - tl) * ((mag - ml) / f)

def _scaling(mean_tau, sd_tau2):
    """
    Returns the chi-2 scaling factor from the mean and variance of the
    uncertainty model, as reported in equation 5.4 of Al Atik (2015)
    """
    return (sd_tau2 ** 2.) / (2.0 * mean_tau ** 2.)

def _dof(mean_tau, sd_tau2):
    """
    Returns the degrees of freedom for the chi-2 distribution from the mean and
    variance of the uncertainty model, as reported in equation 5.5 of Al Atik
    (2015)
    """
    return (2.0 * mean_tau ** 4.) / (sd_tau2 ** 2.)

def _at_percentile(tau, var_tau, percentile):
    """
    Returns the value of the inverse chi-2 distribution at the given
    percentile from the mean and variance of the uncertainty model, as
    reported in equations 5.1 - 5.3 of Al Atik (2015)
    """
    assert (percentile >= 0.0) and (percentile <= 1.0)
    c_val = _scaling(tau, var_tau)
    k_val = _dof(tau, var_tau)
    return np.sqrt(c_val * chi2.ppf(percentile, df=k_val))

# Mean tau values from the global model - Table 5.1
GLOBAL_TAU_MEAN = {
    "PGV": {"tau1": 0.3733, "tau2": 0.3639, "tau3": 0.3434, "tau4": 0.3236},
    "SA": {"tau1": 0.4518, "tau2": 0.4270, "tau3": 0.3863, "tau4": 0.3508}
    }

# Standard deviation of tau values from the global model - Table 5.1
GLOBAL_TAU_SD = {
    "PGV": {"tau1": 0.0558, "tau2": 0.0554, "tau3": 0.0477, "tau4": 0.0449},
    "SA": {"tau1": 0.0671, "tau2": 0.0688, "tau3": 0.0661, "tau4": 0.0491}
}

# Mean tau values from the CENA constant-tau model
CENA_CONSTANT_TAU_MEAN = {"PGV": {"tau": 0.3441}, "SA": {"tau": 0.3695}}

# Standard deviation of tau values from CENA constant-tau model
CENA_CONSTANT_TAU_SD = {"PGV": {"tau": 0.0554}, "SA": {"tau": 0.0688}}

# Mean tau values from the CENA tau model
CENA_TAU_MEAN = {
    "PGV": {"tau1": 0.3477, "tau2": 0.3281, "tau3": 0.3092},
    "SA": {"tau1": 0.3730, "tau2": 0.3375, "tau3": 0.3064}
    }

# Standard deviation of tau values from the CENA tau model
CENA_TAU_SD = {
    "PGV": {"tau1": 0.0554, "tau2": 0.0477, "tau3": 0.0449},
    "SA": {"tau1": 0.0688, "tau2": 0.0661, "tau3": 0.0491}
    }

def global_tau(imt, mag, params):
    """
    'Global' model of inter-event variability, as presented in equation 5.6
    (p103)
    """
    if isinstance(imt, PGV):
        C = params["PGV"]
    else:
        C = params["SA"]
    if mag > 6.5:
        return C["tau4"]
    elif (mag > 5.5) and (mag <= 6.5):
        return ITPL(mag, C["tau4"], C["tau3"], 5.5, 1.0)
    elif (mag > 5.0) and (mag <= 5.5):
        return ITPL(mag, C["tau3"], C["tau2"], 5.0, 0.5)
    elif (mag > 4.5) and (mag <= 5.0):
        return ITPL(mag, C["tau2"], C["tau1"], 4.5, 0.5)
    else:
        return C["tau1"]

def cena_constant_tau(imt, mag, params):
    """
    Returns the inter-event tau for the constant tau case
    """
    if isinstance(imt, PGV):
        return params["PGV"]["tau"]
    else:
        return params["SA"]["tau"]

def cena_tau(imt, mag, params):
    """
    Returns the inter-event standard deviation, tau, for the CENA case
    """
    if isinstance(imt, PGV):
        C = params["PGV"]
    else:
        C = params["SA"]
    if mag > 6.5:
        return C["tau3"]
    elif (mag > 5.5) and (mag <= 6.5):
        return ITPL(mag, C["tau3"], C["tau2"], 5.5, 1.0)
    elif (mag > 5.0) and (mag <= 5.5):
        return ITPL(mag, C["tau2"], C["tau1"], 5.0, 0.5)
    else:
        return C["tau1"]

def get_tau_at_quantile(mean, stddev, quantile):
    """
    Returns the value of tau at a given quantile in the form of a dictionary
    organised by intensity measure
    """
    tau_model = {}
    for imt in mean:
        tau_model[imt] = {}
        for key in mean[imt]:
            if quantile is None:
                tau_model[imt][key] = mean[imt][key]
            else:
                tau_model[imt][key] = _at_percentile(mean[imt][key],
                                                     stddev[imt][key],
                                                     quantile)
    return tau_model

# Gather tau values into dictionary
TAU_SETUP = {
    "cena": {"MEAN": CENA_TAU_MEAN, "STD": CENA_TAU_SD},
    "cena_constant": {"MEAN": CENA_CONSTANT_TAU_MEAN,
                      "STD": CENA_CONSTANT_TAU_SD},
    "global": {"MEAN": GLOBAL_TAU_MEAN, "STD": GLOBAL_TAU_SD}
    }

# Gather tau model implementation functions into dictionary
TAU_EXECUTION = {
    "cena": cena_tau,
    "cena_constant": cena_constant_tau,
    "global": global_tau}

# Phi_ss coefficients for the global model
PHI_SS_GLOBAL = CoeffsTable(sa_damping=5., table="""\
imt     mean_a   var_a  mean_b  var_b
pgv     0.5034  0.0609  0.3585  0.0316
pga     0.5477  0.0731  0.3505  0.0412
0.010   0.5477  0.0731  0.3505  0.0412
0.020   0.5464  0.0727  0.3505  0.0416
0.030   0.5450  0.0723  0.3505  0.0419
0.040   0.5436  0.0720  0.3505  0.0422
0.050   0.5424  0.0716  0.3505  0.0425
0.075   0.5392  0.0707  0.3505  0.0432
0.100   0.5361  0.0699  0.3505  0.0439
0.150   0.5299  0.0682  0.3543  0.0453
0.200   0.5240  0.0666  0.3659  0.0465
0.250   0.5183  0.0651  0.3765  0.0476
0.300   0.5127  0.0637  0.3876  0.0486
0.400   0.5022  0.0611  0.4066  0.0503
0.500   0.4923  0.0586  0.4170  0.0515
0.750   0.4704  0.0535  0.4277  0.0526
1.000   0.4519  0.0495  0.4257  0.0508
1.500   0.4231  0.0439  0.4142  0.0433
2.000   0.4026  0.0405  0.4026  0.0396
3.000   0.3775  0.0371  0.3775  0.0366
4.000   0.3648  0.0358  0.3648  0.0358
5.000   0.3583  0.0353  0.3583  0.0356
7.500   0.3529  0.0350  0.3529  0.0355
10.00   0.3519  0.0350  0.3519  0.0355
""")

# Phi_ss coefficients for the CENA model
PHI_SS_CENA = CoeffsTable(sa_damping=5., table="""\
imt     mean_a   var_a  mean_b   var_b
pgv     0.5636  0.0807  0.4013  0.0468
pga     0.5192  0.0693  0.3323  0.0364
0.010   0.5192  0.0693  0.3323  0.0364
0.020   0.5192  0.0693  0.3331  0.0365
0.030   0.5192  0.0693  0.3339  0.0365
0.040   0.5192  0.0693  0.3348  0.0367
0.050   0.5192  0.0693  0.3355  0.0367
0.075   0.5192  0.0693  0.3375  0.0370
0.100   0.5192  0.0693  0.3395  0.0372
0.150   0.5192  0.0693  0.3471  0.0382
0.200   0.5192  0.0693  0.3625  0.0402
0.250   0.5192  0.0693  0.3772  0.0423
0.300   0.5192  0.0693  0.3925  0.0446
0.400   0.5192  0.0693  0.4204  0.0492
0.500   0.5192  0.0693  0.4398  0.0527
0.750   0.5192  0.0693  0.4721  0.0590
1.000   0.5192  0.0693  0.4892  0.0626
1.500   0.5192  0.0693  0.5082  0.0668
2.000   0.5192  0.0693  0.5192  0.0693
3.000   0.5192  0.0693  0.5192  0.0693
4.000   0.5192  0.0693  0.5192  0.0693
5.000   0.5192  0.0693  0.5192  0.0693
7.500   0.5192  0.0693  0.5192  0.0693
10.00   0.5192  0.0693  0.5192  0.0693
""")

# Phi_ss coefficients for the CENA constant-phi model
PHI_SS_CENA_CONSTANT = CoeffsTable(sa_damping=5., table="""\
imt     mean_a    var_a   mean_b   var_b
pgv     0.5507   0.0678   0.5507  0.0678
pga     0.5132   0.0675   0.5132  0.0675
0.010   0.5132   0.0675   0.5132  0.0675
10.00   0.5132   0.0675   0.5132  0.0675
""")

# Phi_s2ss coefficients for the CENA
PHI_S2SS_CENA = CoeffsTable(sa_damping=5., table="""\
imt       mean      var
pgv     0.4344   0.0200
pga     0.4608   0.0238
0.010   0.4608   0.0238
0.020   0.4617   0.0238
0.030   0.4700   0.0240
0.040   0.4871   0.0260
0.050   0.5250   0.0290
0.075   0.5800   0.0335
0.100   0.5930   0.0350
0.150   0.5714   0.0325
0.200   0.5368   0.0296
0.250   0.5058   0.0272
0.300   0.4805   0.0250
0.400   0.4440   0.0212
0.500   0.4197   0.0182
0.750   0.3849   0.0139
1.000   0.3667   0.0135
1.500   0.3481   0.0157
2.000   0.3387   0.0173
3.000   0.3292   0.0195
4.000   0.3245   0.0211
5.000   0.3216   0.0224
7.500   0.3178   0.0240
10.00   0.3159   0.0240
""")

def get_phi_ss_at_quantile(phi_model, quantile):
    """
    Returns the phi_ss values at the specified quantile as an instance of
    `class`:: openquake.hazardlib.gsim.base.CoeffsTable - applies to the
    magnitude-dependent cases
    """
    # Setup SA coeffs - the backward compatible Python 2.7 way
    coeffs = deepcopy(phi_model.sa_coeffs)
    coeffs.update(phi_model.non_sa_coeffs)
    for imt in coeffs:
        if quantile is None:
            coeffs[imt] = {"a": phi_model[imt]["mean_a"],
                           "b": phi_model[imt]["mean_b"]}
        else:
            coeffs[imt] = {
                "a": _at_percentile(phi_model[imt]["mean_a"],
                                    phi_model[imt]["var_a"],
                                    quantile),
                "b": _at_percentile(phi_model[imt]["mean_b"],
                                    phi_model[imt]["var_b"],
                                    quantile)
                }
    return CoeffsTable(sa_damping=5., table=coeffs)

def get_phi_s2ss_at_quantile(phi_model, quantile):
    """
    Returns the phi_s2ss value for all periods at the specific quantile as
    an instance of `class`::openquake.hazardlib.gsim.base.CoeffsTable
    """
    # Setup SA coeffs - the backward compatible Python 2.7 way
    coeffs = deepcopy(phi_model.sa_coeffs)
    coeffs.update(phi_model.non_sa_coeffs)
    for imt in coeffs:
        if quantile is None:
            coeffs[imt] = {"phi_s2ss": phi_model[imt]["mean"]}
        else:
            coeffs[imt] = {"phi_s2ss": _at_percentile(phi_model[imt]["mean"],
                                                      phi_model[imt]["var"],
                                                      quantile)}
    return CoeffsTable(sa_damping=5., table=coeffs)

# Gather the models to setup the phi_ss values for the given quantile
PHI_SETUP = {
    "cena": PHI_SS_CENA,
    "cena_constant": PHI_SS_CENA_CONSTANT,
    "global": PHI_SS_GLOBAL}

# Phi site-to-site model for th Central & Eastern US
PHI_S2SS_MODEL = {"cena": PHI_S2SS_CENA}

def get_phi_ss(imt, mag, params):
    """
    Returns the single station phi (or it's variance) for a given magnitude
    and intensity measure type according to equation 5.14 of Al Atik (2015)
    """
    C = params[imt]
    if mag <= 5.0:
        phi = C["a"]
    elif mag > 6.5:
        phi = C["b"]
    else:
        phi = C["a"] + (mag - 5.0) * ((C["b"] - C["a"]) / 1.5)
    return phi

class NGAEastBaseGMPE(GMPETable):
    """
    A generalised base class for the implementation of a GMPE in which the
    mean values are determined from tables (input by the user) and the standard
    deviation model taken from Al Atik (2015). Should be common to all
    NGA East ground motion models.

    :param str tau_model:
        Choice of model for the inter-event standard deviation (tau), selecting
        from "global" {default}, "cena" or "cena_constant"

    :param str phi_model:
        Choice of model for the single-station intra-event standard deviation
        (phi_ss), selecting from "global" {default}, "cena" or "cena_constant"

    :param str phi_s2ss_model:
        Choice of station-term s2ss model. Can be either "cena" or None. When
        None is input then the non-ergodic model is used

    :param TAU:
        Inter-event standard deviation model

    :param PHI_SS:
        Single-station standard deviation model

    :param PHI_S2SS:
        Station term for ergodic standard deviation model

    :param bool ergodic:
        True if an ergodic model is selected, False otherwise
    """
    DEFINED_FOR_STANDARD_DEVIATION_TYPES = set((const.StdDev.TOTAL,
                                                const.StdDev.INTER_EVENT,
                                                const.StdDev.INTRA_EVENT))

    def __init__(self, gmpe_table, tau_model="global", phi_model="global",
                 phi_s2ss_model=None, tau_quantile=None,
                 phi_ss_quantile=None, phi_s2ss_quantile=None):
        """
        Instantiates the class with additional terms controlling which
        type of aleatory uncertainty model is preferred ('global',
        'cena_constant', 'cena'), and the quantile of the epistemic uncertainty
        model (float in the range 0 to 1).

        :param float tau_quantile:
            Epistemic uncertainty quantile for the inter-event standard
            deviation models. Float in the range 0 to 1, or None (mean value
            used)

        :param float phi_ss_quantile:
            Epistemic uncertainty quantile for the intra-event standard
            deviation models. Float in the range 0 to 1, or None (mean value
            used)

        :param float phi_s2ss_quantile:
            Epistemic uncertainty quantile for the site-to-site standard
            deviation models. Float in the range 0 to 1, or None (mean value
            used)
        """
        super().__init__(gmpe_table=gmpe_table)
        self.tau_model = tau_model
        self.phi_model = phi_model
        self.phi_s2ss_model = phi_s2ss_model
        self.TAU = None
        self.PHI_SS = None
        self.PHI_S2SS = None
        if self.phi_s2ss_model:
            self.ergodic = True
        else:
            self.ergodic = False
        self._setup_standard_deviations(
            tau_quantile, phi_ss_quantile, phi_s2ss_quantile)
        if os.path.exists(self.GMPE_TABLE):
            with h5py.File(self.GMPE_TABLE, "r") as f:
                self.init(f)

    def init(self, fle):
        """
        Executes the preprocessing steps at the instantiation stage to read in
        the tables from hdf5 and hold them in memory.
        """
        self.distance_type = fle["Distances"].attrs["metric"]
        self.REQUIRES_DISTANCES = set([self.distance_type])
        # Load in magnitude
        self.m_w = fle["Mw"][:]
        # Load in distances
        self.distances = fle["Distances"][:]
        # Load intensity measure types and levels
        self.imls = hdf_arrays_to_dict(fle["IMLs"])
        self.DEFINED_FOR_INTENSITY_MEASURE_TYPES = set(self._supported_imts())
        if "SA" in self.imls.keys() and "T" not in self.imls:
            raise ValueError("Spectral Acceleration must be accompanied by "
                             "periods")
        # Get the standard deviations
        if "Amplification" in fle:
            self._setup_amplification(fle)

    def _setup_standard_deviations(self, tau_quantile, phi_ss_quantile,
                                   phi_s2ss_quantile):
        """
        Reads the standard deviation tables from hdf5 and stores them in
        memory
        :param fle:
            HDF5 Tables as instance of :class:`h5py.File`
        """
        # setup tau
        self.TAU = get_tau_at_quantile(TAU_SETUP[self.tau_model]["MEAN"],
                                       TAU_SETUP[self.tau_model]["STD"],
                                       tau_quantile)
        # setup phi
        self.PHI_SS = get_phi_ss_at_quantile(PHI_SETUP[self.phi_model],
                                             phi_ss_quantile)
        # if required setup phis2ss
        if self.ergodic:
            self.PHI_S2SS = get_phi_s2ss_at_quantile(
                PHI_S2SS_MODEL[self.phi_s2ss_model],
                phi_s2ss_quantile)

    def get_mean_and_stddevs(self, sctx, rctx, dctx, imt, stddev_types):
        """
        Returns the mean and standard deviations
        """
        # Return Distance Tables
        imls = self._return_tables(rctx.mag, imt, "IMLs")
        # Get distance vector for the given magnitude
        idx = np.searchsorted(self.m_w, rctx.mag)
        dists = self.distances[:, 0, idx - 1]
        # Get mean and standard deviations
        mean = self._get_mean(imls, dctx, dists)
        nsites = getattr(dctx, self.distance_type).shape
        stddevs = self.get_stddevs(rctx.mag, imt, stddev_types, nsites)
        if self.amplification:
            # Apply amplification
            mean_amp, sigma_amp = self.amplification.get_amplification_factors(
                imt,
                sctx,
                rctx,
                getattr(dctx, self.distance_type),
                stddev_types)
            mean = np.log(mean) + np.log(mean_amp)
            for iloc in range(len(stddev_types)):
                stddevs[iloc] *= sigma_amp[iloc]
            return mean, stddevs
        else:
            return np.log(mean), stddevs

    def get_stddevs(self, mag, imt, stddev_types, num_sites):
        """
        Returns the standard deviations for either the ergodic or
        non-ergodic models
        """
        tau = self._get_tau(imt, mag)
        phi = self._get_phi(imt, mag)
        sigma = np.sqrt(tau ** 2. + phi ** 2.)
        stddevs = []
        for stddev_type in stddev_types:
            assert stddev_type in self.DEFINED_FOR_STANDARD_DEVIATION_TYPES
            if stddev_type == const.StdDev.TOTAL:
                stddevs.append(sigma + np.zeros(num_sites))
            elif stddev_type == const.StdDev.INTRA_EVENT:
                stddevs.append(phi + np.zeros(num_sites))
            elif stddev_type == const.StdDev.INTER_EVENT:
                stddevs.append(tau + np.zeros(num_sites))
        return stddevs

    def _get_tau(self, imt, mag):
        """
        Returns the inter-event standard deviation (tau)
        """
        return TAU_EXECUTION[self.tau_model](imt, mag, self.TAU)

    def _get_phi(self, imt, mag):
        """
        Returns the within-event standard deviation (phi)
        """
        phi = get_phi_ss(imt, mag, self.PHI_SS)
        if self.ergodic:
            C = self.PHI_S2SS[imt]
            phi = np.sqrt(phi ** 2. + C["phi_s2ss"] ** 2.)
        return phi

# For the total standard deviation model the magnitude boundaries depend on
# the model selected
MAG_LIMS_KEYS = {
    "cena": {"mag": [5.0, 5.5, 6.5], "keys": ["tau1", "tau2", "tau3"]},
    "cena_constant": {"mag": [np.inf], "keys": ["tau"]},
    "global": {"mag": [4.5, 5.0, 5.5, 6.5],
               "keys": ["tau1", "tau2", "tau3", "tau4"]}
    }

class NGAEastGMPE(NGAEastBaseGMPE):
    """
    For the "core" NGA East set the table is provided in the code in a
    subdirectory fixed to the path of the present file. The GMPE table option
    is therefore no longer needed
    """
    NGA_EAST_TABLE = ""

    def __init__(self, tau_model="global", phi_model="global",
                 phi_s2ss_model=None, tau_quantile=None,
                 phi_ss_quantile=None, phi_s2ss_quantile=None):
        if not self.NGA_EAST_TABLE:
            raise NotImplementedError("NGA East Fixed-Table GMPE requires "
                                      "input table")
        super().__init__(
            gmpe_table=self.NGA_EAST_TABLE,
            tau_model=tau_model, phi_model=phi_model,
            phi_s2ss_model=phi_s2ss_model, tau_quantile=tau_quantile,
            phi_ss_quantile=phi_ss_quantile,
            phi_s2ss_quantile=phi_s2ss_quantile)

class NGAEastBaseGMPETotalSigma(NGAEastBaseGMPE):
    """
    The Al Atik (2015) standard deviation model defines mean and quantiles
    for the inter- and intra-event residuals. However, it also defines
    separately a total standard deviation model with expectation and quantiles.
    As the inter- and intra-event quantile cannot be recovered unambiguously
    from the total standard deviation quantile this form of the model is
    defined only for total standard deviation. Most likely it is this form
    that would be used for seismic hazard analysis.

    :param SIGMA:
        Total standard deviation model at quantile

    :param list magnitude_limits:
        Magnitude limits corresponding to the selected standard deviation
        model

    :param list tau_keys:
        Keys for the tau values corresponding to the selected standard
        deviation model
    """
    DEFINED_FOR_STANDARD_DEVIATION_TYPES = set((const.StdDev.TOTAL,))

    def __init__(self, gmpe_table, tau_model="global", phi_model="global",
                 phi_s2ss_model=None, sigma_quantile=None):
        """
        Instantiates the model call the BaseNGAEastModel to return the
        expected TAU, PHI_SS and PHI_S2SS values then uses these to
        calculate the expected total standard deviation and its variance
        according to equations 5.15, 5.16 and/or 5.18 and 5.19 of Al Atik
        (2015)

        :param float sigma_quantile:
            Quantile of the epistemic uncertainty model for the total
            standard deviation. Should be float between 0 and 1, or None (mean
            value taken)
        """
        super().__init__(gmpe_table=gmpe_table,
                         tau_model=tau_model, phi_model=phi_model,
                         phi_s2ss_model=phi_s2ss_model, tau_quantile=None,
                         phi_ss_quantile=None, phi_s2ss_quantile=None)
        # Upon instantiation the TAU, PHI_SS, and PHI_S2SS objects contain
        # the mean values
        self.SIGMA = None
        self.magnitude_limits = []
        self.tau_keys = []
        self._get_sigma_at_quantile(sigma_quantile)

    def get_stddevs(self, mag, imt, stddev_types, num_sites):
        """
        Returns the total standard deviation
        """
        stddevs = []
        for stddev_type in stddev_types:
            assert stddev_type in self.DEFINED_FOR_STANDARD_DEVIATION_TYPES
            if stddev_type == const.StdDev.TOTAL:
                sigma = self._get_total_sigma(imt, mag)
                stddevs.append(sigma + np.zeros(num_sites))
        return stddevs

    def _get_sigma_at_quantile(self, sigma_quantile):
        """
        Calculates the total standard deviation at the specified quantile
        """
        # Mean mean is found in self.TAU. Get the variance in tau
        tau_std = TAU_SETUP[self.tau_model]["STD"]
        # Mean phiss is found in self.PHI_SS. Get the variance in phi
        phi_std = deepcopy(self.PHI_SS.sa_coeffs)
        phi_std.update(self.PHI_SS.non_sa_coeffs)
        for key in phi_std:
            phi_std[key] = {"a": PHI_SETUP[self.phi_model][key]["var_a"],
                            "b": PHI_SETUP[self.phi_model][key]["var_b"]}
        if self.ergodic:
            # IMT list should be taken from the PHI_S2SS_MODEL
            imt_list = list(
                PHI_S2SS_MODEL[self.phi_s2ss_model].non_sa_coeffs.keys())
            imt_list += \
                list(PHI_S2SS_MODEL[self.phi_s2ss_model].sa_coeffs.keys())
        else:
            imt_list = phi_std.keys()
        phi_std = CoeffsTable(sa_damping=5, table=phi_std)
        tau_bar, tau_std = self._get_tau_vector(self.TAU, tau_std, imt_list)
        phi_bar, phi_std = self._get_phi_vector(self.PHI_SS, phi_std, imt_list)
        sigma = {}
        # Calculate the total standard deviation
        for imt in imt_list:
            sigma[imt] = {}
            for i, key in enumerate(self.tau_keys):
                # Calculates the expected standard deviation
                sigma_bar = np.sqrt(tau_bar[imt][i] ** 2. +
                                    phi_bar[imt][i] ** 2.)
                # Calculated the variance in the standard deviation
                sigma_std = np.sqrt(tau_std[imt][i] ** 2. +
                                    phi_std[imt][i] ** 2.)
                # The keys swap from tau to sigma
                new_key = key.replace("tau", "sigma")
                if sigma_quantile is not None:
                    sigma[imt][new_key] =\
                        _at_percentile(sigma_bar, sigma_std, sigma_quantile)
                else:
                    sigma[imt][new_key] = sigma_bar
                self.tau_keys[i] = new_key
        self.SIGMA = CoeffsTable(sa_damping=5, table=sigma)

    def _get_tau_vector(self, tau_mean, tau_std, imt_list):
        """
        Gets the vector of mean and variance of tau values corresponding to
        the specific model and returns them as dictionaries
        """
        self.magnitude_limits = MAG_LIMS_KEYS[self.tau_model]["mag"]
        self.tau_keys = MAG_LIMS_KEYS[self.tau_model]["keys"]
        t_bar = {}
        t_std = {}
        for imt in imt_list:
            t_bar[imt] = []
            t_std[imt] = []
            for mag, key in zip(self.magnitude_limits, self.tau_keys):
                t_bar[imt].append(
                    TAU_EXECUTION[self.tau_model](imt, mag, tau_mean))
                t_std[imt].append(
                    TAU_EXECUTION[self.tau_model](imt, mag, tau_std))
        return t_bar, t_std

    def _get_phi_vector(self, phi_mean, phi_std, imt_list):
        """
        Gets the vector of mean and variance of phi values corresponding to
        the specific model and returns them as dictionaries
        """
        p_bar = {}
        p_std = {}

        for imt in imt_list:
            p_bar[imt] = []
            p_std[imt] = []
            for mag in self.magnitude_limits:
                phi_ss_mean = get_phi_ss(imt, mag, phi_mean)
                phi_ss_std = get_phi_ss(imt, mag, phi_std)
                if self.ergodic:
                    # Add on the phi_s2ss term according to Eqs. 5.15 and 5.16
                    # of Al Atik (2015)
                    phi_ss_mean = np.sqrt(
                        phi_ss_mean ** 2. +
                        PHI_S2SS_MODEL[self.phi_s2ss_model][imt]["mean"] ** 2.
                        )
                    phi_ss_std = np.sqrt(
                        phi_ss_std ** 2. +
                        PHI_S2SS_MODEL[self.phi_s2ss_model][imt]["var"] ** 2.
                        )
                p_bar[imt].append(phi_ss_mean)
                p_std[imt].append(phi_ss_std)
        return p_bar, p_std

    def _get_total_sigma(self, imt, mag):
        """
        Returns the estimated total standard deviation for a given intensity
        measure type and magnitude
        """
        C = self.SIGMA[imt]
        if mag <= self.magnitude_limits[0]:
            # The CENA constant model is always returned here
            return C[self.tau_keys[0]]
        elif mag > self.magnitude_limits[-1]:
            return C[self.tau_keys[-1]]
        else:
            # Needs interpolation
            for i in range(len(self.tau_keys) - 1):
                l_m = self.magnitude_limits[i]
                u_m = self.magnitude_limits[i + 1]
                if mag > l_m and mag <= u_m:
                    return ITPL(mag,
                                C[self.tau_keys[i + 1]],
                                C[self.tau_keys[i]],
                                l_m,
                                u_m - l_m)

class NGAEastGMPETotalSigma(NGAEastBaseGMPETotalSigma):
    """
    Subclass of the :class:`NGAEastBaseGMPETotalSigma` for the cases when the
    GMPE table is fixed. This forms the main base-class for the total sigma
    version of the core set of NGA East models
    """
    NGA_EAST_TABLE = ""

    def __init__(self, tau_model="global", phi_model="global",
                 phi_s2ss_model=None, sigma_quantile=None):
        """
        Instantiates the GMPE without the hdf5 table fort the median values
        """
        if not self.NGA_EAST_TABLE:
            raise NotImplementedError("NGA East Fixed-Table GMPE requires "
                                      "input table")
        super().__init__(
            self.NGA_EAST_TABLE, tau_model=tau_model, phi_model=phi_model,
            phi_s2ss_model=phi_s2ss_model, sigma_quantile=sigma_quantile)

# /////////////////////////////////////////////////////////////////////////////
# Now to start adding the actual NGA East GMPEs
# /////////////////////////////////////////////////////////////////////////////
BASE_PATH = os.path.join(os.path.dirname(__file__), "nga_east_tables")

# Boore (2015) suite

class Boore2015NGAEastA04(NGAEastGMPE):
    """
    Boore (2015) NGA East GMPE using the Atkinson (2004) attenuation model

    Boore, DM (2015) "Point-Source Stochastic-Method Simulations of
    Ground Motions for the PEER NGA-East Project", in "NGA-East: Median
    Ground Motion Models for the Central and Eastern North America Region",
    PEER Report 2015/04, Pacific Earthquake Engineering Research Center
    """
    NGA_EAST_TABLE = os.path.join(BASE_PATH,
                                  "NGAEast_BOORE_A04_J15_Adjusted.hdf5")

class Boore2015NGAEastA04TotalSigma(NGAEastGMPETotalSigma):
    """
    Boore (2015) NGA East GMPE using the Atkinson (2004) attenuation model
    for use with the total sigma aleatory uncertainty model
    """
    NGA_EAST_TABLE = os.path.join(BASE_PATH,
                                  "NGAEast_BOORE_A04_J15_Adjusted.hdf5")

class Boore2015NGAEastAB14(NGAEastGMPE):
    """
    Boore (2015) NGA East GMPE using the Atkinson & Boore (2014) attenuation
    model
    """
    NGA_EAST_TABLE = os.path.join(BASE_PATH,
                                  "NGAEast_BOORE_AB14_J15_Adjusted.hdf5")

class Boore2015NGAEastAB14TotalSigma(NGAEastGMPETotalSigma):
    """
    Boore (2015) NGA East GMPE using the Atkinson & Boore (2014) attenuation
    model for use with the total sigma aleatory uncertainty model
    """
    NGA_EAST_TABLE = os.path.join(BASE_PATH,
                                  "NGAEast_BOORE_AB14_J15_Adjusted.hdf5")

class Boore2015NGAEastAB95(NGAEastGMPE):
    """
    Boore (2015) NGA East GMPE using the Atkinson & Boore (1995) attenuation
    model
    """
    NGA_EAST_TABLE = os.path.join(BASE_PATH,
                                  "NGAEast_BOORE_AB95_J15_Adjusted.hdf5")

class Boore2015NGAEastAB95TotalSigma(NGAEastGMPETotalSigma):
    """
    Boore (2015) NGA East GMPE using the Atkinson & Boore (1995) attenuation
    model for use with the total sigma aleatory uncertainty model
    """
    NGA_EAST_TABLE = os.path.join(BASE_PATH,
                                  "NGAEast_BOORE_AB95_J15_Adjusted.hdf5")

class Boore2015NGAEastBCA10D(NGAEastGMPE):
    """
    Boore (2015) NGA East GMPE using the Boore et al (2010) attenuation
    model
    """
    NGA_EAST_TABLE = os.path.join(BASE_PATH,
                                  "NGAEast_BOORE_BCA10D_J15_Adjusted.hdf5")

class Boore2015NGAEastBCA10DTotalSigma(NGAEastGMPETotalSigma):
    """
    Boore (2015) NGA East GMPE using the Boore et al (2010) attenuation
    model for use with the total sigma aleatory uncertainty model
    """
    NGA_EAST_TABLE = os.path.join(BASE_PATH,
                                  "NGAEast_BOORE_BCA10D_J15_Adjusted.hdf5")

class Boore2015NGAEastBS11(NGAEastGMPE):
    """
    Boore (2015) NGA East GMPE using the Boatwright and Seekins (2011)
    attenuation model
    """
    NGA_EAST_TABLE = os.path.join(BASE_PATH,
                                  "NGAEast_BOORE_BS11_J15_Adjusted.hdf5")

class Boore2015NGAEastBS11TotalSigma(NGAEastGMPETotalSigma):
    """
    Boore (2015) NGA East GMPE using the Boatwright and Seekins (2011)
    attenuation model for use with the total sigma aleatory uncertainty model
    """
    NGA_EAST_TABLE = os.path.join(BASE_PATH,
                                  "NGAEast_BOORE_BS11_J15_Adjusted.hdf5")

class Boore2015NGAEastSGD02(NGAEastGMPE):
    """
    Boore (2015) NGA East GMPE using the Silva et al (2002) attenuation model
    """
    NGA_EAST_TABLE = os.path.join(BASE_PATH,
                                  "NGAEast_BOORE_SGD02_J15_Adjusted.hdf5")

class Boore2015NGAEastSGD02TotalSigma(NGAEastGMPETotalSigma):
    """
    Boore (2015) NGA East GMPE using the Silva et al (2002) attenuation model
    for use with the total sigma aleatory uncertainty model
    """
    NGA_EAST_TABLE = os.path.join(BASE_PATH,
                                  "NGAEast_BOORE_SGD02_J15_Adjusted.hdf5")

# Darragh, Abrahamson, Silva and Gregor (2015) suite

class DarraghEtAl2015NGAEast1CCSP(NGAEastGMPE):
    """
    NGA East model of Darragh et al. (2015) adopting the single-corner
    constant stress parameter (1CCSP)

    Darragh, RB, Abrahamson, NA, Wilva, WJ, Gregor, N (2015) "Development of
    Hard Rock Ground Motion Models for Region 2 of Central and Eastern
    North America" in ...
    """
    NGA_EAST_TABLE = os.path.join(BASE_PATH,
                                  "NGAEast_DARRAGH_1CCSP.hdf5")

class DarraghEtAl2015NGAEast1CCSPTotalSigma(NGAEastGMPETotalSigma):
    """
    NGA East model of Darragh et al. (2015) adopting the single-corner
    constant stress parameter (1CCSP) for use with the total sigma aleatory
    uncertainty model
    """
    NGA_EAST_TABLE = os.path.join(BASE_PATH,
                                  "NGAEast_DARRAGH_1CCSP.hdf5")

class DarraghEtAl2015NGAEast1CVSP(NGAEastGMPE):
    """
    NGA East model of Darragh et al. (2015) adopting the single-corner
    variable stress parameter (1CVSP)
    """
    NGA_EAST_TABLE = os.path.join(BASE_PATH,
                                  "NGAEast_DARRAGH_1CVSP.hdf5")

class DarraghEtAl2015NGAEast1CVSPTotalSigma(NGAEastGMPETotalSigma):
    """
    NGA East model of Darragh et al. (2015) adopting the single-corner
    variable stress parameter (1CVSP) for use with the total sigma aleatory
    uncertainty model
    """
    NGA_EAST_TABLE = os.path.join(BASE_PATH,
                                  "NGAEast_DARRAGH_1CVSP.hdf5")

class DarraghEtAl2015NGAEast2CCSP(NGAEastGMPE):
    """
    NGA East model of Darragh et al. (2015) adopting the two-corner
    constant stress parameter (2CCSP)
    """
    NGA_EAST_TABLE = os.path.join(BASE_PATH,
                                  "NGAEast_DARRAGH_2CCSP.hdf5")

class DarraghEtAl2015NGAEast2CCSPTotalSigma(NGAEastGMPETotalSigma):
    """
    NGA East model of Darragh et al. (2015) adopting the two-corner
    constant stress parameter (2CCSP) for use with the total sigma aleatory
    uncertainty model
    """
    NGA_EAST_TABLE = os.path.join(BASE_PATH,
                                  "NGAEast_DARRAGH_2CCSP.hdf5")

class DarraghEtAl2015NGAEast2CVSP(NGAEastGMPE):
    """
    NGA East model of Darragh et al. (2015) adopting the two-corner
    variable stress parameter (1CVSP)
    """
    NGA_EAST_TABLE = os.path.join(BASE_PATH,
                                  "NGAEast_DARRAGH_2CVSP.hdf5")

class DarraghEtAl2015NGAEast2CVSPTotalSigma(NGAEastGMPETotalSigma):
    """
    NGA East model of Darragh et al. (2015) adopting the two-corner
    variable stress parameter (2CVSP) for use with the total sigma aleatory
    uncertainty model
    """
    NGA_EAST_TABLE = os.path.join(BASE_PATH,
                                  "NGAEast_DARRAGH_2CVSP.hdf5")

# Yenier & Atkinson (2015)

class YenierAtkinson2015NGAEast(NGAEastGMPE):
    """
    NGA East Model of Yenier & Atkinson (2015)
    Yenier, E and Atkinson, GA (2015) "Regionally-Adjustable Generic Ground-
    Motion Prediction Equation based on Equivalent Point-Source Simulations:
    Application to Central and Eastern North America" in PEER 2015/04, Chapter
    4
    """
    NGA_EAST_TABLE = os.path.join(BASE_PATH,
                                  "NGAEast_YENIER_ATKINSON.hdf5")

class YenierAtkinson2015NGAEastTotalSigma(NGAEastGMPETotalSigma):
    """
    NGA East Model of Yenier & Atkinson (2015) for use with the total sigma
    aleatory uncertainty model
    """
    NGA_EAST_TABLE = os.path.join(BASE_PATH,
                                  "NGAEast_YENIER_ATKINSON.hdf5")

# Pezeschk et al. (2015)

class PezeschkEtAl2015NGAEastM1SS(NGAEastGMPE):
    """
    NGA East Model of Pezeschk et al (2015) for the large-M simulation scaling

    Pezeschk, S., Zandieh, A., Campbell, KW and Tavakoli B (2015) "Ground-
    Motion Prediction Equations for Eastern North America using a Hybrid
    Empirical Method" in PEER 2015/04, Chapter 5
    """
    NGA_EAST_TABLE = os.path.join(BASE_PATH,
                                  "NGAEast_PEZESCHK_M1SS.hdf5")

class PezeschkEtAl2015NGAEastM1SSTotalSigma(NGAEastGMPETotalSigma):
    """
    NGA East Model of Pezeschk et al (2015) for the large-M simulation scaling
    for use with the total sigma aleatory uncertainty model
    """
    NGA_EAST_TABLE = os.path.join(BASE_PATH,
                                  "NGAEast_PEZESCHK_M1SS.hdf5")

class PezeschkEtAl2015NGAEastM2ES(NGAEastGMPE):
    """
    NGA East Model of Pezeschk et al (2015) for the large-M empirical scaling
    """
    NGA_EAST_TABLE = os.path.join(BASE_PATH,
                                  "NGAEast_PEZESCHK_M2ES.hdf5")

class PezeschkEtAl2015NGAEastM2ESTotalSigma(NGAEastGMPETotalSigma):
    """
    NGA East Model of Pezeschk et al (2015) for the large-M empirical scaling
    for use with the total sigma aleatory uncertainty model
    """
    NGA_EAST_TABLE = os.path.join(BASE_PATH,
                                  "NGAEast_PEZESCHK_M2ES.hdf5")

# Frankel (2015)

class Frankel2015NGAEast(NGAEastGMPE):
    """
    NGA East Model of Frankel (2015) for application to Central & Eastern
    United States

    Frankel, A (2015) "Ground-Motion Predictions for Eastern North American
    Earthquakes Using Hybrid Broadband Seismograms from Finite-Fault
    Simulation with Constant Stress-Drop Scaling" in PEER 2015/04, Chapter 6
    """
    NGA_EAST_TABLE = os.path.join(BASE_PATH,
                                  "NGAEast_FRANKEL_J15.hdf5")

class Frankel2015NGAEastTotalSigma(NGAEastGMPETotalSigma):
    """
    NGA East Model of Frankel (2015) for application to Central & Eastern
    United States for use with the total sigma aleatory uncertainty model
    """
    NGA_EAST_TABLE = os.path.join(BASE_PATH,
                                  "NGAEast_FRANKEL_J15.hdf5")

# Shahjouei & Pezeschk (2015)

class ShahjoueiPezeschk2015NGAEast(NGAEastGMPE):
    """
    NGA East Model of Shahjouei & Pezeschk (2015) for application to Central &
    Eastern United States

    Shajouei, A and Pezeschk, S (2015) "Hybrid Empirical Ground-Motion Model
    for Central and Eastern North America using Hybrid Broadband Simulations
    and NGA-West2 GMPEs" in PEER 2015/04, Chapter 7
    """
    NGA_EAST_TABLE = os.path.join(BASE_PATH,
                                  "NGAEast_SHAHJOUEI_PEZESCHK.hdf5")

class ShahjoueiPezeschk2015NGAEastTotalSigma(NGAEastGMPETotalSigma):
    """
    NGA East Model of Shahjouei & Pezeschk (2015) for application to Central &
    Eastern United States, for use with the total sigma aleatory uncertainty
    model
    """
    NGA_EAST_TABLE = os.path.join(BASE_PATH,
                                  "NGAEast_SHAHJOUEI_PEZESCHK.hdf5")

# Al Noman and Cramer (2015)

class AlNomanCramer2015NGAEast(NGAEastGMPE):
    """
    NGA East Model of Al Noman & Cramer (2015) for application to Central &
    Eastern United States

    Al Noman & Cramer (2015) "Empirical Ground-Motion Prediction Equations for
    Eastern North America" in PEER 2015/04, Chapter 8
    """
    NGA_EAST_TABLE = os.path.join(BASE_PATH,
                                  "NGAEast_ALNOMAN_CRAMER.hdf5")

class AlNomanCramer2015NGAEastTotalSigma(NGAEastGMPETotalSigma):
    """
    NGA East Model of Al Noman & Cramer (2015) for application to Central &
    Eastern United States, for use with the total sigma aleatory uncertainty
    model
    """
    NGA_EAST_TABLE = os.path.join(BASE_PATH,
                                  "NGAEast_ALNOMAN_CRAMER.hdf5")

# Graizer (2015)

class Graizer2015NGAEast(NGAEastGMPE):
    """
    NGA East Model of Graizer (2015) for application to Central & Eastern
    United States

    Graizer, V (2015) "Ground-Motion Prediction Equations for the Central and
    Eastern United States" in PEER 2015/04, Chapter 9
    """
    NGA_EAST_TABLE = os.path.join(BASE_PATH,
                                  "NGAEast_GRAIZER.hdf5")

class Graizer2015NGAEastTotalSigma(NGAEastGMPETotalSigma):
    """
    NGA East Model of Graizer (2015) for application to Central & Eastern
    United States, for use with the total sigma aleatory uncertainty
    model
    """
    NGA_EAST_TABLE = os.path.join(BASE_PATH,
                                  "NGAEast_GRAIZER.hdf5")

# Hassani & Atkinson (2015)

class HassaniAtkinson2015NGAEast(NGAEastGMPE):
    """
    NGA East Model of Hassani & Atkinson (2015) for application to Central &
    Eastern United States

    Hassani, B & Atkinson, GA (2015) "Referenced Empirical Ground-Motion Model
    for Eastern North America" in PEER 2015/04, Chapter 10
    """
    NGA_EAST_TABLE = os.path.join(BASE_PATH,
                                  "NGAEast_HASSANI_ATKINSON.hdf5")

class HassaniAtkinson2015NGAEastTotalSigma(NGAEastGMPETotalSigma):
    """
    NGA East Model of Hassani & Atkinson (2015) for application to Central &
    Eastern United States, for use with the total sigma aleatory uncertainty
    model
    """
    NGA_EAST_TABLE = os.path.join(BASE_PATH,
                                  "NGAEast_HASSANI_ATKINSON.hdf5")

# Hollenback et al. (2015)

class HollenbackEtAl2015NGAEastGP(NGAEastGMPE):
    """
    NGA East Model of Hollenback et al (2015) for application to Central &
    Eastern United States using the GP Finite-Fault model

    Hollenback, J, Keuhn, N, Goulet, CA and Abrahamson, NA (2015) "PEER NGA-
    East Median Ground Motion Models" in PEER 2015/04, Chapter 11
    """
    NGA_EAST_TABLE = os.path.join(BASE_PATH,
                                  "NGAEast_PEER_GP.hdf5")

class HollenbackEtAl2015NGAEastGPTotalSigma(NGAEastGMPETotalSigma):
    """
    NGA East Model of Hollenback et al (2015) for application to Central &
    Eastern United States, for use with the total sigma aleatory uncertainty
    model
    """
    NGA_EAST_TABLE = os.path.join(BASE_PATH,
                                  "NGAEast_PEER_GP.hdf5")

class HollenbackEtAl2015NGAEastEX(NGAEastGMPE):
    """
    NGA East Model of Hollenback et al (2015) for application to Central &
    Eastern United States using the EXSIM Finite-Fault model

    Hollenback, J, Keuhn, N, Goulet, CA and Abrahamson, NA (2015) "PEER NGA-
    East Median Ground Motion Models" in PEER 2015/04, Chapter 11
    """
    NGA_EAST_TABLE = os.path.join(BASE_PATH,
                                  "NGAEast_PEER_EX.hdf5")

class HollenbackEtAl2015NGAEastEXTotalSigma(NGAEastGMPETotalSigma):
    """
    NGA East Model of Hollenback et al (2015) for application to Central &
    Eastern United States, for use with the total sigma aleatory uncertainty
    model
    """
    NGA_EAST_TABLE = os.path.join(BASE_PATH,
                                  "NGAEast_PEER_EX.hdf5")


import io
import os
import random

import boto3
import tornado.testing

from service.s3_object import S3Object

class TestS3Object(tornado.testing.AsyncTestCase):
    def setUp(self):
        super().setUp()

        self.s3_object = S3Object(os.environ["AWS_S3_BUCKET"], os.environ["AWS_S3_OBJECT"])

        self.expected_contents = f"data-{random.random()}".encode("utf8")

    @tornado.testing.gen_test
    async def test_download_returns_contents_of_s3_object(self):
        boto3.resource("s3").Bucket(os.environ["AWS_S3_BUCKET"]).upload_fileobj(
            Key=os.environ["AWS_S3_OBJECT"],
            Fileobj=io.BytesIO(self.expected_contents))

        contents = await self.s3_object.download()

        self.assertEqual(self.expected_contents, contents)

    @tornado.testing.gen_test
    async def test_upload_sets_contents_of_s3_object(self):
        await self.s3_object.upload(self.expected_contents)

        contents = io.BytesIO()
        boto3.resource("s3").Bucket(os.environ["AWS_S3_BUCKET"]).download_fileobj(
            Key=os.environ["AWS_S3_OBJECT"],
            Fileobj=contents)
        self.assertEqual(self.expected_contents, contents.getvalue())


import requests
from bs4 import BeautifulSoup
import sys
import json

q = str(sys.stdin.readline()) + " stock share news "
# q = "lic stock share news"
url = f"https://www.google.com/search?q={q}&start=0"

response = requests.get(url)
soup = BeautifulSoup(response.content, "html.parser")

news = ["livemint.com", "zeebiz.com", "moneycontrol.com"]

links = {"livemint.com": [], "moneycontrol.com": [], "zeebiz.com": []}

isMoneyControl= False
#isFirstLiveMint= True
for anchor in soup.find_all("a"):
    link = anchor.get("href")
    for i in news :
        if link and i in link:
            l = link[7:]
            if i=="livemint.com" :
                # if isFirstLiveMint :
                #     isFirstLiveMint= False
                #     continue
                html_index = l.find(".html")
                if html_index!=-1:
                    l = l[:html_index + 5]
                else:
                    continue
                links[i].append(l)
            elif i=="zeebiz.com" :
                and_index = l.find("&")
                l = l[:and_index]
                links[i].append(l)
            elif i=="moneycontrol.com":
                if(isMoneyControl):
                    continue
                isMoneyControl=True
                links[i].append(l)

json_str = json.dumps(links)
sys.stdout.write(json_str)
sys.stdout.flush()


import cv2
import numpy as np

from function import to_gray

def _calc_var_in_class(binary: np.ndarray, gray: np.ndarray):
    w0 = (binary == 0).mean()
    w1 = (binary == 255).mean()

    mask = (binary == 0)
    masked_img = gray[mask]
    if len(masked_img) > 0:
        M0 = masked_img.mean()
    else:
        M0 = 0
    M1 = gray[~mask].mean()
    return w0 * w1 * (M0 - M1)**2

def threshold_search(gray: np.ndarray):
    candidates = np.arange(0, 255, 1)
    scores = []
    for th in candidates:
        binary = gray.copy()
        binary[binary >= th] = 255
        binary[binary < th] = 0

        score = _calc_var_in_class(binary, gray)
        scores.append(score)
    idx = np.argmax(scores)
    return idx

img = cv2.imread("imori.jpg")
gray = to_gray(img)
threshold = threshold_search(gray)
img2 = gray.copy()
img2[img2 < threshold] = 0
img2[img2 >= threshold] = 255

cv2.imwrite("answer_4.jpg", img2)
cv2.imshow("otsu-binary", img2)
cv2.waitKey(0)
cv2.destroyAllWindows()


"""Utility functions for testing neurodsp functions."""

from functools import wraps

import numpy as np
import matplotlib.pyplot as plt

from neurodsp.utils.data import compute_nsamples

from neurodsp.tests.settings import N_SECONDS, FS

###################################################################################################
###################################################################################################

def check_sim_output(sig, n_seconds=None, fs=None):
    """Helper function to check some basic properties of simulated signals."""

    n_seconds = N_SECONDS if not n_seconds else n_seconds
    fs = FS if not fs else fs
    exp_n_samples = compute_nsamples(n_seconds, fs)

    assert isinstance(sig, np.ndarray)
    assert len(sig) == exp_n_samples
    assert sum(np.isnan(sig)) == 0

def plot_test(func):
    """Decorator for simple testing of plotting functions.

    Notes
    -----
    This decorator closes all plots prior to the test.
    After running the test function, it checks an axis was created with data.
    It therefore performs a minimal test - asserting the plots exists, with no accuracy checking.
    """

    @wraps(func)
    def wrapper(*args, **kwargs):

        plt.close('all')

        func(*args, **kwargs)

        ax = plt.gca()
        assert ax.has_data()

    return wrapper

def check_exponent(xs, offset, exp):
    return xs*exp + offset


import json
import pandas as pd
import requests
import csv
from pprint import pprint
import sqlite3
import bmt
from oaklib.implementations.ubergraph.ubergraph_implementation import UbergraphImplementation
from sqlalchemy import create_engine

oi = UbergraphImplementation()
conn = sqlite3.connect('test.db')
tk = bmt.Toolkit()

def run():

    engine = create_engine('sqlite://',
                           echo=False)

    r = requests.get('https://smart-api.info/api/metakg')
    metakg = r.json()['associations']
    metakg_small = []
    for association in metakg:
        if association.get('api').get('x-translator').get('component') == 'KP':
            assoc = {
                "subject": association.get('subject'),
                "object": association.get('object'),
                "predicate": association.get("predicate"),
                "provided_by": association.get("provided_by"),
                "api_name": association.get("api").get("name"),
                "api_id": association.get("api").get("smartapi").get("id")
            }
            metakg_small.append(assoc)

    metakg_df = pd.DataFrame.from_dict(metakg_small)
    engine.execute("drop table if exists metakg_table")
    metakg_df.to_sql('metakg_table',
                     con=engine)

    df = pd.read_sql(
        'SELECT distinct subject, object, provided_by, predicate, api_name, '
        'api_id from metakg_table where predicate = "ameliorates" or '
        'predicate = "approved_to_treat" or predicate = "treats"',
        engine)
    # write DataFrame to CSV file
    # df.to_csv('metakg.csv', index=False)
    rows = df.to_dict('records')
    return rows

def query_endpoint():
    rows = run()
    for row in rows:
        trapi = make_trapi(row.get('subject'), row.get('object'), row.get('predicate'))
        print(trapi)
    return None

def make_trapi(subject: str, object: str, predicate: str):
    return ""

def get_id_prefixes():
    rows = run()
    for row in rows:
        subject = "biolink:"+row.get('subject')
        element = tk.get_element(subject)
        id_prefixes = []
        if element is None:
            print(subject + " isn't a valid biolink item?")
        elif "id_prefixes" not in element:
            ancestors = tk.get_ancestors(element.name)
            for ancestor in ancestors:
                if tk.get_element(ancestor).id_prefixes is not None:
                    id_prefixes = id_prefixes + tk.get_element(ancestor).id_prefixes
        elif len(element.id_prefixes) != 0:
            resource = id_prefixes[0]
            print(resource)
        else:
            print("id prefixes is empty for: " + subject)


from deap import base, creator, tools, algorithms
import random
import numpy

class Deap:
    def execute():
        NBR_ITEMS = 10
        MAX_ITEM = 10
        MAX_WEIGHT = 10
        IND_INIT_SIZE = 10

        creator.create("Fitness", base.Fitness, weights=(-1.0, 1.0))
        creator.create("Individual", set, fitness=creator.Fitness)

        items = {}
        # Create random items and store them in the items' dictionary.
        for i in range(NBR_ITEMS):
            items[i] = (random.randint(1, 10), random.uniform(0, 100))

        toolbox = base.Toolbox()
        toolbox.register("attr_item", random.randrange, NBR_ITEMS)
        toolbox.register("individual", tools.initRepeat, creator.Individual, toolbox.attr_item, IND_INIT_SIZE)
        toolbox.register("population", tools.initRepeat, list, toolbox.individual)

        def evalKnapsack(individual):
            weight = 0.0
            value = 0.0
            for item in individual:
                weight += items[item][0]
                value += items[item][1]
            if len(individual) > MAX_ITEM or weight > MAX_WEIGHT:
                return 10000, 0             # Ensure overweighted bags are dominated
            return weight, value

        def cxSet(ind1, ind2):
            """Apply a crossover operation on input sets. The first child is the
            intersection of the two sets, the second child is the difference of the
            two sets.
            """
            temp = set(ind1)                # Used in order to keep type
            ind1 &= ind2                    # Intersection (inplace)
            ind2 ^= temp                    # Symmetric Difference (inplace)
            return ind1, ind2

        def mutSet(individual):
            """Mutation that pops or add an element."""
            if random.random() < 0.5:
                if len(individual) > 0:     # We cannot pop from an empty set
                    individual.remove(random.choice(sorted(tuple(individual))))
            else:
                individual.add(random.randrange(NBR_ITEMS))
            return individual,

        toolbox.register("evaluate", evalKnapsack)
        toolbox.register("mate", cxSet)
        toolbox.register("mutate", mutSet)
        toolbox.register("select", tools.selNSGA2)

        NGEN = 50
        MU = 50
        LAMBDA = 100
        CXPB = 0.7
        MUTPB = 0.2

        pop = toolbox.population(n=MU)
        hof = tools.ParetoFront()
        stats = tools.Statistics(lambda ind: ind.fitness.values)
        stats.register("avg", numpy.mean, axis=0)
        stats.register("std", numpy.std, axis=0)
        stats.register("min", numpy.min, axis=0)
        stats.register("max", numpy.max, axis=0)

        algorithms.eaMuPlusLambda(pop, toolbox, MU, LAMBDA, CXPB, MUTPB, NGEN, stats, halloffame=hof)

        # print(pop, stats, hof)
        print(pop)
        # print(stats)
        # print(hof)


#!/usr/bin/env python3

import os
import sys
import json
import base64
import argparse
import binascii
import pymongo
from dotenv import load_dotenv
from blockchain_parser.block import Block
import bitcoin.rpc
import tna
import util

load_dotenv()

btc_conf_file = os.path.expanduser(os.getenv("BTC_CONF_FILE"))

mongo = pymongo.MongoClient(os.getenv('MONGO_URL'))
db = mongo[os.getenv('MONGO_NAME')]

parser = argparse.ArgumentParser(description="import transactions to bitdb")
parser.add_argument("--start-block", type=int, help="block to start, if not given will go from db.meta.last_block")
parser.add_argument("--end-block", type=int, help="block to finish on, if not given will go until rpc fails")
parser.add_argument("--dry", action="store_true", help="dry run (no inserts)")
parser.add_argument("--verbose", action="store_true", help="show json from tna")
args = parser.parse_args()

rpc = bitcoin.rpc.RawProxy(btc_conf_file=btc_conf_file)

last_block_height = util.meta_get_last_block_height(db)
if not args.start_block:
    if last_block_height is not None:
        args.start_block = last_block_height + 1
    else:
        args.start_block = 0

if last_block_height is not None:
    if args.start_block < last_block_height:
        print("start_block is smaller than last block height in db ({})".format(last_block_height))
        print("this means you are rescanning parts of the blockchain you've already scanned")
        print("the entries will be deleted, and this is safe, it just might take longer")
        if input("press enter to continue, anything else to exit: ") != "":
            sys.exit()
    if args.start_block-1 > last_block_height:
        print("start_block-1 is greater than last block height in db ({})".format(last_block_height))
        print("this might mean you are skipping blocks")
        if input("press enter to continue, anything else to exit: ") != "":
            sys.exit()

# this is just used for printing status
percentage_end_block_count = args.end_block

# if end_block not provided we should keep going until an error occurs
if not args.end_block:
    args.end_block = 999999999
    percentage_end_block_count = rpc.getblockcount()

total_blocks = args.end_block - args.start_block

if not args.dry:
    print('deleted {} txs from gte block {}'.format(
        util.delete_txs_gte(db, args.start_block),
        args.start_block
    ))

for height in range(args.start_block, args.end_block):
    try:
        block_hash = rpc.getblockhash(height)
    except bitcoin.rpc.JSONRPCError as e:
        print('cannot get block hash, breaking')
        break

    raw_block  = binascii.unhexlify(rpc.getblock(block_hash, False)) # set to 0 to get raw hex data

    block = Block(raw_block, height=height)

    documents = []
    for tx in block.transactions:
        res = tna.extract(block, tx)
        if args.verbose:
            print(json.dumps(res, indent=4))
        documents.append(res)

    if args.dry:
        inserted = 0
    else:
        inserted = len(db.confirmed.insert_many(documents).inserted_ids)
        util.meta_update_last_block_height(db, height)

    print("{}%\theight={} inserted={}".format(
        round((block.height-args.start_block) / (percentage_end_block_count-args.start_block) * 100, 2),
        block.height,
        inserted
    ))


from django.contrib.auth.mixins import LoginRequiredMixin
from django.shortcuts import render, redirect
from django.views import View
from django.core.paginator import  Paginator

from dashboard.models import Post
from dashboard.forms import PostModelForm

from users.models import get_friendships

def order_post_by_creation(post):
    return post.created

class DashboardView(View, LoginRequiredMixin):

    def get(self, request):
        form = PostModelForm()
        friends = get_friendships(request, email_only=True)
        user_posts = Post.objects.filter(author__email=request.user.email)
        friends_posts = []
        for friend in friends:
            friends_posts += Post.objects.filter(author__email=friend)
        posts = list(user_posts) + friends_posts

        # order posts by creation date
        posts.sort(reverse=True, key=order_post_by_creation)

        # pagination
        paginator = Paginator(posts, 2)
        page_number = request.GET.get('page')
        page_obj = paginator.get_page(page_number)

        context = {
            'form': form,
            'friends': friends,
            'page_obj': page_obj,
        }
        return render(request, 'dashboard/dashboard.html', context)

    def post(self, request):
        form = PostModelForm(request.POST, request.FILES)
        if form.is_valid():
            post = Post()
            post.author = request.user
            post.image = form.cleaned_data['image']
            post.comment = form.cleaned_data['comment']
            post.save()
            return redirect('dashboard:dashboard')
        return redirect('dashboard:dashboard')

def delete_post(request, pk):
    post = Post.objects.get(pk=pk)
    post.delete()
    return redirect('dashboard:dashboard')


#!/usr/bin/env python3
# Ulas Kamaci, Evan Widloski - 2018-08-27

import numpy as np
from mas.block import block_mul, block_herm
from mas.decorators import vectorize, _vectorize
from scipy.stats import poisson
from scipy.signal import fftconvolve
from scipy.signal import convolve2d
from scipy.ndimage.filters import gaussian_filter
from PIL import Image
import logging

log = logging.getLogger(name=__name__)

def image_numpyer(*, inpath, outpath, size, upperleft):
    """
    Read png/jpg image, crop it with the specified size and location, and save
    it as an npy file

    Args:
        inpath (string): Path to the input png/jpg file
        outpath (string): Path to the output npy file
        size (tuple): Integer or tuple of 2 integers indicating the height and width
        of the cropped image, respectively.
        upperleft (tuple): Tuple of 2 integers indicating the coordinate of
        the upper left corner of the cropped image
    """
    if type(size) is int:
        size = (size,size)
    x = np.array(Image.open(inpath).convert('L'))
    x = x[upperleft[0]:upperleft[0]+size[0], upperleft[1]:upperleft[1]+size[1]]
    np.save(outpath, x)

def image_selector(*, inpath, size, upperleft):
    """
    Read png/jpg image, crop it with the specified coordinates and size, and
    show the full image vs cropped image.
    Puts a box around the cropped part for visualization. Box code might need
    adjustment for non-SDO images.

    Args:
        inpath (string): Path to the input png/jpg file
        size (tuple): Integer or tuple of 2 integers indicating the height and
        width of the cropped image, respectively.
        upperleft (tuple): Tuple of 2 integers indicating the coordinate of
        the upper left corner of the cropped image
    """
    if type(size) is int:
        size = (size,size)
    x = np.array(Image.open(inpath).convert('L'))
    x_c = x[upperleft[0]:upperleft[0]+size[0], upperleft[1]:upperleft[1]+size[1]]
    xmax = np.max(x[: int(x.shape[0]*0.94), :]) # to preserve the dynamic range

    # put a frame around the selected part
    t = 10
    x[upperleft[0] - t:upperleft[0],
    upperleft[1]: upperleft[1] + size[1]] = xmax

    x[upperleft[0] + size[0]+1: upperleft[0] + size[0] + t,
    upperleft[1]: upperleft[1] + size[1]] = xmax

    x[upperleft[0] - t: upperleft[0] + size[0] + t,
    upperleft[1] - t: upperleft[1]] = xmax

    x[upperleft[0] - t: upperleft[0] + size[0] + t,
    upperleft[1] + size[1] + 1: upperleft[1] + size[1] + t] = xmax

    f, (ax1, ax2) = plt.subplots(1, 2)
    ax1.imshow(x[: int(x.shape[0]*0.94), :]) # crop the text to preserve dynamic range (for SDO)
    ax2.imshow(x_c)

def rectangle_adder(*, image, size, upperleft):
    """
    Add a rectangle of value one to the specified position of a given image.

    Args:
        image (ndaray): input image on which the rectangle will be added
        size (tuple): Integer or tuple of 2 integers indicating the height and
        width of the rectangle, respectively.
        upperleft (tuple): Tuple of 2 integers indicating the coordinate of
        the upper left corner of the position of the added rectangle
    """
    if type(size) is int:
        size = (size,size)
    image[
        upperleft[0]:upperleft[0]+size[0], upperleft[1]:upperleft[1]+size[1]
    ] += np.ones(size)
    return image

@_vectorize(signature='(i,j,k)->(m,n,o)', included=[0, 'sources'])
def get_measurements(
    *,
    sources,
    psfs,
    mode='circular',
    real=True,
    meas_size=None,
    blur_sigma=None,
    noise_sigma=None,
    drift_amount=None,
    **kwargs
):
    """
    Convolve the sources and psfs to obtain measurements.
    Args:
        sources (ndarray): 4d array of sources
        psfs (PSFs): PSFs object containing psfs and other csbs state data
        mode (string): {'circular', 'valid'} (default='valid') convolution mode.
            `circular`: circular convolution of the source and the psfs. `valid`:
            linearly convolve the sources and psfs, then take the fully overlapping
            part.  `auto`: use `valid` if possible, then fall back to `circular`
        real (bool): (default=True) whether returned measurement should be real
            type of the convolution performed to obtain the measurements from psfs
            and sources

    Optional:
        meas_size (tuple): 2d tuple of size of the detector array
        blur_sigma (float): std deviation of additional blur applied to the PSFs
        noise_sigma (float): std deviation of additional noise applied to the PSFs

    Returns:
        ndarray that is the noisy version of the input
    """

    if mode == 'auto':
        _, source_width, _ = sources.shape
        _, _, psf_width, _ = psfs.psfs.shape

        if source_width > psf_width:
            mode = 'valid'
        else:
            mode = 'circular'
            log.warning('Falling back to circular convolution')

    if mode == 'circular':
        [p, aa, bb] = sources.shape
        [k, p, ss, ss] = psfs.psfs.shape
        if meas_size is not None:
            sources = size_equalizer(sources, ref_size=meas_size)
            [p, aa, bb] = sources.shape

        # reshape psfs and sources
        psfs_ext = np.repeat(
            size_equalizer(psfs.psfs, ref_size=[aa,bb]),
            psfs.copies.astype(int),axis=0
        )

        if blur_sigma is not None:
            for i in range(psfs_ext.shape[0]):
                for j in range(psfs_ext.shape[1]):
                    psfs_ext[i,j] = gaussian_filter(psfs_ext[i,j], sigma=blur_sigma)
                    psfs.psfs_modified = size_equalizer(psfs_ext, ref_size=[ss,ss])

        if drift_amount is not None:
            kernel = np.zeros((drift_amount,drift_amount))
            for i in range(drift_amount):
                for j in range(drift_amount):
                    if i+j == drift_amount - 1:
                        kernel[i,j] = 1 / drift_amount
            for i in range(psfs_ext.shape[0]):
                for j in range(psfs_ext.shape[1]):
                    psfs_ext[i,j] = convolve2d(
                        psfs_ext[i,j], kernel, mode='same')
            psfs.psfs_modified = size_equalizer(psfs_ext, ref_size=[ss,ss])

        if noise_sigma is not None:
            psfs_ext = np.random.normal(loc=psfs_ext, scale=noise_sigma)
            psfs_ext[psfs_ext < 0] = 0
            psfs.psfs_modified = size_equalizer(psfs_ext, ref_size=[ss,ss])

        # ----- forward -----
        measurement = np.fft.fftshift(
            np.fft.ifft2(
                np.einsum(
                    'ijkl,jkl->ikl',
                    np.fft.fft2(psfs_ext),
                    np.fft.fft2(sources)
                )
            ),
            axes=(1, 2)
        )

    if mode == 'valid':
        [p, aa, bb] = sources.shape
        [k, p, ss, ss] = psfs.psfs.shape
        if meas_size is not None:
            [ma, mb] = meas_size
            sources = size_equalizer(sources, ref_size=[ma + ss - 1, mb + ss - 1])
            [p, aa, bb] = sources.shape
            ta, tb = [aa + ss - 1, bb + ss - 1]
            fa, fb = [aa - ss + 1, bb - ss + 1]
        else:
            ta, tb = [aa + ss - 1, bb + ss - 1]
            fa, fb = [aa - ss + 1, bb - ss + 1]

        # reshape psfs and sources
        psfs_ext = np.repeat(
            size_equalizer(psfs.psfs, ref_size=[ta,tb]),
            psfs.copies.astype(int),axis=0
        )

        if blur_sigma is not None:
            for i in range(psfs_ext.shape[0]):
                for j in range(psfs_ext.shape[1]):
                    psfs_ext[i,j] = gaussian_filter(psfs_ext[i,j], sigma=blur_sigma)
                    psfs.psfs_modified = size_equalizer(psfs_ext, ref_size=[ss,ss])

        if drift_amount is not None:
            kernel = np.zeros((drift_amount,drift_amount))
            for i in range(drift_amount):
                for j in range(drift_amount):
                    if i+j == drift_amount - 1:
                        kernel[i,j] = 1 / drift_amount
            for i in range(psfs_ext.shape[0]):
                for j in range(psfs_ext.shape[1]):
                    psfs_ext[i,j] = convolve2d(
                        psfs_ext[i,j], kernel, mode='same')
            psfs.psfs_modified = size_equalizer(psfs_ext, ref_size=[ss,ss])

        if noise_sigma is not None:
            psfs_ext = np.random.normal(loc=psfs_ext, scale=noise_sigma)
            psfs_ext[psfs_ext < 0] = 0
            psfs.psfs_modified = size_equalizer(psfs_ext, ref_size=[ss,ss])

        sources_ext = size_equalizer(sources, ref_size=[ta,tb])

        # ----- forward -----
        measurement = np.fft.fftshift(
            np.fft.ifft2(
                np.einsum(
                    'ijkl,jkl->ikl',
                    np.fft.fft2(psfs_ext),
                    np.fft.fft2(sources_ext)
                )
            ),
            axes=(1, 2)
        )
        measurement = size_equalizer(measurement, ref_size=[fa,fb])

    return measurement.real if real else measurement

def get_contributions(real=True, *, sources, psfs):
    """
    Convolve the sources and psfs to obtain contributions.
    Args:
        sources (ndarray): 4d array of sources
        psfs (PSFs): PSFs object containing psfs and other csbs state data
        real (bool): (default=True) whether returned measurement should be real
            type of the convolution performed to obtain the measurements from psfs
            and sources

    Returns:
        ndarray that is the noisy version of the input
    """
    assert sources.shape[0] == psfs.psfs.shape[1], "source and psf dimensions do not match"

    # reshape psfs
    [p,aa,bb] = sources.shape
    [k,p,ss,ss] = psfs.psfs.shape
    psfs = size_equalizer(psfs.psfs, ref_size=[aa,bb])

    psfs = np.repeat(psfs, psfs.copies.astype(int), axis=0)
    psf_dfts = np.fft.fft2(psfs)

    # ----- forward -----
    measurement = np.fft.fftshift(
        np.fft.ifft2(
            np.einsum(
                'ijkl,jkl->ijkl',
                psf_dfts,
                np.fft.fft2(sources)
            )
        ),
        axes=(2, 3)
    )
    return measurement.real if real else measurement

@vectorize
def add_noise(signal, dbsnr=None, max_count=None, model='Poisson', no_noise=False):
    """
    Add noise to the given signal at the specified level.

    Args:
        (ndarray): noise-free input signal
        dbsnr (float): signal to noise ratio in dB: for Gaussian noise model, it is
        defined as the ratio of variance of the input signal to the variance of
        the noise. For Poisson model, it is taken as the average snr where snr
        of a pixel is given by the square root of its value.
        max_count (int): Max number of photon counts in the given signal
        model (string): String that specifies the noise model. The 2 options are
        `Gaussian` and `Poisson`
        no_noise (bool): (default=False) If True, return the clean signal

    Returns:
        ndarray that is the noisy version of the input
    """
    if no_noise is True:
        return signal
    else:
        assert model.lower() in ('gaussian', 'poisson'), "invalid noise model"
        if model.lower() == 'gaussian':
            var_sig = np.var(signal)
            var_noise = var_sig / 10**(dbsnr / 10)
            out = np.random.normal(loc=signal, scale=np.sqrt(var_noise))
        elif model.lower() == 'poisson':
            if max_count is not None:
                sig_scaled = signal * (max_count / signal.max())
                # print('SNR:{}'.format(np.sqrt(sig_scaled.mean())))
                out = poisson.rvs(sig_scaled) * (signal.max() / max_count)
            else:
                avg_brightness = 10**(dbsnr / 10)**2
                sig_scaled = signal * (avg_brightness / signal.mean())
                out = poisson.rvs(sig_scaled) * (signal.mean() / avg_brightness)
        return out

@vectorize
def size_equalizer(x, ref_size, mode='center'):
    """
    Crop or zero-pad a 2D array so that it has the size `ref_size`.
    Both cropping and zero-padding are done such that the symmetry of the
    input signal is preserved.
    Args:
        x (ndarray): array which will be cropped/zero-padded
        ref_size (list): list containing the desired size of the array [r1,r2]
        mode (str): ('center', 'topleft') where x should be placed when zero padding
    Returns:
        ndarray that is the cropper/zero-padded version of the input
    """
    assert len(x.shape) == 2, "invalid shape for x"

    if x.shape[0] > ref_size[0]:
        pad_left, pad_right = 0, 0
        crop_left = 0 if mode == 'topleft' else (x.shape[0] - ref_size[0] + 1) // 2
        crop_right = crop_left + ref_size[0]
    else:
        crop_left, crop_right = 0, x.shape[0]
        pad_right = ref_size[0] - x.shape[0] if mode == 'topleft' else (ref_size[0] - x.shape[0]) // 2
        pad_left = ref_size[0] - pad_right - x.shape[0]
    if x.shape[1] > ref_size[1]:
        pad_top, pad_bottom = 0, 0
        crop_top = 0 if mode == 'topleft' else (x.shape[1] - ref_size[1] + 1) // 2
        crop_bottom = crop_top + ref_size[1]
    else:
        crop_top, crop_bottom = 0, x.shape[1]
        pad_bottom = ref_size[1] - x.shape[1] if mode == 'topleft' else (ref_size[1] - x.shape[1]) // 2
        pad_top = ref_size[1] - pad_bottom - x.shape[1]

    # crop x
    cropped = x[crop_left:crop_right, crop_top:crop_bottom]
    # pad x
    padded = np.pad(
        cropped,
        ((pad_left, pad_right), (pad_top, pad_bottom)),
        mode='constant'
    )

    return padded

def crop(im, center=None, *, width):
    """
    Return a cropped rectangle from an input image

    Args:
        im (ndarray): input image
        center (tuple): coordinate pair of center of cropped rectangle. defaults to image center
        width (int, tuple): length of each axis of cropped rectangle.  returns square if integer

    Returns:
        cropped rectangle of input image
    """

    if type(width) is int:
        width = (width, width)

    if center is None:
        center = (im.shape[0] // 2, im.shape[1] // 2)

    # assert (
    #     (0 <= center[0] - width[0]) and
    #     (0 <= center[1] - width[1]) and
    #     (im.shape[0] >= center[0] + width[0]) and
    #     (im.shape[1] >= center[1] + width[1])
    # ), "Cropped region falls outside image bounds"

    crop_left = (im.shape[0] - width[0] + 1) // 2
    crop_right = crop_left + width[0]
    crop_top = (im.shape[1] - width[1] + 1) // 2
    crop_bottom = crop_top + width[1]

    return im[crop_left:crop_right, crop_top:crop_bottom]

def size_compressor(signal, energy_ratio=0.9995):
    """
    Compute width of an image such that the given `energy_ratio` is achieved

    Args:
        signal (ndarray): 2D input image
        energy_ratio (float): ratio of output to input signal energies

    Returns:
        width containing 'energy_ratio' percent of total energy, measured
        from the central pixel
    """

    energy0 = sum(sum(signal**2))
    energy_target = energy0 * energy_ratio
    width = int(signal.shape[0] / 2) + 1
    width_change = signal.shape[0] - width
    while width_change > 1:
        energy2 = sum(sum((size_equalizer(signal,[width,width]))**2))
        width_change = int(width_change / 2)
        if energy2 >= energy_target:
            width -= width_change
        else:
            width += width_change
    width = int(width / 2) * 2 + 1
    return width

def downsample(x, factor=2):
    """
    Downsample an image by averaging factor*factor sized patches.  Discards remaining pixels
    on bottom and right edges

    Args:
        x (ndarray): input image to downsample
        factor (int): factor to downsample image by

    Returns:
        ndarray containing downsampled image
    """

    return fftconvolve(
        x,
        np.ones((factor, factor)) / factor**2,
        mode='valid'
    )[::factor, ::factor]

def downsample_mid(x, factor=2):
    """
    Downsample an image by average factor*factor sized patches.
    Shift patches by factor / 2

    Args:
        x (ndarray): input image to downsample
        factor (int): factor to downsample image by

    Returns:
        ndarray containing downsampled image
    """

    X = np.fft.fftn(x)
    kern = size_equalizer(np.ones((factor, factor)) / factor**2, x.shape)
    kern = np.fft.fftshift(kern)
    kern = np.fft.fftn(kern)
    # -factor to prevent overlap from first kernel pos to last kernel pos
    # return np.fft.ifftn(kern * X)[:-factor:factor, :-factor:factor]
    return np.fft.ifftn(kern * X)[::factor, ::factor]

def upsample(x, factor=2):
    """
    Upsample an image by turning 1 pixel into a factor*factor sized patch

    Args:
        x (ndarray): input image to upsample
        factor (int): factor to upsample image by

    Returns:
        ndarray containing upsampled image
    """

    return np.repeat(
        np.repeat(
            x,
            repeats=factor,
            axis=0
        ),
        repeats=factor,
        axis=1
    )

def dof2wavelength(*, dof, base_wavelength, ps):
    """Returns a wavelength relative to base_wavelength, measured in DOF
    for a given photon sieve

    Args:
        dof (float, ndarray): dof(s) of returned wavelength relative
            to base_wavelength
        base_wavelength (float): reference wavelength
        ps (PhotonSieve): photon sieve parameters to compute DOF
    """

    return (
        base_wavelength + 2 * ps.smallest_hole_diameter *
        base_wavelength * dof / ps.diameter
    )

def wavelength2dof(*, wavelength, base_wavelength, ps):
    """Returns dof of a wavelength relative to base_wavelength
    for a given photon sieve

    Args:
        dof (float, ndarray): wavelength(s) with which to compute DOF
        base_wavelength (float): reference wavelength
        ps (PhotonSieve): photon sieve parameters to compute DOF
    """

    return (
        (wavelength - base_wavelength) * ps.diameter /
        (2 * ps.smallest_hole_diameter * base_wavelength)
    )

def modulate(shape, amp=1, width=1, grid=5):

    tile = np.tile(
        np.hstack((
            [.5] * width,
            [-.5] * width
        )),
        (width * grid, grid // 2 + 1)
    )[:, :width * grid]

    modulated = np.vstack((
        np.hstack((tile, tile.T)),
        np.hstack((tile.T, tile)),
    ))

    modulated = np.tile(
        modulated,
        (
            shape[0] // modulated.shape[0] + 1,
            shape[1] // modulated.shape[1] + 1
        )
    )[:shape[0], :shape[1]]

    return amp * modulated


import pygame
import random

from variables import *
import menu

def main():
	global running, playing
	global deltaTime
	global sc_width
	global sc_height
	while running:
		for event in pygame.event.get():
			if event.type == pygame.QUIT:
				running = False
		if playing:
			# Draw
			draw()
			# Modification
			keys = pygame.key.get_pressed()
			modification(keys)
			# Update
			update()
		else:
			result = menu.main(screen, sc_width, sc_height)
			if result == 1:
				restart_everything()
			elif result == 2:
				running = False

def draw():
	global screen
	global player_1, player_2, ball
	screen.fill("black")
	#pygame.draw.rect(screen,"white",(200,150,100,50))
	pygame.draw.rect(screen, player_1.color, player_1.rect)
	pygame.draw.rect(screen, player_2.color, player_2.rect)
	pygame.draw.ellipse(screen, ball.color, ball.rect)

def modification(keys):
	global running, first_move
	global player_1, player_2
	global deltaTime

	if keys[pygame.K_ESCAPE]:
		running = False

	# Player 1
	if keys[pygame.K_w]:
		if player_1.top > 0:
			player_1.top -= player_1.velocity * deltaTime
			first_move = True
	if keys[pygame.K_s]:
		if player_1.bottom < (FIXED_HEIGHT):
			player_1.top += player_1.velocity * deltaTime
			first_move = True
	# Player 2
	if keys[pygame.K_UP]:
		if player_2.top > 0:
			player_2.top -= player_2.velocity * deltaTime
			first_move = True
	if keys[pygame.K_DOWN]:
		if player_2.bottom < (FIXED_HEIGHT):
			player_2.top += player_2.velocity * deltaTime
			first_move = True

def update():
	global sc_width, sc_height, deltaTime, playing, first_move
	global player_1, player_2, ball

	player_1.update()
	player_2.update()
	if first_move:
		ball.update()
		ball.collide_with(player_1)
		ball.collide_with(player_2)
	else:
		ball.update(movement = False)

	if ball.collide_with_the_wall != 0:
		playing = False

	sc_width, sc_height = pygame.display.get_surface().get_size()
	pygame.display.flip()
	deltaTime = clock.tick(60) / 1000

def fixW(number):
	global sc_width
	real_number = (number / FIXED_WIDTH) * sc_width
	return real_number

def fixH(number):
	global sc_height
	real_number = (number / FIXED_HEIGHT) * sc_height
	return real_number

class Base():
	def __init__(self):
		#Rect(left, top, width, height)
		self.width = 25
		self.height = 150
		self.left = 50
		self.top = (FIXED_HEIGHT - self.height) / 2
		self.rect = pygame.Rect(fixW(self.left),fixH(self.top), fixW(self.width), fixH(self.height))
		self.bottom = self.top + self.height
		self.right = self.left + self.width

		self.velocity = 500
		self.color = "white"

	def update(self):
		self.rect = pygame.Rect(fixW(self.left),fixH(self.top), fixW(self.width), fixH(self.height))
		self.bottom = self.top + self.height
		self.right = self.left + self.width
		self.centerW = (self.left + self.right) / 2
		self.centerH = (self.top + self.bottom) / 2
class Player(Base):
	pass
class Ball(Base):
	def __init__(self):
		Base.__init__(self)
		self.width = 40
		self.height = 40
		self.left = (FIXED_WIDTH - self.width) / 2
		self.top = (FIXED_HEIGHT - self.height) / 2
		self.rect = pygame.Rect(fixW(self.left),fixH(self.top), fixW(self.width), fixH(self.height))

		self.velocity = 500

		self.facing_x = random.choice([-1,1])
		self.facing_y = random.choice([-1,1])

		self.collide_with_the_wall = 0

	def update(self, movement = True):
		global deltaTime, sc_width, sc_height
		Base.update(self)
		if movement:
			self.left += self.velocity * self.facing_x * deltaTime
			self.top += self.velocity * self.facing_y * deltaTime
			if self.left <= 0 and self.facing_x == -1:
				self.facing_x = 1
				self.collide_with_the_wall = -1
			elif self.right >= FIXED_WIDTH and self.facing_x == 1:
				self.facing_x = -1
				self.collide_with_the_wall = 1
			if self.top <= 0 and self.facing_y == -1:
				self.facing_y = 1
			elif self.bottom >= FIXED_HEIGHT and self.facing_y == 1:
				self.facing_y = -1
	def collide_with(self, other):

		col_left = other.right > self.left and other.left < self.left
		col_right = other.left < self.right and other.right > self.right
		col_top = other.top < self.top and other.bottom > self.top
		col_bottom =  other.bottom > self.bottom and other.top < self.bottom
		col_centerW = other.right > self.centerW and other.left < self.centerW

		# Collide from the left
		if self.facing_x == -1 and col_left and col_top:
			self.facing_x = 1
		if self.facing_x == -1 and col_left and col_bottom:
			self.facing_x = 1

		# Collide from the right
		if self.facing_x == 1 and col_right and col_top:
			self.facing_x = -1
		if self.facing_x == 1 and col_right and col_bottom:
			self.facing_x = -1

		# Collide from the bottom
		if self.facing_y == 1 and col_bottom and col_centerW:
			self.facing_y = -1
			#doesnt make sense, but to the game
			self.facing_x *= -1

		# Collide from the top
		if self.facing_y == -1 and col_top and col_centerW:
			self.facing_y = 1
			#doesnt make sense, but to the game
			self.facing_x *= -1

clock = pygame.time.Clock()
ball = Ball()
player_1 = Player()
player_2 = Player()
player_1.left = 50
player_2.left = FIXED_WIDTH - player_2.width - 50
first_move = False
sc_width = 1366
sc_height = 720

def restart_everything():
	global playing, ball, player_1, player_2, deltaTime, first_move
	playing = True
	ball.__init__()
	player_1.__init__()
	player_2.__init__()
	player_1.left = 50
	player_2.left = FIXED_WIDTH - player_2.width - 50
	deltaTime = 0
	first_move = False

if __name__ == "__main__":
	main()


#coding:utf-8
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select
from selenium.webdriver.chrome.options import Options
import time
import sys

class WhatsApp :
    "La class du bot whatsapp "
    def connect(self) :

        self.driver = webdriver.Firefox()
        self.driver.get("https://web.whatsapp.com/")
        input('Taper Entrer après le scannage du code Qr')

        self.driver.implicitly_wait(10)
        time.sleep(10.1)
        try :
            self.attente = WebDriverWait(driver, 10)
            self.element = self.attente.until(EC.element_to_be_clickable((By.XPATH,'/html/body/div[1]/div/div/div[3]/div/div[1]/div/button')))
        except :
            time.sleep(10.10)
            print('chargement complet de la page en cours ')
        print("Vous êtes connecté !")

    def SendMessage(self) :

        def Contact() :#SearchContact.send_keys(contact)
            XpathS = "//input[@title='Rechercher ou démarrer une nouvelle discussion']"
            contact = input('Enter le nom de votre contact __-> ')
            print("Tentatif de recherche de votre contact")
            self.driver.implicitly_wait(5)
            SearchContact = self.driver.find_element_by_xpath(XpathS)
            SearchContact.click()
            self.driver.implicitly_wait(30)
            print('typologie')
            SearchContact.send_keys(Keys.CONTROL + contact)
        Contact()

    def personal_Message(self) :

            driver = self.driver

            Message = "c'est gratuit chef  !"
            try:
                attente = WebDriverWait(driver, 125)
                self.element = attente.until(EC.element_to_be_clickable((By.XPATH, '//span[@title = "{}"]'.format('Rostant'))))
            except Exception :
                print("Domm1 !")

            dest = driver.find_element_by_xpath('//span[@title = "{}"]'.format('Rostant')) #Le Destinatair du Message
            dest.click()
            Message_input = driver.find_element_by_xpath('/html/body/div[1]/div/div/div[4]/div/footer/div[1]/div[2]/div/div[2]')
            Message_input.send_keys(Message)
            try:
                elem = WebDriverWait(driver, 125)
                self.poil = attente.until(EC.element_to_be_clickable((By.XPATH, '/html/body/div[1]/div/div/div[4]/div/footer/div[1]/div[3]/button')))
            except Exception :
                print("Domm2 !")
            driver.find_element_by_xpath('/html/body/div[1]/div/div/div[4]/div/footer/div[1]/div[3]/button').click()

    def disconnect(self) :

        driver = self.driver
        driver.close()

bot = WhatsApp()
bot.SendMessage()

"""contact = input("Enter le nom ")
text = input("Entrer le message !")
driver = webdriver.Firefox()
driver.get("https://web.whatsapp.com")
print("Scan QR Code, And then Enter")
input()
print("Logged In")
inp_xpath_search = "//input[@title='Rechercher ou démarrer une nouvelle discussion']"
input_box_search = WebDriverWait(driver,50).until(lambda driver: driver.find_element_by_xpath(inp_xpath_search))
input_box_search.click()
time.sleep(2)
input_box_search.send_keys(contact)
time.sleep(2)
selected_contact = driver.find_element_by_xpath("//span[@title='"+contact+"']")
selected_contact.click()
inp_xpath = '//div[@class="_2S1VP copyable-text selectable-text"][@contenteditable="true"][@data-tab="1"]'
input_box = driver.find_element_by_xpath(inp_xpath)
time.sleep(2)
input_box.send_keys(text + Keys.ENTER)
time.sleep(2)"""


import argparse
from pysdncrypt.encdec import CryptText
parser = argparse.ArgumentParser()
parser.add_argument("inputfile", help="File to be encrypted")
parser.add_argument("outputfile", help='Encrypted output file')

args = parser.parse_args()

def main():

    CryptText.minifyjson(args.inputfile, args.outputfile)
    # # Open inputfile
    # json_input = open(args.inputfile, 'r').read()
    #
    # # Minify
    # json_mini = json.dumps(json.loads(json_input), separators=(',', ':'))
    #
    # # save the encrypted inputed file
    # open(args.outputfile, 'w').write(json_mini)
    #
    print("Successful minification, {} minified to {}".format(args.inputfile, args.outputfile))

if __name__ == "__main__":
    main()


import sys
import re
import nltk

def tokenize_text(text, x):
    '''
    Convert  string `text` into a list of word tokens.
    Removes stopwords and words that are too short.

    Parameters
    ----------
    text : str
        The text string to be tokenized.
    x : interger or variable
        the minimum length of words to be included
    Returns
    -------
    tokens : list of strings[InternetShortcut]
URL=https://github.com/S2DSLondon/Aug20_Ditchley/stargazers

        `text` broken up into tokens.
    '''
    tokens = []

    for sent in nltk.sent_tokenize(text):
        for word in nltk.word_tokenize(sent):
            if (len(word) >= x):
                tokens.append(word)

    return tokens


'''
Deploy a classifier as a service for front-end consumption.

The application contains 3 parts:
    1. Show a random Iris flower
    2. Allow users to challenge tha AI
    3. Show the current scoreboard

The plan is to write a data pipeline that:
    - Gathers and validates Iris flower data
    - Trains and validates a model capable of classifying Iris flowers
    - Deploys that model as a service for front-end consumption
'''
# from utils_app import show_random_iris
import random
import numpy as np
import pandas as pd
import tensorflow as tf
import os
import streamlit as st
import json

column_names = ['sepal_length',
                'sepal_width',
                'petal_length',
                'petal_width',
                'species']

test_url = "https://storage.googleapis.com/download.tensorflow.org/data/iris_test.csv"

test_fp = tf.keras.utils.get_file(
    fname=os.path.basename(test_url),
    origin=test_url
    )

# load data 'flower_app/data/iris_test.csv'
iris_test_data = pd.read_csv(test_fp,
                             skiprows=[0],
                             names=column_names
                             )
# encode 0, 1, 2 label classes to string names
# np.where(condition, [x, y])
iris_test_data['species'] = np.where(iris_test_data['species'] == 0,
                                     'setosa',
                                     np.where(iris_test_data['species'] == 1,
                                              'versicolor',
                                              'virginica'
                                              ))

st.write(iris_test_data.head())

def show_random_iris():
    choices = ['setosa', 'versicolor', 'virginica']
    random_flower = random.choice(choices)
    flowers = {
        'setosa': 'https://upload.wikimedia.org/wikipedia/commons/thumb/5/56/Kosaciec_szczecinkowaty_Iris_setosa.jpg/440px-Kosaciec_szczecinkowaty_Iris_setosa.jpg',
        'versicolor': 'https://upload.wikimedia.org/wikipedia/commons/thumb/4/41/Iris_versicolor_3.jpg/440px-Iris_versicolor_3.jpg',
        'virginica': 'https://upload.wikimedia.org/wikipedia/commons/thumb/9/9f/Iris_virginica.jpg/440px-Iris_virginica.jpg'
    }
    random_url = flowers[random_flower]
    single_type = iris_test_data[(iris_test_data['species'] == random_flower)]
    random_row = single_type.sample(n=1).drop(['species'], axis=1)
    final = random_flower, random_url, random_row
    return(final)

# GET /get_random_flower
random_result = show_random_iris()
res = {
    "name": random_result[0],
    "url": random_result[1],
    "data": {"keys" : list(random_result[2]), "values": random_result[2].iloc[0].tolist()}
}

# st.write(json.dumps(res))
print(json.dumps(res))


import torch
import torch as T
import numpy as np
import torch.nn as nn
from matplotlib import pyplot as plt
from torch.autograd import Variable
from tqdm import tqdm
import os
import seaborn as sns
from src.data_helper import *
import pandas as pd
from src.evaluator import Evaluator
from torch.optim.lr_scheduler import ReduceLROnPlateau
from src.utils import *

class Trainer:
    def __init__(self, model, device=T.device("cpu"),
                 save_dir="./trained_models",
                 save_name="model.pt",
                 verbose=True):
        self.model = model
        self.device = device
        self.save_dir = save_dir
        self.save_name = save_name
        self.verbose = verbose
        self.device = device
        self.evaluator = Evaluator()

        self.train_losses = []
        self.valid_losses = []
        self.test_losses = []
        self.train_acc = []
        self.valid_acc = []
        self.test_acc = []
        self.valid_loss_min = np.Inf
        self.valid_acc_max = -np.Inf

        self.set_model_path(save_dir, save_name)

    def set_model_path(self, save_dir, save_name):
        if not os.path.exists(save_dir):
            os.makedirs(save_dir)

        self.model_path = os.path.join(save_dir, save_name)

    def set_data(self, train_x, train_y, val_x, val_y):
        self.train_x = train_x
        self.train_y = train_y
        self.val_x = val_x
        self.val_y = val_y

    def split_batch(self, x, y, batch_size, shuffle=True):
        batches = []
        if shuffle:
            indices = np.random.permutation(len(x))
            dataset = [x[i] for i in indices]
            labelset = [y[i] for i in indices]
        else:
            dataset = x
            labelset = y
        for i in range(0, len(dataset), batch_size):
            batches.append((np.array(dataset[i:i + batch_size]), np.array(labelset[i:i + batch_size])))

        return batches

    def eval(self, val_loader):
        val_losses = []
        val_accuracies = []

        self.model.eval()
        for inp, lab in val_loader:
            inp = Variable(torch.FloatTensor(inp), requires_grad=False)
            lab = Variable(torch.FloatTensor(lab), requires_grad=False)
            inp, lab = inp.to(self.device).detach(), lab.to(
                self.device).detach()
            out = self.model(inp).detach()
            loss = self.model.loss(out.detach(), lab.detach())
            out = out.cpu().numpy()
            out = [x[::-1][:int(sum(lab[i]).item())]
                   for i, x in enumerate(out.argsort())]
            for i, x in enumerate(out):
                out_vec = np.zeros(self.model.num_classes)
                out_vec[x] = 1
                out[i] = out_vec

            acc = self.evaluator.multilabel_accuracy(out, lab.cpu().numpy())
            val_losses.append(loss.item())
            val_accuracies.append(acc)

        self.model.train()

        return np.mean(val_losses), np.mean(val_accuracies)

    def test(self, audio_file_path=None,
             label_file_path=None, data_config=None,
            k=1, plot=False):
        self.model.to(self.device)
        self.model.eval()
        #Going through each data_filename within a label
        audio, sr = librosa.load(audio_file_path, sr=data_config['sr'])
        data = audio_to_tensor(audio, data_config)

        inp = Variable(torch.FloatTensor(np.array([data])).to(self.device), requires_grad=False)
        prob_out = self.model(inp).detach().cpu().numpy()[0]
        labels = np.array(['E' + ('0' if (i % 44)+1 < 10 else '') + str(i + 1) for i in range(44)]
                          + ['J' + ('0' if (i % 44)+1 < 10 else '') + str(i + 1) for i in range(44)])
        if plot:
            df = pd.DataFrame({'probability': prob_out, 'labels': labels})
            ax = sns.barplot(x='labels', y='probability',
                             data=df, errwidth=0)
            plt.xticks(color='w')
            plt.savefig('images/' +
                        os.path.basename(audio_file_path).replace('wav', 'png'))
            plt.show()
        ans = labels[np.argsort(prob_out)[::-1]][:k]
        strans = '[' + ", ".join(['"%s"' % x for x in ans]) + ']'
        print(strans)

    def train(self, optimizer='adam', training_params=None):
        # utility for running the training process
        self.model.to(self.device)
        epochs = training_params['epochs']
        batch_size = training_params['batch_size']
        self.model.set_loss_function(training_params['loss_function'])
        self.model.set_optimizer(
            optimizer, lr=training_params['learning_rate'])
        # print options
        counter = 0
        print_every = 5
        clip = 5

        self.model.train()
        # scheduler = ReduceLROnPlateau(self.model.optimizer, factor=0.5, patience=0, verbose=True)

        val_loader = self.split_batch(
            self.val_x, self.val_y, batch_size=batch_size, shuffle=False)

        for i in range(epochs):
            train_loader = self.split_batch(
                self.train_x, self.train_y, batch_size=batch_size, shuffle=True)
            t = tqdm(train_loader)
            tot_loss = 0
            counter = 0
            for inputs, labels in t:
                counter += 1
                inputs = Variable(torch.FloatTensor(
                    inputs), requires_grad=True)
                labels = Variable(torch.FloatTensor(labels))
                inputs, labels = inputs.to(self.device), labels.to(self.device)

                self.model.zero_grad()
                output = self.model(inputs)
                loss = self.model.loss(output, labels.detach())
                loss.backward()
                nn.utils.clip_grad_norm_(self.model.parameters(), clip)
                self.model.step()
                tot_loss += loss.item()
                if counter % print_every == 0:
                    t.set_postfix(loss=tot_loss / counter)

            val_loss, val_acc = self.eval(val_loader)

            if self.verbose:
                print('Epoch/batch: {}/{}'.format(i + 1, epochs))
                print('Val accuracy:', val_acc)
                print('Val loss:', val_loss)
            if val_loss <= self.valid_loss_min:
                print('Validation loss decreased ({:.6f} --> {:.6f}).  Saving model ...'.
                      format(self.valid_loss_min, val_loss))
                torch.save(self.model.state_dict(), self.model_path)
                self.valid_loss_min = val_loss
            elif val_acc > self.valid_acc_max:
                self.valid_acc_max = val_acc
                torch.save(self.model.state_dict(), self.model_path)
            # scheduler.step(val_loss)

        return (self.model)

    def load_model_from_path(self, path, device=None):
        if device is None:
            device = "cuda" if torch.cuda.is_available() else "cpu"
        self.model.load_state_dict(T.load(path, map_location=device))
        print("Model loaded sucessful!")

    def save_train_losses(self):
        plt.plot(self.train_losses)
        out_dir = 'output/train_losses'
        if not os.path.exists(out_dir):
            os.makedirs(out_dir)
        plt.savefig("{}/{}_{}".format(out_dir,
                    self.model.name, 'train_losses.png'))