#!/usr/bin/env python
# -*- coding: utf-8 -*-

import itertools
import os
import random
import string

# Built-in imports
import unittest

# 3rd party imports
import numpy as np
import xarray as xr
from ddt import data, ddt, idata, unpack

from pyrfu import mms, pyrf
from pyrfu.mms.psd_moments import _moms

from . import (
    generate_data,
    generate_spectr,
    generate_timeline,
    generate_ts,
    generate_vdf,
)

__author__ = "Louis Richard"
__email__ = "louisr@irfu.se"
__copyright__ = "Copyright 2020-2023"
__license__ = "MIT"
__version__ = "2.4.2"
__status__ = "Prototype"

def generate_feeps(f_s, n_pts, data_rate, dtype, lev, mms_id):
    var = {"tmmode": data_rate, "dtype": dtype, "lev": lev}
    tint = ["2019-01-01T00:00:00.000000000", "2019-01-01T00:10:00.000000000"]
    eyes = mms.feeps_active_eyes(var, tint, mms_id)
    keys = [f"{k}-{eyes[k][i]}" for k in eyes for i in range(len(eyes[k]))]
    feeps_dict = {k: generate_spectr(f_s, n_pts, 16, f"energy-{k}") for k in keys}
    feeps_dict["spinsectnum"] = pyrf.ts_scalar(
        generate_timeline(f_s, n_pts), np.tile(np.arange(12), n_pts // 12 + 1)[:n_pts]
    )

    feeps_alle = xr.Dataset(feeps_dict)
    feeps_alle.attrs = {"mmsId": mms_id, **var}

    return feeps_alle

def generate_eis(f_s, n_pts, data_rate, dtype, lev, specie, data_unit, mms_id):
    pref = f"mms{mms_id:d}_epd_eis"
    pref = f"{pref}_{data_rate}_{lev}_{dtype}"

    if data_rate == "brst":
        pref = f"{pref}_{data_rate}_{dtype}"
    else:
        pref = f"{pref}_{dtype}"

    suf = f"{specie}_P1_{data_unit.lower()}_t"

    keys = [f"{pref}_{suf}{t:d}" for t in range(6)]

    spin_nums = pyrf.ts_scalar(
        generate_timeline(f_s, n_pts),
        np.sort(np.tile(np.arange(n_pts // 12 + 1), (12,)))[1 : n_pts + 1],
    )
    sectors = pyrf.ts_scalar(
        generate_timeline(f_s, n_pts),
        np.tile(np.arange(12), n_pts // 12 + 1)[1 : n_pts + 1],
    )

    if dtype.lower() == "extof":
        energies = np.array(
            [
                47.645324,
                54.928681,
                62.419454,
                70.833554,
                80.315371,
                91.00098,
                103.018894,
                116.554129,
                131.801143,
                148.970297,
                168.295534,
                190.060874,
                214.590996,
                242.245343,
                273.466432,
                308.768669,
                348.73539,
                394.035378,
                445.404668,
                503.597543,
                569.429005,
                643.764143,
                727.683404,
                822.660211,
                930.654627,
            ]
        )
    else:
        energies = np.array(
            [
                10.51516,
                11.509144,
                12.612351,
                13.817409,
                15.111664,
                16.55435,
                18.134081,
                19.857029,
                21.774935,
                23.807037,
                26.021971,
                28.526016,
                31.215776,
                34.228877,
                37.604494,
                41.116729,
                45.29041,
                51.412368,
                58.570702,
                65.951929,
                75.09237,
            ]
        )

    eis_dict = {"spin": spin_nums, "sector": sectors}

    for i, k in enumerate(keys):
        eis_dict[f"t{i:d}"] = generate_spectr(f_s, n_pts, len(energies), "energy")
        eis_dict[f"look_t{i:d}"] = generate_ts(f_s, n_pts, "vector")

    # glob_attrs = {**outdict["spin"].attrs["GLOBAL"], **var}
    glob_attrs = {
        "delta_energy_plus": 0.5 * np.ones(len(energies)),
        "delta_energy_minus": 0.5 * np.ones(len(energies)),
        "species": specie,
        "randattrs": "".join(random.choice(string.ascii_lowercase) for _ in range(10)),
    }

    # Build Dataset
    eis = xr.Dataset(eis_dict, attrs=glob_attrs)
    eis = eis.assign_coords(energy=energies)

    return eis

@ddt
class CalcEpsilonTestCase(unittest.TestCase):
    @data(
        (
            generate_vdf(64.0, 100, (32, 16, 16), energy01=False, species="bazinga"),
            generate_vdf(64.0, 100, (32, 16, 16), energy01=False, species="bazinga"),
            generate_ts(64.0, 100, "scalar"),
            generate_ts(64.0, 100, "scalar"),
        ),
        (
            generate_vdf(64.0, 100, (32, 16, 16), energy01=False, species="ions"),
            generate_vdf(64.0, 100, (32, 16, 16), energy01=False, species="ions"),
            generate_ts(32.0, 100, "scalar"),
            generate_ts(64.0, 100, "scalar"),
        ),
    )
    @unpack
    def test_calc_epsilon_input(self, vdf, model_vdf, n_s, sc_pot):
        with self.assertRaises(ValueError):
            mms.calculate_epsilon(vdf, model_vdf, n_s, sc_pot)

    @data(
        (
            generate_vdf(64.0, 100, (32, 16, 16), energy01=False, species="ions"),
            generate_vdf(64.0, 100, (32, 16, 16), energy01=False, species="ions"),
            {},
        ),
        (
            generate_vdf(64.0, 100, (32, 16, 16), energy01=False, species="electrons"),
            generate_vdf(64.0, 100, (32, 16, 16), energy01=False, species="electrons"),
            {},
        ),
        (
            generate_vdf(64.0, 100, (32, 16, 16), energy01=True, species="ions"),
            generate_vdf(64.0, 100, (32, 16, 16), energy01=True, species="ions"),
            {},
        ),
    )
    @unpack
    def test_calc_epsilon_output(self, vdf, model_vdf, kwargs):
        mms.calculate_epsilon(
            vdf,
            model_vdf,
            generate_ts(64.0, 100, "scalar"),
            generate_ts(64.0, 100, "scalar"),
            **kwargs,
        )

class DbInitTestCase(unittest.TestCase):
    def test_db_init_inpput(self):
        with self.assertRaises(AssertionError):
            mms.db_init("bazinga!")

    def test_db_init_output(self):
        self.assertIsNone(mms.db_init(os.getcwd()))

@ddt
class Def2PsdTestCase(unittest.TestCase):
    @data(
        ("I AM GROOT!!", "s^3/cm^6"),
        ("ions", "bazinga"),
    )
    @unpack
    def test_def2psd_input(self, species, units):
        with self.assertRaises(ValueError):
            mms.def2psd(generate_vdf(64.0, 100, (32, 32, 16), False, species, units))

    @idata(
        itertools.product(
            [
                "ions",
                "ion",
                "protons",
                "proton",
                "alphas",
                "alpha",
                "helium",
                "electrons",
                "e",
            ],
            ["keV/(cm^2 s sr keV)", "eV/(cm^2 s sr eV)", "1/(cm^2 s sr)"],
        )
    )
    @unpack
    def test_def2psd_output(self, species, units):
        vdf = generate_vdf(64.0, 100, (32, 32, 16), False, species, units)
        result = mms.def2psd(vdf)
        self.assertIsInstance(result, xr.Dataset)

        spectr = generate_spectr(64.0, 100, 32, {"species": species, "UNITS": units})
        result = mms.def2psd(spectr)
        self.assertIsInstance(result, xr.DataArray)

@ddt
class Dpf2PsdTestCase(unittest.TestCase):
    @data(
        ("I AM GROOT!!", "s^3/cm^6"),
        ("ions", "bazinga"),
    )
    @unpack
    def test_dpf2psd_input(self, species, units):
        with self.assertRaises(ValueError):
            mms.dpf2psd(generate_vdf(64.0, 100, (32, 32, 16), False, species, units))

    @idata(
        itertools.product(
            [
                "ions",
                "ion",
                "protons",
                "proton",
                "alphas",
                "alpha",
                "helium",
                "electrons",
                "e",
            ],
            ["1/(cm^2 s sr keV)", "1/(cm^2 s sr eV)"],
        )
    )
    @unpack
    def test_dpf2psd_output(self, species, units):
        vdf = generate_vdf(64.0, 100, (32, 32, 16), False, species, units)
        result = mms.dpf2psd(vdf)
        self.assertIsInstance(result, xr.Dataset)

        spectr = generate_spectr(64.0, 100, 32, {"species": species, "UNITS": units})
        result = mms.dpf2psd(spectr)
        self.assertIsInstance(result, xr.DataArray)

@ddt
class Dsl2GseTestCase(unittest.TestCase):
    def test_dsl2gse_input(self):
        with self.assertRaises(TypeError):
            mms.dsl2gse(generate_ts(64.0, 42, "vector"), np.random.random((42, 3)), 1)

    @data(
        xr.Dataset({"z_dec": generate_ts(64.0, 42), "z_ra": generate_ts(64.0, 42)}),
        np.random.random(3),
    )
    def test_dsl2gse_output(self, value):
        result = mms.dsl2gse(generate_ts(64.0, 42, "vector"), value, 1)
        self.assertIsInstance(result, xr.DataArray)
        result = mms.dsl2gse(generate_ts(64.0, 42, "vector"), value, -1)
        self.assertIsInstance(result, xr.DataArray)

@ddt
class Dsl2GsmTestCase(unittest.TestCase):
    def test_dsl2gsm_input(self):
        with self.assertRaises(TypeError):
            mms.dsl2gsm(generate_ts(64.0, 42, "vector"), np.random.random((42, 3)), 1)

    @data(
        xr.Dataset({"z_dec": generate_ts(64.0, 42), "z_ra": generate_ts(64.0, 42)}),
        np.random.random(3),
    )
    def test_dsl2gsm_output(self, value):
        result = mms.dsl2gsm(generate_ts(64.0, 42, "vector"), value, 1)
        self.assertIsInstance(result, xr.DataArray)
        result = mms.dsl2gsm(generate_ts(64.0, 42, "vector"), value, -1)
        self.assertIsInstance(result, xr.DataArray)

@ddt
class EisCombineProtonPadTestCase(unittest.TestCase):
    @idata(
        itertools.product(
            ["srvy", "brst"],
            ["proton", "alpha", "oxygen"],
            ["flux", "cps", "counts"],
        )
    )
    @unpack
    def test_eis_combine_proton_pad_output(self, tmmode, specie, unit):
        mms_id = random.randint(1, 5)
        phxtof_allt = generate_eis(
            64.0, 100, tmmode, "phxtof", "l2", specie, unit, mms_id
        )
        extof_allt = generate_eis(
            64.0, 100, tmmode, "extof", "l2", specie, unit, mms_id
        )
        result = mms.eis_combine_proton_pad(phxtof_allt, extof_allt)
        self.assertIsInstance(result, xr.DataArray)

    @idata(itertools.product([99, 100], repeat=2))
    @unpack
    def test_eis_combine_proton_pad_input(self, n_phxtof, n_extof):
        phxtof_allt = generate_eis(
            64.0, n_phxtof, "brst", "phxtof", "l2", "proton", "flux", 1
        )
        extof_allt = generate_eis(
            64.0, n_extof, "brst", "extof", "l2", "proton", "flux", 1
        )
        result = mms.eis_combine_proton_pad(phxtof_allt, extof_allt)
        self.assertIsInstance(result, xr.DataArray)

    @data(None, [1, 0, 0], generate_ts(64.0, 10, "vector"))
    def test_eis_combine_proton_pad_vec(self, vec):
        phxtof_allt = generate_eis(
            64.0, 100, "brst", "phxtof", "l2", "proton", "flux", 1
        )
        extof_allt = generate_eis(64.0, 100, "brst", "extof", "l2", "proton", "flux", 1)
        result = mms.eis_combine_proton_pad(phxtof_allt, extof_allt, vec)
        self.assertIsInstance(result, xr.DataArray)

    def test_eis_combine_proton_pad_options(self):
        phxtof_allt = generate_eis(
            64.0, 100, "brst", "phxtof", "l2", "proton", "flux", 1
        )
        extof_allt = generate_eis(64.0, 100, "brst", "extof", "l2", "proton", "flux", 1)
        result = mms.eis_combine_proton_pad(phxtof_allt, extof_allt, None, despin=True)
        self.assertIsInstance(result, xr.DataArray)

@ddt
class EisCombineProtonSpecTestCase(unittest.TestCase):
    @idata(
        itertools.product(
            ["srvy", "brst"],
            ["proton", "alpha", "oxygen"],
            ["flux", "cps", "counts"],
        )
    )
    @unpack
    def test_eis_combine_proton_spec_output(self, tmmode, specie, unit):
        mms_id = random.randint(1, 5)
        phxtof_allt = generate_eis(
            64.0, 100, tmmode, "phxtof", "l2", specie, unit, mms_id
        )
        extof_allt = generate_eis(
            64.0, 100, tmmode, "extof", "l2", specie, unit, mms_id
        )
        result = mms.eis_combine_proton_spec(phxtof_allt, extof_allt)
        self.assertIsInstance(result, xr.Dataset)

    @idata(itertools.product([99, 100], repeat=2))
    @unpack
    def test_eis_combine_proton_spec_ctimes(self, n_phxtof, n_extof):
        phxtof_allt = generate_eis(
            64.0, n_phxtof, "brst", "phxtof", "l2", "proton", "flux", 1
        )
        extof_allt = generate_eis(
            64.0, n_extof, "brst", "extof", "l2", "proton", "flux", 1
        )
        result = mms.eis_combine_proton_spec(phxtof_allt, extof_allt)
        self.assertIsInstance(result, xr.Dataset)

@ddt
class EisOmniTestCase(unittest.TestCase):
    @idata(
        itertools.product(
            ["srvy", "brst"],
            ["extof", "phxtof"],
            ["proton", "alpha", "oxygen"],
            ["flux", "cps", "counts"],
        )
    )
    @unpack
    def test_eis_omni_output(self, tmmode, dtype, specie, unit):
        eis = generate_eis(
            64.0, 100, tmmode, dtype, "l2", specie, unit, random.randint(1, 4)
        )
        result = mms.eis_omni(eis, "mean")
        self.assertIsInstance(result, xr.DataArray)

@ddt
class EisPadTestCase(unittest.TestCase):
    @data(None, [1, 0, 0], generate_ts(64.0, 10, "vector"))
    def test_eis_pad_output(self, vec):
        eis = generate_eis(64.0, 100, "brst", "extof", "l2", "proton", "flux", 1)
        result = mms.eis_pad(eis, vec)
        self.assertIsInstance(result, xr.DataArray)

@ddt
class EisPadSpinAvgTestCase(unittest.TestCase):
    @idata(
        itertools.product(
            ["srvy", "brst"],
            ["extof", "phxtof"],
            ["proton", "alpha", "oxygen"],
            ["flux", "cps", "counts"],
        )
    )
    @unpack
    def test_eis_pad_spin_avg_output(self, tmmode, dtype, specie, unit):
        eis = generate_eis(
            64.0, 100, tmmode, dtype, "l2", specie, unit, random.randint(1, 4)
        )
        eis_pad = mms.eis_pad(eis)
        result = mms.eis_pad_spinavg(eis_pad, eis.spin)
        self.assertIsInstance(result, xr.DataArray)

@ddt
class EisProtonCorrectionTestCase(unittest.TestCase):
    def test_eis_proton_correction_dataarray(self):
        flux_eis = generate_spectr(64, 100, 16, "energy")
        result = mms.eis_proton_correction(flux_eis)
        self.assertIsInstance(result, xr.DataArray)

    @idata(
        itertools.product(
            ["srvy", "brst"],
            ["extof", "phxtof"],
            ["proton", "alpha", "oxygen"],
            ["flux", "cps", "counts"],
        )
    )
    @unpack
    def test_eis_proton_correction_dataset(self, tmmode, dtype, specie, unit):
        flux_eis = generate_eis(
            64.0, 100, tmmode, dtype, "l2", specie, unit, random.randint(1, 4)
        )
        result = mms.eis_proton_correction(flux_eis)
        self.assertIsInstance(result, xr.Dataset)

@ddt
class EisSpinAvgTestCase(unittest.TestCase):
    @idata(
        itertools.product(
            ["mean", "sum"],
            ["srvy", "brst"],
            ["extof", "phxtof"],
            ["proton", "alpha", "oxygen"],
            ["flux", "cps", "counts"],
        )
    )
    @unpack
    def test_eis_spin_avg_output(self, method, tmmode, dtype, specie, unit):
        eis_allt = generate_eis(
            64.0, 100, tmmode, dtype, "l2", specie, unit, random.randint(1, 4)
        )
        result = mms.eis_spin_avg(eis_allt, method)
        self.assertIsInstance(result, xr.Dataset)

@ddt
class MakeModelVDFTestCase(unittest.TestCase):
    @data(
        (generate_vdf(64.0, 100, (32, 16, 16), species="ions"), False),
        (generate_vdf(64.0, 100, (32, 16, 16), species="electrons"), False),
        (generate_vdf(64.0, 100, (32, 16, 16), species="ions"), True),
    )
    @unpack
    def test_make_Model_vdf_output(self, vdf, isotropic):
        result = mms.make_model_vdf(
            vdf,
            generate_ts(64.0, 100, "vector"),
            generate_ts(64.0, 100, "scalar"),
            generate_ts(64.0, 100, "scalar"),
            generate_ts(64.0, 100, "vector"),
            generate_ts(64.0, 100, "tensor"),
            isotropic,
        )
        self.assertIsInstance(result, xr.Dataset)

class HpcaEnergiesTestCase(unittest.TestCase):
    def test_hpca_energies_output(self):
        result = mms.hpca_energies()
        self.assertIsInstance(result, list)

@ddt
class MakeModelKappaTestCase(unittest.TestCase):
    @data(
        (generate_vdf(64.0, 100, (32, 16, 16), species="bazinga"), random.random()),
    )
    @unpack
    def test_make_model_kappa_input(self, vdf, kappa):
        with self.assertRaises(ValueError):
            mms.make_model_kappa(
                vdf,
                generate_ts(64.0, 100, "scalar"),
                generate_ts(64.0, 100, "vector"),
                generate_ts(64.0, 100, "scalar"),
                kappa,
            )

    @data(
        (generate_vdf(64.0, 100, (32, 16, 16), species="ions"), random.random()),
        (generate_vdf(64.0, 100, (32, 16, 16), species="electrons"), random.random()),
        (generate_vdf(64.0, 100, (32, 16, 16), species="ions"), random.random()),
    )
    @unpack
    def test_make_model_kappa_output(self, vdf, kappa):
        result = mms.make_model_kappa(
            vdf,
            generate_ts(64.0, 100, "scalar"),
            generate_ts(64.0, 100, "vector"),
            generate_ts(64.0, 100, "scalar"),
            kappa,
        )

        self.assertIsInstance(result, xr.Dataset)

@ddt
class Psd2DefTestCase(unittest.TestCase):
    @data(
        ("I AM GROOT!!", "s^3/cm^6"),
        ("ions", "bazinga"),
    )
    @unpack
    def test_psd2def_input(self, species, units):
        with self.assertRaises(ValueError):
            mms.psd2def(generate_vdf(64.0, 100, (32, 32, 16), False, species, units))

    @idata(
        itertools.product(
            [
                "ions",
                "ion",
                "protons",
                "proton",
                "alphas",
                "alpha",
                "helium",
                "electrons",
                "e",
            ],
            ["s^3/cm^6", "s^3/m^6", "s^3/km^6"],
        )
    )
    @unpack
    def test_psd2def_output(self, species, units):
        vdf = generate_vdf(64.0, 100, (32, 32, 16), False, species, units)
        result = mms.psd2def(vdf)
        self.assertIsInstance(result, xr.Dataset)

        spectr = generate_spectr(64.0, 100, 32, {"species": species, "UNITS": units})
        result = mms.psd2def(spectr)
        self.assertIsInstance(result, xr.DataArray)

@ddt
class Psd2DpfTestCase(unittest.TestCase):
    @data(
        ("I AM GROOT!!", "s^3/cm^6"),
        ("ions", "bazinga"),
    )
    @unpack
    def test_psd2dpf_input(self, species, units):
        with self.assertRaises(ValueError):
            mms.psd2dpf(generate_vdf(64.0, 100, (32, 32, 16), False, species, units))

    @idata(
        itertools.product(
            [
                "ions",
                "ion",
                "protons",
                "proton",
                "alphas",
                "alpha",
                "helium",
                "electrons",
                "e",
            ],
            ["s^3/cm^6", "s^3/m^6", "s^3/km^6"],
        )
    )
    @unpack
    def test_psd2dpf_output(self, species, units):
        vdf = generate_vdf(64.0, 100, (32, 32, 16), False, species, units)
        result = mms.psd2dpf(vdf)
        self.assertIsInstance(result, xr.Dataset)

        spectr = generate_spectr(64.0, 100, 32, {"species": species, "UNITS": units})
        result = mms.psd2dpf(spectr)
        self.assertIsInstance(result, xr.DataArray)

@ddt
class PsdMomentsTestCase(unittest.TestCase):
    @data(
        (generate_vdf(64.0, 100, (32, 32, 16)), "brst"),
        (generate_vdf(64.0, 100, (32, 32, 16), species="electrons"), "brst"),
        (generate_vdf(64.0, 100, (32, 32, 16), energy01=False), "brst"),
        (generate_vdf(64.0, 100, (32, 32, 16), energy01=False), "fast"),
        (generate_vdf(64.0, 100, (32, 32, 16), energy01=True), "brst"),
    )
    @unpack
    def test_psd_moments_input(self, vdf, data_rate):
        delta_theta = 0.5 * np.ones(vdf.data.shape[3])
        vdf.attrs["delta_theta_minus"] = delta_theta
        vdf.attrs["delta_theta_plus"] = delta_theta

        delta_phi = 0.5 * np.ones((vdf.data.shape[0], vdf.data.shape[2]))
        vdf.attrs["delta_phi_minus"] = delta_phi
        vdf.attrs["delta_phi_plus"] = delta_phi
        vdf.data.attrs["FIELDNAM"] = f"MMS1 FPI/DIS {data_rate}SkyMap dist"
        mms.psd_moments(vdf, generate_ts(64.0, 100, "scalar"))

    @data({"energy_range": [1, 1000]}, {"no_sc_pot": True})
    def test_psd_moments_options(self, options):
        vdf = generate_vdf(64.0, 100, (32, 32, 16))
        vdf.data.attrs["FIELDNAM"] = "MMS1 FPI/DIS brstSkyMap dist"
        mms.psd_moments(vdf, generate_ts(64.0, 100, "scalar"), **options)

    @data(
        (
            np.random.random((100, 32)),  # energy
            np.random.random((10000, 32)),  # delta_v
            random.random(),  # q_e
            np.random.random(100),  # sc_pot
            random.random(),  # p_mass
            random.choice([True, False]),  # flag_inner_electron
            random.random(),  # w_inner_electron
            np.random.random((100, 32, 16)),  # phi
            np.random.random((100, 32, 16)),  # theta
            np.arange(32),  # int_energies
            np.random.random((100, 32, 32, 16)),  # vdf
            np.random.random((100, 32, 16)),  # delta_ang
        )
    )
    def test_moms(self, value):
        result = _moms.__wrapped__(*value)
        self.assertIsInstance(result[0], np.ndarray)
        self.assertIsInstance(result[1], np.ndarray)
        self.assertIsInstance(result[2], np.ndarray)
        self.assertIsInstance(result[3], np.ndarray)

@ddt
class PsdRebinTestCase(unittest.TestCase):
    @data(generate_vdf(64.0, 100, (32, 32, 16), energy01=True, species="ions"))
    def test_psd_rebin_output(self, vdf):
        result = mms.psd_rebin(
            vdf,
            vdf.phi,
            vdf.attrs["energy0"],
            vdf.attrs["energy1"],
            vdf.attrs["esteptable"],
        )
        self.assertIsInstance(result[0], np.ndarray)
        self.assertEqual(len(result[0]), 50)
        self.assertIsInstance(result[1], np.ndarray)
        self.assertListEqual(list(result[1].shape), [50, 64, 32, 16])
        self.assertIsInstance(result[2], np.ndarray)
        self.assertEqual(len(result[2]), 64)
        self.assertIsInstance(result[3], np.ndarray)
        self.assertListEqual(list(result[3].shape), [50, 32])

@ddt
class FeepsActiveEyesTestCase(unittest.TestCase):
    @idata(itertools.product(["srvy", "brst"], ["electron", "ion"], ["sitl", "l2"]))
    @unpack
    def test_feeps_active_eyes_output(self, data_rate, dtype, lev):
        tint = ["2019-01-01T00:00:00.000000000", "2019-01-01T00:10:00.000000000"]
        result = mms.feeps_active_eyes(
            {"tmmode": data_rate, "dtype": dtype, "lev": lev},
            tint,
            random.randint(1, 4),
        )
        self.assertIsInstance(result, dict)

        result = mms.feeps_active_eyes(
            {"tmmode": data_rate, "dtype": dtype, "lev": lev},
            pyrf.iso86012datetime64(np.array(tint)),
            str(random.randint(1, 4)),
        )
        self.assertIsInstance(result, dict)

@ddt
class FeepsCorrectEnergiesTestCase(unittest.TestCase):
    @idata(itertools.product(["srvy", "brst"], ["electron", "ion"]))
    @unpack
    def test_feeps_correct_energies_output(self, data_rate, dtype):
        # Generate fake FEEPS data
        feeps_alle = generate_feeps(
            64.0, 100, data_rate, dtype, "l2", random.randint(1, 4)
        )

        result = mms.feeps_correct_energies(feeps_alle)
        self.assertIsInstance(result, xr.Dataset)

@ddt
class FeepsFlatFieldCorrectionsTestCase(unittest.TestCase):
    @idata(itertools.product(["srvy", "brst"], ["electron", "ion"]))
    @unpack
    def test_feeps_flat_field_corrections_output(self, data_rate, dtype):
        # Generate fake FEEPS data
        feeps_alle = generate_feeps(
            64.0, 100, data_rate, dtype, "l2", random.randint(1, 4)
        )

        result = mms.feeps_flat_field_corrections(feeps_alle)
        self.assertIsInstance(result, xr.Dataset)

@ddt
class FeepsOmniTestCase(unittest.TestCase):
    @idata(itertools.product(["srvy", "brst"], ["electron", "ion"]))
    @unpack
    def test_feeps_omni_output(self, data_rate, dtype):
        # Generate fake FEEPS data
        feeps_alle = generate_feeps(
            64.0, 100, data_rate, dtype, "l2", random.randint(1, 4)
        )
        feeps_alle, _ = mms.feeps_split_integral_ch(feeps_alle)

        result = mms.feeps_omni(feeps_alle)
        self.assertIsInstance(result, xr.DataArray)

@ddt
class FeepsPadTestCase(unittest.TestCase):
    @idata(itertools.product(["srvy", "brst"], ["electron", "ion"]))
    @unpack
    def test_feeps_pad_ouput(self, data_rate, dtype):
        # Generate fake FEEPS data
        feeps_alle = generate_feeps(
            64.0, 100, data_rate, dtype, "l2", random.randint(1, 4)
        )
        feeps_alle, _ = mms.feeps_split_integral_ch(feeps_alle)

        result = mms.feeps_pad(feeps_alle, generate_ts(64.0, 100, "vector"))
        self.assertIsInstance(result, xr.DataArray)

@ddt
class FeepsPadSpinAvgTestCase(unittest.TestCase):
    @idata(itertools.product(["srvy", "brst"], ["electron", "ion"]))
    @unpack
    def test_feeps_pad_spin_avg(self, data_rate, dtype):
        # Generate fake FEEPS data
        feeps_alle = generate_feeps(
            64.0, 100, data_rate, dtype, "l2", random.randint(1, 4)
        )
        feeps_alle, _ = mms.feeps_split_integral_ch(feeps_alle)

        feeps_pad = mms.feeps_pad(feeps_alle, generate_ts(64.0, 100, "vector"))
        result = mms.feeps_pad_spinavg(feeps_pad, feeps_alle.spinsectnum)
        self.assertIsInstance(result, xr.DataArray)

@ddt
class FeepsPitchAnglesTestCase(unittest.TestCase):
    @idata(itertools.product(["srvy", "brst"], ["electron", "ion"]))
    @unpack
    def test_feeps_pitch_angles_output(self, data_rate, dtype):
        # Generate fake FEEPS data
        feeps_alle = generate_feeps(
            64.0, 100, data_rate, dtype, "l2", random.randint(1, 4)
        )
        feeps_alle, _ = mms.feeps_split_integral_ch(feeps_alle)

        result = mms.feeps_pitch_angles(feeps_alle, generate_ts(64.0, 100, "vector"))
        self.assertIsInstance(result[0], xr.DataArray)

@ddt
class FeepsRemoveBadDataTestCase(unittest.TestCase):
    @idata(itertools.product(["srvy", "brst"], ["electron", "ion"]))
    @unpack
    def test_feeps_remove_bad_data_output(self, data_rate, dtype):
        # Generate fake FEEPS data
        feeps_alle = generate_feeps(
            64.0, 100, data_rate, dtype, "l2", random.randint(1, 4)
        )

        result = mms.feeps_remove_bad_data(feeps_alle)
        self.assertIsInstance(result, xr.Dataset)

@ddt
class FeepsRemoveSunTestCase(unittest.TestCase):
    @idata(itertools.product(["srvy", "brst"], ["electron", "ion"]))
    @unpack
    def test_feeps_remove_sun(self, data_rate, dtype):
        # Generate fake FEEPS data
        feeps_alle = generate_feeps(
            64.0, 100, data_rate, dtype, "l2", random.randint(1, 4)
        )
        feeps_alle, _ = mms.feeps_split_integral_ch(feeps_alle)

        result = mms.feeps_remove_sun(feeps_alle)
        self.assertIsInstance(result, xr.Dataset)

@ddt
class FeepsSpinAvgTestCase(unittest.TestCase):
    @idata(itertools.product(["srvy", "brst"], ["electron", "ion"]))
    @unpack
    def test_feeps_spin_avg(self, data_rate, dtype):
        # Generate fake FEEPS data
        feeps_alle = generate_feeps(
            64.0, 100, data_rate, dtype, "l2", random.randint(1, 4)
        )
        feeps_alle, _ = mms.feeps_split_integral_ch(feeps_alle)
        feeps_omni = mms.feeps_omni(feeps_alle)

        result = mms.feeps_spin_avg(feeps_omni, feeps_alle.spinsectnum)
        self.assertIsInstance(result, xr.DataArray)

@ddt
class FeepsSplitIntegralChTestCase(unittest.TestCase):
    @idata(itertools.product(["srvy", "brst"], ["electron", "ion"]))
    @unpack
    def test_feeps_split_integral_ch(self, data_rate, dtype):
        feeps_alle = generate_feeps(
            64.0, 100, data_rate, dtype, "l2", random.randint(1, 4)
        )
        mms.feeps_split_integral_ch(feeps_alle)

@ddt
class FkPowerSpectrum4scTestCase(unittest.TestCase):
    @data((None, None), (random.random(), None), (None, [0.1, 1]))
    @unpack
    def test_fk_power_spectrum_4sc(self, df, f_range):
        e_mms = [generate_ts(64.0, 100, "scalar") for _ in range(4)]
        r_mms = [generate_ts(64.0, 100, "vector") for _ in range(4)]
        b_mms = [generate_ts(64.0, 100, "vector") for _ in range(4)]
        tint = ["2019-01-01T00:00:00.000000000", "2019-01-01T00:10:00.000000000"]

        result = mms.fk_power_spectrum_4sc(
            e_mms, r_mms, b_mms, tint, df=df, f_range=f_range
        )
        self.assertIsInstance(result, xr.Dataset)

@ddt
class ReduceTestCase(unittest.TestCase):
    @data("s^3/cm^6", "s^3/m^6", "s^3/km^6")
    def test_reduce_units(self, value):
        vdf = generate_vdf(64.0, 42, [32, 32, 16], energy01=True, species="ions")
        vdf.data.attrs["UNITS"] = value
        result = mms.reduce(vdf, np.eye(3), "1d", "cart")
        self.assertIsInstance(result, xr.DataArray)

    @data(
        (False, "ions", np.eye(3), "1d", "cart"),
        (False, "electrons", np.eye(3), "1d", "cart"),
        (True, "ions", np.eye(3), "1d", "cart"),
        (True, "electrons", np.eye(3), "1d", "cart"),
        (False, "electrons", generate_ts(64.0, 42, "tensor"), "1d", "cart"),
        (False, "ions", np.eye(3), "1d", "pol"),
        # (False, "ions", np.eye(3), "2d", "pol"), mc_pol_2d NotImplementedError
    )
    @unpack
    def test_reduce_output(self, energy01, species, xyz, dim, base):
        vdf = generate_vdf(64.0, 42, [32, 32, 16], energy01, species)
        result = mms.reduce(vdf, xyz, dim, base)
        self.assertIsInstance(result, xr.DataArray)

    @data(
        ("1d", "cart", {"vg": np.linspace(-1, 1, 42)}),
        ("1d", "cart", {"lower_e_lim": generate_ts(64.0, 42)}),
        ("1d", "cart", {"vg_edges": np.linspace(-1.01, 1.01, 102)}),
    )
    @unpack
    def test_reduce_options(self, dim, base, options):
        vdf = generate_vdf(64.0, 42, [32, 32, 16], energy01=False, species="ions")
        xyz = np.eye(3)
        result = mms.reduce(vdf, xyz, dim, base, **options)
        self.assertIsInstance(result, xr.DataArray)

    @data(
        ("ions", "s^3/m^6", np.array([1, 0, 0]), "1d", "pol", {}),
        ("I AM GROOT", "s^3/m^6", np.eye(3), "1d", "pol", {}),
        ("ions", "bazinga", np.eye(3), "1d", "pol", {}),
        ("ions", "s^3/m^6", np.eye(3), "2d", "pol", {}),
        ("ions", "s^3/m^6", np.eye(3), "1d", "pol", {"lower_e_lim": generate_data(42)}),
    )
    @unpack
    def test_reduce_input(self, species, units, xyz, dim, base, options):
        vdf = generate_vdf(64.0, 42, [32, 32, 16], energy01=True, species=species)
        vdf.data.attrs["UNITS"] = units
        with self.assertRaises((TypeError, ValueError, NotImplementedError)):
            mms.reduce(vdf, xyz, dim, base, **options)

@ddt
class RotateTensorTestCase(unittest.TestCase):
    @data(("rot", generate_data(100, "vector")), ("gse", None))
    @unpack
    def test_rotate_tensor_input(self, flag, vec):
        with self.assertRaises((TypeError, NotImplementedError)):
            mms.rotate_tensor(generate_ts(64.0, 100, "tensor"), flag, vec)

    @data(
        ("fac", generate_ts(64.0, 100, "vector"), "pp"),
        ("fac", generate_ts(64.0, 100, "vector"), "qq"),
        ("rot", np.random.random(3), "pp"),
        ("rot", np.random.random((3, 3)), "pp"),
    )
    @unpack
    def test_rotate_tensor_output(self, rot_flag, vec, perp):
        result = mms.rotate_tensor(
            generate_ts(64.0, 100, "tensor"), rot_flag, vec, perp
        )
        self.assertIsInstance(result, xr.DataArray)

@ddt
class SpectrToDatasetTestCase(unittest.TestCase):
    @data(generate_spectr(64.0, 100, 10))
    def test_spectr_to_dataset_output(self, spectr):
        result = mms.spectr_to_dataset(spectr)
        self.assertIsInstance(result, xr.Dataset)

@ddt
class Scpot2NeTestCase(unittest.TestCase):
    @data(None, generate_ts(64.0, 100, "scalar"))
    def test_scpot2ne_output(self, i_aspoc):
        result = mms.scpot2ne(
            generate_ts(64.0, 100, "scalar"),
            generate_ts(64.0, 100, "scalar"),
            generate_ts(64.0, 100, "tensor"),
            i_aspoc,
        )
        self.assertIsInstance(result[0], xr.DataArray)
        self.assertIsInstance(result[1], float)
        self.assertIsInstance(result[2], float)
        self.assertIsInstance(result[3], float)
        self.assertIsInstance(result[4], float)

@ddt
class TokenizeTestCase(unittest.TestCase):
    @idata(
        itertools.product(
            mms.tokenize.__globals__["PARAMS_SCALARS"],
            mms.tokenize.__globals__["INSTRUMENTS"],
            mms.tokenize.__globals__["SAMPLING_RATES"],
            mms.tokenize.__globals__["DATA_LVLS"],
        )
    )
    def test_tokenize_scalar(self, var):
        result = mms.tokenize("_".join(var))
        self.assertIsInstance(result, dict)
        self.assertListEqual(
            list(result.keys()), ["param", "to", "cs", "inst", "tmmode", "lev"]
        )
        self.assertEqual(result["to"], 0)

    @idata(
        itertools.product(
            mms.tokenize.__globals__["PARAMS_VECTORS"],
            mms.tokenize.__globals__["COORDINATE_SYSTEMS"],
            mms.tokenize.__globals__["INSTRUMENTS"],
            mms.tokenize.__globals__["SAMPLING_RATES"],
            mms.tokenize.__globals__["DATA_LVLS"],
        )
    )
    def test_tokenize_vector(self, var):
        result = mms.tokenize("_".join(var))
        self.assertIsInstance(result, dict)
        self.assertListEqual(
            list(result.keys()), ["param", "to", "cs", "inst", "tmmode", "lev"]
        )
        self.assertEqual(result["to"], 1)

    @idata(
        itertools.product(
            mms.tokenize.__globals__["PARAMS_TENSORS"],
            mms.tokenize.__globals__["COORDINATE_SYSTEMS"],
            mms.tokenize.__globals__["INSTRUMENTS"],
            mms.tokenize.__globals__["SAMPLING_RATES"],
            mms.tokenize.__globals__["DATA_LVLS"],
        )
    )
    def test_tokenize_tensor(self, var):
        result = mms.tokenize("_".join(var))
        self.assertIsInstance(result, dict)
        self.assertListEqual(
            list(result.keys()), ["param", "to", "cs", "inst", "tmmode", "lev"]
        )

@ddt
class VdfElimTestCase(unittest.TestCase):
    @data(
        random.randint(0, 15),
        random.randint(0, 15) + 0.4,
        [random.randint(0, 15), random.randint(16, 31)],
    )
    def test_vdf_elim_output(self, e_int):
        result = mms.vdf_elim(
            generate_vdf(64.0, 42, [32, 32, 16], energy01=True), e_int
        )
        self.assertIsInstance(result, xr.Dataset)

@ddt
class VdfOmniTestCase(unittest.TestCase):
    @data("mean", "sum")
    def test_vdf_omni_output(self, method):
        result = mms.vdf_omni(generate_vdf(64.0, 100, (32, 32, 16)), method)
        self.assertIsInstance(result, xr.DataArray)

if __name__ == "__main__":
    unittest.main()


import requests
from django.shortcuts import render

def home(request):
    response = requests.get('https://www.mubasher.info/api/1/stocks/prices/all?country=eg')
    data = response.json()
    print(data)
    return render(request, 'egx/home.html', {'lastUpdate': data['lastUpdate'], 'prices': data['prices']})

def companies(request):
    response = requests.get('https://www.mubasher.info/api/1/listed-companies?country=eg&size=400')
    data = response.json()
    print(data['rows'])
    return render(request, 'egx/companies.html', {'companies': data['rows']})


#!/usr/bin/env python

import os
import sys

from setuptools import setup, Extension
from setuptools.command.build_py import build_py
from setuptools.command.sdist import sdist
from distutils.spawn import find_executable
from glob import glob

sources = ['../functions/MLX90640_API.cpp', 'mlx90640-python.cpp']
# If we have swig, use it.  Otherwise, use the pre-generated
# wrapper from the source distribution.
if find_executable('swig'):
    sources += ['MLX90640.i']
elif os.path.exists('MLX90640_wrap.cxx'):
    sources += ['MLX90640_wrap.cxx']
elif os.path.exists('MLX90640_wrap.c'):
    sources += ['MLX90640_wrap.c']
else:
    print("Error:  Building this module requires either that swig is installed\n"
          "        (e.g., 'sudo apt install swig') or that MLX90640_wrap.c from the\n"
          "        source distribution (on pypi) is available.")
    sys.exit(1)

# Fix so that build_ext runs before build_py
# Without this, wiringpi.py is generated too late and doesn't
# end up in the distribution when running setup.py bdist or bdist_wheel.
# Based on:
#  https://stackoverflow.com/a/29551581/7938656
#  and
#  https://blog.niteoweb.com/setuptools-run-custom-code-in-setup-py/
class build_py_ext_first(build_py):
    def run(self):
        self.run_command("build_ext")
        return build_py.run(self)

# Make sure wiringpi_wrap.c is available for the source dist, also.
class sdist_ext_first(sdist):
    def run(self):
        self.run_command("build_ext")
        return sdist.run(self)

classifiers = ['Development Status :: 4 - Beta',
               'Operating System :: POSIX :: Linux',
               'License :: OSI Approved :: MIT License',
               'Intended Audience :: Developers',
               'Programming Language :: Python :: 2.6',
               'Programming Language :: Python :: 2.7',
               'Programming Language :: Python :: 3',
               'Topic :: Software Development',
               'Topic :: System :: Hardware']

_MLX90640 = Extension(
    '_MLX90640',
    include_dirs=['../headers'],
    sources=sources,
    swig_opts=['-threads'],
    extra_link_args=['-lbcm2835']
)

setup(
    name = 'MLX90640',
    version = '0.0.2',
    classifiers = classifiers,
    ext_modules = [ _MLX90640 ],
    py_modules = ["MLX90640"],
    install_requires=[],
    cmdclass = {'build_py' : build_py_ext_first, 'sdist' : sdist_ext_first},
)


from django.shortcuts import render
from bookstore.models import BookInfo
from django.contrib.auth.models import User

# Create your views here.

def homepage(request):
    books = None
    if request.user.is_authenticated:
        user = request.user
        books = BookInfo.objects.filter(department=user.profile.department)[:10]
    else:
         books = BookInfo.objects.all()[:10]

    return render(request,"index.html",{"page":"home","books":books})

def aboutpage(request):

    user = User.objects.get(username="sale")

    return render(request,"about.html",{"page":"about","author":user})


# -*- coding: utf-8 -*-

from __future__ import annotations

from collections.abc import Callable, Mapping

import torch

def get_pooling(method: str) -> Callable[[Mapping, torch.Tensor], torch.Tensor]:
    methods = {
        "cls": cls_pooling,
        "mean": mean_pooling,
        "max": max_pooling,
    }
    pooling = methods.get(method)
    if pooling is None:
        raise ValueError(f"Pooling `method` should be one of {list(methods)}")

    return pooling

def mean_pooling(model_output: Mapping, attention_mask: torch.Tensor) -> torch.Tensor:
    token_embeddings = model_output["last_hidden_state"]
    input_mask_expanded = attention_mask.unsqueeze(dim=-1).float()
    sum_embeddings = torch.sum(token_embeddings * input_mask_expanded, dim=1)
    sum_mask = torch.clamp(input_mask_expanded.sum(dim=1), min=1e-9)

    return sum_embeddings / sum_mask

def max_pooling(model_output: Mapping, attention_mask: torch.Tensor) -> torch.Tensor:
    token_embeddings = model_output["last_hidden_state"]
    input_mask_expanded = (
        attention_mask.unsqueeze(dim=-1).expand(token_embeddings.size()).float()
    )
    token_embeddings[input_mask_expanded == 0] = -1e9
    max_over_time = torch.max(token_embeddings, dim=1)[0]

    return max_over_time

def cls_pooling(model_output: Mapping, attention_mask: torch.Tensor) -> torch.Tensor:
    return model_output["last_hidden_state"][:, 0]


import jsbeautifier
import numpy as np
import pandas as pd
from tqdm import tqdm
from pattern.text.en import polarity, positive, subjectivity
from gensim.parsing.preprocessing import remove_stopwords
import string
from src.preprocess import preprocessing, SymSpell, remove_blanc
from nltk.corpus import stopwords
import spacy
from gensim.models import KeyedVectors
import matplotlib.pyplot as plt
from pattern.text.en import parse
import multiprocessing
import pathos.pools as pp
spacy_nlp = spacy.load('en_core_web_lg')
spacy_stopwords = spacy.lang.en.stop_words.STOP_WORDS
stop_words = set(list(set(stopwords.words('english'))) + list(spacy_stopwords))

def get_word2vec_vocab():
    """
    Gets the vocabulary from pretrained word2vec and writes a new file with all the vocabulary in a new file
    The file contains a single word per line. Over 2 billion words
    :return: Nothing
    """
    print("Extracting word2vec vocabulary")
    model = KeyedVectors.load_word2vec_format("data/GoogleNews-vectors-negative300.bin", binary=True)
    vocab = model.wv.vocab
    w = open("extracted/lists/GoogleNews-vectors-negative300", "w+")
    with tqdm(total=len(vocab)) as pbar:
        for v in vocab:
            if len(parse(v + "\n")) > 0:
                if parse(v + "\n").split("/")[1] == "NNP" and " ".join(v.lower().split("_")).title() == " ".join(v.split("_")):
                    w.write(v + "\n")
            pbar.update(1)
    w.close()

def count_words(filepath ='extracted/lists/australian_user_reviews.txt',
                names_file="extracted/lists/games_abbreviations.txt"):
    """
    Looks for games in the filepath and writes the
    list of games that were found extracted/lists/game_lines.csv"

    :param filepath: data in which the words are counted
    :param names_file: file containing the name in a csv file. In if a single column of words it should work as a single coulmn csv
    :return:
    """
    games_dict = {}
    with open(filepath) as fp:
        for cnt, line in enumerate(fp):
            with open(names_file) as n:
                for i, name in enumerate(n):
                    name = name.strip("\n")
                    name, abbr = name.split(",")
                    if name in line or (abbr in line and len(abbr) > 1):
                        if name not in games_dict:
                            games_dict.update({name: [cnt]})
                        else:
                            games_dict[name] += [cnt]
    w = open("extracted/lists/game_lines.csv", "w+")
    for game in games_dict.keys():
        w.write("{},{}\n".format(game, games_dict[game]))
    w.close()

# Other startegies:
# - If the First few words with Caps Might Be Games is... ( . League of Legend is ... )
# - Words with only caps might be accronyms. people i think go much more with accronyms if name is long ()
#       * Will need to verify what it is an accronym for, or at least what it is not
#       * faire un dictionnaire de vocabulaire avec texte des tp precedents. Les mots ALL CAPS trouvés devront être
#         vérifiés s'ils sont des vrais mots (e.g. AAAAAH -> ah, WWWWHHHHYYYYYYY -> why, FFIV -> ffiv is not a word,
#         thus a game accronym)

def look_for_words(filepath ='extracted/lists/australian_user_reviews.txt',
                   write_to="extracted/lists/game_names.txt",
                   write_rejected="extracted/lists/game_names_rejected_for_length.txt",
                   what_is_it=" is a ",
                   what_it_is="game",
                   cutoff=5,
                   this_list=['Now', 'Game', 'game', 'the game', 'Now', 'now', "There", "there", "This", "this",
                                      "this game", "", "it", "It"]):
    """
    Looks for games in filepath and writes the list of games that were found extracted/lists/game_lines.csv"

    :param filepath: Path to the file containing the documents to analyse (1 doc per line)
    :param write_to: Name of the file the patterns found
    :param write_rejected: list of words that were rejected, but made it close (for further inspection)
    :param what_is_it: e.g. " is a " to be followed by "game" . In a corpus about hockey, it might often be "is a * player"
    :param what_it_is: e.g. game
    :param cutoff: If the sequence if too long the word is rejected
    :param this_list: list of words that might be identified to be avoided
    :return:
    """
    print("Finding games...")

    w = open(write_to, "w+")
    w2 = open(write_rejected, "w+")
    with open(filepath) as fp:
        for cnt, line in enumerate(fp):
            line_split = line.split(what_is_it)
            if len(line_split) > 1:
                line_split1 = line_split[1].split(".")[-1].split(",")[-1].split("!")[-1].split("?")[-1].replace("'",
                                                                                "").replace(
                    '"', "").replace("-", "").replace("_", "").replace("*", "")
                line_split = line_split[0].split(".")[-1].split(",")[-1].split("!")[-1].split("?")[-1].replace("'",
                                                                                "").replace(
                    '"', "").replace("-", "").replace("_", "").replace("*", "")
                if line_split is "":
                    continue
                if line_split not in this_list and (
                        what_it_is in line_split1.split(" ")):
                    flag = False
                    for word in line_split.split(" "):
                        if word in ["This", "this", "", "it", "It"]:
                            flag = True
                    if flag is False:
                        if len(line_split.split(" ")) <= cutoff:
                            line_split = " ".join([l.capitalize() for l in line_split.split(" ")])
                            w.write(line_split + '\n')
                        else:
                            w2.write(line_split + '\n')

    w.close()
    w2.close()

def unique_words(filepath="extracted/lists/game_names.txt", write_to="extracted/lists/games_unique.txt"):
    """
    Takes the list in extracted/lists/game_names.txt and writes the file extracted/lists/games_unique.txt
    that will have all the unique games names in it, one name per line
    """
    gamenames = {}
    with open(filepath) as fp:
        for cnt, line in enumerate(fp):
            if line not in gamenames:
                gamenames.update({line: 0})
            else:
                gamenames[line] += 1
    w = open(write_to, "w+")
    for game in gamenames.keys():
        w.write("{}".format(game))
    w.close()

def words_abbreviation(filepath="extracted/lists/games_unique.txt", write_to="extracted/lists/games_abbreviations.txt"):
    """
    Takes the list in  and writes the file games_unique.txt that will have all the
    unique games names in it
    """
    gamenames = {}
    with open(filepath) as fp:
        for cnt, game in enumerate(fp):
            game = game.split("\n")[0]
            abb = "".join(''.join(x for x in game.replace("'", "") if not x.islower()).split(" "))
            if game not in gamenames:
                gamenames.update({game: abb})
            else:
                exit("whoops")
    w = open(write_to, "w+")
    for game in gamenames.keys():
        w.write("{},{}\n".format(game, gamenames[game]))
    w.close()

def get_user_ids(line):
    """
    Finds the id of a user in a line of a json file
    :param line: The line containing the id
    :return:
    """
    try:
        id = line.split("'user_id': ")[1]
        if id[0] == "'":
            id = id.split(",")[0].strip("'")
        elif id[0] == '"':
            id = id.split('\"\n')[0]
        else:
            exit("unexpected start of user_id")
    except:
        # print(cnt, "has an empty user_id section")
        id = ''
    return id

def get_funny(funny):
    try:
        if funny[0] == "'":
            funny = funny.split(",")[0].strip("'")
        elif funny[0] == '"':
            funny = funny.split('\"\n')[0]
        else:
            exit("unexpected start of funny")
    except:
        # print(cnt, "has an empty funny section")
        funny = ''
    return funny

def get_helpful(helpful):
    try:
        # helpful = line.split("'helpful': ")[1]
        if helpful[0] == "'":
            helpful = helpful.split(",")[0].strip("'")
        elif helpful[0] == '"':
            helpful = helpful.split('\"\n')[0]
        else:
            exit("unexpected start of helpful")
    except:
        # print(cnt, "has an empty helpful section")
        helpful = ''
    return helpful

def get_recommend(recommend):
    try:
        # recommend = line.split("'recommend': ")[1]
        recommend = recommend.split(",")[0]
        if recommend == 'True':
            recommend = 1
        else:
            recommend = 0
    except:
        # print(cnt, "has an empty recommend section")
        recommend = ''
    return recommend

def complete_game_list_with_word2vec():
    from src.vectorize import EmbeddingsModel
    my_model = EmbeddingsModel()
    direc = "saved_models/"
    checkpoint = "gensim_model_pretrainedGoogleNews-vectors-negative300.bin.model"
    my_model.load_model(direc + checkpoint)

def get_review(line, cnt):
    helpfuls = line.split("'reviews': ")[1].split("'helpful': ")[1:]
    reviews = line.split("'reviews': ")[1].split("'review': ")[1:]
    funnys = line.split("'reviews': ")[1].split("'funny': ")[1:]
    recommends = line.split("'reviews': ")[1].split("'recommend': ")[1:]
    revs, polars, positivs, subjs, funs, helps, recoms = [], [], [], [], [], [], []
    for review, fun, recomm, help in zip(reviews, funnys, recommends, helpfuls):
        try:
            if review[0] == "'":
                review = review.split("\'\n")[0]
                review = review[1:]
            elif review[0] == '"':
                review = review.split('\"\n')[0]
                review = review[1:]
            else:
                exit("unexpected start of review")
        except:
            # print(cnt, "has an empty review section")
            review = ''
        revs += [review]
        polars += [polarity(review)]
        positivs += [positive(review)]
        subjs += [subjectivity(review)]
        funs += [get_funny(fun)]
        helps += [get_helpful(help)]
        recoms += [get_recommend(recomm)]
    return revs, polars, positivs, subjs, funs, helps, recoms

def build_dataframe(json_fname="australian_user_reviews", csv_fname="steam_australia", get_more_infos=False):
    """
    Builds the dataframe to do cool data science with
    :param json_fname: json file containing the information to scrape for infos
    :param csv_fname: name of the csv filename that will contain the dataframe
    :param get_more_infos: Get more infos?
    :return:
    """
    print("Building the dataframe in extracted/dataframes...")
    filepath = 'data/{}.json'.format(json_fname)
    w = open("extracted/lists/{}.txt".format(json_fname), "w+")
    reviews = []
    ids = []
    funnys = []
    helpfuls = []
    recommends = []
    polarities = []
    positives = []
    subjectivities = []
    n_reviews_per_user = []
    n_positive_reviews_per_user = []
    n_negative_reviews_per_user = []
    n_reviews_per_user_with_at_least_1neg = []
    n_reviews_per_user_with_at_least_1pos = []
    n_positive_reviews_per_user_with_at_least_1neg = []
    n_negative_reviews_per_user_with_at_least_1neg = []
    n_positive_reviews_per_user_with_at_least_1pos = []
    n_negative_reviews_per_user_with_at_least_1pos = []
    with tqdm(total=25799) as pbar:
        with open(filepath) as fp:
            for cnt, line in enumerate(fp):
                line = jsbeautifier.beautify(line)
                revs, polars, positivs, subjs, funs, helps, recoms = get_review(line, cnt)
                reviews.extend(revs)
                polarities.extend(polars)
                positives.extend(positivs)
                subjectivities.extend(subjs)
                ids.extend([get_user_ids(line) for _ in range(len(revs))])
                funnys.extend(funs)
                helpfuls.extend(helps)
                recommends.extend(recoms)
                n_reviews_per_user += [len(revs)]
                if get_more_infos:
                    if len([x for x in recoms if str(x) == "0"]) > 0:
                        n_reviews_per_user_with_at_least_1neg += [len(revs)]
                        n_negative_reviews_per_user_with_at_least_1neg += [len([x for x in recoms if str(x) == "0"])]
                        n_positive_reviews_per_user_with_at_least_1neg += [len([x for x in recoms if str(x) == "1"])]
                    if len([x for x in recoms if str(x) == "1"]) > 0:
                        n_reviews_per_user_with_at_least_1pos += [len(revs)]
                        n_positive_reviews_per_user_with_at_least_1pos += [len([x for x in recoms if str(x) == "1"])]
                        n_negative_reviews_per_user_with_at_least_1pos += [len([x for x in recoms if str(x) == "0"])]
                n_positive_reviews_per_user += [len([x for x in recoms if str(x) == "1"])]
                n_negative_reviews_per_user += [len([x for x in recoms if str(x) == "0"])]
                w.write(reviews[-1] + '\n')
                pbar.update(1)
    w.close()
    n_positive_reviews_per_user = np.array(n_positive_reviews_per_user)
    n_negative_reviews_per_user = np.array(n_negative_reviews_per_user)
    n_negative_reviews_per_user_with_at_least_1pos = np.array(n_negative_reviews_per_user_with_at_least_1pos)
    n_negative_reviews_per_user_with_at_least_1neg = np.array(n_negative_reviews_per_user_with_at_least_1neg)
    n_reviews_per_user = np.array(n_reviews_per_user)
    plt.boxplot(n_reviews_per_user)
    plt.savefig("images/boxplot_reviews_per_id")
    plt.boxplot([n_positive_reviews_per_user, n_negative_reviews_per_user])
    plt.savefig("images/boxplot_negpos_reviews_per_id")
    plt.boxplot([n_negative_reviews_per_user_with_at_least_1pos, n_negative_reviews_per_user_with_at_least_1neg])
    plt.savefig("images/boxplot_negpos_reviews_per_id")
    print("Max n_reviews for 1 user:", np.max(n_reviews_per_user))
    print("Max n positive reviews for 1 user:", np.max(n_positive_reviews_per_user),
          "/", n_reviews_per_user[np.argmax(n_positive_reviews_per_user)])
    print("Max n negative reviews for 1 user:", np.max(n_negative_reviews_per_user),
          "/", n_reviews_per_user[np.argmax(n_negative_reviews_per_user)])

    print("Mean n_reviews for 1 user:", np.mean(n_reviews_per_user))
    print("Mean n positive reviews for 1 user:", np.mean(n_positive_reviews_per_user))
    print("Mean n negative reviews for 1 user:", np.mean(n_negative_reviews_per_user))

    print("Median n_reviews for 1 user:", np.median(n_reviews_per_user))
    print("Median n positive reviews for 1 user:", np.median(n_positive_reviews_per_user))
    print("Median n negative reviews for 1 user:", np.median(n_negative_reviews_per_user))

    if get_more_infos:
        print("\n\nWITH at least 1 positive review\n\n")
        print("Number of reviewers", len(n_reviews_per_user_with_at_least_1pos), "/25799")
        print("Max n_reviews for 1 user:", np.max(n_reviews_per_user_with_at_least_1pos))
        print("Max n positive reviews for 1 user:", np.max(n_positive_reviews_per_user_with_at_least_1pos),
              "/", n_reviews_per_user[np.argmax(n_positive_reviews_per_user_with_at_least_1pos)])
        print("Max n negative reviews for 1 user:", np.max(n_negative_reviews_per_user_with_at_least_1pos),
              "/", n_reviews_per_user[np.argmax(n_negative_reviews_per_user_with_at_least_1pos)])

        print("Mean n_reviews for 1 user:", np.mean(n_reviews_per_user_with_at_least_1pos))
        print("Mean n positive reviews for 1 user:", np.mean(n_positive_reviews_per_user_with_at_least_1pos))
        print("Mean n negative reviews for 1 user:", np.mean(n_negative_reviews_per_user_with_at_least_1pos))

        print("Median n_reviews for 1 user:", np.median(n_reviews_per_user_with_at_least_1pos))
        print("Median n positive reviews for 1 user:", np.median(n_positive_reviews_per_user_with_at_least_1pos))
        print("Median n negative reviews for 1 user:", np.median(n_negative_reviews_per_user_with_at_least_1pos))

        print("\n\nWITH at least 1 negative review\n\n")
        print("Number of reviewers", len(n_reviews_per_user_with_at_least_1neg),"/25799")
        print("Max n_reviews for 1 user:", np.max(n_reviews_per_user_with_at_least_1neg))
        print("Max n positive reviews for 1 user:", np.max(n_positive_reviews_per_user_with_at_least_1neg),
              "/", n_reviews_per_user[np.argmax(n_positive_reviews_per_user_with_at_least_1neg)])
        print("Max n negative reviews for 1 user:", np.max(n_negative_reviews_per_user_with_at_least_1neg),
              "/", n_reviews_per_user[np.argmax(n_negative_reviews_per_user_with_at_least_1neg)])

        print("Mean n_reviews for 1 user:", np.mean(n_reviews_per_user_with_at_least_1neg))
        print("Mean n positive reviews for 1 user:", np.mean(n_positive_reviews_per_user_with_at_least_1neg))
        print("Mean n negative reviews for 1 user:", np.mean(n_negative_reviews_per_user_with_at_least_1neg))

        print("Median n_reviews for 1 user:", np.median(n_reviews_per_user_with_at_least_1neg))
        print("Median n positive reviews for 1 user:", np.median(n_positive_reviews_per_user_with_at_least_1neg))
        print("Median n negative reviews for 1 user:", np.median(n_negative_reviews_per_user_with_at_least_1neg))

    ids = np.array(ids)
    funnys = np.array(funnys)
    helpfuls = np.array(helpfuls)
    recommends = np.array(recommends)
    reviews_arr = np.array(reviews, dtype='<U32')
    polarities = np.array(polarities)
    positives = np.array(positives)
    subjectivities = np.array(subjectivities)
    df = pd.DataFrame(columns=["id",
                               "funny",
                               "helpful",
                               "recommend",
                               "polarity",
                               "positive",
                               "subjectivity",
                               "review"],
                      data=np.concatenate((ids.reshape(-1, 1),
                                           funnys.reshape(-1, 1),
                                           helpfuls.reshape(-1, 1),
                                           recommends.reshape(-1, 1),
                                           polarities.reshape(-1, 1),
                                           positives.reshape(-1, 1),
                                           subjectivities.reshape(-1, 1),
                                           reviews_arr.reshape(-1, 1)), 1))
    df["review"] = reviews
    df.to_csv("extracted/dataframes/{}.csv".format(csv_fname), index=False)
    print()

def normalize_csv(df_name="steam_australia.csv", norm_fname="steam_australia_norm", spell_check=True, correct=False):
    """
    Care must be taken to remove the lowercases at the good time or it can ruin the normalization of named entities
    1- Usual text normalization. Lowercase is applied here only!
    2- names of games are turned into Capitalized version and underscored version (e.g. Team_Fortress_2)
    3- Words composed of more than one word are in word2vec are replace with the Capitalized and underscored version (e.g. New_York)
    """
    print("Normalizing csv...")

    names_file = "games_abbreviations.txt"
    names = pd.read_csv("extracted/lists/" + names_file, names=["names", "abbr"])
    df = pd.read_csv("extracted/dataframes/" + df_name)
    w = open("extracted/lists/normalized_names.csv", "w+")
    words_dict = None
    ss = None
    t = str.maketrans(dict.fromkeys(string.punctuation, " "))
    if spell_check:
        print("Normalizing text...")
        if correct:
            ss = SymSpell(max_edit_distance=2)
            vocabulary = "data/english_words_479k.txt"
            with open(vocabulary) as f:
                words = f.readlines()
            eng_words = [word.strip() for word in words]
            print('Creating dictionary for symspell')
            _ = ss.create_dictionary_from_arr(eng_words, token_pattern=r'.+')
            words_dict = {k: 0 for k in eng_words}
        with tqdm(total=len(df)) as pbar:
            for l, line in enumerate(df["review"]):
                # Stop words might need to be removed later if a classification task is done.
                # We need them for normalize_csv_for_word2vec, which is done after normalization
                # to impove the number of hits in that next step. Stop words are part of expressions of named
                # entities, e.g. The_Hague is a city
                line = str(line).lower().translate(t)
                line = preprocessing(line, stop_words, ss, words_dict, remove_stopwords=False, correct=False)
                df.set_value(l, "review", line)
                pbar.update(1)
    with tqdm(total=len(df["review"])) as pbar:
        for l, line in enumerate(df["review"]):
            line = str(line) #.lower()
            for i, (name, abbr) in enumerate((zip(names["names"], names["abbr"]))):
                if " " + name.lower() + " " in line:
                    # Replace the name, lowercase or not, not Capitalized
                    line = line.replace(" " + name.lower() + " ", " " + name.replace(" ", "_") + " ")
                    w.write(name.replace(" ", "_") + ',"' + line + '"\n')
                if " " + name.lower() in line:
                    # Replace the name, lowercase or not, not Capitalized
                    line = line.replace(" " + name.lower(), " " + name.replace(" ", "_"))
                    w.write(name.replace(" ", "_") + ',"' + line + '"\n')
                if name.lower() + " " in line:
                    # Replace the name, lowercase or not, not Capitalized
                    line = line.replace(name.lower() + " ", name.replace(" ", "_") + " ")
                    w.write(name.replace(" ", "_") + ',"' + line + '"\n')
                # if abbr.lower() in line and len(abbr) > 1:
                # Replace the abbreviations, lowercase or not with Capitalized full name
                #    line = line.replace(abbr.lower(), name.replace(" ", "_"))
                #    w.write(name.replace(" ", "_") + ',"' + line + '"\n')

                df.set_value(l, "review", line)
            pbar.update(1)

    w.close()
    df.to_csv("extracted/dataframes/{}.csv".format(norm_fname), index=False)

def remove_stop_words_from_list(list_vocab):
    # Do not remove stop words in "composed" words like New_York
    print("Removing stopwords from vocabulary...")
    cleared_list = []
    stopwords_removed = []
    list_vocab = list(np.array(list_vocab).reshape(-1))
    with tqdm(total=len(list_vocab)) as pbar:
        for vocab in list_vocab:
            if len(remove_stopwords(str(vocab).lower())) > 0:
                cleared_list += [str(vocab)]
            else:
                stopwords_removed += [str(vocab)]
            pbar.update(1)
    print("Removed", len(stopwords_removed), "stopwords")
    assert len(stopwords_removed) + len(cleared_list) == len(list_vocab)
    return cleared_list, stopwords_removed

def normalize_csv_for_word2vec(df_name="steam_australia_norm.csv", w2v_norm_fname="steam_australia_norm_w2v", multiple_cores=True):
    """
    word2vec has a lot of vocabulary that cannot be detected in the sentences we will use to fine-tune word2vec

    :param df_name:
    :param w2v_norm_fname:
    :return:
    """
    print("Normalizing name entities in accordance to the word2vec vocabulary")
    vocab = "extracted/lists/GoogleNews-vectors-negative300"
    df = pd.read_csv("extracted/dataframes/" + df_name)
    voc = np.array(pd.read_table(vocab))
    voc, stopwords = remove_stop_words_from_list(voc)
    w = open("extracted/lists/normalized_words_w2v.txt", "w+")

    print("Normalisation for pretrained word2vec started. This will take a while...")
    with tqdm(total=len(voc)) as pbar:
        for word in voc:
            word = str(word.strip("\n"))
            if (  # and len(word.split("_")) > 1
                    "_".join([w.lower().capitalize() for w in word.split("_")]) == word or "_".join(
                [w.lower() for w in word.split("_")]) == word):
                if word != word.lower():
                    if len(word) > 1 and "" not in word.split("_"):
                        for l, line in enumerate(df["review"]):
                            line = str(line)
                            # The spaces ensure it is tow complete words, not the end of one and begining of the other
                            expr = " ".join(word.lower().split("_"))
                            if " " + expr + " " in line:
                                # Replace the name, lowercase or not, not Capitalized
                                line = line.replace(" " + expr + " ", " " + word + " ")
                                w.write(word + '\n')
                                df.set_value(l, "review", line)
                            # start of sentence
                            if expr + " " in line and line.split(expr)[0] == "":
                                # Replace the name, lowercase or not, not Capitalized
                                line = line.replace(expr + " ", word + " ")
                                w.write(word + '\n')
                                df.set_value(l, "review", line)
                            # end of sentence
                            if " " + expr in line and line.split(expr)[-1] == "":
                                # Replace the name, lowercase or not, not Capitalized
                                line = line.replace(" " + expr, " " + word)
                                w.write(word + '\n')
                                df.set_value(l, "review", line)
            pbar.update(1)
    w.close()
    df.to_csv("extracted/dataframes/{}.csv".format(w2v_norm_fname), index=False)

def normalize_csv_for_word2vec_parallel(df_name="steam_australia_norm.csv", w2v_norm_fname="steam_australia_norm_w2v"):
    """
    word2vec has a lot of vocabulary that cannot be detected in the sentences we will use to fine-tune word2vec

    :param df_name:
    :param w2v_norm_fname:
    :return:
    """
    print("Normalizing name entities in accordance to the word2vec vocabulary")
    vocab = "extracted/lists/GoogleNews-vectors-negative300"
    df = pd.read_csv("extracted/dataframes/" + df_name)
    voc = np.array(pd.read_table(vocab))
    voc, stopwords = remove_stop_words_from_list(voc)
    w1 = open("extracted/lists/normalized_reviews_w2v.txt", "w+")
    print("Using", multiprocessing.cpu_count()-1, "cores")
    pool = pp.ProcessPool(multiprocessing.cpu_count()-1)
    with tqdm(total=int(len(df["review"]) / (multiprocessing.cpu_count()-1))) as pbar:
        def parallel_words_normalization(line):
            pbar.update(1)
            line = str(line)
            # w1.write(line + "\n")
            for word in voc:
                word = str(word.strip("\n"))
                if (  # and len(word.split("_")) > 1
                        "_".join([w.lower().capitalize() for w in word.split("_")]) == word or "_".join(
                    [w.lower() for w in word.split("_")]) == word):
                    if word != word.lower():
                        if len(word) > 1 and "" not in word.split("_"):
                            # The spaces ensure it is tow complete words, not the end of one and begining of the other
                            expr = " ".join(word.lower().split("_"))
                            if " " + expr + " " in line:
                                # Replace the name, lowercase or not, not Capitalized
                                line = line.replace(" " + expr + " ", " " + word + " ")
                                # w.write(word + '\n')
                                # df.set_value(l, "review", line)
                            # start of sentence
                            if expr + " " in line and line.split(expr)[0] == "":
                                # Replace the name, lowercase or not, not Capitalized
                                line = line.replace(expr + " ", word + " ")
                                #w.write(word + '\n')
                                # df.set_value(l, "review", line)
                            # end of sentence
                            if " " + expr in line and line.split(expr)[-1] == "":
                                # Replace the name, lowercase or not, not Capitalized
                                line = line.replace(" " + expr, " " + word)
                                #w.write(word + '\n')
            w1.write(line + "\n")
            return line
        print("Normalisation for pretrained word2vec started. This will take a while...")
        #with tqdm(total=len(df["review"])) as pbar:
            #for l, line in enumerate(df["review"]):
                # line = str(line)
        lines = pool.map(parallel_words_normalization, df["review"])
    df["review"] = lines
    #df.set_value(l, "review", line)
    #w.write(line + '\n')
    #pbar.update(1)
    w1.close()
    df.to_csv("extracted/dataframes/{}.csv".format(w2v_norm_fname), index=False)

if __name__ == "__main__":
    look_for_words(filepath ='extracted/lists/australian_user_reviews.txt',
                   write_to="extracted/lists/game_names.txt",
                   write_rejected="extracted/lists/game_names_rejected_for_length.txt",
                   what_is_it=" is a ",
                   what_it_is="game",
                   cutoff=5,
                   this_list=['Now', 'Game', 'game', 'the game', 'Now', 'now', "There", "there", "This", "this",
                                      "this game", "", "it", "It"])
    look_for_words(filepath='extracted/lists/australian_user_reviews.txt',
                   write_to="extracted/lists/features_names.txt",
                   write_rejected="extracted/lists/features_names_rejected_for_length.txt",
                   what_is_it=" is a ",
                   what_it_is="feature",
                   cutoff=3,
                   this_list=['Now', 'Game', 'game', 'the game', 'Now', 'now', "There", "there", "This", "this",
                                      "this game", "", "it", "It"])

    #unique_words()
    #words_abbreviation()
    #count_words()
    #build_dataframe(get_more_infos=True)
    normalize_csv(spell_check=True, correct=False)
    #get_word2vec_vocab()
    # # normalize_csv_for_word2vec()
    normalize_csv_for_word2vec_parallel()


from flask import Flask
import ml
import api

app = Flask(__name__)

rainRecord = [18.5, 0.7, 0, 0, 0, 0, 0, 0, 103, 19.1]
rainRecordIdx = 0

predRecord = [1.5, 1.3, 1.9,  2.3,  3.4,  5.4,  1.2, 1.3, 5.6,   8.1,  9.3,  10.2, 11.4,  9.5,  8.7,  7.3, 6.5, 4.3, 2.1, 1.3]

@app.route("/rain", methods=['GET'])
def getRain():
  #########
  global rainRecordIdx
  res = api.getRainFromDummy(rainRecordIdx)
  rainRecordIdx += 1
  #########
  # res = api.getRainFromApi()
  rainRecord.pop(0)
  rainRecord.append(res)

  # prediction = ml.fl.predict(rainRecord)
  prediction = [predRecord[rainRecordIdx%20]]
  return str(res) + "," + str(prediction[0])

@app.route("/waterlevel", methods=['GET'])
def getWaterlevel():
  prediction = ml.fl.predict(rainRecord)
  print(rainRecord)
  return str(prediction[0])


"""Implementation of PipelineEnhancer class."""
from typing import List, Union, cast, TypeVar

from py4ai.analytics.ml.core import Enhancer
from py4ai.analytics.ml.core.pipeline import Pipeline, Step
from py4ai.analytics.ml.eda import compose
from py4ai.data.model.text import Document, DocumentsUtilsMixin

TDocumentsUtilsMixin = TypeVar("TDocumentsUtilsMixin", bound=DocumentsUtilsMixin)

class PipelineEnhancer(Enhancer, Pipeline):
    """Enhancer constituted by a sequence of Enhancers."""

    def __init__(self, steps: List[Step]) -> None:
        """
        Class initializer.

        :param steps: list of Enhancer instances that compose the pipeline.
        """
        super(PipelineEnhancer, self).__init__(steps=steps)

    def _validate_steps(self) -> None:
        """
        Check if all steps are Enhancers.

        :raises TypeError: if any of the steps is not an Enhancer
        """
        if any([not isinstance(stage[1], Enhancer) for stage in self.steps]):
            raise TypeError("Each step of the pipeline must be an Enhancer")

    def apply(
        self, documents: Union[Document, TDocumentsUtilsMixin]
    ) -> Union[Document, TDocumentsUtilsMixin]:
        """
        Sequentially enhance documents.

        :param documents: documents to enhance
        :return: enhanced documents
        """
        composer = compose(
            *list(
                map(
                    lambda x: lambda y: cast(Enhancer, x).enhance(y),
                    reversed(self.steps),
                )
            )
        )
        return composer(object)


# [SublimeLinter @python:3]

from ..utils import echo

def patch(csv_file, patch_file, reverse=False, dry_run=False, force=False):
    read_count = 0
    applied_count = 0

    with open(csv_file, "r+b") as fp_csv, \
            open(patch_file, "r") as fp_patch:
        for line in fp_patch:
            if line.startswith("#"):
                continue

            try:
                offset, expected_code, replace_code = map(int, line.rstrip().split(":"))
            except ValueError:
                echo("ERROR: invalid patch file")
                return 1

            if reverse:
                expected_code, replace_code = replace_code, expected_code

            fp_csv.seek(offset)
            char = fp_csv.read(1)

            if not char:
                echo("ERRPR: patch is not applicable")
                return 1

            char_code = ord(char)
            read_count += 1

            if char_code == replace_code:
                continue

            if char_code != expected_code:
                echo("ERROR: unexpected char at", offset)

                if not force:
                    return 1

            fp_csv.seek(offset)

            if not dry_run:
                fp_csv.write(bytes((replace_code, )))

            applied_count += 1

    echo("Replacements applied: {} / {}".format(applied_count, read_count))
    return 0


import tkinter as tk
import requests
import config

window = tk.Tk()

window.geometry(f"{window.winfo_screenwidth()}x{window.winfo_screenheight()}")

def format_response(weather):
    try:
        name = weather["name"]
        desc = weather["weather"][0]["description"]
        temp = weather["main"]["temp"]
        final = f"""City: {name}\nConditions: {desc}\nTemperature(°C): {str(temp)}"""
    except:
        final = "There was a problem retrieving the data"
    return final

def get_weather(entry):
    url = "https://api.openweathermap.org/data/2.5/weather"
    params = {"APPID": config.key, "q": entry, "units": "metric"}
    response = requests.get(url, params=params)
    weather = response.json()
    label["text"] = format_response(weather)

background_image = tk.PhotoImage(file="mountains5.png")
background_Label = tk.Label(window, image=background_image)
background_Label.pack()

frame = tk.Frame(window, bg="#fcdf03", bd=5)
frame.place(relx=0.1, rely=0.15, relheight=0.07, relwidth=0.8)

entry = tk.Entry(frame, font=("System", 15))
entry.place(relheight=1, relwidth=0.8)

button = tk.Button(frame, text="Get Weather",
                   command=lambda: get_weather(entry.get()))
button.place(relx=0.8, relheight=1, relwidth=0.2)

lower_frame = tk.Frame(window, bg="#fcdf03", bd=5)
lower_frame.place(relx=0.1, rely=0.3, relheight=0.6, relwidth=0.8)

label = tk.Label(lower_frame, font=("Microsoft Himalaya", 30),
                 anchor="nw", justify="left", bd=5)
label.place(relheight=1, relwidth=1)

window.mainloop()


from math import ceil
from math import floor
from math import log
from math import sin
from math import sqrt
from random import randint
from random import uniform
from random import sample
from random import choice
from statistics import median

class f1():

    def __init__(self):
        self.calls = 0

    def calc(self,x):

        self.calls += 1

        return 100*(x[1]-x[0]**2)**2 + (1-x[0])**2

class f2():

    def __init__(self):
        self.calls = 0

    def calc(self,x):

        self.calls += 1

        result = 0

        for index in range(len(x)):
            result += (x[index]-(index+1))**2

        return result

class f3():

    def __init__(self):
        self.calls = 0

    def calc(self,x):

        self.calls += 1

        result = 0

        for index in range(len(x)):
            result += x[index]**2

        return 0.5 + ((sin(sqrt(result)))**2-0.5)/(1+0.001*result)**2

class f4():

    def __init__(self):
        self.calls = 0

    def calc(self,x):

        self.calls += 1

        result = 0

        for index in range(len(x)):
            result += x[index]**2

        return (result**0.25)*(1+(sin(50*(result**0.1)))**2)

class k_turnir_elimination_GA():

    def __init__(self, k = 3, display = "bin", dg = -50, gg = 150, variables = 2, precision = 3, popSize = 10, pm = 0.1, evaluations = 5000, function = None, crossoverType = 1):

        self.k = k
        self.display = display
        self.dg = dg
        self.gg = gg
        self.variables = variables
        self.precision = precision
        self.popSize = popSize
        self.pm = pm
        self.evaluations = evaluations
        self.function = function
        self.crossoverType = crossoverType

        if self.display == "bin":

            self.numberOfBits = self.variables * ceil((log(floor((1+(self.gg-self.dg)*pow(10,self.precision))))/(log(2))))

    class Chromosome:

        def __init__(self, display, variables, dg, gg, size = None):

            self.display = display
            self.size = size
            self.variables = variables
            self.dg = dg
            self.gg = gg

            if display == "bin":
                self.value = []

                for _ in range(size):
                    self.value.append(randint(0,1))

            elif display == "dec":
                self.value = []

                for _ in range(variables):
                    self.value.append(uniform(dg,gg))
            else:
                raise Exception("Wrong display!")

        def decode(self):

            decoded_vars = []
            to_Decode = []
            binaries = []

            for index in range(self.variables):
                to_Decode.append(self.value[int(index*(self.size/self.variables)):int((index+1)*(self.size/self.variables))])

            for binary in to_Decode:
                b = 1
                value = 0
                binary.reverse()
                for bit in binary:
                    value += b*bit
                    b *= 2
                binaries.append(value)

            for b in binaries:
                decoded_vars.append(self.dg+ b/(pow(2,self.size/self.variables)-1)*(self.gg-self.dg))

            return decoded_vars

        def evaluate(self,func):

            if self.display == "bin":
                self.fitnessValue = func.calc(self.decode())
            elif self.display == "dec":
                self.fitnessValue = func.calc(self.value)
            else:
                raise Exception("Wrong display!")

    def crossover(self,parent1,parent2,child):

        if self.display == "bin":
            #Binarno s jednom točkom prekida
            if self.crossoverType == 1:
                value = []
                stop = randint(1,self.numberOfBits)
                for index in range(self.numberOfBits):
                    if index < stop:
                        value.append(parent1.value[index])
                    else:
                        value.append(parent2.value[index])
                child.value = value
            #Binarno uniformno
            elif self.crossoverType == 2:
                value = []
                for index in range(self.numberOfBits):
                    value.append(choice([parent1.value[index],parent2.value[index]]))
                child.value = value
            else:
                raise Exception("Wrong crossover type!")
        elif self.display == "dec":
            #Decimalno aritmetičko
            if self.crossoverType == 1:
                value = []
                a = uniform(0,1)
                for index in range(self.variables):
                    value.append(a*parent1.value[index]+(1-a)*parent2.value[index])
                child.value = value
            #Decimalno heurističko
            elif self.crossoverType == 2:
                value = []
                a = uniform(0,1)
                if parent1.fitnessValue > parent2.fitnessValue:
                    for index in range(self.variables):
                        value.append(a*(parent2.value[index]-parent1.value[index])+parent2.value[index])
                else:
                    for index in range(self.variables):
                        value.append(a*(parent1.value[index]-parent2.value[index])+parent1.value[index])
                for index in range(self.variables):
                    if value[index] < self.dg:
                        value[index] = self.dg
                    elif value[index] > self.gg:
                        value[index] = self.gg
                child.value = value

    def mutation(self, unit):

        mutation = uniform(0,1)

        if mutation <= self.pm:

            #Jednolika binarna mutacija
            if self.display == "bin":
                for index in range(self.numberOfBits):
                    if unit.value[index] == 0:
                        unit.value[index] = 1
                    elif unit.value[index] == 1:
                        unit.value[index] = 0
                    else:
                        raise Exception("Illegal value!")

            #Jednolika aritmetička mutacija
            elif self.display == "dec":
                for index in range(self.variables):
                    unit.value[index] = uniform(self.dg,self.gg)

        else:
            pass

    def run(self):

        #stvori populaciju
        self.population = []
        if self.display == "bin":
            for _ in range(self.popSize):
                self.population.append(self.Chromosome(display = self.display, size = self.numberOfBits, variables = self.variables, dg = self.dg, gg = self.gg))
        elif self.display == "dec":
            for _ in range(self.popSize):
                self.population.append(self.Chromosome(display = self.display, variables = self.variables, dg = self.dg, gg = self.gg))
        else:
            raise Exception("Wrong display!")

        #evaluairaj populaciju
        for example in self.population:
            example.evaluate(self.function)

        #ponavljaj dok nije zadovoljen uvjet zaustavljanja
        while(self.function.calls <= self.evaluations):

            #odaberi slucajno k jedniki
            k_samples = sample(self.population,self.k)

            #pronađi najlosiju
            k_samples.sort(key = lambda x: x.fitnessValue, reverse = True)

            #nova = krizanje neke preostale 2
            toCrossover = sample(k_samples[1:],2)
            self.crossover(toCrossover[0],toCrossover[1],k_samples[0])

            #mutacija
            self.mutation(k_samples[0])

            #evaluacija nove
            k_samples[0].evaluate(self.function)
             #dodaj novu u populaciju -> nova samo zamjeni vrijednost stare

        self.population.sort(key = lambda x: x.fitnessValue)

        if self.display == "bin":
            return [self.population[0].decode(),self.population[0].fitnessValue]
        else:
            return [self.population[0].value,self.population[0].fitnessValue]

zadatak = 0

if zadatak == 1 or zadatak == 0:
    print("1.Zadatak\n")

    f1dec = k_turnir_elimination_GA(k = 3, display = "dec", dg = -50, gg = 150, variables = 2, precision = 3, popSize = 200, pm = 0.3, evaluations = 1000000, function = f1(), crossoverType = 2)
    x,fx = f1dec.run()
    print("Funkcija f1 - DECIMLANI PRIKAZ: x = {}, f(x) = {}".format(x,fx))
    f1bin = k_turnir_elimination_GA(k = 3, display = "bin", dg = -50, gg = 150, variables = 2, precision = 3, popSize = 200, pm = 0.3, evaluations = 1000000, function = f1(), crossoverType = 2)
    x,fx = f1bin.run()
    print("Funkcija f1 - BINARNI PRIKAZ: x = {}, f(x) = {}".format(x,fx))
    print("\n")

    f2dec = k_turnir_elimination_GA(k = 3, display = "dec", dg = -50, gg = 150, variables = 5, precision = 3, popSize = 200, pm = 0.3, evaluations = 1000000, function = f2(), crossoverType = 2)
    x,fx = f2dec.run()
    print("Funkcija f2 - DECIMLANI PRIKAZ: x = {}, f(x) = {}".format(x,fx))
    f2bin = k_turnir_elimination_GA(k = 3, display = "bin", dg = -50, gg = 150, variables = 5, precision = 3, popSize = 200, pm = 0.3, evaluations = 1000000, function = f2(), crossoverType = 1)
    x,fx = f2bin.run()
    print("Funkcija f2 - BINARNI PRIKAZ: x = {}, f(x) = {}".format(x,fx))
    print("\n")

    f3dec = k_turnir_elimination_GA(k = 3, display = "dec", dg = -50, gg = 150, variables = 2, precision = 3, popSize = 200, pm = 0.3, evaluations = 1000000, function = f3(), crossoverType = 2)
    x,fx = f3dec.run()
    print("Funkcija f3 - DECIMLANI PRIKAZ: x = {}, f(x) = {}".format(x,fx))
    f3bin = k_turnir_elimination_GA(k = 3, display = "bin", dg = -50, gg = 150, variables = 2, precision = 3, popSize = 200, pm = 0.3, evaluations = 1000000, function = f3(), crossoverType = 2)
    x,fx = f3bin.run()
    print("Funkcija f3 - BINARNI PRIKAZ: x = {}, f(x) = {}".format(x,fx))
    print("\n")

    f4dec = k_turnir_elimination_GA(k = 3, display = "dec", dg = -50, gg = 150, variables = 2, precision = 3, popSize = 200, pm = 0.3, evaluations = 1000000, function = f4(), crossoverType = 2)
    x,fx = f4dec.run()
    print("Funkcija f4 - DECIMLANI PRIKAZ: x = {}, f(x) = {}".format(x,fx))
    f4bin = k_turnir_elimination_GA(k = 3, display = "bin", dg = -50, gg = 150, variables = 2, precision = 3, popSize = 200, pm = 0.3, evaluations = 1000000, function = f4(), crossoverType = 1)
    x,fx = f4bin.run()
    print("Funkcija f4 - BINARNI PRIKAZ: x = {}, f(x) = {}".format(x,fx))
    print("\n")

if zadatak == 2 or zadatak == 0:
    print("Zadatak 2.\n")

    dimensions = [1,3,6,10]

    for index in range(len(dimensions)):

        f3dec = k_turnir_elimination_GA(k = 3, display = "dec", dg = -50, gg = 150, variables = dimensions[index], precision = 3, popSize = 200, pm = 0.3, evaluations = 100000, function = f3(), crossoverType = 2)
        x,fx = f3dec.run()
        print("Funkcija f3 sa {} varijabla - DECIMLANI PRIKAZ: x = {}, f(x) = {}".format(dimensions[index],x,fx))

    print("\n")

    for index in range(len(dimensions)):

        f4dec = k_turnir_elimination_GA(k = 3, display = "dec", dg = -50, gg = 150, variables = dimensions[index], precision = 3, popSize = 200, pm = 0.3, evaluations = 100000, function = f4(), crossoverType = 2)
        x,fx = f4dec.run()
        print("Funkcija f3 sa {} varijabla - DECIMLANI PRIKAZ: x = {}, f(x) = {}".format(dimensions[index],x,fx))

    print("\n")

if zadatak == 3 or zadatak == 0:
    print("Zadatak 3\n")

    dimensions = [3,6]

    f3dec3var = []
    f3bin3var = []
    f3dec6var = []
    f3bin6var = []

    f4dec3var = []
    f4bin3var = []
    f4dec6var = []
    f4bin6var = []

    for index in range(len(dimensions)):

        for _ in range(10):

            f3dec = k_turnir_elimination_GA(k = 3, display = "dec", dg = -50, gg = 150, variables = dimensions[index], precision = 4, popSize = 200, pm = 0.3, evaluations = 100000, function = f3(), crossoverType = 2)
            x,fx = f3dec.run()
            if index == 0:
                f3dec3var.append(fx)
            else:
                f3dec6var.append(fx)
            f3dec = k_turnir_elimination_GA(k = 3, display = "bin", dg = -50, gg = 150, variables = dimensions[index], precision = 4, popSize = 200, pm = 0.3, evaluations = 100000, function = f3(), crossoverType = 2)
            x,fx = f3dec.run()
            if index == 0:
                f3bin3var.append(fx)
            else:
                f3bin6var.append(fx)

            f4dec = k_turnir_elimination_GA(k = 3, display = "dec", dg = -50, gg = 150, variables = dimensions[index], precision = 4, popSize = 200, pm = 0.3, evaluations = 100000, function = f4(), crossoverType = 2)
            x,fx = f4dec.run()
            if index == 0:
                f4dec3var.append(fx)
            else:
                f4dec6var.append(fx)
            f4dec = k_turnir_elimination_GA(k = 3, display = "bin", dg = -50, gg = 150, variables = dimensions[index], precision = 4, popSize = 200, pm = 0.3, evaluations = 100000, function = f4(), crossoverType = 2)
            x,fx = f4dec.run()
            if index == 0:
                f4bin3var.append(fx)
            else:
                f4bin6var.append(fx)

    print("Pronađeni minimumi za f3 - DECIMALNO - 3 VAR")
    print(f3dec3var,"\n")
    print("Pronađeni minimumi za f3 - BINARNO - 3 VAR")
    print(f3bin3var,"\n")
    print("Pronađeni minimumi za f3 - DECIMALNO - 6 VAR")
    print(f3dec6var,"\n")
    print("Pronađeni minimumi za f3 - BINARNO - 6 VAR")
    print(f3bin6var,"\n")
    print("Pronađeni minimumi za f4 - DECIMALNO - 3 VAR")
    print(f4dec3var,"\n")
    print("Pronađeni minimumi za f4 - BINARNO - 3 VAR")
    print(f4bin3var,"\n")
    print("Pronađeni minimumi za f4 - DECIMALNO - 6 VAR")
    print(f4dec6var,"\n")
    print("Pronađeni minimumi za f4 - BINARNO - 6 VAR")
    print(f4bin6var,"\n")

if zadatak == 4 or zadatak == 0:
    print("Zadatak4\n")

    popSizes = [30,50,100,200]
    pm = [0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1]

    meidanDict = dict()
    valuesDict = dict()

    for index1 in range(len(popSizes)):

        for index2 in range(len(pm)):

            meidanDict[str(popSizes[index1])+"---"+str(pm[index2])] = 0
            valuesDict[str(popSizes[index1])+"---"+str(pm[index2])] = []

    for index1 in range(len(popSizes)):

        for index2 in range(len(pm)):

            for _ in range(10):

                f3dec = k_turnir_elimination_GA(k = 3, display = "dec", dg = -50, gg = 150, variables = 10, precision = 4, popSize = popSizes[index1], pm = pm[index2], evaluations = 100000, function = f3(), crossoverType = 2)
                x,fx = f3dec.run()

                valuesDict[str(popSizes[index1])+"---"+str(pm[index2])].append(fx)

            meidanDict[str(popSizes[index1])+"---"+str(pm[index2])] = median(valuesDict[str(popSizes[index1])+"---"+str(pm[index2])])

    print(meidanDict)
    print(valuesDict)
    print("\n")

if zadatak == 5 or zadatak == 0:
    print("Zadatak 5\n")

    k_size = [3,5,10,20]

    meidanDict = dict()
    valuesDict = dict()

    for index in range(len(k_size)):

        meidanDict[str(k_size[index])] = 0
        valuesDict[str(k_size[index])] = []

    for index in range(len(k_size)):

        for _ in range(10):

            f3dec = k_turnir_elimination_GA(k = k_size[index], display = "dec", dg = -50, gg = 150, variables = 10, precision = 4, popSize = 200, pm = 0.6, evaluations = 100000, function = f3(), crossoverType = 2)
            x,fx = f3dec.run()

            valuesDict[str(k_size[index])].append(fx)

        meidanDict[str(k_size[index])] = median(valuesDict[str(k_size[index])])

    print(meidanDict)
    print(valuesDict)
    print("\n")


import sys
from collections import deque
input = sys.stdin.readline

n, m = map(int, input().split())
graph = []

for i in range(n):
    graph.append(list(map(int, input().strip('\n'))))

visited = [[0 for j in range(m)] for i in range(n)]
d = [[1, 0], [-1, 0], [0, 1], [0, -1]]

q = deque([[0, 0]])
visited[0][0] = 1

while q:

    x, y = map(int, q.popleft())

    for i in range(len(d)):

        dx = x + d[i][0]
        dy = y + d[i][1]

        if dx < 0 or dy < 0 or dx >= n or dy >= m:
            continue

        if visited[dx][dy] >= 1:
            continue

        if graph[dx][dy] == 1:
            visited[dx][dy] = visited[x][y] + 1
            q.append([dx, dy])

print(visited[n-1][m-1])


from architectures.celeba_net import CelebaNet
from architectures.fashion_mnist import FashionMnistNet
from architectures.mnist_net import MnistNet
# from architectures.mnist_net_pate import MnistNetPate
from architectures.mnist_net_pate_timing import MnistNetPate
from architectures.resnet import ResNet10, ResNet12, ResNet14, ResNet16, \
    ResNet18
from architectures.retino_net import RetinoNet, SimpleRetinoNet
from architectures.small_resnet import ResNet8
from architectures.tiny_resnet import ResNet6
from architectures.vggs import VGG
from datasets.deprecated.chexpert.model.classifier import SingleClassClassifier
from datasets.deprecated.coco.models.tresnet import tresnet_m
from datasets.deprecated.coco.models.tresnet import tresnet_l
from datasets.deprecated.coco.models.tresnet import tresnet_xl
from architectures.densenet import densenet121_capc
from architectures.densenet_xray import get_densenet121_xray
from architectures.densenet_pre import densenetpre
from architectures.resnet_pre import resnetpre
from models.utils_models import get_model_type_by_id, get_model_name_by_id
from architectures.PascalNetwork import *

def get_private_model(name, model_type, args):
    """Private model held by each party."""
    if args.use_pretrained_models == True:
        if model_type == 'densenet121_cxpert':
            model = densenetpre()
            model.dataset = 'cxpert'
        elif model_type == 'resnet50':
            model = resnetpre()
            model.dataset == 'pascal'
    elif model_type.startswith('VGG'):
        model = VGG(name=name, args=args, model_type=model_type)
    elif model_type == 'ResNet6':
        model = ResNet6(name=name, args=args)
    elif model_type == 'ResNet8':
        model = ResNet8(name=name, args=args)
    elif model_type == 'ResNet10':
        model = ResNet10(name=name, args=args)
    elif model_type == 'ResNet12':
        model = ResNet12(name=name, args=args)
    elif model_type == 'ResNet14':
        model = ResNet14(name=name, args=args)
    elif model_type == 'ResNet16':
        model = ResNet16(name=name, args=args)
    elif model_type == 'ResNet18':
        model = ResNet18(name=name, args=args)
    elif model_type == 'resnet50':
        return ResNet(Bottleneck, num_classes=args.num_classes)
    elif model_type.startswith('chexpert'):
        model = SingleClassClassifier(name=name, args=args)
    elif model_type == 'MnistNet':
        model = MnistNet(name=name, args=args)
    elif model_type == 'MnistNetPate':
        model = MnistNetPate(name=name, args=args)
    elif model_type == 'FashionMnistNet':
        model = FashionMnistNet(name=name, args=args)
    elif model_type == 'RetinoNet':
        model = RetinoNet(name=name, args=args)
    elif model_type == 'SimpleRetinoNet':
        model = SimpleRetinoNet(name=name, args=args)
    elif model_type == 'CelebaNet':
        model = CelebaNet(name=name, args=args)
    elif model_type == 'tresnet_m':
        model = tresnet_m(name=name, args=args)
    elif model_type == 'tresnet_l':
        model = tresnet_l(name=name, args=args)
    elif model_type == 'tresnet_xl':
        model = tresnet_xl(name=name, args=args)
    elif model_type == 'densenet121_capc':
        model = densenet121_capc(name=name, args=args)
    elif model_type == 'densenet121_mimic':
        model = get_densenet121_xray(name=name, args=args)
        model.dataset = 'mimic'
    elif model_type == 'densenet121_cxpert':
        model = get_densenet121_xray(name=name, args=args)
        model.dataset = 'cxpert'
    elif model_type == 'densenet121_padchest':
        model = get_densenet121_xray(name=name, args=args)
        model.dataset = 'padchest'
    elif model_type == 'densenet121_vin':
        model = get_densenet121_xray(name=name, args=args)
        model.dataset = 'vin'
    elif model_type in ['densenet121_' + str(x) for x in args.xray_datasets]:
        assert args.dataset in args.xray_datasets
        model = get_densenet121_xray(name=name, args=args)
    else:
        raise Exception(f'Unknown architecture: {model_type}')

    # Set the attributes if not already set.
    if getattr(model, 'dataset', None) == None:
        model.dataset = args.dataset
    if getattr(model, 'model_type', None) == None:
        model.model_type = model_type

    return model

def get_private_model_by_id(args, id=0):
    model_type = get_model_type_by_id(args=args, id=id)
    name = get_model_name_by_id(id=id)
    model = get_private_model(name=name, args=args, model_type=model_type)
    return model