import numpy
import statsmodels.regression
import statsmodels.tools
import scipy.optimize as opti
import scipy.interpolate as interp
import scipy.signal as signal
import matplotlib.pyplot as plt

class PVP:
    def __init__(self, sampling_period=0.01):
        self.sampling_period = sampling_period
        self._kinematic_profile = None
        self.fit_params = [None, None, None]
        self._pvp_params = None
        self._removed_outliers = 0

    def __repr__(self):
        _str = f"{self.__class__.__name__}\n"
        try:
            _str += f"Duration: \t {self.timestamps[-1]:.2f} seconds \n"
        except AttributeError:
            pass
        try:
            _str += f"Number of trajectories: \t {self.kinematic_profile.shape[0]}\n"
        except AttributeError:
            pass
        try:
            _str += f"Outliers Removed: \t {self._removed_outliers}\n"
        except AttributeError:
            pass
        try:
            _str += "PVP stats: \t tau={:.3f}, sigma0={:.2e}, Dtau={:.3f}\n".format(
                *tuple(self._pvp_params.values())[1:]
            )
        except AttributeError:
            pass
        try:
            _str += "PVP fit: \t C={:.2f}, Omega = {:.3f}".format(
                -self.fit_params[1], self.fit_params[2]
            )
        except (AttributeError, TypeError):
            pass
        return _str

    @property
    def timestamps(self):
        return [
            self.sampling_period * i for i in range(self._kinematic_profile.shape[0])
        ]

    @property
    def pvp_params(self):
        self._pvp_params = {
            "tau_index": numpy.argmax(self.std_prof),
            "tau": numpy.argmax(self.std_prof) * self.sampling_period,
            "sigma_0": numpy.max(self.std_prof),
            "Dtau": self.mean_prof[numpy.argmax(self.std_prof)],
        }
        return self._pvp_params

    @property
    def kinematic_profile(self):
        return self._kinematic_profile.T

    @kinematic_profile.setter
    def kinematic_profile(self, item):
        if len(item.shape) != 3:
            raise ValueError(
                f"The shape of kinematic profiles should be of length 3 (time, dimension, number of movements) but it has length {len(item.shape)}"
            )
        self._kinematic_profile = item

    @property
    def _reference_signal(self):
        return self.kinematic_profile

    def _remove_outliers(self, remove_outliers_k_sigma_away=3.5):
        """_remove_outliers

        Remove trajectory outliers, by removing all trajectories that are outside of the range (m +- k sigma), where m is the mean trajectory and sigma is the standard deviation of the set of trajectories.

        .. note:
            The formula above is based on the confidence interval for a Gaussian, and we apply it component per component. A true multivariate approach would use the confidence interval for a multivariate Gaussian see e.g. https://stats.stackexchange.com/questions/29860/confidence-interval-of-multivariate-gaussian-distribution

        :param remove_outliers_k_sigma_away: k, defaults to 3.5
        :type remove_outliers_k_sigma_away: float, optional
        :return: (index of removed trajectories in old array, new array)
        :rtype: tuple(list, array)
        """
        _indx = []
        k = remove_outliers_k_sigma_away
        for ncomp in range(self.kinematic_profile.shape[1]):
            mean = numpy.mean(self.kinematic_profile[:, ncomp, :], axis=0)
            std = numpy.std(self.kinematic_profile[:, ncomp, :], axis=0)
            for n, _traj in enumerate(self.kinematic_profile[:, ncomp, :]):
                if (_traj > mean + k * std).any() or (_traj < mean - k * std).any():
                    _indx.append(n)
        _indx = list(set(_indx))
        self._kinematic_profile = numpy.delete(self._kinematic_profile, _indx, axis=2)
        self.compute_profiles()
        self._removed_outliers += len(_indx)
        return _indx, self._kinematic_profile

    def plot_std_profiles(self, ax=None, fit=True, prof_kwargs=None, fit_kwargs=None):
        """plot_std_profiles

        Plots the standard deviation profiles on the provided axis. If not provided, will create a new figure from scratch.
        If fit is True, will also compute the spline fit to the second and third phase. Keyword arguments to the std plotter (prof_kwargs) and to the fit plotter (fit_kwargs) can also be given.

        .. note::

            You should have fitted the profiles prior to plotting them.

        :param ax: axis on which to draw, defaults to None. If None, creates a new figure and axis to draw on.
        :type ax: plt.axis, optional
        :param fit: whether to plot the spline fit, defaults to True
        :type fit: bool, optional
        :param prof_kwargs: keyword arguments are passed to plotter for the standard deviation profile, defaults to None
        :type prof_kwargs: dict, optional
        :param fit_kwargs: keyword arguments are passed to plotter for the spline fit, defaults to None
        :type fit_kwargs: dict, optional
        """

        if ax is None:
            _, ax = plt.subplots(1, 1)

        prof_kwargs = {} if prof_kwargs is None else prof_kwargs
        fit_kwargs = {} if fit_kwargs is None else fit_kwargs

        y = self.std_prof
        x = self.timestamps
        ax.semilogy(x, y, "k-", lw=3, label="PVP", **prof_kwargs)
        if fit:
            ax.semilogy(
                self.pvp_fit_x,
                self.pvp_fit_y,
                "r-",
                lw=3,
                label="Spline fit",
                **fit_kwargs,
            )
            ax.set_title(
                r"$\tau = {:.3f}, C = {{{:.1f}}}, \Omega = {:.1f}$".format(
                    self.pvp_params["tau"], -self.fit_params[1], self.fit_params[2]
                )
            )
        ax.grid(visible=True, which="minor", linestyle="--")
        ax.set_xlabel("Time (s)")
        ax.set_ylabel(r"$\sigma\mathrm{(t) (m)}$")

    def _compute_mean_profile(self):
        for k in range(self._reference_signal.shape[1]):
            self._mean_prof_incr = numpy.mean(self._reference_signal[:, k, :], axis=0)
            if k == 0:
                self.mean_prof = self._mean_prof_incr ** 2
            else:
                self.mean_prof += self._mean_prof_incr ** 2

        self.mean_prof = numpy.sqrt(self.mean_prof)

        return self.mean_prof

    def _compute_std_profile(self):
        for k in range(self._reference_signal.shape[1]):
            self._std_prof_incr = numpy.std(self._reference_signal[:, k, :], axis=0)
            if k == 0:
                self.std_prof = self._std_prof_incr ** 2
            else:
                self.std_prof += self._std_prof_incr ** 2

        self.std_prof = numpy.sqrt(self.std_prof)
        return self.std_prof

    def compute_profiles(self):
        """compute_profiles

        Computes the mean and standard deviation profiles for a set of trajectories.

        :return: (mean profile, standard deviation profile)
        :rtype: tuple(array, array)
        """
        self._compute_mean_profile()
        self._compute_std_profile()
        return self.mean_prof, self.std_prof

    def compute_pvp(self, remove_outliers_k_sigma_away=3.5):
        """compute_pvp

        Run the full PVP routine:

            + compute profiles
            + remove outliers k sigma away
            + fit profiles

        :param remove_outliers_k_sigma_away: remove outliers k sigma away, defaults to 3.5
        :type remove_outliers_k_sigma_away: float, optional
        :return: standard deviation profile, x and y values of the fit, kinematic profiles
        :rtype: tuple(array, array, array, array)
        """

        _, std_prof = self.compute_profiles()
        _, kinematic_profiles = self._remove_outliers(
            remove_outliers_k_sigma_away=remove_outliers_k_sigma_away
        )
        _, fit_x, fit_y = self._fit_profiles()
        return std_prof, fit_x, fit_y, kinematic_profiles

    def _fit_profiles(self, **optim_kwargs):

        ### Define cost function for optimization procedure
        def monospline(THETA, *args):
            x = args[0]
            y = args[1]
            a, b, mt = THETA
            out = 0
            for i, v in enumerate(x):
                if v < mt:
                    out += (a + b * v - y[i]) ** 2
                else:
                    out += (a + b * mt - y[i]) ** 2
            return out

        ## Once Omega has been determined, run a classical LR on the second phase to get LR diagnostics
        def get_fit_second_phase(y, indx_omega):
            x = [self.sampling_period * i for i in range(indx_omega)]
            yy = y[:indx_omega]
            xx = statsmodels.tools.add_constant(x)
            model = statsmodels.regression.linear_model.OLS(yy, xx)
            self.second_phase_fit = model.fit()
            return self.second_phase_fit

        indx_tau, tau, sigma0, Dtau = tuple(self.pvp_params.values())

        ### Initialize optimization algorithm - Data and start parameters
        theta0 = optim_kwargs.pop(
            "spline_param_guess", [sigma0, -5, 1]
        )  # should work for most cases
        if indx_tau:
            x = self.timestamps[0:-indx_tau]
            y = numpy.log2(self.std_prof[indx_tau:])
        else:
            x = self.timestamps
            y = numpy.log2(self.std_prof)

        ## Global Optimization
        n_iter = optim_kwargs.pop("basinhopping_n_iter", 50)

        res = opti.basinhopping(
            func=monospline,
            x0=theta0,
            niter=n_iter,
            minimizer_kwargs={
                "method": "Nelder-Mead",
                "args": (x, y),
                "options": {"maxiter": 1000, "disp": 0},
            },
        )

        a, b, c = res.x
        c0 = int(numpy.ceil(c / self.sampling_period))
        self.omega = c
        a, b = get_fit_second_phase(y, c0).params

        _yy = [a + b * i * self.sampling_period for i in range(0, c0)] + [
            a + c * b for i in range(c0, len(y[indx_tau:]))
        ]
        _yy = [2 ** v for v in _yy]
        t_fit = [
            self.pvp_params["tau"] + i * self.sampling_period
            for i in range(0, len(_yy))
        ]
        self.pvp_fit_x = t_fit
        self.pvp_fit_y = _yy
        self.fit_params = [a, b, c + self.pvp_params["tau"]]
        return self.fit_params, self.pvp_fit_x, self.pvp_fit_y

    def _extend(self, trajectory, extend_to=3):
        """_extend extend trajectories

        Extends the self._kinematic_profile buffer with a new trajectory while ensuring the series in the buffer always have the same size as the trajectory. For example, if the buffer has shape (X, Y) and the trajectory series has length (Z):

            + if Z > Y, then the buffer is filled with the last values to reach shape (X, Z)
            + if Z < Y, then the trajectory is filled with the last value to reach shape (1, Y)

        The minimum duration of the series can be set with extend_to.

        :param trajectory: trajectory to add to the self._kinematic_profile buffer
        :type trajectory: array_like
        :param extend_to: minimum duration of the series in seconds, defaults to 3
        :type extend_to: int, optional
        """
        if len(trajectory.shape) == 1:
            trajectory = trajectory.reshape(-1, 1)
        if self._kinematic_profile is None:  # First traj
            Nmin = 1 + int(numpy.ceil(extend_to / self.sampling_period))
            if trajectory.shape[0] < Nmin:
                fill = numpy.full(
                    shape=(Nmin - trajectory.shape[0], trajectory.shape[1]),
                    fill_value=trajectory[-1, :],
                )
                trajectory = numpy.concatenate((trajectory, fill), axis=0)
            self._kinematic_profile = numpy.expand_dims(trajectory, axis=2)
        else:
            if self._kinematic_profile.shape[0] < trajectory.shape[0]:
                fill = numpy.full(
                    shape=(
                        -self._kinematic_profile.shape[0] + trajectory.shape[0],
                        self._kinematic_profile.shape[1],
                        self._kinematic_profile.shape[2],
                    ),
                    fill_value=self._kinematic_profile[-1, :, :],
                )

                self._kinematic_profile = numpy.concatenate(
                    (self._kinematic_profile, fill), axis=0
                )

            elif self._kinematic_profile.shape[0] > trajectory.shape[0]:
                fill = numpy.full(
                    shape=(
                        self._kinematic_profile.shape[0] - trajectory.shape[0],
                        self._kinematic_profile.shape[1],
                    ),
                    fill_value=trajectory[-1, :],
                )
                trajectory = numpy.concatenate((trajectory, fill), axis=0)
            self._kinematic_profile = numpy.concatenate(
                (self._kinematic_profile, numpy.expand_dims(trajectory, axis=2)), axis=2
            )
        return self._kinematic_profile

    def _correct_edges(self, container, method="speed_threshold", edges = ['start', 'stop'], thresholds = [1,5], **kwargs):
        """_find_start correct start

        Trajectories may not always be consistently segmented. This function performs a correction for the start point, as indicated by the method.

            + method = 'speed_threshold' :
                Computes a threshold for speed as x_percent * max speed. All points the target and the first time when the threshold is crossed are removed.
                \*\*kwargs = {'percent' : x_percent}

        :param container: output from add_traj
        :type container: numpy.ndarray
        :param method: method to correct start, defaults to "speed_threshold"
        :type method: str, optional
        :return: trajectory with correction for speed
        :rtype: numpy.ndarray
        """
        time, traj, speed = container

        indx = 1
        stp_index = len(traj)-1
        if method == "speed_threshold":
            ### Removes points until having reached a speed that is 1% of the max speed.
            max_speed = numpy.max(numpy.abs(speed[1:]))

            if 'start' in edges:
                percent = thresholds[0]
                while abs(speed[indx]) < max_speed * percent / 100:
                    indx += 1
            if 'stop' in edges:
                try:
                    percent = thresholds[1]
                except IndexError:
                    percent = thresholds[0]

                while abs(speed[stp_index]) < max_speed * percent / 100: # find first bump
                    stp_index -= 1
                while abs(speed[stp_index]) > max_speed * percent / 100: # find start of decrease
                    stp_index -= 1

        else:
            raise NotImplementedError(
                "Only method speed_threshold is implemented for now."
            )

        container = numpy.concatenate(
            (time[indx:stp_index].reshape(1, -1), traj[indx:stp_index].reshape(1, -1)), axis=0
        )
        container = numpy.concatenate((container, speed[indx:stp_index].reshape(1, -1)), axis=0)

        return container, indx, stp_index

    def plot_kinematic_profiles(self, ax=None, **kwargs):
        """plot_kinematic_profiles

        Plots the kinematic profiles on the provided axis. If not provided, will create a new figure from scratch.

        :param ax: axis on which to draw, defaults to None. If None, creates a new figure and axis to draw on.
        :type ax: plt.axis, optional
        :param **kwargs: keyword arguments are passed to plt.plot()
        :type **kwargs: key-values
        """
        if ax is None:
            fig, ax = plt.subplots(1, 2)

        x = self.timestamps
        for k in range(self.kinematic_profile.shape[1]):
            for y in self.kinematic_profile[:, k, :]:
                ax[k].plot(x, y, "-", **kwargs)
                ax[k].set_xlabel("Time (s)")
                ax[k].set_ylabel("Position")

    def add_trajectory(self, t, *args, extend_to=3, target=None, correct_edges=False, correct_edges_kwargs = None):
        """Add trajectory to the set from which PVPs are computed

        Pass the time series, and any number of positional series. For example in dim3 with x, y, z, you would call (with defaults kwargs)

        .. code-block:: python

            pvp.add_trajectory(t, x, y, z, extend_to = 3, target = None, correct_start = False)

        You control the duration of the PVP (e.g. how far in time trajectories are extended). You also need to specify the target location for each trajectory. You can optionally synchronize the trajectories by pre-processing them (correct_start). Currently, a simple thresholding rule takes care of this synchronization.

        :param t: time series
        :type t: numpy.array like
        :param args: positional series
        :type args: numpy.array like
        :param extend_to: minimal PVP duration, defaults to 3
        :type extend_to: int, optional
        :param target: target location, defaults to None. If None, will use the null vector as target.
        :type target: iterable, optional
        :param correct_start: whether to correct the location of the start of the movement for synchronization, defaults to False
        :type correct_start: bool, optional
        """

        default_correct_edges_kwargs = dict(method="speed_threshold", edges = ['start'], percent=[2, 5])
        if correct_edges_kwargs is not None:
            default_correct_edges_kwargs.update(correct_edges_kwargs)

        target = [0 for i in args] if target is None else target
        projections = self._project(target, *args)

        container = self._interp_filt(
            numpy.array(t),
            *projections,
            deriv=0,
            resampling_period=self.sampling_period,
        )

        indx = 0
        if correct_edges:
            _norm = numpy.sqrt(numpy.sum(container[1, :, :] ** 2, axis=1))
            tmp_container = self._interp_filt(
                container[0, :, 0],
                _norm,
                deriv=1,
                resampling_period=self.sampling_period,
            )
            _, indx, stp_indx = self._correct_edges(
                tmp_container, **default_correct_edges_kwargs
            )

        self._extend(container[1, :, :], extend_to)

    def _get_orthonormal_basis(self, target, x0):
        target = numpy.asarray(target).squeeze()
        x0 = numpy.atleast_1d(numpy.asarray(x0).squeeze())
        if x0.shape[0] == 1:
            return self._bon1(target, x0)
        elif x0.shape[0] == 2:
            return self._bon2(target, x0)
        elif x0.shape[0] == 3:
            return self._bon3(target, x0)
        else:
            raise NotImplementedError("Dimensions above 3 are not supported yet. ")

    # below does not reliably produce an orthonormal basis
    # switching to a manual cse disjunction up to 3D for now

    # def _get_orthonormal_basis(self, target, x0):
    #     target = numpy.asarray(target).squeeze()
    #     x0 = numpy.asarray(x0).squeeze()
    #     random_basis = numpy.array(
    #         [
    #             (target - x0),
    #             *[
    #                 -1 + 2 * numpy.random.random(x0.shape[0])
    #                 for v in range(x0.shape[0] - 1)
    #             ],
    #         ]
    #     ).T
    #     self.Q, _ = numpy.linalg.qr(random_basis)
    #     return self.Q

    def _bon1(self, target, x0):
        return normalize(target - x0).reshape(-1, 1)

    def _bon2(self, target, x0):
        v1 = normalize(target - x0)
        v2 = numpy.array([-v1[1], v1[0]])
        return numpy.array([[v1], [v2]]).T

    def _bon3(self, target, x0):
        array = self._bon2(target, x0).T
        vec3 = numpy.cross(array[0], array[1])
        return numpy.array([[array[0]], [array[1]], [vec3]]).T

    def _project_x(self, Q, target, x):
        u = (numpy.asarray(x) - numpy.asarray(target)).reshape(-1, 1)
        return (Q.T @ u).squeeze()

    def _project(self, target, *args):
        dim = len(args)
        output = numpy.zeros(shape=(dim, len(args[0])))
        args = numpy.array(args).T
        Q = self._get_orthonormal_basis(target, args[0, :])
        for n, arg in enumerate(args):
            output[:, n] = self._project_x(Q, target, arg)
        return output

    def _interp_filt(
        self,
        t,
        *args,
        resampling_period=0.01,
        filter_kwargs={"filtername": "kaiser", "fc": 10, "rdb": 10, "width": 5},
        deriv=2,
    ):
        """_interp_filt interpolates and filters a 1D trajectory

        Takes a trajectory, resamples it with the chosen resampling_period and filters it with the given filter. Also provides the unfiltered derivatives up to order "deriv".

        :param t: trajectory time
        :type t: array like
        :param x: trajectory position
        :type x: array like
        :param resampling_period: timestep at which the trajectory will be down/over sampled, defaults to 0.01
        :type resampling_period: float, optional
        :param filter_kwargs: scipy.signal filter description, defaults to {"filtername": "kaiser", "fc": 10, "rdb": 10, "width": 5}
        :type filter_kwargs: dict, optional
        :param deriv: order for the trajectory derivatives, defaults to 2
        :type deriv: int, optional
        :return: an array, where the first line is the time vector, and all other lines are the nth derivatives of the trajectory (0 <= n <= deriv).
        :rtype: numpy.ndarray
        """

        t = numpy.asarray(t)
        t = t - t[0]  # set null time target
        Ts = resampling_period
        output_container = None
        for n, x in enumerate(args):
            x = numpy.asarray(x)
            interpolator = interp.interp1d(t, x, fill_value="extrapolate")
            resampling_instants = numpy.linspace(
                0,
                t[-1] + (Ts - t[-1] % Ts),
                num=1 + int((t[-1] + (Ts - t[-1] % Ts)) / Ts),
            )
            x_interp = interpolator(resampling_instants)

            if filter_kwargs["filtername"] == "kaiser":
                N, beta = signal.kaiserord(
                    filter_kwargs["rdb"], filter_kwargs["width"] * 2 * Ts
                )
                taps = signal.firwin(
                    N, filter_kwargs["fc"] * 2 * Ts, window=("kaiser", beta)
                )
                filtered_x = signal.filtfilt(taps, 1, x_interp)
            else:
                b, a = filter_kwargs["b"], filter_kwargs["a"]
                filtered_x = signal.filtfilt(b, a, x_interp)

            container = numpy.concatenate(
                (resampling_instants.reshape(1, -1), filtered_x.reshape(1, -1)), axis=0
            )

            ## Compute derivatives
            resampling_instants = numpy.append(
                resampling_instants, resampling_instants[-1] + Ts
            )
            for i in range(deriv):
                filtered_x = numpy.concatenate(
                    (
                        filtered_x.reshape(-1)[0].reshape(1, -1),
                        filtered_x.reshape(1, -1),
                    ),
                    axis=1,
                )
                filtered_x = numpy.divide(
                    numpy.diff(filtered_x), numpy.diff(resampling_instants)
                )
                container = numpy.concatenate((container, filtered_x), axis=0)

            if n == 0:
                output_container = numpy.expand_dims(container, axis=2)
            else:
                container = numpy.expand_dims(container, axis=(2))
                output_container = numpy.concatenate(
                    (output_container, container), axis=2
                )
        return output_container

class PVP_alpha(PVP):
    @property
    def _reference_signal(self):
        return numpy.expand_dims(self.kinematic_profile[:, 0, :], 1)

class PVP_total(PVP):
    @property
    def _reference_signal(self):
        return self.kinematic_profile

class PVP_generalized(PVP):
    @property
    def _reference_signal(self):
        return self.kinematic_profile

    def _compute_std_profile(self):
        std_prof = numpy.empty(shape=(self._kinematic_profile.shape[0]))
        for nt, t in enumerate(self._kinematic_profile):
            cov = numpy.cov(t)
            if len(cov.shape) >= 2:
                std_prof[nt] = (numpy.linalg.det(cov)) ** (
                    1 / 2 / self._kinematic_profile.shape[1]
                )
            else:
                std_prof[nt] = (cov.squeeze()) ** (
                    1 / 2 / self._kinematic_profile.shape[1]
                )
        self.std_prof = std_prof
        return self.std_prof

def normalize(a):
    return a / numpy.linalg.norm(a, 2)


from json import loads
from logging_tools import Logger
from population import create_sample
from requests import get
from requests import post
from requests.auth import HTTPBasicAuth
from string import Template
from time import sleep
from uuid import uuid1

""" Template for the whisk rest api. """
whisk_rest_api = Template(
    '$APIHOST/api/v1/namespaces/$NAMESPACE/$ENDPOINT/$VALUE'
)

def evolution_parameters(function=3, instance=1, dim=3, population_size=20):
    """ Returns the evolution parameters. """

    return {
        'id': str(uuid1()),
        'problem': {
            'name': 'BBOB',
            'function': function,
            'instance': instance,
            'search_space': [-5, 5],
            'dim': dim,
            'error': 1e-8
        },
        'population': [],
        'population_size': population_size,
        'experiment': {
            'experiment_id': 'dc74efeb-9d64-11e7-a2bd-54e43af0c111',
            'owner': 'mariosky',
            'type': 'benchmark'
        },
        'algorithm': {
            'name': 'GA',
            'iterations': 5,
            'selection': {
                'type': 'tools.selTournament',
                'tournsize': 12
            },
            'crossover': {
                'type': 'cxTwoPoint',
                'CXPB': [0, 0.2]
            },
            'mutation': {
                'type': 'mutGaussian',
                'mu': 0,
                'sigma': 0.5,
                'indpb' : 0.05,
                'MUTPB':0.5
            }
        }
    }

def create_parameters(settings, population=None):
    """ Creates the evolution parameters with a population. """

    parameters = evolution_parameters(
        settings.function,
        settings.instance,
        settings.dim,
        settings.population_size
    )
    parameters['population'] = population or create_sample(parameters)
    return parameters

def get_request_data(settings, endpoint, value, json={}):
    """ Gets the request data. """

    auth = settings.auth.split(':')
    return {
        'url': whisk_rest_api.safe_substitute(
            APIHOST=settings.apihost,
            NAMESPACE=settings.namespace,
            ENDPOINT=endpoint,
            VALUE=value,
        ),
        'json': json,
        'params': {
            'blocking': str(settings.blocking),
            'result': 'True'
        },
        'auth': HTTPBasicAuth(auth[0], auth[1]),
        'verify': not settings.insecure
    }

def crossover_migration(pop1, pop2, key = lambda p: p['fitness']['score']):
    """ Does the crossover migration. """

    pop1.sort(key=key)
    pop2.sort(key=key)
    size = min(len(pop1), len(pop2))

    cxpoint = int(size / 2)

    pop1[cxpoint:] = pop2[:cxpoint + size % 2]

    return pop1

def request_evolution(settings, population):
    """ Gets the population using blocking. """

    data = get_request_data(settings, 'actions', 'gaService', population)

    logger = Logger(settings.verbose)
    logger.log('POST request to ' + data['url'])

    response = post(**data).json()
    logger.log('POST complete!')
    return response

def request_evolution_id(settings, population):
    """ Evolves a population and returns it's OpenWhisk activationid. """

    response = request_evolution(settings, population)
    return response['activationId']

def request_evolved(settings, id):
    """ Gets the population data with it's OpenWhisk activation id. """

    data = get_request_data(settings, 'activations', id)

    logger = Logger(settings.verbose)
    logger.log('Polling activationId ' + str(id))

    for _ in range(0, settings.timeout):
        logger.log('GET request to ' + data['url'])

        response = get(**data).json()
        logger.log('GET complete!')

        if 'error' not in response:
            return loads(response['response']['result']['value'])

        sleep(1)

    raise ValueError('Timeout exception.')

def evolve(settings, population):
    """ Evolves the population with the given settings. """

    response = request_evolution(settings, population)
    print(response)
    if 'activationId' in response:
        return request_evolved(settings, response['activationId'])
    else:
        return loads(response['value'])


import os
from pathlib import Path
import pandas as pd
import numpy as np
from matplotlib import pyplot as plt
from shutil import copyfile

config = {
    # General
    'symbol': 'spy',
    'symbol_name': 'S&P500',
    'category': {'unespecified': ['spy']},  # 'gld', 'spy','xle', 'emb','dia', 'qqq', 'ewp'
    'extension': '.csv',
    'separator': ';',
    'desired_length': 550,   # for mahab states
    'pattern': '_no_missing_data',  # besides the name, gaps have been handled (the name it's a typo)
    'cols': ['open', 'high', 'low', 'close', 'volume', 'datetime', 'gap', 'timestamp'],
    'names_per_set': {
        'dev': 'devset',
        'train': 'train',
        'mah': 'mahalanobis_state'
    },

    # ############ For sec level
    # 'years_to_explore': ['2016', '2017', '2018', '2019', '2020'],
    # 'output_path': 'C:\\Users\\suare\\data\\tmp\\spy_seeds_seconds',
    # 'name': 'spy-seconds',
    # 'lvl_str': 's',
    # 'path': 'C:\\Users\\suare\\data\\analysis\\quantquote\\',
    # 'resample': False,  # resampling done in a previous script
    # 'ms_field': 'timestamp',  # time
    # 'dt_field': 'datetime',   # date
    # 'desired_abs_mean_tresh': 0.01,
    # 'desired_abs_min_tresh': 0.00000000000001,
    # 'allowed_outliers_pct': 0.01,
    # 'pattern': 'csv',  # no pattern needed here by 05/04/2020
    # 'prefix': 'table_'
    #### get largets and get smallest applied to 1 (min and maxs)

    # ############ For minute level
    'years_to_explore': ['2001', '2002', '2003', '2004', '2005', '2006', '2007', '2008', '2009', '2010',
                         '2011', '2012', '2013', '2014', '2015', '2016', '2017', '2018', '2019', '2020'],
    # IMP. don't do 2020 alone or period id == 1
    'output_path': 'C:\\Users\\suare\\data\\tmp\\spy_seeds_minutes',
    'name': 'spy-minutes',
    # 'lvl_str': 'm',
    # 'desired_abs_mean_tresh': 0.1,
    'lvl_str': 'm',
    'desired_abs_mean_tresh': 1,
    'path': 'C:\\Users\\suare\\data\\analysis\\quantquote\\',
    'resample': True,  # resampling done in a previous script
    'ms_field': 'timestamp',  # time
    'dt_field': 'datetime',   # date
    'desired_abs_min_tresh': 0.00000000000001,
    'allowed_outliers_pct': 0.01,
    'prefix': ''
}

def read_file(filename: str) -> pd.DataFrame:
    path = os.sep.join([config['path'], filename])
    filepath = os.sep.join([path, config['prefix'] + config['symbol'] + config['extension']])
    if os.path.isfile(filepath):
        print(f'Reading {filename}')
        df = pd.read_csv(filepath, names=config['cols'])
        # print(df.head())
        return df

def get_pretraining_states(mahabset_df: pd.DataFrame, config: dict) -> dict:
    """
    This function get the dataframes for pretraining
    :param mahabset_df:
    :param config:
    :return:
    """
    mahabset_df = remove_non_trading_hours_minimised(df=mahabset_df)  # remove non trading hours

    # Generate close price returns and moving average to select the period with a mean close to 0
    log_ret = False  # otherwise percentual
    if log_ret:
        mahabset_df['close_returns'] = np.log(mahabset_df['close'] / mahabset_df['close'].shift(1))
        # mahabset_df['close_returns'] = np.log(1 + mahabset_df['close'].pct_change())
    else:
        mahabset_df['close_returns'] = mahabset_df['close'] / mahabset_df['close'].shift(1) - 1
        # mahabset_df['close_returns'] = mahabset_df['close'].pct_change(1)  # same result
    mahabset_df.dropna(inplace=True)
    sma_col = f"SMA_{config['desired_length']}"
    mahabset_df[sma_col] = mahabset_df['close_returns'].rolling(window=config['desired_length']).mean()
    mahabset_df[sma_col+'_abs'] = mahabset_df[sma_col].abs()
    mahabset_df['roll_std'] = mahabset_df['close_returns'].rolling(window=config['desired_length']).std()
    mahabset_df['SMA_start_date'] = mahabset_df[config['dt_field']].shift(config['desired_length'])
    mahabset_df['SMA_start_ms'] = mahabset_df[config['ms_field']].shift(config['desired_length'])

    # Drop first rows as the moving average is NaN
    len_with_nans = len(mahabset_df)

    # Prepare extra cols
    # mahabset_df['datetime'] = mahabset_df.index.astype(str)   #  maybe for min level is relevant
    mahabset_df.set_index(config['dt_field'], drop=False, inplace=True)
    mahabset_df.dropna(inplace=True)
    assert (len_with_nans - len(mahabset_df)) == config['desired_length'], 'There are non expected NaNs'

    states_dict = dict()
    for i in range(1, 4):  # States 1, 2 and 3 (hardcoded by now)
        selected_df = identify_state(config, mahabset_df, sma_col, state_id=i)
        # assert len(selected_df) == 1, "The maximum value is not a unique value."
        print(len(selected_df))
        print(selected_df)

        for idx, rw in selected_df.iterrows():
            print(f"Current selection from {rw['SMA_start_date']} to {rw[config['dt_field']]} with mean {sma_col}")
            mah_state = \
                mahabset_df[(mahabset_df['SMA_start_date'].between(rw['SMA_start_date'], rw[config['dt_field']]))]

            # avoid overwriting an state with another which len is <=35 (it has to be greater because of the indicators)
            if len(mah_state) >= 35:
                states_dict[i] = mah_state
                states_dict[i].sort_index(ascending=True, inplace=True)

    assert len(states_dict) == 3, f"State missing or diff than 3 states? states ok:{states_dict.keys()}"
    return states_dict

def identify_state(config: dict, mahabset_df: pd.DataFrame, sma_col: str, state_id: int) -> pd.DataFrame:
    """
    This function identifies the last row of a mahab state depending on a logic defined manually for that state id.
    Values 35, 20 and 20 were given manually to be able to have enough records hourly for all mahab states.
        These have been left as values by default after.
    """
    if state_id == 1:  # Select one close to 0 (the closest)
        # Select one close to 0
        # (not the closest cos there are period = 0 due to lack of liquidity at certain frequencies)
        selected_df = mahabset_df[mahabset_df[sma_col + '_abs'] <= config['desired_abs_mean_tresh']]
        # selected_df = selected_df[selected_df[sma_col + '_abs'].max() == selected_df[sma_col + '_abs']]
        # Filter by desired mean fpr the lateral movement (the one with greatest STDEV)
        # selected_df = selected_df[selected_df['roll_std'].max() == selected_df['roll_std']]
        # inst.of max, t.g.m.
        selected_df = selected_df[selected_df['roll_std'].isin(list(selected_df['roll_std'].nlargest(35)))]
    elif state_id == 2:  # Select one negative (min)
        # get the min from a boundary. filter periods with 0 return at all. it may be due to lack of liquidity at s lvl
        selected_df = mahabset_df[mahabset_df[sma_col + '_abs'] >= (config['desired_abs_min_tresh'])]
        # selected_df = selected_df[selected_df[sma_col].min() == selected_df[sma_col]]
        # inst.of min, to get many
        selected_df = selected_df[selected_df[sma_col].isin(list(selected_df[sma_col].nsmallest(20)))]
    elif state_id == 3:  # Select one positive (max)
        # selected_df = mahabset_df[mahabset_df[sma_col].nlargest(3) == mahabset_df[sma_col]]
        # inst.of max, to get many
        selected_df = mahabset_df[mahabset_df[sma_col].isin(list(mahabset_df[sma_col].nlargest(20)))]
    else:
        assert False, "The trend/pattern for this state has not been specified"
    return selected_df

def remove_non_trading_hours_minimised(df) -> pd.DataFrame:
    trading_dates = pd.read_csv('trading_dates_Q11998_to_Q32021.csv')  # this list has format: dd/MM/yyyy
    # inverting date format (hardcoded)
    trading_dates = trading_dates.trading_dates.astype(str).apply(lambda x: x[6:10]+'-'+x[3:5]+'-'+x[:2])
    df.index = pd.to_datetime(df.datetime)
    df = df.between_time('09:31', '15:59')
    df = df[df.index.astype(str).str[:10].isin(trading_dates)]
    return df

def remove_non_trading_hours(df, config: dict(), level: str = None) -> pd.DataFrame:
    # Parse cols, dates and sort
    # this may be useful at minute level
    # df['date'] = pd.to_datetime(df['date'], format='%Y%m%d').dt.strftime('%Y-%m-%d')
    # df['time'] = (pd.to_datetime(df['time'].astype(str).str.strip(), format='%H%M').dt.strftime('%H:%M'))
    # df['datetime'] = df.date.astype(str) + ' ' + df.time.astype(str)
    # trading_dates = df.datetime.str[:10].unique()  # list of market days
    trading_dates = pd.read_csv('trading_dates_Q11998_to_Q32021.csv')  # this list has format: dd/MM/yyyy
    # inverting date format (hardcoded)
    trading_dates = trading_dates.trading_dates.astype(str).apply(lambda x: x[6:10]+'-'+x[3:5]+'-'+x[:2])
    df.index = pd.to_datetime(df.datetime)
    df.drop(columns=['date', 'time', 'datetime', 'splits', 'earnings', 'dividends'],
            errors='ignore', inplace=True)
    df.sort_index(inplace=True, ascending=True)

    # Resample (but not fill gaps. This should have been done already)
    print(f' - Original size: {len(df)}')
    if config['resample']:
        ohlc_dict = {'open': 'first', 'high': 'max', 'low': 'min', 'close': 'last', 'volume': 'sum'}
        df = df.resample(level.split('-')[0]).agg(ohlc_dict)
        print(f' - Size after resampling at {level}: {len(df)}')

    # Remove non-trading hours and non-trading days
    df = df.between_time('09:31', '15:59')  # This is potentially the only important line from the function
    print(f' - Size after filtering out non-market hours: {len(df)}')
    df = df[df.index.astype(str).str[:10].isin(trading_dates)]
    # (low impact..) df = df.between_time('09:31','13:00') for half days (shortened sessions / early closing)
    print(f' - Size after filtering out non-trading days: {len(df)}')

    if config['resample']:
        # Fill gaps
        df['volume'] = df['volume'].fillna(0)
        df['close'] = df['close'].ffill()
        df['open'] = df['open'].fillna(df['close'])
        df['low'] = df['low'].fillna(df['close'])
        df['high'] = df['high'].fillna(df['close'])

    return df

def parse_and_save(file_dict: dict, level: str, period_id: int,
                   setid: str, setname: str, config: dict, all_files: list()) -> list():
    file_path = file_dict[setid]
    if setid == 'mah':
        # For the mahalanobis set, it creates a moving average of x examples over the previous period to the devset.
        states_dict = get_pretraining_states(mahabset_df=pd.read_csv(file_path, sep=config['separator']),
                                             config=config)
        for k in states_dict.keys():
            # non trading hours have been removed in get_pretraining_states
            state_filepath = \
                os.sep.join([config['output_path'], level, str(period_id), f'{config["symbol"]}_{setname}_{k}.csv'])
            states_dict[k] = \
                remove_non_trading_hours(df=states_dict[k], config=config, level=level)  # for the sake of standarisation
            assert len(states_dict[k]) >= 35, "Mahalanobis set too small (must be >40 for technical indicators)"
            states_dict[k].to_csv(state_filepath, sep=';')
            all_files.append(state_filepath)
    else:
        set_filepath = os.sep.join([config['output_path'], level, str(period_id), f'{config["symbol"]}_{setname}.csv'])
        print(file_path)
        df = remove_non_trading_hours(df=pd.read_csv(file_path, sep=config['separator']), config=config, level=level)
        assert len(df) >=35 , "Dataset set too small (must be >=35 for technical indicators)"
        df.to_csv(set_filepath, sep=';')
        all_files.append(set_filepath)
    return all_files

def compute(files: dict, periods: (), period_id: int, files_for_indicators: list, config: dict) -> (dict, list):
    """
    This function orchestrates the whole process in both levels
    """
    period, mahab_period, dev_period = periods
    files[period] = dict()

    # 1. it picks a period and changes the name.
    for level in os.listdir(config['path']):
        files[period][level] = dict()

        # Process second and minute level data. For min level, filter only files with pattern (as there are many others)
        if config['lvl_str'] in level and '.' not in level:  # and '30' in level:  # IMP!! comment last condition out
            print('=========='+level+'\n'+'==========')
            lvl_path = config['path'] + level + os.sep + config['symbol_name']
            for file in os.listdir(lvl_path):  # all of these loops are not efficient at all
                if '.csv' in file and config['pattern'] in file:
                    if mahab_period in file:
                        files[period][level]['mah'] = lvl_path + os.sep + file
                    if dev_period in file:
                        files[period][level]['dev'] = lvl_path + os.sep + file
                    if period in file:
                        files[period][level]['train'] = lvl_path + os.sep + file

            # 2. Export all sets in a folder with a period number (like a seed)
            Path(os.sep.join([config['output_path'], level, str(period_id)])).mkdir(parents=True, exist_ok=True)
            for setid, setname in config['names_per_set'].items():
                print(f'set id: {setid}')
                files_for_indicators = \
                    parse_and_save(file_dict=files[period][level],
                                   level=level, period_id=period_id, setid=setid, setname=setname,
                                   config=config, all_files=files_for_indicators)
            # Debug
            print(files[period][level]['dev'])
            print(os.sep.join([config['output_path'], level, str(period_id), f'{config["symbol"]}_devset.csv']))

    return files, files_for_indicators

##########################################################
# Specific handling for second or minute level frequencies

def compute_seconds() -> (pd.DataFrame, list):
    mahab_m = '07'
    dev_m = '10'
    first = True
    period_id = 1
    files = dict()
    files_for_indicators = list()

    for yr in config['years_to_explore']:
        # this will need to be refactored/changed (maybe to an iterator) for min level
        for q_month in ['01', '04', '07', '10']:  # just for s level
            # it picks the prior period as a devset
            if not first:
                mahab_period = dev_period
                dev_period = period
            else:
                mahab_period = f'{int(yr) - 1}-{mahab_m}'
                dev_period = f'{int(yr) - 1}-{dev_m}'
                first = False
            period = yr + '-' + q_month

            # Compute periods in order
            files, files_for_indicators = \
                compute(files=files, periods=(period, mahab_period, dev_period), period_id=period_id,
                        files_for_indicators=files_for_indicators, config=config)
            period_id = period_id + 1
    return files, files_for_indicators

def compute_minutes() -> (pd.DataFrame, list):
    files = dict()
    files_for_indicators = list()
    period_id = 1

    for yr in config['years_to_explore']:
        period = str(yr)
        dev_period = str(int(yr) - 1)
        mahab_period = str(int(yr) - 2)
        files[period] = dict()

        # if period_id == 6:
        # Compute periods in order
        files, files_for_indicators = \
            compute(files=files, periods=(period, mahab_period, dev_period), period_id=period_id,
                    files_for_indicators=files_for_indicators, config=config)

        period_id = period_id + 1
    return files, files_for_indicators

compute_func = {
    's': compute_seconds,
    'm': compute_minutes,
    'h': compute_minutes,
}

if __name__ == "__main__":
    # Difference handling periods at the second and minutes level due to data granularity and volume
    all_files_dict, file_list = compute_func[config['lvl_str']]()

    # Let's generate /txt files too in a TMP location
    pd.DataFrame({'files': file_list}).to_csv(f'tmp/files_for_indicators_lvl-{config["lvl_str"]}.csv')
    # Then trigger from here the whole convertion (technical indicators).


from tkinter import *
from tkinter import messagebox
from tkinter import ttk #css for tkinter
from configparser import ConfigParser

# import io
# import urllib.request
# import base64

import time

import ssl
ssl._create_default_https_context = ssl._create_unverified_context

import requests

weather_url = 'http://api.openweathermap.org/data/2.5/weather?q={},{}&appid={}'

bg_url = 'https://api.unsplash.com/search/photos?query={}&client_id={}'

config_file = 'config.ini'
config = ConfigParser()
config.read(config_file)
weather_api_key = config['weather_api_key']['key']
unsplash_access_key = config['unsplash_api_key']['access_key']

def get_image(city):
    image_index = 0
    result = requests.get(bg_url.format(city, unsplash_access_key))
    if result:
        json = result.json()
        first_url = json['results'][image_index]['urls']['raw']
        return first_url
        u = urllib.request.urlopen(first_url)
        image_byt = u.read()
        u.close()
        # photo = PhotoImage(data=base64.encodestring(image_byt))
        # return photo
    else:
        return None

def get_weather(city, country):
    result = requests.get(weather_url.format(city, country, weather_api_key))
    if result:
        json = result.json()
        # (City, Country, temp_celsius, temp_fahrenheit, icon, weather)
        city = json['name']
        country = json['sys']['country']
        temp_kelvin = json['main']['temp']
        temp_celsius = temp_kelvin - 273.15
        temp_fahrenheit = temp_celsius * 9/5 + 32
        icon = json['weather'][0]['icon']
        weather = json['weather'][0]['main']
        final = (city, country, temp_celsius, temp_fahrenheit, icon, weather)
        return final
    else:
        return None

def search():
    city = city_text.get()
    country = country_text.get()
    weather = get_weather(city, country)
    photo = get_image(city)
    if weather and city and country:
        location_lbl['text'] = '{}, {}'.format(weather[0], weather[1])
        image['bitmap'] = 'weather_icons/{}.png'.format(weather[4])
        weather_lbl['text'] = weather[5]
        temp_lbl['text'] = '{:.2f}°C \n {:.2f}°F'.format(weather[2], weather[3])
        url_lbl['text'] = photo

    elif not city or not country:
        messagebox.showerror('Error', 'Cannot find city: {} in country: {}'.format(city, country))
    else:
        messagebox.showerror('Error', 'Error Occured')

app = Tk()
app.title("Weather App")
app.geometry('900x700')

# city_image = Tk()

#Top Frame
top_frame = LabelFrame(app, text='Search', padx=50, pady=5)
top_frame.pack(side='top',padx=10, pady=10)

##Search Field
city_text = StringVar()
city_entry = ttk.Entry(top_frame, textvariable=city_text)
city_entry.pack(pady=2)

##Country Field
country_text = StringVar()
country_entry = ttk.Entry(top_frame, textvariable=country_text)
country_entry.pack(pady=2)

##Search Button
search_btn = ttk.Button(top_frame, text="Search by City, Country", width=20, command=search)
search_btn.pack(pady=10)

#Bottom Frame
bottom_frame = LabelFrame(app, text='Details', height=500, padx=100, pady=5)
bottom_frame.pack(side='top', padx=10, pady=10)

##Location
location_lbl = ttk.Label(bottom_frame, text='--', font=('bold', 20))
location_lbl.pack()

##Image
image = Label(bottom_frame, bitmap='--', relief='sunken')
image.pack(pady=10)

##Weather
weather_lbl = ttk.Label(bottom_frame, text='--')
weather_lbl.pack()

##Temperature
temp_lbl = ttk.Label(bottom_frame, text='--', font=('bold', 30))
temp_lbl.pack(padx=10, pady=10)

url_lbl = ttk.Label(bottom_frame, text='--')
url_lbl.pack(padx=10, pady=10)

#Bottom Frame
def bottom():
    statusbar = ttk.Label(app, text='Application Opened: {}'.format(time.asctime(time.localtime())), relief='sunken', anchor='w', font=('Italic', 15))
    statusbar.pack(side='bottom', fill='x')
bottom()

app.mainloop()


import pymongo
import os
import pandas as pd
import json

def main():
    client = pymongo.MongoClient("mongodb://localhost:27017/")
    databases = client.list_database_names()

    if "fifa" not in databases:
        db = client["fifa"]

        players_collection = db["players"]
        ultimate_team_collection = db["ultimate_teams"]

        for file in os.listdir("data/players"):
            data = pd.read_csv("data/players/" + file)
            data["year"] = "20" + file.split(".")[0][-2:]
            if "female" in file:
                data["gender"] = "F"
            else:
                data["gender"] = "M"
            data_json = json.loads(data.to_json(orient='records'))

            for player in data_json:
                columns_to_format = ["ls", "st", "rs", "lw", "lf",	"cf", "rf", "rw", "lam", "cam",	"ram", "lm", "lcm",	"cm", "rcm", "rm", "lwb", "ldm", "cdm", "rdm", "rwb", "lb", "lcb", "cb", "rcb", "rb", "gk"]
                for column in columns_to_format:
                    if isinstance(player[column], str):
                        if "+" in player[column]:
                            split = player[column].split("+")
                            player[column] = int(split[0]) + int(split[1])
                        elif "-" in player[column]:
                            split = player[column].split("-")
                            player[column] = int(split[0]) - int(split[1])
                list_columns = ["player_positions", "player_tags", "player_traits"]
                for column in list_columns:
                    if player[column] is not None:
                        player[column] = [x.strip() for x in player[column].split(',')]
            players_collection.insert_many(data_json)
            print("Successfully loaded data for", file)

        print("Creating Indices for Faster Searching")
        players_collection.create_index([('year', pymongo.ASCENDING), ('gender', pymongo.ASCENDING)])
        players_collection.create_index([('year', pymongo.ASCENDING), ('gender', pymongo.ASCENDING), ('short_name', pymongo.ASCENDING)])
        players_collection.create_index([('year', pymongo.ASCENDING), ('gender', pymongo.ASCENDING), ('overall', pymongo.DESCENDING)])

        ultimate_team_collection.create_index([('year', pymongo.ASCENDING), ('username', pymongo.ASCENDING), ('team_name', pymongo.ASCENDING)])

    else:
        print("Data has been previously loaded.")

if __name__ == "__main__":
    main()


#! /usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import unicode_literals
import logging
import unittest

from tutorons.regex.extract import ApacheConfigRegexExtractor, JavascriptRegexExtractor,\
    GrepRegexExtractor, SedRegexExtractor
from tutorons.common.htmltools import HtmlDocument

logging.basicConfig(level=logging.INFO, format="%(message)s")

'''
TODO consider implementing regular expression checking for these languages:
1. tcl shell
2. Python regular expression methods
3. Java methods
'''

class ExtractRegexFromModRewriteTest(unittest.TestCase):

    def setUp(self):
        self.extractor = ApacheConfigRegexExtractor()

    def test_extract_regex_for_rewrite_rule(self):
        node = HtmlDocument('\n'.join([
            "",
            "RewriteRule ^.*$ index.php",
            "",
        ]))
        regions = self.extractor.extract(node)
        self.assertEqual(len(regions), 1)
        r = regions[0]
        self.assertEqual(r.node, node)
        self.assertEqual(r.start_offset, 13)
        self.assertEqual(r.end_offset, 16)

    def test_extract_regex_for_rewrite_condition(self):
        node = HtmlDocument('\n'.join([
            "",
            "RewriteCond  %{HTTP_USER_AGENT}  ^Mozilla",
            "",
        ]))
        regions = self.extractor.extract(node)
        self.assertEqual(len(regions), 1)
        r = regions[0]
        self.assertEqual(r.start_offset, 34)
        self.assertEqual(r.end_offset, 41)

    def test_allow_whitespace_before_directive(self):
        node = HtmlDocument('\n'.join([
            "",
            "    RewriteCond  %{HTTP_USER_AGENT}  ^Mozilla",
            "",
        ]))
        regions = self.extractor.extract(node)
        r = regions[0]
        self.assertEqual(r.start_offset, 38)
        self.assertEqual(r.end_offset, 45)

    def test_case_insensitive_directive_detected(self):
        node = HtmlDocument('\n'.join([
            "",
            "REWRITEcOnD  %{HTTP_USER_AGENT}  ^Mozilla",
            "",
        ]))
        regions = self.extractor.extract(node)
        r = regions[0]
        self.assertEqual(r.start_offset, 34)
        self.assertEqual(r.end_offset, 41)

class ExtractRegexFromJavascriptTest(unittest.TestCase):

    def setUp(self):
        self.extractor = JavascriptRegexExtractor()

    def test_extract_regex_from_variable_declaration(self):
        node = HtmlDocument('\n'.join([
            '',
            "var pattern = /regular-expression/g;",
            '',
        ]))
        regions = self.extractor.extract(node)
        self.assertEqual(len(regions), 1)
        r = regions[0]
        self.assertEqual(r.node, node)
        self.assertEqual(r.start_offset, 16)
        self.assertEqual(r.end_offset, 33)

    def test_skip_code_that_doesnt_pass_javascript_parser(self):
        node = HtmlDocument('\n'.join([
            '',
            "<>/regex/;",
            '',
        ]))
        regions = self.extractor.extract(node)
        self.assertEqual(len(regions), 0)

    def test_skip_regex_with_repeated_flags(self):
        node = HtmlDocument('\n'.join([
            '',
            "var pattern = /regular-expression/gg;",
            '',
        ]))
        regions = self.extractor.extract(node)
        self.assertEqual(len(regions), 0)

    def test_skip_regex_with_invalid_flags(self):
        node = HtmlDocument('\n'.join([
            '',
            "var pattern = /regular-expression/x;",
            '',
        ]))
        regions = self.extractor.extract(node)
        self.assertEqual(len(regions), 0)

class ExtractRegexFromGrepTest(unittest.TestCase):

    def setUp(self):
        self.extractor = GrepRegexExtractor()

    def test_extract_regex_from_variable_declaration(self):
        node = HtmlDocument('\n'.join([
            '',
            "grep pattern *",
            '',
        ]))
        regions = self.extractor.extract(node)
        self.assertEqual(len(regions), 1)
        r = regions[0]
        self.assertEqual(r.node, node)
        self.assertEqual(r.start_offset, 6)
        self.assertEqual(r.end_offset, 12)

    def test_extract_same_pattern_from_multiple_greps_in_one_element(self):
        node = HtmlDocument('\n'.join([
            '',
            "grep pattern *",
            "grep pattern *",
            '',
        ]))
        regions = self.extractor.extract(node)
        self.assertEqual(len(regions), 2)
        r1 = regions[0]
        self.assertEqual(r1.start_offset, 6)
        self.assertEqual(r1.end_offset, 12)
        r2 = regions[1]
        self.assertEqual(r2.start_offset, 21)
        self.assertEqual(r2.end_offset, 27)

    def test_extract_pattern_containing_spaces(self):
        node = HtmlDocument('\n'.join([
            '',
            "grep 'Pattern with spaces' *",
            '',
        ]))
        regions = self.extractor.extract(node)
        r = regions[0]
        self.assertEqual(r.start_offset, 7)
        self.assertEqual(r.end_offset, 25)

    def test_extract_pattern_from_option(self):
        node = HtmlDocument('\n'.join([
            '',
            "grep -e pattern *",
            '',
        ]))
        regions = self.extractor.extract(node)
        r = regions[0]
        self.assertEqual(r.start_offset, 9)
        self.assertEqual(r.end_offset, 15)

    def test_extract_patterns_from_multiple_options(self):
        node = HtmlDocument('\n'.join([
            '',
            "grep -e pattern1 -e pattern2 *",
            '',
        ]))
        regions = self.extractor.extract(node)
        self.assertEqual(len(regions), 2)
        self.assertTrue(any([r.start_offset == 9 and r.end_offset == 16 for r in regions]))
        self.assertTrue(any([r.start_offset == 21 and r.end_offset == 28 for r in regions]))

class ExtractRegexFromSedTest(unittest.TestCase):

    def setUp(self):
        self.extractor = SedRegexExtractor()

    def test_extract_regexes_from_address_range(self):
        node = HtmlDocument('\n'.join([
            '',
            'sed "/addr1/,/addr2/p" file',
            '',
        ]))
        regions = self.extractor.extract(node)
        self.assertEqual(len(regions), 2)
        r1 = regions[0]
        self.assertEqual(r1.node, node)
        self.assertEqual(r1.start_offset, 7)
        self.assertEqual(r1.end_offset, 11)
        r2 = regions[1]
        self.assertEqual(r2.start_offset, 15)
        self.assertEqual(r2.end_offset, 19)

    def test_ignore_addresses_that_arent_regex(self):
        node = HtmlDocument('\n'.join([
            '',
            'sed "0,1p" file',
            '',
        ]))
        regions = self.extractor.extract(node)
        self.assertEqual(len(regions), 0)

    def test_extract_regex_from_substitute_pattern(self):
        node = HtmlDocument('\n'.join([
            '',
            'sed "s/patt/replace/" file',
            '',
        ]))
        regions = self.extractor.extract(node)
        self.assertEqual(len(regions), 1)
        r = regions[0]
        self.assertEqual(r.start_offset, 8)
        self.assertEqual(r.end_offset, 11)

    def test_extract_regex_from_multiple_substitute_patterns(self):
        node = HtmlDocument('\n'.join([
            '',
            'sed -e "s/patt1/replace/" -e "s/patt2/replace/" file',
            '',
        ]))
        regions = self.extractor.extract(node)
        self.assertEqual(len(regions), 2)
        self.assertTrue(any([r.start_offset == 11 and r.end_offset == 15 for r in regions]))
        self.assertTrue(any([r.start_offset == 33 and r.end_offset == 37 for r in regions]))

    def test_handle_escaped_characters(self):
        node = HtmlDocument('\n'.join([
            '',
            'sed \'s/pa\/tt/replace/\' file',
            '',
        ]))
        regions = self.extractor.extract(node)
        self.assertEqual(len(regions), 1)
        r = regions[0]
        self.assertEqual(r.start_offset, 8)
        self.assertEqual(r.end_offset, 13)
        self.assertEqual(r.pattern, r'pa/tt')

    def test_handle_find_pattern_with_character_class(self):
        '''
        This test case failed earlier as we performed a regex search with the pattern found
        against the original command, and it was being interpreted as a regex, and not a raw string.
        '''
        node = HtmlDocument('sed "s/[A-Z]bc//g" file.txt')
        regions = self.extractor.extract(node)
        self.assertEqual(len(regions), 1)

if __name__ == '__main__':
    unittest.main()


#!/usr/bin/env python
"""
Datapath for QEMU qdisk
"""

import urlparse
import os
import sys
import xapi
import xapi.storage.api.v5.datapath
import xapi.storage.api.v5.volume
import importlib
from xapi.storage.libs.libcow.datapath import QdiskDatapath
from xapi.storage import log

def get_sr_callbacks(dbg, uri):
    u = urlparse.urlparse(uri)
    sr = u.netloc
    sys.path.insert(
        0,
        '/usr/libexec/xapi-storage-script/volume/org.xen.xapi.storage.' + sr)
    mod = importlib.import_module(sr)
    return mod.Callbacks()

class Implementation(xapi.storage.api.v5.datapath.Datapath_skeleton):
    """
    Datapath implementation
    """
    def activate(self, dbg, uri, domain):
        callbacks = get_sr_callbacks(dbg, uri)
        return QdiskDatapath.activate(dbg, uri, domain, callbacks)

    def attach(self, dbg, uri, domain):
        callbacks = get_sr_callbacks(dbg, uri)
        return QdiskDatapath.attach(dbg, uri, domain, callbacks)

    def deactivate(self, dbg, uri, domain):
        callbacks = get_sr_callbacks(dbg, uri)
        return QdiskDatapath.deactivate(dbg, uri, domain, callbacks)

    def detach(self, dbg, uri, domain):
        callbacks = get_sr_callbacks(dbg, uri)
        return QdiskDatapath.detach(dbg, uri, domain, callbacks)

    def open(self, dbg, uri, domain):
        callbacks = get_sr_callbacks(dbg, uri)
        return QdiskDatapath.epc_open(dbg, uri, domain, callbacks)

    def close(self, dbg, uri):
        callbacks = get_sr_callbacks(dbg, uri)
        return QdiskDatapath.epc_close(dbg, uri, callbacks)

if __name__ == "__main__":
    log.log_call_argv()
    CMD = xapi.storage.api.v5.datapath.Datapath_commandline(Implementation())
    CMD_BASE = os.path.basename(sys.argv[0])
    if CMD_BASE == "Datapath.activate":
        CMD.activate()
    elif CMD_BASE == "Datapath.attach":
        CMD.attach()
    elif CMD_BASE == "Datapath.close":
        CMD.close()
    elif CMD_BASE == "Datapath.deactivate":
        CMD.deactivate()
    elif CMD_BASE == "Datapath.detach":
        CMD.detach()
    elif CMD_BASE == "Datapath.open":
        CMD.open()
    else:
        raise xapi.storage.api.v5.datapath.Unimplemented(CMD_BASE)


"""
    USC Spring 2020
    INF 553 Foundations of Data Mining
    Assignment 3

    Student Name: Jiabin Wang
    Student ID: 4778-4151-95
"""
from pyspark import SparkConf, SparkContext, StorageLevel
from trainAuxiliary import *
'''
import os
import re
import json
import time
import sys
import math
import random
import itertools
'''

if __name__ == "__main__":
    time_start = time.time()

    # Get the input parameters
    input_file_path = sys.argv[1] #"./Dataset/train_review.json"
    model_file_path = sys.argv[2]
    cf_type = sys.argv[3]

    # Configure the Spark
    conf = (
        SparkConf()
        .setAppName("task3")
        .set("spark.driver.memory", "4g")
        .set("spark.executor.memory", "4g")
    )
    sc = SparkContext(conf=conf)
    sc.setLogLevel("ERROR")

    if cf_type == "item_based":
        pairCollection = itemBasedTrain(sc, input_file_path, model_file_path)
        itemBasedOutputModel(model_file_path, pairCollection)

    if cf_type == "user_based":
        result = userBasedTrain(sc, input_file_path, model_file_path)
        userBasedOutputModel(model_file_path, result)

    time_end = time.time()
    print("Duration: ", time_end - time_start, "s")


# Take the code from the How To Decode A Website exercise
# (if you didn’t do it or just want to play with some different code, use the code from the solution),
#  and instead of printing the results to a screen, write the results to a txt file.
#  In your code, just make up a name for the file you are saving to.

# Extras:

# Ask the user to specify the name of the output file that will be saved.

import requests
from bs4 import BeautifulSoup

base_url = 'http://www.nytimes.com'
r = requests.get(base_url)
soup = BeautifulSoup(r.text)
titles = []
for story_heading in soup.find_all(class_="story-heading"):
    if story_heading.a:
        # print(story_heading.a.text.replace("\n", " ").strip())
        titles.append(story_heading.a.text.replace("\n", " ").strip())
    else:
        print(story_heading.contents[0].strip())

name_file = input('Rename the file please ')

with open('{}.txt'.format(name_file), 'w') as open_file:
  open_file.write(str(titles))


from django.conf.urls.defaults import *
from django.contrib import admin
import os.path
admin.autodiscover()
MEDIA_ROOT = os.path.join(os.path.abspath(os.path.dirname(__file__)), "media")

urlpatterns = patterns('',
    (r'^admin/doc/', include('django.contrib.admindocs.urls')),
    (r'^admin/', include(admin.site.urls)),

    (r'^media/(?P<path>.*)$', 'django.views.static.serve',
     {'document_root': MEDIA_ROOT}),
    url(r'^p/(\w+)$', 'air.views.object', name='object'),
    url(r'^compare/(\w+)$', 'air.views.compareTo', name='compareTo'),
    url(r'^compare/(\w+)/(\w+)$', 'air.views.compare', name='compare'),
    url(r'^categories/$', 'air.views.categories', name='categories'),
    url(r'^category/$', 'air.views.category', name='category'),
    url(r'^projection/(\w+)/(\w+)$', 'air.views.projection', name='projection'),
    url(r'^add/$', 'air.views.add', name='add'),
    url(r'^reset/$', 'air.views.reset', name='reset'),
    url(r'^$', 'air.views.explore', name='explore'),
)


import eventlet
import msgpack
import random
from copy import copy
from datetime import datetime

from . import cmds
from . import msgs

import os
log_file = open(os.path.join(os.getcwd(), 'client.log'), 'w')

def write_log(msg):
    global log_file
    log_file.write(
        "{0} - {1}\n".format(datetime.now(), str(msg)),
    )

class RaftClient(object):

    def __init__(self, server_address_list):
        self.server_address_list = server_address_list
        self._leader_address = None
        self._leader_sock = None
        self.status = 'init'
        self.cmd_seq = 0
        self.client_id = None

    @classmethod
    def select_server(cls, server_address_list):
        return random.choice(server_address_list)

    def register(self):
        cmd = cmds.get_client_register_cmd()
        cmd_msg = msgs.get_client_register_req_msg(cmd)
        self.cmd_seq = 0
        ret = self.execute_command(cmd_msg)
        self.client_id = ret['resp'][1]
        return ret

    def get_next_seq(self):
        self.cmd_seq += 1
        return self.cmd_seq

    def send_command_req(self, command_msg):
        self._leader_sock.sendall(command_msg)

    def set_value(self, key, value):
        cmd = cmds.get_client_update_cmd(
            key,
            value,
        )
        cmd_msg = msgs.get_client_update_req_msg(
            self.client_id,
            self.get_next_seq(),
            cmd,
        )
        return self.execute_command(cmd_msg)

    def get_value(self, key):
        cmd = cmds.get_client_query_cmd(key)
        cmd_msg = msgs.get_client_query_req_msg(
            self.client_id,
            self.get_next_seq(),
            cmd,
        )
        return self.execute_command(cmd_msg)

    def wait_command_ret(self):
        unpacker = msgpack.Unpacker()
        while True:
            chunk = self._leader_sock.recv(1024)
            if len(chunk) == 0:
                break
            unpacker.feed(chunk)
            try:
                return unpacker.next()
            except StopIteration:
                pass
        return None

    def execute_command(self, command_msg):
        s_addr_list = copy(self.server_address_list)
        while len(s_addr_list) > 0:
            try:
                if self._leader_address is None:
                    self._leader_address = RaftClient.select_server(s_addr_list)
                    write_log(
                        "selected server {0}".format(self._leader_address))
                if self._leader_sock is None:
                    self._leader_sock = eventlet.connect(self._leader_address)
                timeout = eventlet.Timeout(2)
                try:
                    self.send_command_req(command_msg)
                    write_log(
                        "sent {0} - cmd: {1}".format(
                            self._leader_address,
                            msgpack.unpackb(command_msg),
                        )
                    )
                    ret = self.wait_command_ret()
                finally:
                    timeout.cancel()
                if ret is not None:
                    if ret['success']:
                        if ret['resp'][2] < self.cmd_seq:
                            continue
                        return ret
                    else:
                        if 'leader_hint' in ret:
                            self._leader_sock.close()
                            self._leader_sock = None
                            self._leader_address = (
                                ret['leader_hint'][0],
                                ret['leader_hint'][1] + 1000,
                            )
                            continue
            except (eventlet.timeout.Timeout, Exception) as e:
                write_log("hit exception:\n {0}".format(str(e)))
                pass
            if self._leader_address in s_addr_list:
                s_addr_list.remove(self._leader_address)
            if len(s_addr_list) == 0:
                s_addr_list = copy(self.server_address_list)
            self._leader_address = None
            self._leader_sock = None


#!/usr/bin/python

import csv
import json
import pprint
import re
import sys

def replace_if_not_empty(dict, key, value):
    if key not in dict or not dict[key]:
        dict[key] = value

def to_float_or_none(value):
    # lmao lazy
    try:
        return float(value)
    except ValueError:
        return None

def replace_slashes(str):
    return str.replace("/", "\\") if str else None

def strip_or_none(str):
    try:
        return str.strip() if str.strip() else None
    except:
        return None

def write_json(dic, file_name):
    print(f"saving {file_name}")
    with open(file=file_name, mode="w") as f:
        json.dump(dic, f, indent=2)
        f.write("\n")

def main(tsv_name: str):
    print(f"reading {tsv_name}")

    with open(file=tsv_name) as tsv_file:
        # skip first 2 lines
        tsv_file.readline()
        tsv_file.readline()

        # convert file to array of dicts
        # not efficient but good for reuse
        rows = list(csv.DictReader(tsv_file, delimiter="\t"))

        extract_keyboards(rows)
        extract_mousepads(rows)
        extract_mice(rows)
        extract_users(rows)

def extract_keyboards(rows):
    kbs = {}
    for row in rows:
        kb_model = replace_slashes(strip_or_none(row["Keyboard Model"]))
        if not kb_model:
            continue

        if kb_model not in kbs or not kbs[kb_model]:
            kb = {}
        else:
            kb = kbs[kb_model]

        # set keyboard switch
        replace_if_not_empty(kb, "switch", strip_or_none(row["Key Switch"]))
        kbs[kb_model] = kb

    # pprint.PrettyPrinter().pprint(kbs)
    write_json(kbs, "keyboards.json")

def extract_mousepads(rows):
    mps = {}
    for row in rows:
        mp = replace_slashes(strip_or_none(row["Mousepad"]))
        if not mp:
            continue
        if mp not in mps or not mps[mp]:
            mps[mp] = {}

    # pprint.PrettyPrinter().pprint(mps)
    write_json(mps, "mousepads.json")

def extract_mice(rows):
    mice = {}

    # iterate through rows
    for row in rows:
        mm = replace_slashes(strip_or_none(row["Mouse Model"]))
        # skip row
        if not mm:
            continue

        # get mouse model
        if mm not in mice or not mice[mm]:
            mouse = {}
        else:
            mouse = mice[mm]

        # update mouse model
        # using split because sometimes there is a range of values
        # im lazy so i just take first value
        replace_if_not_empty(
            mouse, "sensor", row["Sensor"] if row["Sensor"] else None)
        replace_if_not_empty(mouse, "weight", to_float_or_none(
            row["Weight"].split("g")[0]))
        replace_if_not_empty(mouse, "length", to_float_or_none(
            row["Length"].split("m")[0]))
        replace_if_not_empty(mouse, "width", to_float_or_none(
            row["Width"].split("m")[0]))
        replace_if_not_empty(mouse, "height", to_float_or_none(
            row["Height"].split("m")[0]))
        replace_if_not_empty(
            mouse, "switch", row["Mouse Switch"] if row["Mouse Switch"] else None)

        # set mouse model
        mice[mm] = mouse

    # pprint.PrettyPrinter().pprint(mice)
    write_json(mice, "mice.json")

def extract_users(rows):
    status = "Mouse"
    users = {}

    # iterate through rows
    for row in rows:
        if row["Rank"].startswith("Notable"):
            status = "Notable"
        elif row["Rank"].startswith("Traitors"):
            status = "Traitors"
        elif row["Rank"].startswith("Banned"):
            break

        # skip notable mentions
        if status == "Notable":
            continue

        # skip row if username is not well-formatted
        pattern = "=HYPERLINK\\(\"https:\\/\\/osu\\.ppy\\.sh\\/u\\/(\\d+)\",\"(\\w+)\"\\)"
        result = re.search(pattern, row["Name"])
        if not result or not result.group(1) or not result.group(2):
            continue

        # extract user info
        userID = result.group(1)
        userName = result.group(2)
        is_traitor = status == "Trators"

        # win settings, very safe way
        win_settings = row['=HYPERLINK("http://puu.sh/nJtmY/e2a5589f67.png","OS")'].split()
        if win_settings:
            win_sensitivity = to_float_or_none(win_settings[0].split("/")[0])
            if len(win_settings) >= 2:
                accl_setting = win_settings[1].strip().lower()
                if accl_setting.startswith("off"):
                    win_acceleration = False
                elif accl_setting.startswith("on"):
                    win_acceleration = True
            else:
                win_acceleration = None

        # osu settings, again it is focused on safe
        osu_multiplyer = to_float_or_none(
            row["Multiplier"].strip().split("~")[0].rstrip("xX"))
        if row["Raw"].strip().lower().startswith("on"):
            osu_raw = True
        elif row["Raw"].strip().lower().startswith("off"):
            osu_raw = False
        else:
            osu_raw = None

        # hardware setup info
        screen_resolution = row["Resolution"].strip().split("~")[0].split("x")
        if len(screen_resolution) >= 2:
            screen_width = to_float_or_none(screen_resolution[0])
            screen_height = to_float_or_none(screen_resolution[1])
        else:
            screen_width = None
            screen_height = None

        mousepad = strip_or_none(row["Mousepad"])
        keyboard = strip_or_none(row["Keyboard Model"])

        # mouse playstyle info
        playstyle = strip_or_none(row["Playstyle"])
        mouse = strip_or_none(row["Mouse Model"])
        dpi = to_float_or_none(row["DPI"].strip().rstrip("dpi"))
        polling = to_float_or_none(row["Polling"].lower().rstrip("hz"))

        # possibly calculate mouse area
        if win_sensitivity and osu_multiplyer and dpi and screen_width and screen_height:
            m = [0.00625, 0.0125, 0.25, 0.5, 0.75, 1.0, 1.5, 2.0, 2.5, 3.0, 3.5]
            win_multi = m[int(win_sensitivity - 1)]

            # get play area
            if (screen_width / screen_height) >= (4 / 3):
                play_height = screen_height
                play_width = screen_height * (4 / 3)
            else:
                play_width = screen_width
                play_height = screen_width / (4 / 3)

            # get area
            effective_ppi = dpi * win_multi * osu_multiplyer
            area_width = round(25.4 * play_width / effective_ppi)
            area_height = round(25.4 * play_height / effective_ppi)
        else:
            area_width = None
            area_height = None

        # create new user
        users[userID] = {
            "name": userName,  # dont really need username tho
            "rank": None,
            "pp": None,
            "is_banned": False,
            "is_traitor": is_traitor,
            "windows_sensitivity": win_sensitivity,
            "windows_acceleration": win_acceleration,
            "osu_multiplyer": osu_multiplyer,
            "osu_raw": osu_raw,
            "screen_width": screen_width,
            "screen_height": screen_height,
            "playstyle": playstyle,
            "dpi": dpi,
            "polling": polling,
            "area_width": area_width,
            "area_height": area_height,
            "mouse": mouse,
            "mousepad": mousepad,
            "keyboard": keyboard,
        }

    # pprint.PrettyPrinter().pprint(users)
    write_json(users, "users.json")

if __name__ == "__main__":
    if len(sys.argv) == 2:
        main(sys.argv[1])
    else:
        print("Usage: python ./parse_tsv 'tsv/file/path'")


"""Module with lagrangian decomposition methods."""
# Python packages

# Package modules
import logging as log
from firedecomp.AL import ARPP
from firedecomp.AL import ADPP
from firedecomp.fix_work import utils as _utils
from firedecomp.original import model as _model
from firedecomp.classes import problem as _problem

import time
import math
import gurobipy
import copy

###############################################################################
# CLASS LagrangianRelaxation()
###############################################################################
class AugmentedLagrangian(object):
    def __init__(
            self,
            problem_data,
            min_res_penalty=1000000,
            valid_constraints=None,
            gap=0.01,
            max_iters=100000,
            max_time=10,
            log_level="AL",
            solver_options=None,
    ):

        """Initialize the Lagrangian Relaxation object.

        Args:
            problem_data (:obj:`Problem`): problem data.
            min_res_penalty (:obj:`int`):
            gap (:obj:`float`): GAP tolerance for stop criteria.
                Defaults to 0.01.
            max_iters (:obj:`int`): maximum number of iterations. Defaults to
                10.
            max_time (:obj:`float`): maximum cpu time (in seconds). Defaults to
                3600.
            log_level (:obj:`str`): logging level. Defaults to ``'benders'``.
        """

        # PROBLEM DATA
        if problem_data.period_unit is False:
            raise ValueError("Time unit of the problem is not a period.")
        self.problem_data = problem_data
        self.solution_best = None  # index of DPP best solution
        self.solution_best_original = None
        # GLOBAL VARIABLES
        self.max_iters = max_iters
        self.max_time = max_time
        self.init_time = time.time()
        self.v = 1  # iterations
        self.NL = (1 + len(problem_data.get_names("wildfire")))  # +
        # len(problem_data.get_names("wildfire"))*len(problem_data.get_names("groups"))*2);
        # GUROBI OPTIONS
        if solver_options is None:
            solver_options = {
                'OutputFlag': 0,
                'LogToConsole': 0,
            }
        self.solver_options = solver_options

        # PARAMETERS INNER METHODS
        self.change = []
        self.beta_matrix = []
        self.lambda_matrix = []
        self.lambda_matrix_prev = []
        self.upperbound_matrix = []
        self.lobj_global = float("-inf")
        self.fobj_global = float("inf")
        self.infeas_global = float("inf")
        self.subgradient_global = []
        self.penalties_global = []
        self.index_best = -1
        self.lambda_min = 1e1
        self.lambda_max = 1e5
        self.lamdba_init = 1e3
        self.th_sol = 10

        # INITIALIZE DDecomposedPrimalProblemPP PROBLEM
        # Initialize Decomposite Primal Problem Variables
        self.problem_DPP = []
        self.N = len(self.problem_data.get_names("resources"))
        self.y_master_size = len(
            self.problem_data.get_names("wildfire") + [int(min(self.problem_data.get_names("wildfire"))) - 1])
        self.counterh_matrix = []

        self.lobj_local = []
        self.lobj_local_prev = []
        self.fobj_local = []
        self.infeas_local = []
        self.subgradient_local = []
        self.subgradient_local_prev = []
        self.penalties_local = []
        self.termination_counter = []
        self.best_list_y = []

        for i in range(0, self.y_master_size):
            self.termination_counter.append(0)
            self.lobj_local.append(float("inf"))
            self.lobj_local_prev.append(float("inf"))
            self.fobj_local.append(float("inf"))
            self.infeas_local.append(float("inf"))
            self.subgradient_local.append([])
            self.penalties_local.append([])
            self.upperbound_matrix.append(float("inf"))
        self.termination_counter[self.y_master_size - 1] = self.th_sol + 1

        # INITIALIZE LAMBDA AND BETA
        for i in range(0, self.y_master_size):
            lambda_row = []
            lambda_row_prev = []
            lambda_row_inf = []
            beta_row = []
            subgradient_prev_row = []
            for j in range(0, self.NL):
                lambda_row.append(self.lamdba_init)
                lambda_row_inf.append(float("inf"))
                beta_row.append(0.3)
                lambda_row_prev.append(self.lamdba_init)
                subgradient_prev_row.append(0)
            self.subgradient_local_prev.append(subgradient_prev_row)
            self.lambda_matrix.append(lambda_row)
            self.lambda_matrix_prev.append(lambda_row_prev)
            self.beta_matrix.append(beta_row)
            self.change.append(1.0)

        # CREATE ORIGINAL Problems list
        _utils.get_initial_sol(self.problem_data)
        dict_update = self.problem_data.get_variables_solution()
        print("UPDATE original problem")
        for i in range(0, self.y_master_size - 1):
            self.y_master = dict([(p, 1) for p in range(0, self.y_master_size)])
            for p in range(self.y_master_size - (1 + i), self.y_master_size):
                self.y_master[p] = 0
            print("Create index: " + str(i) + " y: " + str(self.y_master))
            model_DPP = ADPP.DecomposedPrimalProblem(self.problem_data,
                                                     self.lambda_matrix[i], self.beta_matrix[i],
                                                     self.y_master, self.N,
                                                     min_res_penalty=min_res_penalty,
                                                     valid_constraints=valid_constraints)
            self.problem_DPP.append(model_DPP)

    ###############################################################################
    # PUBLIC METHOD subgradient()
    ###############################################################################
    def subgradient(self, subgradient, subgradient_prev, lambda_vector, beta_vector, lambda_matrix_prev, ii):

        lambda_old = lambda_vector.copy()
        beta_old = beta_vector.copy()

        stuck = 0
        if max(subgradient_prev) < 0 and max(subgradient) > 0:
            stuck = 1

        for i in range(0, self.NL):
            LRpen = subgradient[i]
            if stuck == 1 and LRpen < 0:
                new_lambda = lambda_matrix_prev[i]
            else:
                new_lambda = (lambda_old[i] + LRpen * beta_old[i])

            lambda_vector[i] = min(max(self.lambda_min, new_lambda), self.lambda_max)
            beta_vector[i] = beta_vector[i] * 1.2
            # print(str(LRpen) + " -> lambda " + str(lambda_old[i]) + " + " + str(beta_old[i] * LRpen) + " = " + str(
            #    lambda_vector[i]) + " update " + str(beta_old[i]) + " diff " + str(
            #    abs(abs(lambda_vector[i]) - abs(lambda_old[i]))) + " beta " + str(
            #    beta_vector[i]))  # + " change_per "+str(change_per) )
        # print("")
        # print("")
        for i in range(0, self.NL):
            subgradient_prev[i] = subgradient[i]
            lambda_matrix_prev[i] = lambda_old[i]
        del lambda_old
        del beta_old

    ###############################################################################
    # PUBLIC METHOD convergence_checking()
    ###############################################################################
    def convergence_checking(self):
        stop = bool(False)
        result = 0
        optimal_solution_found = 0

        # print("TERMINATION COUNTER"+str(self.termination_counter))

        # CHECK PREVIOUS LAMBDAS CHANGES
        for i in range(0, len(self.lambda_matrix) - 1):
            # print(str(self.termination_counter[i])+"  "+str(self.infeas_local[i]))
            if self.infeas_local[i] > 0:
                self.termination_counter[i] = self.termination_counter[i] + 1

            if self.termination_counter[i] < self.th_sol and self.infeas_local[i] <= 0:
                lobj_diff = abs(
                    (abs(self.lobj_local[i]) - abs(self.lobj_local_prev[i])) / abs(self.lobj_local[i])) * 100
                #        # print(str(i) + "self.lobj_local[i] - self.lobj_local_prev[i] " + str(lobj_diff) + "% ")
                if (lobj_diff < 0.1):
                    self.termination_counter[i] = self.termination_counter[i] + 1
                else:
                    self.termination_counter[i] = 0
                    self.lobj_local_prev[i] = self.lobj_local[i]

        # CHECK TERMINATION COUNTER MATRIX
        counter = 0
        all_termination_counter_finished = 0
        for i in range(0, self.y_master_size):
            if self.termination_counter[i] >= (self.th_sol):
                counter = counter + 1
        if counter == self.y_master_size:
            all_termination_counter_finished = 1
        # print("counter" + str(counter) + " termination_counter" + str(self.y_master_size))

        # STOPPING CRITERIA CASES
        current_time = time.time() - self.init_time
        # check convergence
        if (self.v >= self.max_iters):
            print("[STOP] Max iters achieved!")
            stop = bool(True)
        if (current_time >= self.max_time):
            print("[STOP] Max execution time achieved!")
            stop = bool(True)
        elif (all_termination_counter_finished == 1):
            print("[STOP] Convergence achieved, optimal local point searched!")
            stop = bool(True)
        elif (optimal_solution_found == 1):
            print("[STOP] Convergence achieved, optimal solution found!")
            stop = bool(True)

        return stop

    ###############################################################################
    # PUBLIC METHOD solve()
    ###############################################################################
    def solve(self):
        print("SOLVE ALGORITHM")
        termination_criteria = bool(False)

        while termination_criteria == False:
            # (1) Solve DPP problems
            for i in range(0, self.y_master_size - 1):
                # Show iteration results
                if i == 0:
                    log.info("Iteration # mi lambda f(x) L(x,mi,lambda) penL")
                    print("\n\nIter: " + str(self.v) + " " +
                          "LR(x): " + str(self.lobj_global) + " " +
                          "f(x):" + str(self.fobj_global) + " " +
                          "penL:" + str(self.infeas_global) + " time:" +
                          str(time.time() - self.init_time) + "\n")
                if self.termination_counter[i] < self.th_sol:
                    # print("### Y -> " + str(self.problem_DPP[i].list_y))
                    DPP_sol_row = []
                    DPP_sol_unfeasible = False
                    total_obj_function = []
                    total_unfeasibility = []
                    total_subgradient = []
                    total_obj_function_pen = []
                    total_problem = []
                    self.lobj_local[i] = 0
                    self.fobj_local[i] = 0
                    self.subgradient_local[i] = []
                    for z in range(0, self.NL):
                        self.subgradient_local[i].append(float("-inf"))
                    self.penalties_local[i] = []
                    self.infeas_local[i] = 0
                    for j in range(0, self.N):
                        try:
                            self.problem_DPP[i].change_resource(j, self.lambda_matrix[i], self.beta_matrix[i], self.v)
                            DPP_sol_row.append(self.problem_DPP[i].solve(self.solver_options))

                            if (DPP_sol_row[j].model.Status == 3) or (DPP_sol_row[j].model.Status == 4):
                                DPP_sol_unfeasible = True
                                break
                        except:
                            print("Error Solver: Lambda/beta error")
                            DPP_sol_unfeasible = True
                            break
                        total_problem.append(self.problem_DPP[i].problem_data.copy_problem())
                        subgradient = self.problem_DPP[i].return_LR_obj2()
                        total_obj_function_pen.append(self.problem_DPP[i].return_function_obj_total_pen())
                        total_obj_function.append(self.problem_DPP[i].return_function_obj_total())
                        total_unfeasibility.append(max(subgradient))
                        total_subgradient.append(subgradient)
                        # print(str(j) + " fobj " + str(self.problem_DPP[i].return_function_obj()) + " total " +
                        #      str(self.problem_DPP[i].return_function_obj_total()) + "unfeas " + str(max(subgradient)))

                    if DPP_sol_unfeasible:
                        self.termination_counter[i] = self.th_sol + 1
                    else:
                        bestid = self.problem_DPP[i].return_best_candidate(total_obj_function, total_unfeasibility)
                        self.lobj_local[i] = total_obj_function_pen[bestid]
                        self.fobj_local[i] = total_obj_function[bestid]
                        self.infeas_local[i] = total_unfeasibility[bestid]
                        self.subgradient_local[i] = total_subgradient[bestid]
                        #print("TOTAL" + str(i) +
                        #      " LR " + str(self.lobj_local[i]) +
                        #      " fobj " + str(self.fobj_local[i]) +
                        #      " Infeas " + str(self.infeas_local[i]))

                        self.subgradient(self.subgradient_local[i], self.subgradient_local_prev[i],
                                         self.lambda_matrix[i], self.beta_matrix[i],
                                         self.lambda_matrix_prev[i], i)
                        self.change[i] = 0
                        if self.fobj_global > self.fobj_local[i] and (self.infeas_local[i] <= 0):
                            self.problem_DPP[i].problem_data = total_problem[bestid]
                            self.lobj_global = self.lobj_local[i]
                            self.fobj_global = self.fobj_local[i]
                            self.subgradient_global = self.subgradient_local[i]
                            self.infeas_global = self.infeas_local[i]

                            self.solution_best_original = total_problem[
                                bestid].copy_problem()  # self.update_problem_data_sol(self.problem_DPP[i])
                            self.solution_best_original.constrvio = self.infeas_global
                            self.solution_best_original.solve_status = 2
                            # print("New Solution:")
                            # print(self.solution_best_original.get_solution_info())
                            self.change[i] = 1
                        self.problem_DPP[i].update_original_values(DPP_sol_row[bestid], self.change[i])

                    DPP_sol_row.clear()
            # (3) Check termination criteria
            termination_criteria = self.convergence_checking()
            self.v = self.v + 1

        # DESTROY DPP
        # print(self.solution_best_original.get_solution_info())
        if self.solution_best_original is None:
            self.problem_DPP[0].change_resource(0, self.lambda_matrix[i], self.beta_matrix[i], self.v)
            self.problem_DPP[0].solve(self.solver_options)
            min_fobj = self.problem_DPP[0].return_function_obj_total()
            min_feasibility = max(self.problem_DPP[0].return_LR_obj2())
            best_index = 0
            for i in range(1, self.N):
                self.problem_DPP[i].change_resource(0, self.lambda_matrix[i], self.beta_matrix[i], self.v)
                self.problem_DPP[i].solve(self.solver_options)
                fobj = self.problem_DPP[i].return_function_obj_total()
                feas = max(self.problem_DPP[i].return_LR_obj2())
                if min_feasibility >= feas:
                    if min_feasibility == feas:
                        if min_fobj > fobj:
                            best_index = i
                    else:
                        best_index = i
            self.solution_best_original = self.problem_DPP[best_index].problem_data.copy_problem()
            self.solution_best_original.constrvio = max(self.problem_DPP[i].return_LR_obj2())
            self.solution_best_original.solve_status = 2

        self.problem_data = self.solution_best_original
        return self.solution_best_original

    ###############################################################################
    # PRIVATE extract_infeasibility()
    ###############################################################################
    def extract_infeasibility(self, subgradient):
        infeas = 0
        for i in range(0, len(subgradient)):
            if (subgradient[i] > 0):
                infeas = infeas + subgradient[i]
        return infeas

    ###############################################################################
    # PRIVATE destroy_DPP_set()
    ###############################################################################
    def destroy_DPP_set(self):
        for i in range(0, len(self.problem_DPP)):
            len_p = len(self.problem_DPP[i])
            if (len_p > 0):
                for j in range(0, len_p):
                    del self.problem_DPP[i][0]
                self.problem_DPP[i] = []
        self.problem_DPP = []
        # print("DESTROY")

    ###############################################################################
    # PRIVATE METHOD __log__()
    ###############################################################################
    #    def __log__(self, level="AL"):
    #        log.addLevelName(80, "AL")
    #        log.Logger.LR = logging.LR

    #        if level != 'AL':
    #            log_level = getattr(log, level)
    #            logger = log.getLogger('AL_logging')
    #            logger.setLevel(log_level)
    #            logger.addFilter(logging.LRFilter())
    #            if len(logger.handlers) == 0:
    #                ch = log.StreamHandler()
    #                ch.setLevel(log_level)
    #                # create formatter and add it to the handlers
    #                formatter = log.Formatter("%(levelname)8s: %(message)s")
    #                ch.setFormatter(formatter)
    #                logger.addHandler(ch)
    #        else:
    #            log_level = 80
    #            logger = log.getLogger('AL')
    #            logger.setLevel(log_level)
    #            if len(logger.handlers) == 0:
    #                ch = log.StreamHandler()
    #                ch.setLevel(log_level)
    #                # create formatter and add it to the handlers
    #                formatter = log.Formatter("%(message)s")
    #                ch.setFormatter(formatter)
    #                logger.addHandler(ch)
    #
    #        self.log = logger
    #        return 1

    def update_problem_data_sol(self, solution):

        problem = self.problem_data.copy_problem()
        problem = self.solution_to_problem(problem, solution)

        return problem

    def solution_to_problem(self, problem, solution):
        problem.mipgap = None
        problem.mipgapabs = None
        problem.constrvio = self.infeas_global
        problem.solve_status = 2

        variables = solution
        data = problem.data

        problem.resources.update(
            {i: {'select': round(variables.z[i].getValue()) == 1}
             for i in data.I})

        s = {(i, t): round(variables.s[i, t].x) == 1
             for i in data.I for t in data.T}
        u = {(i, t): round(variables.u[i, t].getValue()) == 1
             for i in data.I for t in data.T}
        e = {(i, t): round(variables.e[i, t].x) == 1
             for i in data.I for t in data.T}
        w = {(i, t): round(variables.w[i, t].getValue()) == 1
             for i in data.I for t in data.T}
        r = {(i, t): round(variables.r[i, t].x) == 1
             for i in data.I for t in data.T}
        er = {(i, t): round(variables.er[i, t].x) == 1
              for i in data.I for t in data.T}
        tr = {(i, t): round(variables.tr[i, t].x) == 1
              for i in data.I for t in data.T}

        problem.resources_wildfire.update(
            {(i, t): {
                'start': s[i, t],
                'use': u[i, t],
                'end': e[i, t],
                'work': w[i, t],
                'travel': tr[i, t],
                'rest': r[i, t],
                'end_rest': er[i, t]
            }
                for i in data.I for t in data.T})

        problem.groups_wildfire.update(
            {(g, t): {'num_left_resources': variables.mu[g, t].x}
             for g in data.G for t in data.T})

        contained = {t: variables.y[t].x == 0
                     for t in data.T}
        contained_period = [t for t, v in contained.items()
                            if v is True]

        if len(contained_period) > 0:
            first_contained = min(contained_period) + 1
        else:
            first_contained = data.max_t + 1

        problem.wildfire.update(
            {t: {'contained': False if t < first_contained else True}
             for t in data.T})

        return problem

    def create_init_solution(self, problem_data):

        T = problem_data.get_names("wildfire")

        min_t = int(min(T))
        list_y = dict([(p, 1) for p in range(0, len(T + [min_t - 1]))])
        list_y[len(list_y) - 1] = 0
        print(list_y)

        problem_data_copy = problem_data.copy_problem()
        init_problem = _model.InputModel(problem_data_copy)
        solver_options = {
            'OutputFlag': 1,
            'LogToConsole': 1,
            'TimeLimit': 10,
        }
        for i in range(0, len(list_y)):
            init_problem.y[i].UB = list_y[i]
            init_problem.y[i].LB = list_y[i]
        init_problem.m.update()
        print("COMPUTE INIT SOLUTION")
        init_problem.solve(solver_options)
        print("END COMPUTE INIT SOLUTION")

        return init_problem