#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Apr 13 11:34:20 2022

@author: chris
"""
import numpy as np
from scipy.interpolate import interp1d

from model.eddington import ERDF
from model.helper import make_array, invert_function

def physics_model(physics_name, log_m_c, initial_guess, bounds,
                  fixed_m_c=True, eddington_erdf_params=None):
    '''
    Return physics model that relates SMF and HMF, including model function,
    model name, initial guess and physical parameter bounds for fitting,
    that related SMF and HMF.
    The model function parameters are left free and can be obtained via fitting.
    Three models implemented:
        none                : no feedback adjustment
        sn                  : supernova feedback
        both                : supernova and black hole feedback
        quasar              : black hole growth model
        eddington_free_ERDF : bolometric luminosity model using free ERDF
        eddington           : bolometric luminosity model using fixed ERDF
                              (need erdf parameter as additional input)
    '''
    if physics_name not in ['none', 'stellar', 'stellar_blackhole', 'quasar',
                            'eddington_free_m_c_free_ERDF',
                            'eddington_free_ERDF', 'eddington']:
        raise NameError('physics_name not known.')

    if physics_name == 'none':
        return(NoFeedback(log_m_c, initial_guess[:1],
                          bounds[:, :1]))

    if physics_name == 'eddington':
        if eddington_erdf_params is None:
            raise ValueError('Eddington model with fixed ERDF needs ERDF '
                             'parameter passed via eddington_erdf_params '
                             'argument.')
        return(QuasarLuminosity(log_m_c, initial_guess[:2], bounds[:, :2],
               eddington_erdf_params))

    if fixed_m_c:
        if physics_name == 'stellar':
            physics = StellarFeedback(log_m_c, initial_guess[:-1],
                                      bounds[:, :-1])
        elif physics_name == 'stellar_blackhole':
            physics = StellarBlackholeFeedback(log_m_c, initial_guess,
                                               bounds)
        elif physics_name == 'quasar':
            physics = QuasarGrowth(log_m_c, initial_guess,
                                   bounds)
        elif physics_name == 'eddington_free_ERDF':
            physics = QuasarLuminosity_free_ERDF(log_m_c, initial_guess,
                                                 bounds)
        elif physics_name == 'eddington':
            if eddington_erdf_params is None:
                raise ValueError('Eddington model with fixed ERDF needs ERDF '
                                 'parameter passed via eddington_erdf_params '
                                 'argument.')
            physics = QuasarLuminosity(log_m_c, initial_guess[:-2],
                                    bounds[:, :-2], eddington_erdf_params)

    else:
        # add log_m_c to initial guess and bounds
        initial_guess = np.insert(initial_guess, 0, log_m_c)
        log_m_c_range = 3  # +- range in which log_m_c is expected
        bounds = np.insert(bounds, 0, [10, log_m_c+log_m_c_range], axis=1)

        if physics_name == 'stellar':
            physics = StellarFeedback_free_m_c(log_m_c,
                                               initial_guess[:-1],
                                               bounds[:, :-1])
        elif physics_name == 'stellar_blackhole':
            physics = StellarBlackholeFeedback_free_m_c(log_m_c,
                                                        initial_guess,
                                                        bounds)
        elif physics_name == 'quasar':
            physics = QuasarGrowth_free_m_c(log_m_c, initial_guess,
                                            bounds)
        elif physics_name == 'eddington_free_ERDF':
            physics = QuasarLuminosity_free_m_c_free_ERDF(log_m_c,
                                                          initial_guess,
                                                          bounds)
        elif physics_name in 'eddington':
            raise NotImplementedError('physics model with free m_c but fixed '
                                      'ERDF not implemented.')
    return(physics)

################ MODEL WITH FREE CRITICAL MASS ################################

class StellarBlackholeFeedback_free_m_c(object):
    def __init__(self, log_m_c, initial_guess, bounds):
        '''
        Model for stellar + black hole feedback with free m_c.

        '''
        self.name = 'stellar_blackhole_free_m_c'
        self.log_m_c = log_m_c               # critical mass for feedback
        self.initial_guess = initial_guess   # initial guess for least_squares
        # fit
        # parameter (A, alpha, beta) bounds
        self.bounds = bounds

        # parameter used compared to the unconstrained model
        self.parameter_used = [0,1,2,3]
        self.fixed_m_c_flag = False

        # max halo mass (just used to avoid overflows)
        self._upper_m_h = 50
        # latest parameter used
        self._current_parameter = None
        self.log_m_h_function = None

    def calculate_log_quantity(self, log_m_h, log_m_c, log_A, alpha, beta):
        '''
        Calculate observable quantity from input halo mass and model parameter.
        '''
        log_m_h = make_array(log_m_h)
        log_m_h = self._check_overflow(log_m_h)

        sn, bh = self._variable_substitution(log_m_h, log_m_c, alpha, beta)
        log_quantity = np.empty_like(bh)

        # deal with bh becoming infinite sometimes
        inf_mask = np.isfinite(sn+bh)
        sn = sn[inf_mask]
        bh = bh[inf_mask]

        log_quantity[inf_mask] = log_A + log_m_h[inf_mask] - np.log10(sn + bh)
        log_quantity[np.logical_not(inf_mask)] = np.inf
        return(log_quantity)

    def calculate_log_halo_mass(self, log_quantity, log_m_c, log_A, alpha,
                                beta, num=500):
        '''
        Calculate halo mass from input quantity quantity and model parameter.
        Do this by calculating table of quantity for halo masses near critical
        mass and linearly interpolating inbetween (and extrapolating beyond
        range). Once interpolated function is created, it's saved and called
        for new calculations until new parameter are passed.
        num controls number of points that halo mass is initially calculated for,
        higher values make result more accurate but also more expensive.

        '''
        if alpha == beta == 0:
            return(log_quantity - log_A + np.log10(2))

        if self._current_parameter == [log_m_c, log_A, alpha, beta]:
            pass
        else:
            self._current_parameter = [log_m_c, log_A, alpha, beta]

            # create lookup tables of halo mass and quantities
            log_m_h_lookup = self._make_m_h_space(log_m_c, log_A, alpha,
                                                  beta, num=num)
            log_quantity_lookup = StellarBlackholeFeedback_free_m_c.\
                calculate_log_quantity(
                    self, log_m_h_lookup, log_m_c, log_A,
                    alpha, beta)

            # use lookup table to create callable spline
            self.log_m_h_function = interp1d(log_quantity_lookup,
                                             log_m_h_lookup,
                                             bounds_error=False,
                                             fill_value=([-np.inf], [np.inf]))

        log_m_h = self.log_m_h_function(log_quantity)
        log_m_h = make_array(log_m_h)
        log_m_h = self._check_overflow(log_m_h)  # deal with very large m_h
        return(log_m_h)

    def calculate_log_halo_mass_alternative(self, log_quantity, log_m_c, log_A,
                                            alpha, beta, xtol=0.1):
        '''
        Calculate halo mass from input observable quantity and model paramter.
        This is done by numerical inverting the relation for the quantity as
        function of halo mass (making this a very expensive operation).
        Absolute tolerance for inversion can be adjusted using xtol.
        This method is more accurate, but more time consuming (also has
        problems if beta is close to 1).
        '''
        log_m_h = invert_function(func=self.calculate_log_quantity,
                                  fprime=self.calculate_dlogquantity_dlogmh,
                                  fprime2=self.calculate_d2logquantity_dlogmh2,
                                  x0_func=self._initial_guess,
                                  y=log_quantity,
                                  args=(log_m_c, log_A, alpha, beta),
                                  xtol=xtol)
        return(log_m_h)

    def calculate_dlogquantity_dlogmh(self, log_m_h, log_m_c, log_A, alpha,
                                      beta):
        '''
        Calculate d/d(log m_h) log_quantity. High mass end for beta near
        one treated as special case, where value apporaches zero.
        '''
        log_m_h = make_array(log_m_h)
        log_m_h = self._check_overflow(log_m_h)

        sn, bh = self._variable_substitution(log_m_h, log_m_c, alpha, beta)
        first_derivative = np.empty_like(bh)

        # deal with bh becoming infinite sometimes
        inf_mask = np.isfinite(bh+sn)
        sn = sn[inf_mask]
        bh = bh[inf_mask]

        first_derivative[inf_mask] = 1 - (-alpha * sn + beta * bh) / (sn + bh)
        first_derivative[np.logical_not(inf_mask)] = 0
        return(first_derivative)

    def calculate_d2logquantity_dlogmh2(self, log_m_h, log_m_c, log_A, alpha,
                                        beta):
        '''
        Calculate d^2/d(log m_h)^2 log_quantity. High mass end for beta near
        one treated as special case, where value apporaches zero.
        '''
        log_m_h = make_array(log_m_h)
        log_m_h = self._check_overflow(log_m_h)

        x = 10**((alpha+beta) * (log_m_h - log_m_c))
        second_derivative = np.empty_like(x)

        # deal with bh becoming infinite sometimes
        inf_mask = np.isfinite(x)
        numerator = -np.log(10) * (alpha+beta)**2 * x[inf_mask]
        denominator = (x[inf_mask]+1)**2

        second_derivative[inf_mask] = numerator/denominator
        second_derivative[np.logical_not(inf_mask)] = 0
        return(second_derivative)

    def _variable_substitution(self, log_m_h, log_m_c, alpha, beta):
        '''
        Transform input quanitities to more easily handable quantities.
        '''
        log_m_h = self._check_overflow(log_m_h)

        ratio = log_m_h - log_m_c         # m_h/m_c in log space
        log_stellar = - alpha * ratio     # log of sn feedback contribution
        if beta == 0:
            log_black_hole = np.full_like(log_stellar, 0)
        else:
            log_black_hole = beta * ratio     # log of bh feedback contribution
        return(10**log_stellar, 10**log_black_hole)

    def _initial_guess(self, log_quantity, log_m_c, log_A, alpha, beta):
        '''
        Calculate initial guess for halo mass (for function inversion). Do this
        by calculating the high and low mass end approximation of the relation.
        '''
        # turnover quantity, where dominating feedback changes
        log_q_turn = log_A - np.log10(2) + log_m_c

        if log_quantity < log_q_turn:
            x0 = (log_quantity - log_A + alpha * log_m_c) / (1 + alpha)
        else:
            x0 = (log_quantity - log_A - beta * log_m_c) / (1 - beta)
        return(x0)

    def _check_overflow(self, log_m_h):
        '''
        Control for overflows by checking if halo mass exceeds upper limit.
        Also deal with Nans.
        '''
        if np.isnan(log_m_h).any():
            return(np.full_like(log_m_h, np.nan))
        if np.any(log_m_h > self._upper_m_h):
            if np.isscalar(log_m_h):
                log_m_h = np.inf
            else:
                log_m_h[log_m_h > self._upper_m_h] = np.inf
        return(log_m_h)

    def _calculate_feedback_regimes(self, log_m_c, log_A, alpha, beta,
                                   log_epsilon=-2, output='halo_mass'):
        '''
        Calculate halo mass where one of the feedbacks is strongly dominating
        in the sense that (M_h/M_c)^-alpha > epsilon * (M_h/M_c)^beta and
        the other way around. Returns 3 values: [log_m_c, log_m_sn, log_m_ bh].
        If output='halo_mass' (default) return transition
        halo masses, if output='quantity' return quantity values (for this
        halo mass and parameter).
        '''
        # check relative contribution of feedbacks using sn = epsilon*bh to
        # get limiting mass where either contribution drops below epsilon,
        # this is log_m_lim = -1/(alpha+beta) * log(epsilon) + log_m_c

        # mass where sn feedback is dominating
        log_m_h_sn = 1/(alpha+beta)  * log_epsilon + log_m_c
        # mass where bh feedback is dominating
        log_m_h_bh = -1/(alpha+beta) * log_epsilon + log_m_c

        if output == 'halo_mass':
            return(np.array([log_m_c, log_m_h_bh, log_m_h_sn]))
        elif output == 'quantity':
            # calculate corresponding quantity values
            log_q_c = StellarBlackholeFeedback_free_m_c.calculate_log_quantity(
                                                   self, log_m_c, log_m_c,
                                                   log_A, alpha, beta)[0]
            log_q_sn = StellarBlackholeFeedback_free_m_c.calculate_log_quantity(
                                                   self, log_m_h_sn, log_m_c,
                                                   log_A, alpha, beta)[0]
            log_q_bh = StellarBlackholeFeedback_free_m_c.calculate_log_quantity(
                                                   self, log_m_h_bh, log_m_c,
                                                   log_A, alpha, beta)[0]
            return(np.array([log_q_c, log_q_sn, log_q_bh]))
        else:
            raise ValueError('output must be either \'halo_mass\' or '
                             '\'quantity\'.')

    def _make_m_h_space(self, log_m_c, log_A, alpha, beta, num, log_epsilon=-3):
        '''
        Create array of m_h points arranged so that the points are dense where
        q(m_h) changes quickly and sparse where they aren't.
        num controls number of points that are calculated.
        epsilon controls how far out points are created (epsilon is the
        ration between the strength of the two feedback types, i.e.
        sn/bh = epsilon).

        '''

        # calculate halo masses where one of the feedbacks strongly dominates
        _, log_m_bh, log_m_sn = StellarBlackholeFeedback_free_m_c.\
                                 _calculate_feedback_regimes(
                                    self, log_m_c, log_A, alpha, beta,
                                    log_epsilon=log_epsilon)

        # create high density space
        dense_log_m_h = np.linspace(log_m_sn, log_m_bh, int(num*0.8))
        # create low density spaces
        sparse_log_m_lower = np.linspace(log_m_sn/2, log_m_sn, int(num*0.1))
        sparse_log_m_upper = np.linspace(log_m_bh, 2*log_m_bh, int(num*0.1))

        log_m_h_table = np.sort(np.concatenate([dense_log_m_h,
                                                sparse_log_m_lower,
                                                sparse_log_m_upper]))
        return(np.unique(log_m_h_table))

class StellarFeedback_free_m_c(StellarBlackholeFeedback_free_m_c):
    def __init__(self, log_m_c, initial_guess, bounds):
        '''
        Model for stellar feedback with free m_c.
        '''
        super().__init__(log_m_c, initial_guess, bounds)
        self.name = 'stellar_free_m_c'
        self.parameter_used = [0,1,2]
        self.fixed_m_c_flag = False

    def calculate_log_quantity(self, log_m_h, log_m_c, log_A, alpha):
        return(StellarBlackholeFeedback_free_m_c.
               calculate_log_quantity(self, log_m_h,
                                      log_m_c,
                                      log_A,
                                      alpha,
                                      beta=0))

    def calculate_log_halo_mass(self, log_quantity, log_m_c, log_A, alpha,
                                num=500):
        return(StellarBlackholeFeedback_free_m_c.
               calculate_log_halo_mass(self, log_quantity,
                                       log_m_c,
                                       log_A,
                                       alpha,
                                       beta=0,
                                       num=num))

    def calculate_dlogquantity_dlogmh(self, log_m_h, log_m_c, log_A, alpha):
        return(StellarBlackholeFeedback_free_m_c.
               calculate_dlogquantity_dlogmh(self, log_m_h,
                                             log_m_c,
                                             log_A,
                                             alpha,
                                             beta=0))

    def calculate_d2logquantity_dlogmh2(self, log_m_h, log_m_c, log_A, alpha):
        return(StellarBlackholeFeedback_free_m_c.
               calculate_d2logquantity_dlogmh2(self, log_m_h,
                                               log_m_c,
                                               log_A,
                                               alpha,
                                               beta=0))

    def _calculate_feedback_regimes(self, log_m_c, log_A, alpha,
                                   log_epsilon=-2, output='halo_mass'):
        return(StellarBlackholeFeedback_free_m_c.
               _calculate_feedback_regimes(self, log_m_c,
                                                log_A,
                                                alpha,
                                                beta=0,
                                                log_epsilon=log_epsilon,
                                                output=output))

class QuasarGrowth_free_m_c(StellarBlackholeFeedback_free_m_c):
    def __init__(self, log_m_c, initial_guess, bounds):
        '''
        Black hole growth model with free m_c.

        '''
        super().__init__(log_m_c, initial_guess, bounds)
        self.name = 'quasargrowth_free_m_c'
        self.parameter_used = [0,1,2]
        self.fixed_m_c_flag = False

    def calculate_log_quantity(self, log_m_h, log_m_c, log_B, eta):
        '''
        Calculate (log of) bolometric luminosity from input halo mass,
        model parameter and chosen log_eddington_ratio.
        '''
        log_m_h = make_array(log_m_h)
        return(log_B + eta*(log_m_h - log_m_c))

    def calculate_log_halo_mass(self, log_quantity, log_m_c, log_B,
                                eta):
        '''
        Calculate (log of) halo mass from input bolometric luminosity,
        model parameter and chosen log_eddington_ratio.
        '''
        log_quantity = make_array(log_quantity)
        return((log_quantity-log_B)/eta + log_m_c)

    def calculate_dlogquantity_dlogmh(self, log_m_h, log_m_c,
                                      log_B, eta):
        '''
        Calculate d/d(log m_h) log_quantity.
        '''
        return(np.full_like(log_m_h, eta))

class QuasarLuminosity_free_m_c_free_ERDF(object):
    def __init__(self, log_m_c, initial_guess, bounds):
        '''
        Black hole bolometric luminosity model with free m_c and ERDF.

        '''
        self.name = 'eddington_free_m_c_free_ERDF'
        self.initial_guess = initial_guess   # initial guess for least_squares
        # fit
        self.bounds = bounds                 # parameter bounds

        self.log_m_c = log_m_c

        # parameter used compared to the unconstrained model
        self.parameter_used = [0,1,2,3,4]
        self.fixed_m_c_flag = False

        # latest parameter used
        self.parameter = None
        self.eddington_distribution = None

    def calculate_log_quantity(self, log_m_h, log_eddington_ratio, log_m_c,
                               log_C, eta):
        '''
        Calculate (log of) bolometric luminosity from input halo mass,
        model parameter and chosen log_eddington_ratio.
        '''
        log_m_h = make_array(log_m_h)
        return(log_eddington_ratio + log_C + eta*(log_m_h - log_m_c))

    def calculate_log_halo_mass(self, log_L, log_eddington_ratio, log_m_c,
                                log_C, eta):
        '''
        Calculate (log of) halo mass from input bolometric luminosity,
        model parameter and chosen log_eddington_ratio.
        '''
        log_L = make_array(log_L)
        return((log_L-(log_eddington_ratio + log_C))/eta + log_m_c)

    def calculate_dlogquantity_dlogmh(self, log_m_h, log_eddington_ratio,
                                      log_m_c, log_C, eta):
        '''
        Calculate d/d(log m_h) log_quantity.
        '''
        return(np.full_like(log_m_h, eta))

    def calculate_log_erdf(self, log_eddington_ratio, log_eddington_star,
                           rho):
        '''
        Calculate (log) value of ERDF (probability of given eddington ratio),
        given input log_eddingtion_ratio and parameter (log_eddington_star,
        rho).

        '''
        self._make_distribution(log_eddington_star, rho)
        log_erdf = self.eddington_distribution.log_probability(
            log_eddington_ratio)
        return(log_erdf)

    def calculate_mean_log_eddington_ratio(self, log_eddington_star,
                                           rho):
        '''
        Calculate mean eddington ratio for the given parameter.

        '''
        self._make_distribution(log_eddington_star, rho)
        mean = self.eddington_distribution.mean()
        return(mean)

    def draw_eddington_ratio(self, log_eddington_star, rho, num=1):
        '''
        Draw random sample of Eddingtion ratio from distribution defined by
        parameter (log_eddington_star, rho). Can draw num samples at
        once.

        '''
        self._make_distribution(log_eddington_star, rho)
        log_eddington_ratio = self.eddington_distribution.rvs(size=num)
        return(log_eddington_ratio)

    def _make_distribution(self, log_eddington_star, rho):
        '''
        Check if distribution function with the given parameter was already
        created and stored, otherwise create it.

        '''
        if self.parameter == [log_eddington_star, rho]:
            pass
        else:
            self.parameter = [log_eddington_star, rho]
            self.eddington_distribution = ERDF(log_eddington_star, rho)
        return(self.eddington_distribution)

################ MODEL WITH FIXED CRITICAL MASS ###############################

class StellarBlackholeFeedback(StellarBlackholeFeedback_free_m_c):
    def __init__(self, log_m_c, initial_guess, bounds):
        '''
        Model for stellar + black hole feedback.

        '''
        super().__init__(log_m_c, initial_guess, bounds)
        self.name = 'stellar_blackhole'
        self.parameter_used = [1,2,3]
        self.fixed_m_c_flag = True

    def calculate_log_quantity(self, log_m_h, log_A, alpha, beta):
        return(StellarBlackholeFeedback_free_m_c.
               calculate_log_quantity(self, log_m_h,
                                      log_m_c=self.log_m_c,
                                      log_A=log_A,
                                      alpha=alpha,
                                      beta=beta))

    def calculate_log_halo_mass(self, log_quantity, log_A, alpha, beta,
                                num=500):
        return(StellarBlackholeFeedback_free_m_c.
               calculate_log_halo_mass(self, log_quantity,
                                       log_m_c=self.log_m_c,
                                       log_A=log_A,
                                       alpha=alpha,
                                       beta=beta,
                                       num=num))

    def calculate_dlogquantity_dlogmh(self, log_m_h, log_A, alpha, beta):
        return(StellarBlackholeFeedback_free_m_c.
               calculate_dlogquantity_dlogmh(self, log_m_h,
                                             log_m_c=self.log_m_c,
                                             log_A=log_A,
                                             alpha=alpha,
                                             beta=beta))

    def calculate_d2logquantity_dlogmh2(self, log_m_h, log_A, alpha, beta):
        return(StellarBlackholeFeedback_free_m_c.
               calculate_d2logquantity_dlogmh2(self, log_m_h,
                                               log_m_c=self.log_m_c,
                                               log_A=log_A,
                                               alpha=alpha,
                                               beta=beta))

    def _calculate_feedback_regimes(self, log_A, alpha, beta, log_epsilon=-2,
                                   output='halo_mass'):
        return(StellarBlackholeFeedback_free_m_c.
               _calculate_feedback_regimes(self,
                                          log_m_c=self.log_m_c,
                                          log_A=log_A,
                                          alpha=alpha,
                                          beta=beta,
                                          log_epsilon=log_epsilon,
                                          output=output))

class StellarFeedback(StellarBlackholeFeedback):
    def __init__(self, log_m_c, initial_guess, bounds):
        '''
        Model for stellar feedback.
        '''
        super().__init__(log_m_c, initial_guess, bounds)
        self.name = 'stellar'
        self.parameter_used = [1,2]
        self.fixed_m_c_flag = True

    def calculate_log_quantity(self, log_m_h, log_A, alpha):
        return(StellarBlackholeFeedback_free_m_c.
               calculate_log_quantity(self, log_m_h,
                                      log_m_c=self.log_m_c,
                                      log_A=log_A,
                                      alpha=alpha,
                                      beta=0))

    def calculate_log_halo_mass(self, log_quantity, log_A, alpha, num=500):
        return(StellarBlackholeFeedback_free_m_c.
               calculate_log_halo_mass(self, log_quantity,
                                       log_m_c=self.log_m_c,
                                       log_A=log_A,
                                       alpha=alpha,
                                       beta=0,
                                       num=num))

    def calculate_dlogquantity_dlogmh(self, log_m_h, log_A, alpha):
        return(StellarBlackholeFeedback_free_m_c.
               calculate_dlogquantity_dlogmh(self, log_m_h,
                                             log_m_c=self.log_m_c,
                                             log_A=log_A,
                                             alpha=alpha,
                                             beta=0))

    def calculate_d2logquantity_dlogmh2(self, log_m_h, log_A, alpha):
        return(StellarBlackholeFeedback_free_m_c.
               calculate_d2logquantity_dlogmh2(self, log_m_h,
                                               log_m_c=self.log_m_c,
                                               log_A=log_A,
                                               alpha=alpha,
                                               beta=0))

    def _calculate_feedback_regimes(self, log_A, alpha, log_epsilon=-2,
                                   output='halo_mass'):
        return(StellarBlackholeFeedback_free_m_c.
               _calculate_feedback_regimes(self, log_m_c=self.log_m_c,
                                          log_A=log_A,
                                          alpha=alpha,
                                          beta=0,
                                          log_epsilon=log_epsilon,
                                          output=output))

class NoFeedback(StellarBlackholeFeedback):
    def __init__(self, log_m_c, initial_guess, bounds):
        '''
        Model without feedback.
        '''
        super().__init__(log_m_c, initial_guess, bounds)
        self.name = 'none'
        self.log2 = np.log10(2)  # so it doesn't need to be calc everytime

    def calculate_log_quantity(self, log_m_h, log_A):
        '''
        Calculate observable quantity from input halo mass and model parameter.
        '''
        log_m_h = make_array(log_m_h)
        return(log_A - self.log2 + log_m_h)

    def calculate_log_halo_mass(self, log_quantity, log_A):
        '''
        Calculate halo mass from input observable quantity and model parameter.
        '''
        log_quantity = make_array(log_quantity)
        return(log_quantity - log_A + self.log2)

    def calculate_dlogquantity_dlogmh(self, log_m_h, log_A):
        log_m_h = make_array(log_m_h)
        return(np.full_like(log_m_h, 1))

    def calculate_d2logquantity_dlogmh2(self, log_m_h, log_A):
        log_m_h = make_array(log_m_h)
        return(np.full_like(log_m_h, 0))

class QuasarGrowth(QuasarGrowth_free_m_c):
    def __init__(self, log_m_c, initial_guess, bounds):
        '''
        Black hole growth model.

        '''
        super().__init__(log_m_c, initial_guess, bounds)
        self.name = 'quasargrowth'
        self.parameter_used = [1,2]
        self.fixed_m_c_flag = True

    def calculate_log_quantity(self, log_m_h, log_B, eta):
        return(QuasarGrowth_free_m_c.
               calculate_log_quantity(self, log_m_h,
                                      log_m_c=self.log_m_c,
                                      log_B=log_B,
                                      eta=eta))

    def calculate_log_halo_mass(self, log_quantity, log_B, eta):
        return(QuasarGrowth_free_m_c.
               calculate_log_halo_mass(self, log_quantity,
                                       log_m_c=self.log_m_c,
                                       log_B=log_B,
                                       eta=eta))

    def calculate_dlogquantity_dlogmh(self, log_m_h, log_B, eta):
        return(QuasarGrowth_free_m_c.
               calculate_dlogquantity_dlogmh(self, log_m_h,
                                             log_m_c=self.log_m_c,
                                             log_B=log_B,
                                             eta=eta))

class QuasarLuminosity_free_ERDF(QuasarLuminosity_free_m_c_free_ERDF):
    def __init__(self, log_m_c, initial_guess, bounds):
        '''
        Black hole bolometric luminosity model with free ERDF.

        '''
        self.name = 'eddington_free_ERDF'
        self.initial_guess = initial_guess   # initial guess for least_squares
        # fit
        self.bounds = bounds                 # parameter bounds

        self.log_m_c = log_m_c

        # parameter used compared to the unconstrained model
        self.parameter_used = [1,2,3,4]
        self.fixed_m_c_flag = True

        # latest parameter used
        self.parameter = None
        self.eddington_distribution = None

    def calculate_log_quantity(self, log_m_h, log_eddington_ratio,
                               log_C, eta):
        return(QuasarLuminosity_free_m_c_free_ERDF.calculate_log_quantity(
                                self,
                                log_m_h=log_m_h,
                                log_eddington_ratio=log_eddington_ratio,
                                log_m_c=self.log_m_c,
                                log_C=log_C,
                                eta=eta))

    def calculate_log_halo_mass(self, log_L, log_eddington_ratio,
                                log_C, eta):
        return(QuasarLuminosity_free_m_c_free_ERDF.calculate_log_halo_mass(
                                self,
                                log_L=log_L,
                                log_eddington_ratio=log_eddington_ratio,
                                log_m_c=self.log_m_c,
                                log_C=log_C,
                                eta=eta))

    def calculate_dlogquantity_dlogmh(self, log_m_h, log_eddington_ratio,
                                      log_C, eta):
        return(QuasarLuminosity_free_m_c_free_ERDF.calculate_dlogquantity_dlogmh(
                                self,
                                log_m_h=log_m_h,
                                log_eddington_ratio=log_eddington_ratio,
                                log_m_c=self.log_m_c,
                                log_C=log_C,
                                eta=eta))

class QuasarLuminosity(QuasarLuminosity_free_ERDF):
    def __init__(self, log_m_c, initial_guess, bounds, eddington_erdf_params):
        '''
        Black hole bolometric luminosity model with fixed ERDF.

        '''
        self.name = 'eddington'
        self.initial_guess = initial_guess   # initial guess for least_squares
        # fit
        self.bounds = bounds                 # parameter bounds

        self.log_m_c = log_m_c

        self.parameter_used = [0,1]
        self.fixed_m_c_flag = True

        # parameter used
        self.parameter = eddington_erdf_params
        self.eddington_distribution = ERDF(*self.parameter)

    def calculate_log_erdf(self, log_eddington_ratio):
        log_erdf = self.eddington_distribution.log_probability(
            log_eddington_ratio)
        return(log_erdf)

    def calculate_mean_log_eddington_ratio(self):
        mean = self.eddington_distribution.mean()
        return(mean)

    def draw_eddington_ratio(self, num=1):
        log_eddington_ratio = self.eddington_distribution.rvs(size=num)
        return(log_eddington_ratio)

    def _make_distribution(self, log_eddington_star, rho):
        raise NotImplementedError('Cannot make new distribution for fixed ERDF'
                                  ' model.')


# best with larger screen or phone sideways

import requests
from io import BytesIO

from PIL import Image

response = requests.get("https://66.media.tumblr.com/b412951f278ecbf5cffe39a6eeb12f57/tumblr_mhrl362GJt1s376ugo1_1280.jpg")
image_url = Image.open(BytesIO(response.content))

print(image_url)

ASCII_CHARS = ['0', '?', '%', '.', 'X', '+', '.', 'Z', ':', '1', '@']

def scale_image(image, new_width=100):
    """Resize image preserving the aspect ratio."""
    original_width, original_height = image.size
    aspect_ratio = original_height / float(original_width)
    new_height = int(aspect_ratio * new_width)
    new_image = image.resize((new_width, new_height))
    return new_image

def convert_to_grayscale(image):
    return image.convert('L')

def map_pixels_to_ascii_chars(image, range_width=25):
    """Map each pixel to an ascii char.

    Based on the range in which the pixel lies. 0-255
    is divided into 11 ranges of 25 pixels each.
    """
    pixels_in_image = list(image.getdata())
    pixels_to_chars = [
        ASCII_CHARS[pixel_value // range_width]
        for pixel_value in pixels_in_image
    ]
    return "".join(pixels_to_chars)

def convert_image_to_ascii(image, new_width=100):
    image = scale_image(image)
    image = convert_to_grayscale(image)
    pixels_to_chars = map_pixels_to_ascii_chars(image)
    len_pixels_to_chars = len(pixels_to_chars)
    image_ascii = [
        pixels_to_chars[index: index + new_width]
        for index in range(0, len_pixels_to_chars, new_width)
    ]
    return "\n".join(image_ascii)

local_image_filename = image_url

image = local_image_filename
image_ascii = convert_image_to_ascii(image)
print(image_ascii)
write: str = image_ascii
text_file = open("/Users/Y-S/Desktop/Output.txt", "w")
text_file.write(str(write))
text_file.close()


from flask import Flask,request,render_template,redirect
import os
app = Flask(__name__, template_folder='template')
from PIL import Image, ImageOps
import io
import base64

app.config["IMAGE_UPLOADS"] = "C:/Users/91932/Downloads/Flask_assign/test_app/test_app/static"

@app.route('/convert',methods = ["GET","POST"])
def upload_convert_image():
	if request.method == "POST":
		image = request.files['file']

		if image.filename == '':
			print("Image must have a file name")
			return redirect(request.url)

		file_name = image.filename

		basedir = os.path.abspath(os.path.dirname(__file__))

		image.save(os.path.join(basedir,"static", file_name))

		img = Image.open(app.config["IMAGE_UPLOADS"] + "/" + file_name)
		gray_image = img.convert('1')
		data = io.BytesIO()
		gray_image.save(data, "JPEG")
		encode_img_data = base64.b64encode(data.getvalue())

		return render_template("main.html",filename = encode_img_data.decode("UTF-8"))

	return render_template('main.html')

app.run(debug=True,port=2000)


# spark function
from pyspark.sql import SparkSession, DataFrame
from pyspark.sql.functions import udf, col, desc, asc
import pyspark.sql.functions as F
from pyspark import Row
from pyspark.sql.types import IntegerType

# python function
import re
import copy
import subprocess
from functools import reduce
import argparse

def hadoop_list(length, hdfs):
    args = "hdfs dfs -ls "+ hdfs +" | awk '{print $8}'"
    proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
    s_output, s_err = proc.communicate()
    all_dart_dirs = s_output.split()
    return all_dart_dirs[:length]

def headerVCF(hdfs, spark):
    col_name = ["key", "value"]
    vcf = spark.sparkContext.textFile(hdfs)
    vcf = vcf.filter(lambda x : re.match("^##", x))\
            .map(lambda x : x.split("=", 1))\
            .toDF(col_name)
    return vcf

def gatkVCF(hdfs, spark):
    vcf = spark.sparkContext.textFile(hdfs)
    #header_contig = vcf.filter(lambda x : re.match("^#", x))
    col_name = vcf.filter(lambda x : x.startswith("#CHROM")).first().split("\t")
    vcf_data = vcf.filter(lambda x : re.match("[^#][^#]", x))\
                    .map(lambda x : x.split("\t"))\
                    .toDF(col_name)
    return vcf_data

def headerWrite(vcf, vcf_list, index, spark):
    contig = vcf_list[index].toPandas()
    contig_dup = contig["key"].drop_duplicates().tolist()
    contig_len = list(range(len(contig_dup)))
    contig_DF = spark.createDataFrame(list(zip(contig_dup, contig_len)), ["key", "index"])

    return vcf.dropDuplicates(["value"]).join(contig_DF, ["key"], "inner")\
            .select(F.concat_ws("=", F.col("key"), F.col("value")).alias("meta"))\
            .coalesce(1)

def unionAll(*dfs):
    return reduce(DataFrame.unionByName, dfs)

def preVCF(hdfs, flag, spark): # hdfs://, flag 0 == lhs, 1 == rhs
    vcf = spark.sparkContext.textFile(hdfs)
    col_name = vcf.filter(lambda x : x.startswith("#CHROM")).first().split("\t")
    vcf_data = vcf.filter(lambda x : re.match("[^#][^#]", x))\
                        .map(lambda x : x.split("\t"))

    step1 = vcf_data.map(alt_filter).toDF(col_name)
    return_vcf = step1.select(chr_remove_udf(step1["#CHROM"]).cast("Integer").alias("CHROM"), "*")\
                    .drop(col("#CHROM")).filter(col("FILTER") == "PASS")
    if flag == 1:
        for index in range(len(return_vcf.columns[:9])):
            return_vcf = return_vcf.withColumnRenamed(return_vcf.columns[index], return_vcf.columns[index] + "_temp")
    return return_vcf

def chr_remove(chrom):
    chrom = re.sub("chr", "", chrom) # "chr" to ""
    if chrom == "X":
        chrom = "23"
    elif chrom == "Y":
        chrom = "24"
    elif chrom == "XY" or chrom == "M":
        chrom = "-99"
    return chrom
chr_remove_udf = udf(chr_remove)

def alt_filter(row):
    if "," in row[4]:
        temp = row[4].split(",")
        temp.sort()
        row_new = [",".join(temp), ]
        return row[:4] + row_new + row[5:]
    else:
        return row

def rowTodict(format_, row):
    return_col = []
    for ref in row:
        temp_dict = dict()
        temp = ref.split(":")
        for index in range(len(temp)):
            temp_dict[format_[index]] = temp[index]
        return_col.append(temp_dict)
    return return_col

def dictToFormat(col_value, d_format):
    result_return = []
    for temp in col_value:
        temp_col = []
        for keys in d_format:
            if keys in temp:
                temp_col.append(temp[keys])
            else:
                temp_col.append(".")
        result_return.append(":".join(temp_col))
    return tuple(result_return)

def selectCol(row, lhs_len, rhs_len, missing = "."):
    # INFO re
    AC, AN = 0, 0
    if row[9] == None :
        GT = row[lhs_len + 9:]
    elif row[lhs_len + 9] == None :
        GT = row[9:lhs_len]
    else:
        GT = row[9:lhs_len]+row[lhs_len + 9:]

    for temp in GT:
        if temp == None:
            break
        else:
            if "0/1:" in temp:
                AC += 1
                AN += 1
            elif "1/1:" in temp:
                AC += 2
                AN += 1
            elif "0/0:" in temp:
                AN += 1
    AN *= 2

    # rhs가 null
    if row.CHROM_temp == None:
        temp = tuple()
        for ref in range(rhs_len - 9):
            temp += (missing,) # GC

        if row[7] != ".":
            info = row[7].split(";")
            del info[0]
            del info[1]
            info.insert(1, "SF=0")
            info = ";".join(info)
            info = ("AC="+str(AC)+";AN="+str(AN)+";"+info,)
        else :
            info = ("AC="+str(AC)+";AN="+str(AN)+";SF=0", )
        return row[:5] + (float(row.QUAL),) + (row.FILTER, ) + info + (row[8],) + row[9:lhs_len] + temp
    # lhs가 null
    elif row.CHROM == None :
        temp = tuple()
        for ref in range(lhs_len - 9):
            temp += (missing,) # GC
        if row[lhs_len + 7] != ".":
            info = row[lhs_len + 7].split(";")
            del info[0]
            del info[1]
            info.insert(1, "SF=1")
            info = ";".join(info)
            info = ("AC="+str(AC)+";AN="+str(AN)+";"+info,)
        else :
            info = ("AC="+str(AC)+";AN="+str(AN)+";SF=1",)

        return row[lhs_len:lhs_len + 5] + (float(row.QUAL_temp), ) + (row.FILTER_temp, ) + info + (row.FORMAT_temp,) + temp + row[lhs_len + 9:]
    # case, control 둘다 존재
    else:
        # QUAL re-calculation
        format_, lhs_format, rhs_format = row[8].split(":") + row[lhs_len + 8].split(":"), row[8].split(":"), row[lhs_len + 8].split(":")
        dup_format, lhs_col, rhs_col = [], rowTodict(lhs_format, row[9:lhs_len]), rowTodict(rhs_format, row[lhs_len + 9:])
        # format duplicate
        for dup in format_:
            if dup not in dup_format:
                dup_format.append(dup)

        result_lhs, result_rhs = dictToFormat(lhs_col, dup_format), dictToFormat(rhs_col, dup_format)
        # qual re-calcualtion # 100
        col_total = lhs_len + rhs_len - 18
        lhs_QUAL = float(row.QUAL) * ((lhs_len - 9) / col_total)
        rhs_QUAL = float(row.QUAL_temp) * ((rhs_len - 9) / col_total)
        QUAL = lhs_QUAL + rhs_QUAL

        if row[lhs_len + 7] == ".":
            info = row[7]
        elif row[7] == ".":
            info = row[lhs_len + 7]
        else:
            info = row[7]
        info = info.split(";")
        del info[0]
        del info[1]
        info.insert(1, "SF=0,1")
        info = ";".join(info)
        info = ("AC="+str(AC)+";AN="+str(AN)+";"+info,)
        #return row[:5]+(QUAL,)+(row[6],)+info+(row[8],)+row[9:lhs_len]+row[lhs_len + 9:]
        return row[:5]+(QUAL,)+(row[6],)+info+(":".join(dup_format), )+result_lhs + result_rhs


import gym
import gym_connect4_twist_n_turn
import random
from agents.random import RandomAgent
from agents.human import HumanAgent
from agents.monde import MondeAgent
from agents.minimax import MinimaxAgent
from agents.monte_DQN import MonteDQNAgent
env = gym.make('Connect4_Twist_n_Turn-v0') # default board size is height=5, width=6
obs = env.reset()
done = False
player = 0
players = [MonteDQNAgent(3000),MinimaxAgent(4)]
for i in players:
    i.game_starts(obs[0])
legal_actions = env.get_moves()
env.render()
while not done:
    # print(f"obs 0 {obs[0]}\n======\nobs 1 {obs[1]}")
    action = players[player].make_move(obs[player],legal_actions)
    obs, reward, done, info = env.step(action)
    legal_actions = info['legal_actions']

    players[1-player].opponent_move(obs[1-player])

    print(f"player {player+1}'s action {action}")
    env.render()
    player = 1 - player # player index 0 or 1 alternates
for i in range(2):
    players[i].game_terminates(reward[i])

print(f"Winner is player {players[env.winner-1]}")
players[0].plot_loss()


import csv
from time import time
from lzss import inflate, deflate

pairs = [
    #['corpus/customer.xml', 'results/lzss-customer.csv'],
    ['corpus/bible.txt',    'results/lzss-bible.csv'],
    ['corpus/image.jpg',    'results/lzss-image.csv'],
    #['corpus/zeros.txt',    'results/lzss-zeros.csv'],
]

W = [2**i-1 for i in [6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]]
L = [2**i-1 for i in [8]]

for input_file, results_file in pairs:
    b = open(input_file, mode="rb").read()
    out = csv.writer(open(results_file, mode="w"))
    out.writerow(["W", "L", "Compression Ratio", "Encode Time", "Decode Time"])

    for w in W:
        for l in L:
            print("{input_file} {w} {l}".format(input_file=input_file, w=w, l=l))
            deflate_t0 = time()
            x = deflate(b, W=w, L=l)
            deflate_t1 = time()
            inflate_t0 = time()
            inflate(x, W=w, L=l)
            inflate_t1 = time()
            x.fill()
            out.writerow([
                w,
                l,
                (len(x) // 8) / len(b),
                deflate_t1 - deflate_t0,
                inflate_t1 - inflate_t0,
            ])


"""Script for updating github workflows for all our integration forks that have to be synchronized with original repository"""

import os
import json
import subprocess
import shutil
import glob

from context import UserRepoContext
from scripts import FogRepoManager, BOT_USER, FOG_USER

def dump_readme(repo_dir, man: FogRepoManager):
    title = man.fork.name
    url = man.parent.html_url
    license_type = man.get_parent_license().key
    if license_type == 'mit':
        cp = ''
    else:
        owner = man.parent.owner
        cp = f'Copyright {man.parent.created_at.year} [{owner.name or owner.login}]({owner.html_url})'

    with open(os.path.join('templates', 'README.md'), 'r') as f:
        readme = f.read().format(title=title, url=url, copyright=cp)
        with open(os.path.join(repo_dir, 'README.md'), 'w') as g:
            g.write(readme)

def copy_workflows(repo_dir):
    target = os.path.join(repo_dir, '.github', 'workflows')
    os.makedirs(target, exist_ok=True)
    for file_ in glob.glob(r'templates/.github/workflows/*.yml'):
        shutil.copy(file_, target)

if __name__ == "__main__":

    with open('config.json', 'r') as f:
        names = json.load(f)['forks_to_sync']

    tkn = os.environ['GITHUB_TOKEN']
    proc = subprocess.run(['git', 'show', '-s', '--format=%B', 'HEAD'], text=True, capture_output=True)
    last_commit_msg = proc.stdout.strip()

    for repo_name in names:
        man = FogRepoManager(tkn, f'{FOG_USER.login}/{repo_name}')

        with UserRepoContext(tkn, FOG_USER.login, BOT_USER.login, BOT_USER.email, repo_name) as c:
            print('> copying workflow files')
            copy_workflows(repo_dir=c.cwd)
            dump_readme(repo_dir=c.cwd, man=man)
            c.run(f'git commit -a -m "{last_commit_msg}"')
            c.run(f'git push origin master')


"""
Translation of SEMAFOR 3.0's edu.cmu.cs.lti.ark.fn.segmentation.RoteSegmenter.java into Python.
This uses a whitelist of n-grams extracted from FrameNet gold frame annotations,
and filters out some of the unigrams according to heuristics.

Argument: path to CoNLL-format file for the input

Legacy output format (default): tab-separated lines like

9_10#true    1#true    11#true    13#true    15#true    0

(underscore-spearated token IDs for each target; last column is sentence ID)

Pass -j to produce JSON output.

We observe a couple of differences from SEMAFOR's RoteSegmenter:
1) In SEMAFOR 3.0, lemmas were not lowercased before being looked up in the whitelist,
   so some additional targets are predicted here. This is now fixed in the Java codebase.
2) The Java WordNet library (JWNL) performs aggressive simplification of words
   during lemmatization, for example stripping out prefixes/suffixes separated by a
   hyphen or period until it finds a dictionary match. More of those lemmas will be
   found in the whitelist than when using NLTK's morphy, which only removes
   inflectional morphology. (For instance, "scud-B missile" matches whitelist entry
   "scud_N missle_N" with the Java implementation, whereas here only "missle" is
   marked as a target.) Note that we are using the same whitelist file, which was
   presumably created with the JWNL lemmatization.

@author: Nathan Schneider (nschneid@cs.cmu.edu)
@since: 2013-05-22
"""

from __future__ import print_function
import sys, os, codecs, json

from semafor.utils.malt_to_conll import read_conll

def get_coarse_pos(pos):
    return pos[0]   # this seems to be what was in the whitelist

with codecs.open(os.path.dirname(os.path.abspath(__file__))+'/semafor3.0_targets_whitelist.txt', 'r', 'utf-8') as whitelistF:
    WHITELIST = frozenset(whitelistF.read().splitlines())

MAX_LEN = 4 # the maximum length of ngrams we'll look for

FORBIDDEN_WORDS = frozenset('a an as for i it so the with'.split())

# if these words precede "of", "of" should not be discarded
PRECEDING_WORDS_OF = frozenset('''%
                                all
                                face
                                few
                                half
                                majority
                                many
                                member
                                minority
                                more
                                most
                                much
                                none
                                one
                                only
                                part
                                proportion
                                quarter
                                share
                                some
                                third'''.split())

# if these words follow "of", "of" should not be discarded
FOLLOWING_WORDS_OF = frozenset('all group their them us'.split())

# all prepositions should be discarded
LOC_PREPS = frozenset('above against at below beside by in on over under'.split())

TEMPORAL_PREPS = frozenset('after before'.split())
DIR_PREPS = frozenset('into through to'.split())
FORBIDDEN_POS_PREFIXES = frozenset(["PR", "CC", "IN", "TO"])

def get_segmentation(sentence):
    numTokens = len(sentence)

    # start indices that we haven't used yet
    remainingStartIndices = set(range(numTokens))

    lemmas = [token.lemma + "_" + get_coarse_pos(token.postag) for token in sentence]

    # look for ngrams, backing off to smaller n
    for n in range(MAX_LEN, 0, -1):
        for start in range(numTokens-n+1):
            if start not in remainingStartIndices:
                continue
            end = start + n
            ngramSpan = slice(start, end)
            ngramLemmas = ' '.join(lemmas[start:end])
            if ngramLemmas in WHITELIST:
                # found a good ngram, add it to results, and remove it from startIndices so we don't overlap later
                candidate_target = sentence[ngramSpan]
                if shouldIncludeToken(candidate_target, sentence):
                    yield candidate_target
                    remainingStartIndices -= set(range(ngramSpan.start, ngramSpan.stop))

def shouldIncludeToken(candidate_target_tokens, sentence):
    """
    Determines whether an ngram should be kept as a target or discarded based on hand-built rules.
    """
    # always include ngrams, n > 1
    if len(candidate_target_tokens)>1: return True

    token = candidate_target_tokens[0]
    idx = token.id-1    # token ID is 1-based
    assert token is sentence[idx]
    # look up the word in pData
    form = token.form.lower()
    pos = token.postag
    lemma = token.lemma
    # look up the preceding and following words in pData, if they exist
    prevToken = sentence[idx-1] if idx-1>=0 else None
    nextToken = sentence[idx+1] if idx+1<len(sentence) else None

    if form in FORBIDDEN_WORDS: return False
    if form in LOC_PREPS: return False
    if form in DIR_PREPS: return False
    if form in TEMPORAL_PREPS: return False
    if pos[:2] in FORBIDDEN_POS_PREFIXES: return False
    # skip "of course" and "in particular"
    if prevToken and (prevToken.form.lower(), form) in [('of','course'), ('in','particular')]: return False

    if form=="of":
        if prevToken.lemma in PRECEDING_WORDS_OF: return True
        if nextToken.form.lower() in FOLLOWING_WORDS_OF: return True
        if prevToken.postag[:2] in ("JJ", "CD"): return True
        if nextToken.postag[:2]=="CD": return True
        if nextToken.postag[:2]=="DT":
            if idx+2<len(sentence):
                nextNextToken = sentence[idx+2]
                if nextNextToken.postag[:2]=="CD": return True
        return nextToken.ne.startswith("GPE") or nextToken.ne.startswith("LOCATION") or nextToken.ne.startswith("CARDINAL")

    if form=="will": return pos!="MD"
    if lemma=="have": return any(1 for t in sentence if t.head==token.id and t.deprel=='OBJ') # 'have' is a target iff it has an object
    return lemma!="be"

def main(fileP, output_format='legacy' or 'json'):
    with codecs.open(fileP, 'r', 'utf-8') as inF:
        for sentId, sentence in enumerate(read_conll(inF, lookup_lemmas=True)):
            assert None not in sentence

            targets = list(get_segmentation(sentence))

            if output_format.lower()=='json':
                sentJ = {"tokens": [tkn.form for tkn in sentence],
                         "frames": []}
                for target in targets:
                    spansJ = []
                    for tkn in sorted(target, key=lambda x: x.id):
                        # group contiguous tokens into spans
                        if spansJ and spansJ[-1]["end"]==tkn.id:
                            spansJ["end"]+=1
                            spansJ["text"]+= ' ' + tkn.form
                        else:
                            spansJ.append({"start": tkn.id-1, "end": tkn.id, "text": tkn.form})
                    sentJ["frames"].append({"target": {"spans": spansJ}})
                print(json.dumps(sentJ))
            else:
                print('\t'.join('_'.join(str(token.id-1) for token in target_tokens)+'#true' for target_tokens in targets) + '\t' + str(sentId))
            # output format: 9_10#true    1#true    11#true    13#true    15#true    0
            # underscore-spearated token IDs for each target; last column is sentence ID
            # (if no targets, will simply be a tab followed by sentence number)

if __name__=='__main__':
    args = sys.argv[1:]
    fmt = 'legacy'
    if args[0]=='-j':
        fmt = 'json'
        args.pop(0)
    fileP, _ = args+[None]
    main(fileP, output_format=fmt)


import pytest
import logging
import time
import threading
import parametrize

logger = logging.getLogger(__name__)
logger.setLevel(level=logging.INFO)

@pytest.mark.describe("Client object")
class TestConnectDisconnect(object):
    @pytest.mark.it("Can disconnect and reconnect")
    @pytest.mark.parametrize(*parametrize.connection_retry_disabled_and_enabled)
    @pytest.mark.parametrize(*parametrize.auto_connect_disabled_and_enabled)
    @pytest.mark.quicktest_suite
    def test_sync_connect_disconnect(self, brand_new_client, leak_tracker):
        leak_tracker.set_initial_object_list()

        client = brand_new_client

        client.connect()
        assert client.connected

        client.disconnect()
        assert not client.connected

        client.connect()
        assert client.connected

        leak_tracker.check_for_leaks()

    @pytest.mark.it(
        "Can do a manual connect in the `on_connection_state_change` call that is notifying the user about a disconnect."
    )
    @pytest.mark.parametrize(*parametrize.connection_retry_disabled_and_enabled)
    @pytest.mark.parametrize(*parametrize.auto_connect_disabled_and_enabled)
    # see "This assert fails because of initial and secondary disconnects" below
    @pytest.mark.skip(reason="two stage disconnect causes assertion in test code")
    def test_sync_connect_in_the_middle_of_disconnect(
        self, brand_new_client, service_helper, random_message, leak_tracker
    ):
        """
        Explanation: People will call `connect` inside `on_connection_state_change` handlers.
        We have to make sure that we can handle this without getting stuck in a bad state.
        """
        leak_tracker.set_initial_object_list()

        client = brand_new_client
        assert client

        reconnected_event = threading.Event()

        def handle_on_connection_state_change():
            nonlocal reconnected_event

            if client.connected:
                logger.info("handle_on_connection_state_change connected.  nothing to do")
            else:
                logger.info("handle_on_connection_state_change disconnected.  reconnecting.")
                client.connect()
                assert client.connected
                reconnected_event.set()
                logger.info("reconnect event set")

        client.on_connection_state_change = handle_on_connection_state_change

        # connect
        client.connect()
        assert client.connected

        # disconnect.
        reconnected_event.clear()
        logger.info("Calling client.disconnect.")
        client.disconnect()

        # wait for handle_on_connection_state_change to reconnect
        logger.info("waiting for reconnect_event to be set.")
        reconnected_event.wait()

        logger.info(
            "reconnect_event.wait() returned.  client.connected={}".format(client.connected)
        )
        # This assert fails because of initial and secondary disconnects
        assert client.connected

        # sleep a while and make sure that we're still connected.
        time.sleep(3)
        assert client.connected

        # finally, send a message to makes reu we're _really_ connected
        client.send_message(random_message)
        event = service_helper.wait_for_eventhub_arrival(random_message.message_id)
        assert event

        leak_tracker.check_for_leaks()

    @pytest.mark.it(
        "Can do a manual disconnect in the `on_connection_state_change` call that is notifying the user about a connect."
    )
    @pytest.mark.parametrize(*parametrize.connection_retry_disabled_and_enabled)
    @pytest.mark.parametrize(*parametrize.auto_connect_disabled_and_enabled)
    @pytest.mark.parametrize(
        "first_connect",
        [pytest.param(True, id="First connection"), pytest.param(False, id="Second connection")],
    )
    def test_sync_disconnect_in_the_middle_of_connect(
        self, brand_new_client, service_helper, random_message, first_connect, leak_tracker
    ):
        """
        Explanation: This is the inverse of `test_connect_in_the_middle_of_disconnect`.  This is
        less likely to be a user scenario, but it lets us test with unusual-but-specific timing
        on the call to `disconnect`.
        """
        leak_tracker.set_initial_object_list()

        client = brand_new_client
        assert client
        disconnect_on_next_connect_event = False

        disconnected_event = threading.Event()

        def handle_on_connection_state_change():
            nonlocal disconnected_event

            if client.connected:
                if disconnect_on_next_connect_event:
                    logger.info("connected.  disconnecting now")
                    client.disconnect()
                    disconnected_event.set()
                else:
                    logger.info("connected, but nothing to do")
            else:
                logger.info("disconnected.  nothing to do")

        client.on_connection_state_change = handle_on_connection_state_change

        if not first_connect:
            # connect
            client.connect()
            assert client.connected

            # disconnect.
            client.disconnect()

        assert not client.connected

        # now, connect (maybe for the second time), and disconnect inside the on_connected handler
        disconnect_on_next_connect_event = True
        disconnected_event.clear()
        client.connect()

        # and wait for us to disconnect
        disconnected_event.wait()
        assert not client.connected

        # sleep a while and make sure that we're still disconnected.
        time.sleep(3)
        assert not client.connected

        # finally, connect and make sure we can send a message
        disconnect_on_next_connect_event = False
        client.connect()
        assert client.connected

        client.send_message(random_message)
        event = service_helper.wait_for_eventhub_arrival(random_message.message_id)
        assert event

        leak_tracker.check_for_leaks()

@pytest.mark.dropped_connection
@pytest.mark.describe("Client object with dropped connection")
@pytest.mark.keep_alive(5)
class TestConnectDisconnectDroppedConnection(object):
    @pytest.mark.it("disconnects when network drops all outgoing packets")
    def test_sync_disconnect_on_drop_outgoing(self, client, dropper, leak_tracker):
        """
        This test verifies that the client will disconnect (eventually) if the network starts
        dropping packets
        """
        leak_tracker.set_initial_object_list()

        client.connect()
        assert client.connected
        dropper.drop_outgoing()

        while client.connected:
            time.sleep(1)

        # we've passed the test. Now wait to reconnect before we check for leaks. Otherwise we
        # have a pending ConnectOperation floating around and this would get tagged as a leak.
        dropper.restore_all()
        while not client.connected:
            time.sleep(1)

        leak_tracker.check_for_leaks()

    @pytest.mark.it("disconnects when network rejects all outgoing packets")
    def test_sync_disconnect_on_reject_outgoing(self, client, dropper, leak_tracker):
        """
        This test verifies that the client will disconnect (eventually) if the network starts
        rejecting packets
        """
        leak_tracker.set_initial_object_list()

        client.connect()
        assert client.connected
        dropper.reject_outgoing()

        while client.connected:
            time.sleep(1)

        # we've passed the test. Now wait to reconnect before we check for leaks. Otherwise we
        # have a pending ConnectOperation floating around and this would get tagged as a leak.
        dropper.restore_all()
        while not client.connected:
            time.sleep(1)

        leak_tracker.check_for_leaks()


# -*- coding: utf-8 -*-
"""
Created on Mon Jun  6 23:03:49 2016

@author: yanxin
"""
from sqlalchemy import create_engine
from selenium import webdriver
import time
import datetime
from bs4 import BeautifulSoup
import pandas as pd
import re

email, passwd,db_name,db_user,db_pswd = open('login.secret').read().strip().split(',')

cities = ["Boston, MA","Chicago, IL","Seattle, WA","DALLAS, TX"]

browser = webdriver.Chrome('/home/yanxin/Desktop/chromedriver')
browser.get('https://app.strava.com/login')
browser.find_element_by_name('email').send_keys(email)
browser.find_element_by_name('password').clear()
browser.find_element_by_name('password').send_keys(passwd)
browser.find_element_by_id('login-button').submit()

A=[]
B=[]
C=[]
D=[]
E=[]
F=[]
G=[]
H=[]
I=[]
J=[]
stat1=[]
stat2=[]
stat3=[]
stat4=[]
stat5=[]
Loca=[]
ath_page=[]
latitude = []
longitude = []

for i in xrange(len(cities)):
    browser.get('https://www.strava.com/segments/search')
    browser.find_element_by_name('keywords').clear()
    browser.find_element_by_name('keywords').send_keys(cities[i])
    browser.find_element_by_id('climb-search-button').submit()
    time.sleep(2)

    html_code = browser.page_source.encode('utf-8')
    Soup = BeautifulSoup(html_code, 'html.parser')
    segmentList = []
    urlList = []

    for link in Soup.find_all('div'):
        segment = link.get('data-segment-id')
        if segment:
           segmentList.append(segment)
           urlList.append('https://www.strava.com/segments/'+segment)

    for r in xrange(len(segmentList)-1):

        filename='data/'+cities[i]+'+'+segmentList[r]+'.html'
        f = open(filename,'rU')
        soup = BeautifulSoup(f, 'html.parser')

        statsGroup=soup.find('ul',class_="inline-stats list-stats")

        states = statsGroup.find_all('b',class_="stat-text")
        stata  = states[0].text.strip('mi')
        statb  = states[1].text.strip('%')
        statc  = states[2].text.strip('ft')
        statd  = states[3].text.strip('ft')
        state  = states[4].text.strip('ft')

        jsgroups= soup.find_all('script')
        regex =  re.compile(r'"lat_lng":\[(\d+.\d+),(\-\d+.\d+)]')

        lat= None
        lng= None
        for js in jsgroups:
            text = js.text
            js_text = re.search(regex, text)
            if js_text:
                found=js_text
                lat = float(found.group(1))
                lng = float(found.group(2))

        locations=soup.find_all('span')
        for li in locations:
            if li.get('data-full-name') is not None:
                location = li.get('data-full-name')

        right_table=soup.find_all('table', class_='striped')

        for table in right_table:
            for row in table.findAll('tr'):
                cells = row.findAll('td')
                states = row.findAll('th')
                if len(cells)==8: #Only extract table body not heading
                    A.append(cells[0].text.strip('\n'))
                    B.append(cells[1].find('a').text)
                    ath_link=cells[1].find('a',href=True)['href']
                    ath_page.append('https://www.strava.com'+ath_link)
                    day = cells[2].find('a').text
                    date=datetime.datetime.strptime(day,'%b %d, %Y').date
                    C.append(date)
                    D.append(cells[3].text.strip('mi/h'))
                    E.append(cells[4].text.strip('\n').strip('bpm').strip('-'))
                    F.append(cells[5].text.strip('\n').strip('W'))
                    G.append(cells[6].text.strip('\n').strip('-'))
                    H.append(cells[7].text.strip('s'))
                    I.append(cities[i])
                    J.append(segmentList[r])
                    stat1.append(stata)
                    stat2.append(statb)
                    stat3.append(statc)
                    stat4.append(statd)
                    stat5.append(state)
                    Loca.append(location)
                    latitude.append(lat)
                    longitude.append(lng)

browser.quit()

df=pd.DataFrame(A,columns=['rank'])
df['name']=B
df['date']=C
df['pace']=D
df['hr']=E
df['power']=F
df['vam']=G
df['time']=H
df['city']=I
df['segment']=J
df['distance']=stat1
df['avg_grade']=stat2
df['lowest_elev']=stat3
df['highest_elev']=stat4
df['elev_difference']=stat5
df['segment_name']=Loca
df['athlete_link']=ath_page
df['latitude']=latitude
df['longitude']=longitude
df['date'] = pd.to_datetime(df['date'].astype(str))

engine = create_engine('postgresql://%s:%s@localhost/%s'%(db_user,db_pswd,db_name))

## insert data into database from Python
df.to_sql('running_data_table', engine, if_exists='replace')


# 백준 2800번 괄호
import sys
from itertools import combinations

input = sys.stdin.readline().rstrip()
index = -1
stack = []
glst = []
resarr = set()

# 괄호쌍 위치 glst에 넣기
for i in range(len(input)):
    if input[i] == '(':
        stack.append(i)
    elif input[i] == ')':
        s = stack.pop()
        glst.append((s,i))

# 조합을 이용해 괄호 제거
for i in range(1,len(glst)+1):
    for comb in combinations(glst,i):
        tmp = list(input)
        for x in comb:
            tmp[x[0]] = ""
            tmp[x[1]] = ""
            resarr.add("".join(tmp))

for x in sorted(list(resarr)):
    print(x)


'''Create and configure the chatroom and comment form'''

from django import forms
from .models import Chatroom, Comment

class AddChatroomForm(forms.ModelForm):
    ''' Configure the chatroom form '''
    class Meta:
        ''' Form meta properties '''
        model = Chatroom
        fields = ('name', 'subject',)

    def __init__(self, *args, **kwargs):

        super().__init__(*args, **kwargs)
        placeholders = {
            'name': 'Give a name to this chatroom',
            'subject': 'Start the discussion with a question or a statement',
        }

        for field in self.fields:
            placeholder = f'{placeholders[field]}'
            self.fields[field].widget.attrs['placeholder'] = placeholder

class AddCommentForm(forms.ModelForm):

    class Meta:
        model = Comment
        fields = ('content',)

    def __init__(self, *args, **kwargs):

        super().__init__(*args, **kwargs)
        placeholders = {
            'content': 'Participate in the discussion',
        }

        self.fields['content'].widget.attrs['autofocus'] = True
        for field in self.fields:
            placeholder = f'{placeholders[field]} *'
        self.fields[field].widget.attrs['placeholder'] = placeholder
        self.fields[field].label = False


import pickle
import parameters as param
import os
import argparse
import paths

class BuildDataset:
    parser = argparse.ArgumentParser()
    parser.add_argument('-mode', default="", help="\nmode values: train, test, validation, duc_test\n"
                                                  "e.g. python build_dataset.py -mode train -model lg100d5g")
    parser.add_argument('-model', default="", help="\nmodel values: lg100d5, lg200d5, neg100, neg200...\n"
                                                   "e.g. python build_dataset.py -mode test -model neg200g")
    def get_args(self):
        return self.parser.parse_args()

    def __init__(self):
        args = self.get_args()
        #if args.mode == '':  # for running in IDE
        #    args.mode = 'test'
        #    args.model = 'bg'

        if args.mode == 'train':
            self.build_whole_train_dataset_and_dictionary(paths.train_data_dir,
                                                          paths.train_article_file_path,
                                                          paths.train_title_file_path,
                                                          paths.x_chunks_dir,
                                                          paths.y_chunks_dir,
                                                          paths.x_txt_dir,
                                                          paths.y_txt_dir,
                                                          paths.x_chunks_sample_dir,
                                                          paths.y_chunks_sample_dir,
                                                          paths.word2int_file_path, paths.word2int_txt_file_path,
                                                          paths.int2word_file_path, paths.int2word_txt_file_path,
                                                          paths.word_freq_file_path, paths.vocab_txt_file_path,
                                                          paths.maxlen_file_path, paths.maxlen_txt_file_path,
                                                          param.lines_per_chunk, param.read_lines)
        elif args.mode == 'validation':
            self.build_validation_or_testing_dataset(paths.validation_article_file_path,
                                                     paths.validation_data_dir,
                                                     paths.validation_rouge_results_dir,
                                                     paths.validation_system_summaries_dir,
                                                     paths.word2int_file_path,
                                                     paths.int2word_file_path,
                                                     paths.maxlen_file_path,
                                                     purpose='validation')
        elif args.mode == 'test':
            self.build_validation_or_testing_dataset(paths.test_article_file_path,
                                                     paths.test_data_dir,
                                                     paths.test_rouge_results_dir,
                                                     paths.test_system_summaries_dir,
                                                     paths.word2int_file_path,
                                                     paths.int2word_file_path,
                                                     paths.maxlen_file_path,
                                                     purpose='test')
        elif args.mode == 'duc_test':
            self.build_validation_or_testing_dataset(paths.test_duc_article_file_path,
                                                     paths.test_duc_data_dir,
                                                     paths.test_duc_rouge_results_dir,
                                                     paths.test_duc_system_summaries_dir,
                                                     paths.word2int_file_path,
                                                     paths.int2word_file_path,
                                                     paths.maxlen_file_path,
                                                     purpose='test')

        else:
            print("\nmode values: train, test, validation, duc_test\n"
                  "e.g. python build_dataset.py -mode train -model lg100d5")

    @staticmethod
    def num_of_lines_of_file(file):
        count = 0
        with open(file, 'r', encoding='utf8') as f:
            for line in f:
                count += 1
        # print("Number of lines: ", str(count))
        return count

    @staticmethod
    def view_some_lines(file=paths.train_article_file_path, from_line=1, to_line=5):
        with open(file, 'r', encoding='utf8') as f:
            line_index = 0
            for line in f:
                line_index += 1
                if from_line <= line_index <= to_line:
                    print(line)

    def build_whole_train_dataset_and_dictionary(self, train_data_dir, train_article_path, train_summary_path,
                                                 x_chunks_dir,
                                                 y_chunks_dir, x_txt_dir, y_txt_dir, x_sample_dir, y_sample_dir,
                                                 word2int_file_path, word2int_txt_file_path,
                                                 int2word_file_path, int2word_txt_file_path,
                                                 word_freq_file_path, vocab_txt_file_path,
                                                 maxlen_file_path, maxlen_txt_file_path,
                                                 lines_per_chunk, read_lines):

        if not os.path.exists(x_chunks_dir):
            os.makedirs(x_chunks_dir)
        else:
            for filename in os.listdir(x_chunks_dir):
                os.remove(x_chunks_dir + filename)
        if not os.path.exists(y_chunks_dir):
            os.makedirs(y_chunks_dir)
        else:
            for filename in os.listdir(y_chunks_dir):
                os.remove(y_chunks_dir + filename)
        if not os.path.exists(x_txt_dir):
            os.makedirs(x_txt_dir)
        else:
            for filename in os.listdir(x_txt_dir):
                os.remove(x_txt_dir + filename)
        if not os.path.exists(y_txt_dir):
            os.makedirs(y_txt_dir)
        else:
            for filename in os.listdir(y_txt_dir):
                os.remove(y_txt_dir + filename)
        if not os.path.exists(x_sample_dir):
            os.makedirs(x_sample_dir)
        else:
            for filename in os.listdir(x_sample_dir):
                os.remove(x_sample_dir + filename)
        if not os.path.exists(y_sample_dir):
            os.makedirs(y_sample_dir)
        else:
            for filename in os.listdir(y_sample_dir):
                os.remove(y_sample_dir + filename)
        print('Directories have been created or old chunks have been removed')
        print('Article file: {}'.format(train_article_path))
        print('Summary file: {}'.format(train_summary_path))
        article_max_len = 0
        summary_max_len = 0
        num_of_lines = self.num_of_lines_of_file(file=train_article_path)
        if num_of_lines > read_lines:
            num_of_lines = read_lines
        # words = list()
        word_freq_dict = dict()
        num_of_chuncks = num_of_lines // lines_per_chunk
        if num_of_lines % lines_per_chunk > 0:
            num_of_chuncks += 1
        print('Lines of file: ' + str(num_of_lines))
        print('Lines per chunk: ' + str(lines_per_chunk))
        print('Chunks: ' + str(num_of_chuncks))
        chunk_index = 0
        line_index = 1

        while chunk_index < num_of_chuncks:
            start_line = line_index
            end_line = start_line + lines_per_chunk - 1
            if end_line > num_of_lines:
                end_line = num_of_lines
            chunk_index += 1
            print(str(chunk_index) + '/' + str(num_of_chuncks) +
                  ' chunk: (start_line, end_line) -> (' + str(start_line) + ', ' + str(end_line) + ')')
            line_index += lines_per_chunk

            train_article_list = self.get_text_list_from_line_range(train_article_path, start_line, end_line)
            # print(train_article_list)
            print('  (1/8) The articles have been assigned to list.')
            train_summary_list = self.get_text_list_from_line_range(train_summary_path, start_line, end_line)
            print('  (2/8) The summaries have been assigned to list.')
            x = []
            y = []
            for sentence in train_article_list:
                # print(sentence)
                # tokenized_sentence = nltk.tokenize.word_tokenize(sentence)
                tokenized_sentence = sentence.split()
                x.append(tokenized_sentence)
                sent_len = len(tokenized_sentence)
                if sent_len > article_max_len:
                    article_max_len = sent_len
                for word in tokenized_sentence:
                    if word not in word_freq_dict:
                        word_freq_dict[word] = 1
                    else:
                        word_freq = word_freq_dict[word] + 1
                        word_freq_dict[word] = word_freq

            print('  (3/8) The words of articles have been added to the words dictionary.')
            print('  (4/8) X set has been created.')
            chunk_index_str = self.int_to_three_digits_str(chunk_index)
            with open(x_chunks_dir + 'x_' + chunk_index_str + '.pickle', 'wb') as f:
                pickle.dump(x, f)
            print('  (5/8) X_' + chunk_index_str + ' chunk has been written to file.')
            del x
            del train_article_list

            for sentence in train_summary_list:
                # print(sentence)
                # tokenized_sentence = nltk.tokenize.word_tokenize(sentence)
                tokenized_sentence = sentence.split()
                y.append(tokenized_sentence)
                sent_len = len(tokenized_sentence)
                if sent_len > summary_max_len:
                    summary_max_len = sent_len
                for word in tokenized_sentence:
                    if word not in word_freq_dict:
                        word_freq_dict[word] = 1
                    else:
                        word_freq = word_freq_dict[word] + 1
                        word_freq_dict[word] = word_freq
                    # words.append(word)

            print('  (6/8) The words of summaries have been appended to the list of words.')
            print('  (7/8) Y set has been created.')

            with open(y_chunks_dir + 'y_' + chunk_index_str + '.pickle', 'wb') as f:
                pickle.dump(y, f)
                print('  (8/8) Y_' + chunk_index_str + ' chunk has been written to file.')
            del y
            del train_summary_list

        print('(1/12) the whole dataset has been obtained')

        with open(word_freq_file_path, "wb") as f:
            pickle.dump(word_freq_dict, f)

        word2int_dict = dict()
        int2word_dict = dict()
        word2int_dict['<PAD>'] = 0
        word2int_dict['<UNK>'] = 1
        word2int_dict['<S>'] = 2
        word2int_dict['</S>'] = 3
        int2word_dict[0] = '<PAD>'
        int2word_dict[1] = '<UNK>'
        int2word_dict[2] = '<S>'
        int2word_dict[3] = '</S>'
        index_word_dict = 4
        for word, freq in word_freq_dict.items():
            word2int_dict[word] = index_word_dict
            int2word_dict[index_word_dict] = word
            index_word_dict += 1
            # word_dict[word] = len(word_dict)
        print('(2/12) word2int dictionary has been created.')
        # int2word_dict = dict(zip(word2int_dict.values(), word2int_dict.keys()))
        print('(3/12) int2word dictionary has been created.')

        with open(word2int_file_path, "wb") as f:
            pickle.dump(word2int_dict, f)
        print('(4/12) word2int has been written to binary file.')
        with open(int2word_file_path, "wb") as f:
            pickle.dump(int2word_dict, f)
        print('(5/12) Int2word has been written to binary file.')

        ######################
        with open(word2int_txt_file_path, "w+", encoding='utf8') as f:
            for k, v in word2int_dict.items():
                f.write('{} {}\n'.format(k, v))
        with open(int2word_txt_file_path, "w+", encoding='utf8') as f:
            for k, v in int2word_dict.items():
                f.write('{} {}\n'.format(k, v))
        print('(6/12) word2int and Int2word has been written to txt file.')
        #############################3

        for filename in os.listdir(x_chunks_dir):
            # print(filename)
            path_to_file = x_chunks_dir + filename
            x = self.read_pickle_file(path_to_file)
            # print(str(len(x[0])) + " " + str(x))
            x = list(map(lambda d: list(map(lambda w: word2int_dict.get(w, word2int_dict["<UNK>"]), d)), x))
            # print(str(len(x[0])) + " " + str(x))
            x = list(map(lambda d: d[:article_max_len], x))
            # print(str(len(x[0])) + " " + str(x))
            x = list(map(lambda d: d + (article_max_len - len(d)) * [word2int_dict["<PAD>"]], x))
            # print(str(len(x[0])) + " " + str(x))
            # ##############3
            # print(x[0])
            # print(x[1])
            with open(path_to_file, 'wb') as f:
                pickle.dump(x, f)
            del x

        print('(7/12) X set has been written to chunked files')

        for filename in os.listdir(y_chunks_dir):
            # print(filename)
            path_to_file = y_chunks_dir + filename
            y = self.read_pickle_file(y_chunks_dir + filename)
            # print(str(len(y[0])) + " " + str(y))
            y = list(map(lambda d: list(map(lambda w: word2int_dict.get(w, word2int_dict["<UNK>"]), d)), y))
            # print(str(len(y[0])) + " " + str(y))
            # y = list(map(lambda d: d[:summary_max_len], y))
            # print(str(len(y[0])) + " " + str(y))
            # ############33
            # print(y[0])
            # print(y[1])
            with open(path_to_file, 'wb') as f:
                pickle.dump(y, f)
            del y

        print('(8/12) Y set has been written to chunked files')

        maxlen_dict = dict()
        maxlen_dict['article_max_len'] = article_max_len
        maxlen_dict['summary_max_len'] = summary_max_len
        maxlen_dict['vocabulary_len'] = index_word_dict
        maxlen_dict['number_of_articles'] = num_of_lines
        maxlen_dict['number_of_chunks'] = num_of_chuncks
        maxlen_dict['articles_per_chunk'] = lines_per_chunk
        with open(maxlen_file_path, 'wb') as f:
            pickle.dump(maxlen_dict, f)

        with open(maxlen_txt_file_path, 'w') as f:
            for k, v in maxlen_dict.items():
                f.write(str(k) + ' ' + str(v) + '\n')
        print('(9/12) The dataset parameters have been written to file.')

        # Vocabulary
        vocab_list = []
        for key, value in word_freq_dict.items():
            temp = (key, value)
            vocab_list.append(temp)
        vocab_list = sorted(vocab_list, key=lambda tup: -tup[1])
        with open(vocab_txt_file_path, 'w+', encoding='utf8') as f:
            for w in vocab_list:
                f.write(str(w[0]) + ' ' + str(w[1]) + '\n')
        print('(10/12) The vocabulary has been written to file.')
        del vocab_list
        # return x, y, word_dict, reversed_dict, article_max_len, summary_max_len, maxlen_dict
        self.write_txt_files_for_binary_files(x_chunks_dir, x_txt_dir,
                                              int2word_file_path)
        # print('(11/9) The txt x chunk files have been written.')
        self.write_txt_files_for_binary_files(y_chunks_dir, y_txt_dir,
                                              int2word_file_path)
        print('(11/12) The txt chunk files have been written.')

        self.create_sample_training_set()
        print('(12/12) The sample dataset has been created')

        print('Process finished.')

    @staticmethod
    def int_to_three_digits_str(number):
        if number < 10:
            return '00' + str(number)
        elif number < 100:
            return '0' + str(number)
        elif number < 1000:
            return str(number)
        else:
            return "Error: the number has more than three digits"

    def get_text_list_from_line_range(self, data_path, start_line, end_line):
        with open(data_path, "r", encoding='utf-8') as f:
            lines_list = []
            start_greater_than = start_line - 1
            end_less_than = end_line + 1
            line_index = 0
            for line in f:
                line_index += 1
                if start_greater_than < line_index < end_less_than:
                    # line = self.clean_str(line.strip())
                    lines_list.append(line)
                    # print(str(line_index) + " " + line)
                if line_index == end_line:
                    break
        return lines_list

    def build_validation_or_testing_dataset(self, articles_data_file,
                                            data_dir, rouge_results_dir, system_summaries_dir,
                                            word2int_dict_pickle_file_path,
                                            int2word_dict_pickle_file_path,
                                            maxlen_file_path,
                                            purpose='validataion', simple_to_full=False):  # or purpose='test'
        # for filename in os.listdir(data_dir):
        #    os.remove(data_dir + filename)
        print('Old files from {}_data have been removed'.format(purpose))
        print('Article file: {}'.format(articles_data_file))
        print('data dir: {}'.format(data_dir))
        print('word2int_dict file: {}'.format(word2int_dict_pickle_file_path))
        print('int2word_dict file: {}'.format(int2word_dict_pickle_file_path))
        print('maxlen  file: {}'.format(maxlen_file_path))
        print('validataion: {}'.format(purpose))

        if not os.path.exists(data_dir):
            os.makedirs(data_dir)
        if not os.path.exists(rouge_results_dir):
            os.makedirs(rouge_results_dir)
        if not os.path.exists(system_summaries_dir):
            os.makedirs(system_summaries_dir)
        # if not os.path.exists(rouge_files_dir):
        #    os.makedirs(rouge_files_dir)

        word2int_dict = self.read_pickle_file(word2int_dict_pickle_file_path)
        print('(1/8) Word2int dictionary has been assigned to the variable')
        article_max_len = self.read_pickle_file(maxlen_file_path)['article_max_len']
        if simple_to_full:
            article_max_len = self.read_pickle_file(maxlen_file_path)['article_max_len']
        ################
        #print('article_max_len ', article_max_len)
        test_article_list = self.get_text_list(articles_data_file)
        print('(2/8) The {} articles has been assigned to list'.format(purpose))
        # x = list(map(lambda d: nltk.tokenize.word_tokenize(d), test_article_list))
        x = list(map(lambda d: d.split(), test_article_list))
        count_aritcles = 0
        for i in x:
            count_aritcles += 1
        #    if len(i) > article_max_len:
        #        # print(len(i))
        #        article_max_len = len(i)
        print('(3/8) The maximum article length has been obtained')
        # print(article_max_len)
        x = list(map(lambda d: list(map(lambda w: word2int_dict.get(w, word2int_dict["<UNK>"]), d)), x))
        x = list(map(lambda d: d[:article_max_len], x))
        x = list(map(lambda d: d + (article_max_len - len(d)) * [word2int_dict["<PAD>"]], x))
        print('(4/8) X has been created')
        with open(data_dir + "x_" + purpose + ".pickle", "wb") as f:
            pickle.dump(x, f)
        print('(5/8) X has been written to file: {}'.format(data_dir + "x_" + purpose + ".pickle"))
        maxlen_dict = dict()
        maxlen_dict[purpose + '_article_max_len'] = article_max_len
        maxlen_dict['number_of_articles'] = count_aritcles
        with open(data_dir + purpose + "_article_max_len.pickle", "wb") as f:
            pickle.dump(maxlen_dict, f)
        with open(data_dir + purpose + "_article_max_len.txt", 'w') as f:
            for k, v in maxlen_dict.items():
                f.write(str(k) + ' ' + str(v) + '\n')
        print('(6/8) The {} set has been created.'.format(purpose))
        self.write_txt_file_of_binary_file(data_dir + "x_" + purpose + ".pickle",
                                           data_dir + "x_" + purpose + ".txt",
                                           int2word_dict_pickle_file_path)
        print('(7/8) X has been written to txt file: {}.'.format(data_dir + "x_" + purpose + ".txt"))

        if simple_to_full:
            dir = paths.test_dir
            prefix = 'test_subset'
            if purpose == 'validation':
                dir = paths.validation_dir
                prefix = purpose

            read_file = dir + '{}_{}_titles.txt'.format(prefix, param.full_summary_model_id)
            write_file = dir + '{}_{}_titles.txt'.format(prefix, param.simple_to_full_model_id)
            r = open(read_file, 'r', encoding='utf8')
            w = open(write_file, 'w', encoding='utf8')
            for line in r:
                w.write(line)
            r.close()
            w.close()
            print('(8/8) File {} has been copied to {}.'.format(read_file, write_file))

        print('process finished.')

    @staticmethod
    def read_pickle_file(path_to_pickle_file):
        with open(path_to_pickle_file, "rb") as f:
            b = pickle.load(f)
        return b

    @staticmethod
    # for a text file, it returns a list of lines
    def get_text_list(data_path):
        with open(data_path, "r", encoding='utf-8') as f:
            lines_list = []
            for line in f:
                lines_list.append(line)
        return lines_list

    def create_sample_training_set(self, x_dir=paths.x_chunks_dir, y_dir=paths.y_chunks_dir,
                                   x_sample_dir=paths.x_chunks_sample_dir, y_sample_dir=paths.y_chunks_sample_dir,
                                   training_size=100, num_of_files=3):
        x_chunked_filenames_list = os.listdir(x_dir)
        y_chunked_filenames_list = os.listdir(y_dir)
        x_chunked_filenames_list.sort()
        y_chunked_filenames_list.sort()
        for xf, yf in zip(x_chunked_filenames_list[:num_of_files], y_chunked_filenames_list[:num_of_files]):
            print('{}  {}'.format(xf, yf))
            x = self.read_pickle_file(x_dir + xf)
            y = self.read_pickle_file(y_dir + yf)
            x_new = []
            y_new = []
            counter = 1
            for i, j in zip(x, y):
                x_new.append(i)
                y_new.append(j)
                counter += 1
                if counter > training_size:
                    break
            with open(x_sample_dir + xf, "wb") as f:
                pickle.dump(x_new, f)
            print('X set has been written to file: {}'.format(x_sample_dir + xf))

            with open(y_sample_dir + yf, "wb") as f:
                pickle.dump(y_new, f)
            print('Y set has been written to file:{}'.format(y_sample_dir + yf))

    # It write a txt file from a binary file
    def write_txt_file_of_binary_file(self, binary_file_path, txt_file_path, int2word_file_path):
        int2word = self.read_pickle_file(int2word_file_path)
        text_int_list = self.read_pickle_file(binary_file_path)
        with open(txt_file_path, 'w+', encoding='utf8') as f:
            for int_line_list in text_int_list:
                line_str = ''
                for int_word in int_line_list:
                    line_str += int2word[int_word] + " "
                f.write(line_str + '\n')

    # it writes txt files from a list of binary files
    def write_txt_files_for_binary_files(self, binary_file_dir, txt_file_dir, int2word_file_path):
        binary_files_list = os.listdir(binary_file_dir)
        int2word = self.read_pickle_file(int2word_file_path)
        for b in binary_files_list:
            text_int_list = self.read_pickle_file(binary_file_dir + b)
            with open(txt_file_dir + b + '.txt', 'w+', encoding='utf8') as f:
                for int_line_list in text_int_list:
                    line_str = ''
                    for int_word in int_line_list:
                        line_str += int2word[int_word] + " "
                    f.write(line_str + '\n')

if __name__ == "__main__":
    BuildDataset()


import collections
from collections import Counter
import statistics
numbers= input('Input your list of numbers:')
number=[ int(x) for x in numbers.split(" ") ]
print('')
mean=0
for x in range(len(number)):
  mean+=number[x]
print('mean:', mean/len(number))

values = sorted(number)
if len(number)%2 ==1:
  print('median:',values[len(values)//2])
if len(number)%2 ==0:
  print('median:',values[(len(values)-1)//2],values[(len(values))//2])

def mode(numbers):
    largestCount = 0
    modes = []
    for x in numbers:
        if x in modes:
            continue
        count = numbers.count(x)
        if count > largestCount:
            del modes[:]
            modes.append(x)
            largestCount = count
        elif count == largestCount:
            modes.append(x)
    return modes

try:
  x= (mode(number)[0])
  print('mode:',x)

except statistics.StatisticsError:
  print('mode: no mode')