#!/usr/bin/env python

"""
Doc-string from learn_model_class:
A script which produces linear energy matrix models for a given data set.
"""
from __future__ import division
# Our standard Modules
import numpy as np
import scipy as sp
import sys
import scipy.sparse
# Our miscellaneous functions
import pandas as pd
import utils as utils
from sklearn import linear_model
import EstimateMutualInfoforMImax as EstimateMutualInfoforMImax
import pymc
import stepper as stepper
import os
from mpathic import SortSeqError
import io_local as io
import gauge as gauge
import qc as qc
import pdb
from mpathic import shutthefuckup
import numerics as numerics
from sklearn.preprocessing import StandardScaler
import cvxopt
from cvxopt import solvers, matrix, spdiag, sqrt, div, exp
import warnings
#import profile_mut_old as profile_mut
from profile_mut import ProfileMut
from utils import handle_errors,check,ControlledError

import fast

class LearnModel:
    """
     Constructor for the learn model class. Models can be learnt via the matrix model or
     the neighbor model. Matrix models assume independent contributions to activity
     from characters at a particular position whereas neighbor model assume near contributions
     to activity from all possible adjacent characters.

     Parameters
     ----------

     df: (pandas data frame)
         Dataframe containing several columns representing \n
         bins and sequence column. The integer values in bins \n
         represent the occurrence of the sequence that bin.

     lm: (str)
         Learning model. Possible values include {'ER','LS','IM', 'PR'}. \n
         'ER': enrichment ratio inference. 'LS': least squares \n
         optimization. 'IM' : mutual information maximization \n
         (similar to maximum likelihood inference in the large data limit). \n
         'PR' stands for Poisson Regression.

     modeltype: (string)
         Type of model to be learned. Valid choices include "MAT" \n
         and "NBR", which stands for matrix model and neigbhour model, \n
         respectively. Matrix model assumes mutations at a location are \n
         independent and neighbour model assumes epistatic effects for \n
         mutations.

     LS_means_std: (pandas dataframe)
         For the least-squares method, this contains \n
         the user supplied mean and standard deviation. \n
         The order of the columns is ['bin', 'mean', 'std'].

     db: (string)
         File name for a SQL script; it could be passed \n
         in to the function MaximizeMI_memsaver

     iteration: (int)
         Total number of MCMC iterations to do. Passed \n
         in the sample method from MCMC.py which may be \n
         part of pymc.

     burnin: (int)
          Variables will not be tallied until this many \n
          iterations are complete (thermalization).

     thin: (int)
         Similar to parameter burnin, but with smaller \n
         default value.

     runnum: (int)
         Run number, used to determine the correct sql \n
         script extension in MaximizeMI_memsaver

     initialize: (string)
         Variable for initializing the learn model class \n
         constructor. Valid values include "rand", \n
         "LS", "PR". rand is MCMC, LS is least squares \n
          and PR and poisson regression.

     start: (int)
         Starting position of the sequence.

     end: (int)
         end position of the sequence.

     foreground: (int)
         Indicates column number representing foreground \n
         (E.g. can be passed to Berg_Von_Hippel method).

     background: (int)
         Indicates column number representing background.

     alpha : (float)
         Regularization strength; must be a positive float. Regularization \n
         improves the conditioning of the problem and reduces the variance of \n
         the estimates. Larger values specify stronger regularization. \n
         Alpha corresponds to ``C^-1`` in other linear models such as \n
         LogisticRegression or LinearSVC. (this snippet taken from ridge.py \n
         written by Mathieu Blondel)

     pseudocounts: (int)
         A artificial number added to bin counts where counts are \n
         really low. Needs to be Non-negative.

     verbose: (bool)
         A value of false for this parameter suppresses the \n
         output to screen.

     tm: (int)
         Number bins. DOUBLE CHECK.
     """

    @handle_errors
    def __init__(self,
                 df,
                 lm='ER',
                 modeltype='MAT',
                 LS_means_std = None,
                 db=None,
                 iteration=30000,
                 burnin=1000,
                 thin=10,
                 runnum=0,
                 initialize='LS',
                 start=0,
                 end = None,
                 foreground=1,
                 background=0,
                 alpha=0.0,
                 pseudocounts=1,
                 drop_library=False,
                 verbose=False,
                 tm=None):

        # set attributes
        self.df = df
        self.lm = lm
        self.modeltype = modeltype
        self.LS_means_std = LS_means_std
        self.db = db
        self.iteration = iteration
        self.burnin = burnin
        self.thin = thin
        self.runnum = runnum
        self.initialize = initialize
        self.start = start
        self.end = end
        self.foreground = foreground
        self.background = background
        self.alpha = alpha
        self.pseudocounts = pseudocounts
        self.drop_library = drop_library
        self.verbose = verbose
        self.tm = tm

        # output df
        self.output_df = None

        # validate parameters
        self._input_checks()

        # Determine dictionary
        seq_cols = qc.get_cols_from_df(df, 'seqs')
        if not len(seq_cols) == 1:
            raise SortSeqError('Dataframe has multiple seq cols: %s' % str(seq_cols))
        dicttype = qc.colname_to_seqtype_dict[seq_cols[0]]

        seq_dict, inv_dict = utils.choose_dict(dicttype, modeltype=modeltype)

        '''Check to make sure the chosen dictionary type correctly describes
             the sequences. An issue with this test is that if you have DNA sequence
             but choose a protein dictionary, you will still pass this test bc A,C,
             G,T are also valid amino acids'''
        # set name of sequences column based on type of sequence
        type_name_dict = {'dna': 'seq', 'rna': 'seq_rna', 'protein': 'seq_pro'}
        seq_col_name = type_name_dict[dicttype]
        lin_seq_dict, lin_inv_dict = utils.choose_dict(dicttype, modeltype='MAT')
        # wtseq = utils.profile_counts(df.copy(),dicttype,return_wtseq=True,start=start,end=end)
        # wt_seq_dict_list = [{inv_dict[np.mod(i+1+seq_dict[w],len(seq_dict))]:i for i in range(len(seq_dict)-1)} for w in wtseq]
        par_seq_dict = {v: k for v, k in seq_dict.items() if k != (len(seq_dict) - 1)}
        # drop any rows with ct = 0
        df = df[df.loc[:, 'ct'] != 0]
        df.reset_index(drop=True, inplace=True)

        # If there are sequences of different lengths, then print error but continue
        if len(set(df[seq_col_name].apply(len))) > 1:
            sys.stderr.write('Lengths of all sequences are not the same!')
        # select target sequence region
        df.loc[:, seq_col_name] = df.loc[:, seq_col_name].str.slice(start, end)
        df = utils.collapse_further(df)
        col_headers = utils.get_column_headers(df)
        # make sure all counts are ints
        df[col_headers] = df[col_headers].astype(int)
        # create vector of column names
        val_cols = ['val_' + inv_dict[i] for i in range(len(seq_dict))]
        df.reset_index(inplace=True, drop=True)
        # Drop any sequences with incorrect length
        if not end:
            '''is no value for end of sequence was supplied, assume first seq is
                correct length'''
            seqL = len(df[seq_col_name][0]) - start
        else:
            seqL = end - start
        df = df[df[seq_col_name].apply(len) == (seqL)]
        df.reset_index(inplace=True, drop=True)
        # Do something different for each type of learning method (lm)
        if lm == 'ER':
            if modeltype == 'NBR':
                emat = self.Markov(df, dicttype, foreground=foreground, background=background,
                                   pseudocounts=pseudocounts)
            else:
                emat = self.Berg_von_Hippel(
                    df, dicttype, foreground=foreground, background=background,
                    pseudocounts=pseudocounts)

        if lm == 'PR':
            emat = self.convex_opt(df, seq_dict, inv_dict, col_headers, tm=tm, \
                                   dicttype=dicttype, modeltype=modeltype)
        if lm == 'LS':
            '''First check that is we don't have a penalty for ridge regression,
                that we at least have all possible base values so that the analysis
                will not fail'''
            if LS_means_std:  # If user supplied preset means and std for each bin
                means_std_df = io.load_meanstd(LS_means_std)

                # change bin number to 'ct_number' and then use as index
                labels = list(means_std_df['bin'].apply(self.add_label))
                std = means_std_df['std']
                std.index = labels
                # Change Weighting of each sequence by dividing counts by bin std
                df[labels] = df[labels].div(std)
                means = means_std_df['mean']
                means.index = labels
            else:
                means = None
            # drop all rows without counts
            df['ct'] = df[col_headers].sum(axis=1)
            df = df[df.ct != 0]
            df.reset_index(inplace=True, drop=True)
            ''' For sort-seq experiments, bin_0 is library only and isn't the lowest
                expression even though it is will be calculated as such if we proceed.
                Therefore is drop_library is passed, drop this column from analysis.'''
            if drop_library:
                try:
                    df.drop('ct_0', inplace=True)
                    col_headers = utils.get_column_headers(df)
                    if len(col_headers) < 2:
                        raise SortSeqError(
                            '''After dropping library there are no longer enough
                            columns to run the analysis''')
                except:
                    raise SortSeqError('''drop_library option was passed, but no ct_0
                        column exists''')
            # parameterize sequences into 3xL vectors
            print('init learn model: \n')
            print(par_seq_dict)
            print('dict: ', dicttype)
            raveledmat, batch, sw = utils.genweightandmat(
                df, par_seq_dict, dicttype, means=means, modeltype=modeltype)
            # Use ridge regression to find matrix.
            emat = self.Compute_Least_Squares(raveledmat, batch, sw, alpha=alpha)

        if lm == 'IM':
            seq_mat, wtrow = numerics.dataset2mutarray(df.copy(), modeltype)
            # this is also an MCMC routine, do the same as above.
            if initialize == 'rand':
                if modeltype == 'MAT':
                    emat_0 = utils.RandEmat(len(df[seq_col_name][0]), len(seq_dict))
                elif modeltype == 'NBR':
                    emat_0 = utils.RandEmat(len(df[seq_col_name][0]) - 1, len(seq_dict))
            elif initialize == 'LS':

                emat_cols = ['val_' + inv_dict[i] for i in range(len(seq_dict))]
                emat_0_df = LearnModel(df.copy(), lm='LS', modeltype=modeltype, alpha=alpha, start=0, end=None, verbose=verbose).output_df
                emat_0 = np.transpose(np.array(emat_0_df[emat_cols]))
                # pymc doesn't take sparse mat
            elif initialize == 'PR':
                emat_cols = ['val_' + inv_dict[i] for i in range(len(seq_dict))]
                emat_0_df = LearnModel(df.copy(), lm='PR', modeltype=modeltype, start=0, end=None).output_df
                emat_0 = np.transpose(np.array(emat_0_df[emat_cols]))
            emat = self.MaximizeMI_memsaver(seq_mat, df.copy(), emat_0, wtrow, db=db, iteration=iteration,
                                            burnin=burnin, thin=thin, runnum=runnum, verbose=verbose)

        # We have infered out matrix.
        # now format the energy matrices to get them ready to output
        if (lm == 'IM' or lm == 'memsaver'):
            if modeltype == 'NBR':
                try:
                    emat_typical = gauge.fix_neighbor(np.transpose(emat))
                except:
                    sys.stderr.write('Gauge Fixing Failed')
                    emat_typical = np.transpose(emat)
            elif modeltype == 'MAT':
                try:
                    emat_typical = gauge.fix_matrix(np.transpose(emat))
                except:
                    sys.stderr.write('Gauge Fixing Failed')
                    emat_typical = np.transpose(emat)

        elif lm == 'ER':
            '''the emat for this format is currently transposed compared to other formats
            it is also already a data frame with columns [pos,val_...]'''
            if modeltype == 'NBR':
                emat_cols = ['val_' + inv_dict[i] for i in range(len(seq_dict))]
                emat_typical = emat[emat_cols]
            else:
                emat_cols = ['val_' + inv_dict[i] for i in range(len(seq_dict))]
                emat_typical = emat[emat_cols]
                try:
                    emat_typical = (gauge.fix_matrix((np.array(emat_typical))))
                except:
                    sys.stderr.write('Gauge Fixing Failed')
                    emat_typical = emat_typical

        elif (lm == 'MK'):
            '''The model is a first order markov model and its gauge does not need
                to be changed.'''

        elif lm == 'PR':
            emat_typical = np.transpose(emat)
        else:  # must be Least squares
            emat_typical = utils.emat_typical_parameterization(emat, len(seq_dict))
            if modeltype == 'NBR':
                try:
                    emat_typical = gauge.fix_neighbor(np.transpose(emat_typical))
                except:
                    sys.stderr.write('Gauge Fixing Failed')
                    emat_typical = np.transpose(emat_typical)
            elif modeltype == 'MAT':
                try:
                    emat_typical = gauge.fix_matrix(np.transpose(emat_typical))
                except:
                    sys.stderr.write('Gauge Fixing Failed')
                    emat_typical = np.transpose(emat_typical)
        em = pd.DataFrame(emat_typical)
        em.columns = val_cols
        # add position column
        if modeltype == 'NBR':
            pos = pd.Series(range(start, start - 1 + len(df[seq_col_name][0])), name='pos')
        else:
            pos = pd.Series(range(start, start + len(df[seq_col_name][0])), name='pos')
        output_df = pd.concat([pos, em], axis=1)
        # Validate model and return
        output_df = qc.validate_model(output_df, fix=True)
        self.output_df = output_df

    def _input_checks(self):

        """
        check input parameters for correctness
        """

        # dataset
        if self.df is None:
            raise ControlledError(
                " The Learn Model class requires pandas dataframe as input dataframe. Entered df was 'None'.")

        elif self.df is not None:
            check(isinstance(self.df, pd.DataFrame),
                  'type(df) = %s; must be a pandas dataframe ' % type(self.df))

            # validate dataset
            check(pd.DataFrame.equals(self.df, qc.validate_dataset(self.df)),
                  " Input dataframe failed quality control, \
                  please ensure input dataset has the correct format of an mpathic dataframe ")

        # check lm is of type string
        check(isinstance(self.lm,str), "type(lm) = %s must be a string " % type(self.lm))

        # check lm value is valid
        valid_lm_values = ['ER', 'LS', 'IM', 'PR']
        check(self.lm in valid_lm_values, 'lm = %s; must be in %s' % (self.lm, valid_lm_values))

        # check that model type is of type string
        check(isinstance(self.modeltype, str), "type(modeltype) = %s must be a string " % type(self.modeltype))

        # check that modeltype value is valid
        valid_modeltype_values = ['MAT', 'NBR']
        check(self.modeltype in valid_modeltype_values,
              'modeltype = %s; must be in %s' % (self.modeltype, valid_modeltype_values))

        # validate LS_mean_std
        LS_means_std_valid_col_order = ['bin','mean','std']

        if self.LS_means_std is not None:
            check(pd.DataFrame.equals(self.LS_means_std, qc.validate_meanstd(self.LS_means_std)),
                  " LS_means_std failed quality control, \
                  please ensure input dataset has the correct format for LS_means_std: %s" % LS_means_std_valid_col_order)

        if self.db is not None:
            # check that db is a string
            check(isinstance(self.db,str),"type(db) = %s must be a string " % type(self.db))

        # check that iteration is an integer
        check(isinstance(self.iteration, int),
              'type(iteration) = %s; must be of type int ' % type(self.iteration))

        # check that burnin is an integer
        check(isinstance(self.burnin, int),
              'type(burnin) = %s; must be of type int ' % type(self.burnin))

        # check that thin is an integer
        check(isinstance(self.thin, int),
              'type(thin) = %s; must be of type int ' % type(self.thin))

        # check that runnum is an integer
        check(isinstance(self.runnum, int),
              'type(runnum) = %s; must be of type int ' % type(self.runnum))

        # check that initialize is a string and it's value is valid
        check(isinstance(self.initialize,str),"type(initialize) = %s must be a string " % type(self.initialize))

        valid_initialize_values = ['rand', 'LS', 'PR']
        check(self.initialize in valid_initialize_values,
              'initialize = %s; must be in %s' % (self.initialize, valid_initialize_values))

        # check that start is an integer
        check(isinstance(self.start, int),
              'type(start) = %s; must be of type int ' % type(self.start))

        check(self.start >= 0, "start = %d must be a positive integer " % self.start)

        if self.end is not None:
            check(isinstance(self.end, int),
                  'type(end) = %s; must be of type int ' % type(self.end))

        # check that foreground is an integer
        check(isinstance(self.foreground, int),
              'type(foreground) = %s; must be of type int ' % type(self.foreground))

        # check that background is an integer
        check(isinstance(self.background, int),
              'type(background) = %s; must be of type int ' % type(self.background))

        # check that alpha is a float
        check(isinstance(self.alpha, float),
              'type(alpha) = %s; must be of type float ' % type(self.background))

        # check that pseudocounts is an integer
        check(isinstance(self.pseudocounts, int),
              'type(pseudocounts) = %s; must be of type int ' % type(self.pseudocounts))

        # check that verbose is a boolean
        check(isinstance(self.verbose, bool),
              'type(verbose) = %s; must be of type bool ' % type(self.verbose))

        if self.tm is not None:
            # check that tm is an integer
            check(isinstance(self.tm, int),
                  'type(tm) = %s; must be of type int ' % type(self.tm))

    def weighted_std(self, values, weights):
        '''Takes in a dataframe with seqs and cts and calculates the std'''
        average = np.average(values, weights=weights)
        variance = np.average((values - average) ** 2, weights=weights)
        return (np.sqrt(variance))

    def add_label(self, s):
        return 'ct_' + str(s)

    def MaximizeMI_memsaver(self,
                            seq_mat, df, emat_0, wtrow, db=None, burnin=1000, iteration=30000, thin=10,
                            runnum=0, verbose=False):
        '''Performs MCMC MI maximzation in the case where lm = memsaver'''
        '''
        @pymc.stochastic(observed=True,dtype=sp.sparse.csr_matrix)
        def sequences(value=seq_mat):
            return 0
        '''
        n_seqs = seq_mat.shape[0]

        @pymc.stochastic(observed=True, dtype=pd.DataFrame)
        def pymcdf(value=df):
            return 0

        @pymc.stochastic(dtype=float)
        def emat(p=pymcdf, value=emat_0):
            p['val'] = numerics.eval_modelmatrix_on_mutarray(np.transpose(value), seq_mat, wtrow)
            MI = EstimateMutualInfoforMImax.alt4(p.copy())  # New and improved
            return n_seqs * MI

        if db:
            dbname = db + '_' + str(runnum) + '.sql'
            M = pymc.MCMC([pymcdf, emat], db='sqlite', dbname=dbname)
        else:
            M = pymc.MCMC([pymcdf, emat])
        M.use_step_method(stepper.GaugePreservingStepper, emat)

        if not verbose:
            M.sample = shutthefuckup(M.sample)

        M.sample(iteration, thin=thin)
        emat_mean = np.mean(M.trace('emat')[burnin:], axis=0)
        return emat_mean

    def robls(self, A, b, rho):
        m, n = A.size

        def F(x=None, z=None):
            if x is None: return 0, matrix(0.0, (n, 1))
            y = A * x - b
            w = sqrt(rho + y ** 2)
            f = sum(w)
            Df = div(y, w).T * A
            if z is None: return f, Df
            H = A.T * spdiag(z[0] * rho * (w ** -3)) * A
            return f, Df, H

        return solvers.cp(F)['x']

    def test_iter(self, A, B):
        m, n1 = A.shape
        n2 = B.shape[1]
        Cshape = (m, n1 * n2)
        with warnings.catch_warnings():
            # ignore depreciation warnings
            warnings.simplefilter("ignore")

            # initialize our output sparse matrix data objects
            data = np.empty((m,), dtype=object)
            col = np.empty((m,), dtype=object)
            row = np.empty((m,), dtype=object)
            # do multiplication for each row
            for i, (a, b) in enumerate(zip(A, B)):
                # column indexes
                col1 = a.indices * n2
                col[i] = (col1[:, None] + b.indices).flatten()
                row[i] = np.full((a.nnz * b.nnz,), i)
                # all data will be 1's, as it is only true or false data
                data[i] = np.ones(len(col[i]))
        data = np.concatenate(data)
        col = np.concatenate(col)
        row = np.concatenate(row)
        return scipy.sparse.coo_matrix((data, (row, col)), shape=Cshape)

    def convex_opt_agorithm(self, s, N0, Nsm, tm, alpha=1):
        bins = tm.shape[1]
        N0_matrix = np.matrix(N0)
        tm_matrix = np.matrix(tm)
        Nsm_matrix = np.matrix(Nsm)
        tm_matrix_squared = np.matrix(np.multiply(tm, tm))
        i, c = s.shape
        # create s matrix, the elements of this matrix are delta_s@i * delta s2@j
        # we will need this for hessian.
        s_hessian_mat = sp.sparse.lil_matrix((i, c * c))
        s_hessian_mat = scipy.sparse.csr_matrix(self.test_iter(s, s))

        def F(x=None, z=None):
            if x is None: return 0, matrix(0.0, (c + bins, 1))
            gm = np.transpose(np.array(x[c:]))
            y = s * x[:c]
            w = np.add(gm, y * tm_matrix)
            term2 = np.array(N0) * np.array(np.exp(w))
            term1 = np.multiply(np.array(Nsm), np.array(w))
            #print(alpha / 2 * np.sum(cvxopt.mul(x, x)))
            f = cvxopt.matrix(-np.sum(term1 - term2) + alpha / 2 * np.sum(cvxopt.mul(x, x)))
            #print(f)
            Df = np.zeros((1, c + bins))

            Nm = np.sum(Nsm, axis=0)

            Df_gm = Nm - sum(term2)

            Df_theta = np.transpose((Nsm - term2) * np.transpose(tm)) * s

            Df[0, :c] = Df_theta

            Df[0, c:] = Df_gm
            #print(Df[0, 90:94])
            Df = Df - np.transpose(alpha * x)
            Df_cvx = cvxopt.matrix(-Df)
            if z is None: return f, Df_cvx
            H = np.zeros((c + bins, c + bins))

            # first do the theta_theta terms

            # first do N_0*tm**2, this will produce a ixm matrix where each term is Ni0 * tm**2.
            Inner_term = N0 * tm_matrix_squared
            # now multiply by a column vector form of w, this will do the multiplication by y, and also sum over m.
            # you now have an ix1 matrix
            Inner_term2 = np.sum(np.array(Inner_term) * np.array(np.exp(w)), axis=1)

            # multiply by hessian matrix and sum over sequences...

            H_thetas_temp = Inner_term2 * s_hessian_mat

            # now we need to reshape such to fill in these values
            H[:c, :c] = H_thetas_temp.reshape((c, c), order='F')

            # now do mixed terms with partials of gms and thetas
            # first multiply the N0 counts by the exponential term. This will give you a matrix of ixm
            Inner_term = np.array(N0) * np.array(np.matrix(np.exp(w)))
            # now use element wise multiplication to multiply by the tms, this will broadcasts their value down the rows
            Inner_term2 = np.array(tm) * Inner_term
            # now convert back to matrix form and multiply by s. This sums over sequence while multiplying by si, and
            # yeilds a matrix of dimension mxc
            H_gm_theta_temp = np.transpose(np.matrix(Inner_term2)) * s
            H[c:c + bins, :c] = H_gm_theta_temp

            # now we are going to do the last bit, the partials with respect to the gms

            # we can use the same 'Inner term' as above N0*exp(gm+tm*sum(theta*s))
            # sum over s
            Temp_term = np.sum(Inner_term, axis=0)
            ident = np.identity(bins)
            # use array multiplication to broadcast multiply such that all off diagonal terms are 0 and
            # the on diagonal terms are equal to sum(Ns0*exp(gm+sum(theta*s)))
            H_gms = np.array(Temp_term) * ident
            H[c:c + bins, c:c + bins] = H_gms

            # now insure that H is symmetric across diagonal
            for q in range(c + bins):
                for k in range(q):
                    H[k, q] = H[q, k]
            penalty_term = alpha * np.identity(c + bins)
            H = cvxopt.matrix(z[0] * (H + penalty_term))

            return f, Df_cvx, H

        return solvers.cp(F)['x']

    def reverse_parameterization(self, output, cols_for_keep, wtrow, seq_dict, bins=1,
                                 modeltype='MAT'):
        output = output[:-bins]
        df_out = np.zeros(len(wtrow))
        print(df_out.shape)
        df_out.flat[cols_for_keep] = output
        print(len(seq_dict))
        df_out2 = df_out.reshape( \
            (len(seq_dict), len(df_out) / len(seq_dict)), order='F')

        return df_out2

    def find_second_NBR_matrix_entry(self, s):
        '''this is a function for use with numpy apply along axis.
            It will take in a sequence matrix and return the second nonzero entry'''
        print(np.nonzero(s)[0])
        return np.nonzero(s)[0][1]

    def convex_opt(self, df, seq_dict, inv_dict, columns, tm=None, modeltype='MAT', dicttype='dna'):
        rowsforwtcalc = 1000
        seq_mat, wtrow = numerics.dataset2mutarray(df.copy(), modeltype, rowsforwtcalc=rowsforwtcalc)
        # need to make sure there is at least one representative
        # of each possible entry, otherwise don't fit it.
        no_reps = np.sum(np.matrix(df['ct_0']) * seq_mat, axis=0)
        cols_for_keep = [x for x in range( \
            seq_mat.shape[1]) if x in np.nonzero(no_reps)[1]]
        # if the model is a neighbor model we also need to
        # make sure we only give each mutation one parameter.
        if modeltype == 'NBR':
            mut_df = ProfileMut(df.loc[:rowsforwtcalc, :]).mut_df
            wtseq = ''.join(list(mut_df['wt']))

            single_seq_dict, single_inv_dict = utils.choose_dict(dicttype, modeltype='MAT')
            seqs = []
            # now make each possible single mutation...
            for i, let in enumerate(wtseq[1:-1]):
                for m in range(1, 4):
                    let_for_mutation = single_seq_dict[let]
                    let_for_mutation = single_inv_dict[np.mod(let_for_mutation + m, 4)]
                    mut_seq = list(wtseq)
                    mut_seq[i + 1] = let_for_mutation
                    seqs.append(''.join(mut_seq))
            # now that we have each mutation, we should find
            # what their matrix representation is...
            seqs_df = pd.DataFrame()
            seqs_df['seq'] = seqs
            seq_mat_mutants, wtrow2 = \
                numerics.dataset2mutarray_withwtseq(seqs_df, modeltype, wtseq)
            # these mutants will have 2 entries which indicate
            # that a single mutation away from wt hits 2 parameters
            # which doesn't make sense, so we should fix the second one to zero...
            bad_cols = np.apply_along_axis(self.find_second_NBR_matrix_entry, \
                                           1, seq_mat_mutants.todense())
            cols_for_keep = [cols_for_keep[x] for x in range(len(cols_for_keep)) \
                             if cols_for_keep[x] not in bad_cols]
        seq_mat = seq_mat.tocsc()
        seq_mat2 = seq_mat[:, cols_for_keep]
        columns = [x for x in columns if 'ct_0' != x]
        N0 = np.matrix(df['ct_0']).T
        Nsm = np.matrix(df[columns])
        if tm:
            tm = np.array(tm)
        else:
            tm = np.matrix([x for x in range(1, len(columns) + 1)])
        print(tm)
        output = self.convex_opt_agorithm(seq_mat2, N0, Nsm, tm)
        output_parameterized = self.reverse_parameterization(output, cols_for_keep, wtrow, seq_dict,
                                                             bins=tm.shape[1],
                                                             modeltype=modeltype)
        print(output_parameterized)
        print(output_parameterized.shape)
        return (output_parameterized)

    def Berg_von_Hippel(self, df, dicttype, foreground=1, background=0, pseudocounts=1):
        '''Learn models using berg von hippel model. The foreground sequences are
             usually bin_1 and background in bin_0, this can be changed via flags.'''
        seq_dict, inv_dict = utils.choose_dict(dicttype)
        # check that the foreground and background chosen columns actually exist.
        columns_to_check = {'ct_' + str(foreground), 'ct_' + str(background)}
        if not columns_to_check.issubset(set(df.columns)):
            raise SortSeqError('Foreground or Background column does not exist!')

        # get counts of each base at each position
        foreground_counts = utils.profile_counts(df, dicttype, bin_k=foreground)
        background_counts = utils.profile_counts(df, dicttype, bin_k=background)
        binheaders = utils.get_column_headers(foreground_counts)
        # add pseudocounts to each position
        foreground_counts[binheaders] = foreground_counts[binheaders] + pseudocounts
        background_counts[binheaders] = background_counts[binheaders] + pseudocounts
        # make sure there are no zeros in counts after addition of pseudocounts
        ct_headers = utils.get_column_headers(foreground_counts)
        if foreground_counts[ct_headers].isin([0]).values.any():
            raise SortSeqError('''There are some bases without any representation in\
                the foreground data, you should use pseudocounts to avoid failure \
                of the learning method''')
        if background_counts[ct_headers].isin([0]).values.any():
            raise SortSeqError('''There are some bases without any representation in\
                the background data, you should use pseudocounts to avoid failure \
                of the learning method''')
        # normalize to compute frequencies
        foreground_freqs = foreground_counts.copy()
        background_freqs = background_counts.copy()
        foreground_freqs[binheaders] = foreground_freqs[binheaders].div(
            foreground_freqs[binheaders].sum(axis=1), axis=0)
        background_freqs[binheaders] = background_freqs[binheaders].div(
            background_freqs[binheaders].sum(axis=1), axis=0)

        output_df = -np.log(foreground_freqs / background_freqs)
        # change column names accordingly (instead of ct_ we want val_)
        rename_dict = {'ct_' + str(inv_dict[i]): 'val_' + str(inv_dict[i]) for i in range(len(seq_dict))}
        output_df = output_df.rename(columns=rename_dict)
        return output_df

    def compute_etas_for_markov(self, freqs_neighbor, freqs, seq_dict, inv_dict):
        total_pairs = len(freqs.index)
        # now lets find the eta value for the foreground bin
        seq_dict_length = len(seq_dict)
        eta = np.zeros((total_pairs - 1, seq_dict_length ** 2))
        # first we will do the first position.
        for q in range(seq_dict_length):  # loop through each possible pairing
            for m in range(seq_dict_length):  # loop through bases again to get pairs
                # compute the value
                eta[0, q * seq_dict_length + m] = freqs_neighbor.loc[0, 'ct_' + \
                                                                     inv_dict[q] + inv_dict[m]] * (
                                                      np.sqrt(freqs.loc[0, 'ct_' + \
                                                                        inv_dict[q]] / freqs.loc[1, 'ct_' + \
                                                                                inv_dict[m]]))

        # now lets do the middle values
        # loop through positions
        for i in range(1, total_pairs - 2):
            for q in range(seq_dict_length):  # loop through each possible pairing
                for m in range(seq_dict_length):  # loop through bases again to get pairs
                    # compute the value
                    eta[i, q * seq_dict_length + m] = freqs_neighbor.loc[i, 'ct_' + \
                                                                         inv_dict[q] + inv_dict[m]] / (
                                                          np.sqrt(freqs.loc[i, 'ct_' + \
                                                                            inv_dict[q]] * freqs.loc[i + 1, 'ct_' + \
                                                                                inv_dict[m]]))

        for q in range(seq_dict_length):  # loop through each possible pairing
            for m in range(seq_dict_length):  # loop through bases again to get pairs
                # compute the value
                eta[total_pairs - 2, q * seq_dict_length + m] = freqs_neighbor.loc[total_pairs - 2, 'ct_' + \
                                                                                inv_dict[q] + inv_dict[m]] * (
                                                                    np.sqrt(freqs.loc[total_pairs - 1, 'ct_' + \
                                                                                inv_dict[m]] / freqs.loc[
                                                                                total_pairs - 2, 'ct_' + \
                                                                                inv_dict[q]]))

        # take the log of each entry
        eta = np.log(eta)
        return eta

    def Markov(self, df, dicttype, foreground=1, background=0, pseudocounts=1):
        '''Learn models using berg von hippel model. The foreground sequences are
             usually bin_1 and background in bin_0, this can be changed via flags.'''
        seq_dict, inv_dict = utils.choose_dict(dicttype)
        seq_dict_length = len(seq_dict)
        # check that the foreground and background chosen columns actually exist.
        columns_to_check = {'ct_' + str(foreground), 'ct_' + str(background)}
        if not columns_to_check.issubset(set(df.columns)):
            raise SortSeqError('Foreground or Background column does not exist!')

        # get counts of each base at each position
        foreground_counts = utils.profile_counts(df, dicttype, bin_k=foreground)
        background_counts = utils.profile_counts(df, dicttype, bin_k=background)
        binheaders = utils.get_column_headers(foreground_counts)
        # get counts of each neighbor pair at each position
        foreground_counts_neighbor = utils.profile_counts_neighbor(df, dicttype, bin_k=foreground)
        background_counts_neighbor = utils.profile_counts_neighbor(df, dicttype, bin_k=background)
        binheaders_neighbor = utils.get_column_headers(foreground_counts_neighbor)
        # add pseudocounts to each position
        foreground_counts_neighbor[binheaders_neighbor] = \
            foreground_counts_neighbor[binheaders_neighbor] + pseudocounts
        background_counts_neighbor[binheaders_neighbor] = \
            background_counts_neighbor[binheaders_neighbor] + pseudocounts

        # do the same for the single base counts

        foreground_counts[binheaders] = foreground_counts[binheaders] + pseudocounts * seq_dict_length
        background_counts[binheaders] = background_counts[binheaders] + pseudocounts * seq_dict_length

        # make sure there are no zeros in counts after addition of pseudocounts
        ct_headers = utils.get_column_headers(foreground_counts_neighbor)
        if foreground_counts_neighbor[ct_headers].isin([0]).values.any():
            raise SortSeqError('''There are some bases without any representation in\
                the foreground data, you should use pseudocounts to avoid failure \
                of the learning method''')
        if background_counts_neighbor[ct_headers].isin([0]).values.any():
            raise SortSeqError('''There are some bases without any representation in\
                the background data, you should use pseudocounts to avoid failure \
                of the learning method''')
        # We will now normalize to compute our model values, we will do this by dividing each row by the
        # sum of all the rows (aka, dividing by counts + 16*psuedocounts)
        foreground_freqs_neighbor = foreground_counts_neighbor.copy()
        background_freqs_neighbor = background_counts_neighbor.copy()
        foreground_freqs_neighbor[binheaders_neighbor] = \
            foreground_freqs_neighbor[binheaders_neighbor].div( \
                foreground_freqs_neighbor[binheaders_neighbor].sum(axis=1), axis=0)
        background_freqs_neighbor[binheaders_neighbor] = \
            background_freqs_neighbor[binheaders_neighbor].div( \
                background_freqs_neighbor[binheaders_neighbor].sum(axis=1), axis=0)
        print(foreground_freqs_neighbor)
        # normalize to compute frequencies
        foreground_freqs = foreground_counts.copy()
        background_freqs = background_counts.copy()
        foreground_freqs[binheaders] = foreground_freqs[binheaders].div( \
            foreground_freqs[binheaders].sum(axis=1), axis=0)
        background_freqs[binheaders] = background_freqs[binheaders].div( \
            background_freqs[binheaders].sum(axis=1), axis=0)

        eta_fg = self.compute_etas_for_markov(self, foreground_freqs_neighbor, foreground_freqs, seq_dict, inv_dict)

        # now lets find the eta value for the background bin

        eta_bg = self.compute_etas_for_markov(self, background_freqs_neighbor, background_freqs, seq_dict, inv_dict)
        # subtract etas to create model
        model = eta_fg - eta_bg

        # turn model into data frame.
        model_df = pd.DataFrame(model)
        # label columns
        model_df.columns = ['val_' + inv_dict[q] + inv_dict[m] for q in range(seq_dict_length) for m in
                            range(seq_dict_length)]
        model_df['pos'] = foreground_counts_neighbor['pos']

        return model_df

    def Compute_Least_Squares(self, raveledmat, batch, sw, alpha=0):
        '''Ridge regression is the only sklearn regressor that supports sample
            weights, which will make this much faster'''
        clf = linear_model.Ridge(alpha=alpha)
        clf.fit(raveledmat, batch, sample_weight=sw)
        emat = clf.coef_
        return emat


from __future__ import division, print_function, absolute_import
import dataSet_ts as dt
import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt
import matplotlib.pyplot as plt
import datetime
import sys, time
import os, math
import kellyGetData as kgd
############### HYPERPARAMETERS ################

appliances = [ 'kettle','microwave', 'washing machine', 'dish washer' , 'fridge']#
windows = {1:("2013-02-27", "2014-02-27")}
#To specify more than one building, just add a key in the dict windows
#i.e. 2:("2013-02-27", "2013-04-27")

def main(args):

    lr = float(args['lr']) #0.0001
    nilmkt_fileName = args['nilmkt_fileName']       # Path to ukdale.h5
    n_steps = int(args['n_steps'])                  # Size of input sequence (length of instance)
    n_hidden_units = int(args['hiddenUnitsLayer1']) # First layer number of units
    training_iters = int(args['iterations'])        # Number of iterations or epochs
    stride_train = int(args['strideTrain'])         # Separation between instances of training set
    num_sequences_per_batch = int(args['numSequences']) #based on appliance
    stride_test =  n_steps                          # Separation between instances of test set
    applianceTest =  int(args['applianceToTest'])   # Apliance (order number in the list) of the appliance to evaluate
    period = 6                                      # It is the period that most of other works use
    loadParam = args['loadAsKelly']
    target_inclusion_prob = float(args['target_inclusion_prob'])
    loadAsKelly = True
    period = int(args['period'])
    data_path = args['data_path']
    if(loadParam == 'N' or loadParam == 'n' or loadParam == 'no' or loadParam == 'NO' or loadParam == 'No'):
        loadAsKelly = False
    # Setting the number of nodes in hidden states of the 3 layers
    n_inputs_0 = n_hidden_units   # Number of cells in input vector
    n_inputs_1 = int(n_hidden_units/2)
    n_inputs_2 = n_hidden_units
    n_inputs_3 = n_steps

    ########### FUNCTIONS TO DEFINE WEIGHTS AND BIAS ####################

    def weight_variable(shape, m=0, std=0.5):
        initial = tf.random_normal(shape,mean=m, stddev=std) #truncated_normal, stddev=0.1 / w['in'] = tf.random_normal
        return tf.Variable(initial)

    def bias_variable(shape, c=1.0):
        initial = tf.constant(c, shape=shape)
        return tf.Variable(initial)

    ############ UPLOADING DATA SET ############

    reader = dt.ReaderTS(windows, appliances, n_steps, stride_train, stride_test, period,
                            flgScaling=1, trainPer=0.5, valPer=0.25, testPer=0.25)

    XdataSet,YdataSet = reader.load_csvdata(period, data_path, target_inclusion_prob, nilmkt_fileName, applianceTest, loadAsKelly, num_sequences_per_batch)

    # Depending on the applianceTest (-1 for all): x.shape = [batch, seqLen]
    #                                y.shape = [batch, seqLen] or [batch, seqLen, numAppliances]
    x_train, x_test, x_val  = XdataSet['train'], XdataSet['test'],  XdataSet['val']
    y_train, y_test, y_val  = YdataSet['train'], YdataSet['test'],  YdataSet['val']

    x_train = x_train.reshape((-1, n_steps, 1))
    y_train = y_train.reshape((-1, n_steps, 1))
    x_val = x_val.reshape((-1, n_steps, 1))
    y_val = y_val.reshape((-1, n_steps, 1))
    x_test = x_test.reshape((-1, n_steps, 1))
    y_test = y_test.reshape((-1, n_steps, 1))
    ##############   STARTING TO CONSTRUCT GRAPH: INPUTS AND OPERATIONS #####################

    x = tf.placeholder(tf.float32, [None, n_steps, 1]) #batch_size x length x dimensions
    y = tf.placeholder(tf.float32, [None, n_steps, 1])

    # Size of the batch (it could change in validation and/or test runs)
    initBatch = tf.placeholder(tf.int32, shape=())

    layer_2_units = (n_steps-3)*8

    #layer 1
    conv1 = tf.layers.conv1d(inputs=x, filters=8, kernel_size=4, strides=1,
        padding='valid', activation = tf.nn.relu)
    conv1 = tf.reshape(conv1,[-1, layer_2_units])

    #layer 2
    Win_2 = weight_variable([layer_2_units,128])
    bin_2 = bias_variable([128])
    out_2 = tf.nn.relu(tf.matmul(conv1, Win_2) +  bin_2)

    #layer 3
    Win_3 = weight_variable([128,layer_2_units])
    bin_3 = bias_variable([layer_2_units])
    out_3 = tf.nn.relu(tf.matmul(out_2, Win_3) +  bin_3)

    #layer 4
    Win_4 = weight_variable([layer_2_units,n_steps])
    bin_4 = bias_variable([n_steps])
    out_4 = tf.nn.relu(tf.matmul(out_3, Win_4) +  bin_4)
    out_4 = tf.expand_dims(out_4, 2)

    #layer 5
    pred = tf.layers.conv1d(inputs=out_4, filters=1, kernel_size=4, strides=1,
        padding='same', activation = tf.nn.relu) #for now use same, the paper says valid

    #pred has too many predictions
    print("conv1.shape:",conv1.shape)
    print("out_2.shape:",out_2.shape)
    print("out_3.shape:",out_3.shape)
    print("out_4.shape:",out_4.shape)
    print("predictions.shape:",pred.shape)
    print("x.shape:",x.shape)
    print("y.shape:",y.shape)

    ## Indicating to tensorflow that save the values of the parameters/layers in each iteration##
    tf.summary.histogram('conv1', conv1)
    tf.summary.histogram('Win_2', Win_2)
    tf.summary.histogram('Win_3', Win_3)
    tf.summary.histogram('Win_4', Win_4)
    ### Flatten real Y and prediction to be able to measure the results as other models do
    yForMetric = tf.reshape(y,[-1])
    predForMetric = tf.reshape(pred,[-1])

    ### Calculating metrics (done just in validation/testing. Not part of the optimization)
    #Check what's the difference btw these different calculatinos. Meanwhile, staying with upMae0
    #tf.metrics.mean_absolute_error. =  tf.contrib.metrics.streaming_mean_absolute_error
    mae0, upMae0 = tf.metrics.mean_absolute_error(labels=yForMetric,predictions=predForMetric) # Some problem with initialization
    upMae1 = tf.reduce_mean(tf.abs(yForMetric-predForMetric))

    sum_output = tf.reduce_sum(pred)
    sum_target = tf.reduce_sum(y)
    relativeError = (sum_output - sum_target) / tf.maximum(sum_output, sum_target)

    #### Defining cost function and optimizator
    cost = tf.losses.mean_squared_error(yForMetric,predForMetric)
    tf.summary.scalar('cost', cost) #Track values of cost
    '''
    #Evaluate to add a regularizer
    l1_regularizer = tf.contrib.layers.l1_regularizer(scale=0.5)#scale=0.05, scope=None
    regularization_penalty = tf.contrib.layers.apply_regularization(l1_regularizer, [W_conv])
    '''
    train_op = tf.train.AdamOptimizer(lr, epsilon=0.1).minimize(cost)

    # Show the sizes of the weight matrices to know number of total paramters used
    for v in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES):
        print(v)

    ## Auxiliar structure to track cost and metrics.
    ## Could be done w/ summary.scalar but haven't figured out yet - Tensorboard
    tracking = {'step': [], 'cost': [], 'mae': [], 'relerr':[]}

    ############   START RUNNING   #############

    with tf.Session() as sess:
        init = tf.global_variables_initializer()
        sess.run(init)
        sess.run(tf.local_variables_initializer()) # for mean absolute error, that need local variables initialized
        merged = tf.summary.merge_all()
        # Write values of some variables to check them later in tensorboard
        train_writer = tf.summary.FileWriter('Logs/'+datetime.datetime.now().strftime("%y-%m-%d_%H-%M"),sess.graph)
        test_writer = tf.summary.FileWriter('Logs/LogsTestAE_SepApp/')
        # Print metrics on validation set every (1/10) of the iterations
        step = 0
        while (step < training_iters):
            [summary,op_train] = sess.run([merged, train_op], feed_dict={x: x_train,y: y_train, initBatch: x_train.shape[0]})
            train_writer.add_summary(summary, step)
            if (step % (training_iters/10) == 0):
                trncost, metric_upMae0, metric_upMae1, out_preds, out_re = sess.run(
	                    [cost,upMae0, upMae1, pred, relativeError], feed_dict={x: x_val, y: y_val, initBatch: x_val.shape[0]})
                tracking['step'].append(step)
                tracking['cost'].append(trncost)
                tracking['mae'].append(metric_upMae0)
                tracking['relerr'].append(out_re)
                print("Time {} Step {:5d} Cost {:10.2f} Metric {:6.2f}/{:6.2f} RelError {:3.2f}".format(
                    datetime.datetime.now().strftime("%y-%m-%d_%H-%M"),step,trncost,metric_upMae0, metric_upMae1 ,out_re))
            step+=1
        tstcost, out_preds, metric_upMae0, out_re  =sess.run([cost, pred,upMae0, relativeError], feed_dict={x: x_test, y: y_test,initBatch: x_test.shape[0]})
        print ("\nTEST Cost {} Metric {} Relat err {}".format(tstcost,metric_upMae0, out_re))

        savedFolder = 'Experiments/app{}_{}'.format(applianceTest,datetime.datetime.now().strftime("%y-%m-%d_%H-%M"))
        os.makedirs(savedFolder)
        yFlat = y_test.flatten()
        predFlat = out_preds.flatten()

        # Concatenate results horizontally to have again the one large sequence
        newTest =np.concatenate([y_test[i] for i in range(y_test.shape[0])],0) # tf.concat([y_test[i] for i in range(y_test.shape[0])],0)
        newPred = np.concatenate([out_preds[i]  for i in range(y_test.shape[0])],0) #tf.concat([preds[i] for i in range(preds.shape[0])],0)
        newInput = np.concatenate([x_test[i]  for i in range(y_test.shape[0])],0)

        assert newTest.shape == newPred.shape
        print ("Test size {}".format(newTest.shape))

        fLog = open('{}/{}_output'.format(savedFolder,applianceTest), 'w')
        fLog.write("Step,Cost,MAE, Rel error\n")
        for i , item in enumerate(tracking['step']):
          a = tracking['step'][i]
          b = tracking['cost'][i]
          c = tracking['mae'][i]
          d = tracking['relerr'][i]
          fLog.write("{},{},{},{}\n".format(a,b,c,d))
        fLog.write("Test,{},{},{}\n".format(tstcost,metric_upMae0,out_re))
        f, axarr = plt.subplots(3, 1, sharex=True)

        axarr[0].plot(newTest)
        axarr[0].set_title('Y-real')

        axarr[1].plot(newPred)
        axarr[1].set_title('Y-pred')

        axarr[2].plot(newInput)
        axarr[2].set_title('X-real')

        f.subplots_adjust(top=0.92, bottom=0.05, left=0.10, right=0.95, hspace=0.3, wspace=0.3)
        plt.savefig('{}/{}_plots'.format(savedFolder,applianceTest))#self.savedFolder+'/'+ch.name+str(numfig)
        plt.clf()

if __name__ == "__main__":

    import sys, time
    if len(sys.argv) > 1:
        config_file_name = sys.argv[-1]
    else:
        config_file_name = 'config_kelly.txt'

    f = open(config_file_name, 'r')
    lines = f.readlines()
    params = {}

    for line in lines:
        line = line.split('\n')[0]
        param_list = line.split(' ')
        param_name = param_list[0]
        param_value = param_list[1]
        params[param_name] = param_value

    main(params)


from django.contrib import admin
from django.urls import path, include
from .import views

urlpatterns = [
    path('', views.alllocs, name = 'alllocs'),
    path('add/', views.add, name='add'),
    path('mylocs/',views.mylocs, name = 'mylocs'),
    path('<int:loc_id>/delete_loc/', views.delete_loc, name = 'delete_loc'),
    path('api/', include('locations.api.urls')),
    # path('', HomePageView.as_view(), name='allblogs'),
]


#! /usr/bin/env python

import os
import pymongo
import sys
import time
from subprocess import call

client = pymongo.MongoClient('localhost', 27017)
if sys.argv[1] == u'test':
    db = client.test
    file_name = u'/tmp/iateststatus'
else:
    db = client.account_instance
    file_name = u'/tmp/iastatus'

account_table = db.account_table
timestamp = u'%f' % time.time()

ret = account_table.find({}, {u'_id':1, u'secondarys':1})
accounts = []
for item in ret:
    primary_id = item[u'_id']
    accounts.append(primary_id)
    secondarys = item[u'secondarys']
    for secondary_id in secondarys:
        accounts.append(secondary_id)

with open(file_name, u'w') as f:
    f.write('updating')

os.chdir(os.path.dirname(__file__))
for account_id in accounts:
    call([u'./get.sh', sys.argv[1], timestamp, account_id])

with open(file_name, u'w') as f:
    f.write('normal')


import numpy as np
import json
import os
import pandas as pd

from tqdm.auto import tqdm
from multiprocessing import Pool
from pandarallel import pandarallel
pandarallel.initialize(nb_workers=50)

from xgboost import XGBClassifier

from model import Model
from utils import read_df

all_columns = ['item_domain_id', 'item_id', 'item_title', 'site_id', 'sku',
       'product_id', 'product_family_id', 'count', 'date_first', 'date_last',
       'date_diff', 'date_first_day', 'date_first_month', 'date_last_day',
       'date_last_month', 'sold_quantity_first', 'sold_quantity_last',
       'sold_quantity_sum', 'sold_quantity_mean', 'sold_quantity_std',
       'sold_quantity_min', 'sold_quantity_max', 'sold_quantity_mode',
       'sold_quantity_mode_tx', 'current_price_first', 'current_price_last',
       'current_price_sum', 'current_price_mean', 'current_price_std',
       'current_price_min', 'current_price_max', 'current_price_mode',
       'current_price_mode_tx', 'minutes_active_first', 'minutes_active_last',
       'minutes_active_sum', 'minutes_active_mean', 'minutes_active_std',
       'minutes_active_min', 'minutes_active_max', 'minutes_active_mode',
       'minutes_active_mode_tx', 'currency_first', 'currency_last',
       'currency_mode', 'currency_mode_tx', 'listing_type_first',
       'listing_type_last', 'listing_type_mode', 'listing_type_mode_tx',
       'shipping_logistic_type_first', 'shipping_logistic_type_last',
       'shipping_logistic_type_mode', 'shipping_logistic_type_mode_tx',
       'shipping_payment_first', 'shipping_payment_last',
       'shipping_payment_mode', 'shipping_payment_mode_tx',
       'minutes_active_series', 'current_price_series',
       'sold_quantity_series']

categorical_columns = ['site_id', 'item_id']
categorical_columns_with_nan = ['item_domain_id', 'product_id', 'product_family_id']
categorical_columns_composed = ['currency', 'listing_type', 'shipping_logistic_type', 'shipping_payment']
categorical_composed_suffix = ['_first', '_last', '_mode']

categorical_columns = categorical_columns + categorical_columns_with_nan
for category_name in categorical_columns_composed:
    for suffix in categorical_composed_suffix:
        categorical_columns.append(category_name+suffix)

numeric_columns = ['count', 'date_first_day', 'date_first_month', 'date_last_day', 'date_last_month']
numeric_columns_composed = ['sold_quantity', 'current_price', 'minutes_active']
numeric_composed_suffix = ['_first', '_last', '_mode', '_mode_tx', '_sum', '_mean', '_std', '_min', '_max']

for numeric_name in numeric_columns_composed:
    for suffix in numeric_composed_suffix:
        numeric_columns.append(numeric_name+suffix)

categorical_to_numeric_composed_suffix = ['_mode_tx']

for category_name in categorical_columns_composed:
    for suffix in categorical_to_numeric_composed_suffix:
        numeric_columns.append(category_name+suffix)

series_columns = ['sold_quantity_series', 'current_price_series', 'minutes_active_series']
series_columns_headers = {}

series_columns_max = 29
for series_name in series_columns:
    series_columns_headers[series_name] = [series_name+'_'+str(i) for i in range(series_columns_max)]
    numeric_columns = numeric_columns + series_columns_headers[series_name]

features = numeric_columns + categorical_columns

class XGBoostFeaturesV2(Model):
    model_name = 'xgboost_features_v2_classification'

    def __init__(self, dataset_path):
        Model.__init__(self, self.model_name, dataset_path)

    def prepare_data(self):
        print('Number of features:', len(features), 'Non-used features: ', set(all_columns) - set(features) - set(series_columns))

        df_train_x_features = read_df(os.path.join(self.dataset_path, self.default_paths['train_data_x_processed']))
        df_train_y_features = read_df(os.path.join(self.dataset_path, self.default_paths['train_data_y_processed']))

        #Randomize and order sync train data
        x_df = df_train_x_features.sample(frac=1, random_state=42).reset_index(drop=True).copy()
        y_df = df_train_y_features.set_index('sku').loc[x_df['sku']].reset_index().copy()

        for column in numeric_columns_composed:
            x_df[column+'_std'] = x_df[column+'_std'].fillna(0)
            y_df[column+'_std'] = y_df[column+'_std'].fillna(0)
        for column in categorical_columns_with_nan:
            x_df[column] = x_df[column].astype(str).astype("category")
            y_df[column] = y_df[column].astype(str).astype("category")

        categorical_pandas_order = {}

        for column in categorical_columns:
            x_df[column] = x_df[column].astype("category")
            categorical_pandas_order[column] = x_df[column].cat.categories
            y_df[column] = y_df[column].astype("category")
            y_df[column] = y_df[column].cat.set_categories(categorical_pandas_order[column], ordered=True)

        for column in categorical_columns:
            x_df[column] = x_df[column].cat.codes
            y_df[column] = y_df[column].cat.codes

        for column in series_columns:
            def series_to_df(data):
                result = np.ones(series_columns_max)*-1
                data = json.loads(data)
                index_limit = len(data) if len(data) <= series_columns_max else series_columns_max
                result[-index_limit:] = data[-index_limit:]
                #return result
                return pd.Series(result, index=series_columns_headers[column])

            series_x_df = x_df[column].parallel_apply(series_to_df)
            series_y_df = y_df[column].parallel_apply(series_to_df)

            x_df = x_df.join(series_x_df)
            y_df = y_df.join(series_y_df)

        X = x_df[features].values
        y = np.array(list(y_df['sold_quantity_series'].apply(json.loads).values))
        y = y.cumsum(axis=1)

        #y_target_stock and y_target_date_0
        y_ts = np.zeros(len(y))
        y_td = np.zeros(len(y))

        for i, t in enumerate(y):
            where_non_zero = np.where(t > 0)[0]
            if len(where_non_zero) == 0:
                y_ts[i] = 0
                y_td[i] = 0
                continue
            j = np.random.choice(where_non_zero)
            y_ts[i] = t[j]
            y_td[i] = np.where(t == t[j])[0][0]

        X = np.concatenate((X, np.reshape(y_ts, (-1, 1))), axis=1)
        y = y_td

        self.prepared_dataset = (X, y)
        self.categorical_pandas_order = categorical_pandas_order

    def train(self):
        X, y = self.prepared_dataset
        self.model = XGBClassifier(n_estimators=1000, max_depth=6, learning_rate=0.1,
                    random_state=0, tree_method='gpu_hist',
                    objective='multi:softprob', num_class=30, use_label_encoder=False,
                    gpu_id=3)
        self.model.fit(X, y)

    def predict(self, df_test):
        df_test = read_df(df_test)
        df_test_fromtrain_x_features = read_df(os.path.join(self.dataset_path, self.default_paths['test_fromtrain_data_x_processed']))

        test_x_df = df_test_fromtrain_x_features.set_index('sku').loc[df_test['sku']].reset_index().copy()
        for column in numeric_columns_composed:
            test_x_df[column+'_std'] = test_x_df[column+'_std'].fillna(0)
        for column in categorical_columns_with_nan:
            test_x_df[column] = test_x_df[column].astype(str).astype("category")
        for column in categorical_columns:
            test_x_df[column] = test_x_df[column].astype("category")
            categorical_order = list(self.categorical_pandas_order[column])
            if len(test_x_df[column].cat.categories) != len(categorical_order):
                for cat in set(test_x_df[column].cat.categories) - set(categorical_order):
                        categorical_order.append(cat)
            test_x_df[column] = test_x_df[column].cat.set_categories(categorical_order, ordered=True)
            test_x_df[column] = test_x_df[column].cat.codes
        for column in  series_columns:
            def series_to_df(data):
                result = np.ones(series_columns_max)*-1
                data = json.loads(data)
                index_limit = len(data) if len(data) <= series_columns_max else series_columns_max
                result[-index_limit:] = data[-index_limit:]
                #return result
                return pd.Series(result, index=series_columns_headers[column])

            series_x_test_df = test_x_df[column].parallel_apply(series_to_df)
            test_x_df = test_x_df.join(series_x_test_df)

        X_test = test_x_df[features].values
        X_test = np.concatenate((X_test, np.reshape(df_test['target_stock'].values, (-1, 1))), axis=1)

        preds = self.model.predict_proba(X_test)
        probabilities = (preds/preds.sum(axis=1)[:,None]).round(4)

        return probabilities


import json
import os

import torch
from torch import nn, optim
from tqdm import tqdm

from args import args
from helper import compute_accuracy, count_parameters
from models import ReviewClassifierBiGRU
from datasets import ReviewDataset, get_num_batches, generate_rnn_batches

if os.path.exists(args.model_state_filepath):
    print('Model file already exists. Press enter to override and start new training.')
    input()

if not os.path.exists(args.reviews_json_train_filepath):
    with open(args.reviews_json_filepath, 'r') as source, open(args.reviews_json_train_filepath, 'w') as dest:
        for i, line in enumerate(source):
            if i >= args.reviews_train:
                break
            dest.write(line)

# create dataset and vectorizer
tokenizer = 'basic_english'
dataset = ReviewDataset.load_dataset_and_make_vectorizer(args.reviews_json_train_filepath, args.words_limit, args.max_review_words, tokenizer)
dataset.save_vectorizer(args.vectorizer_filepath)
vectorizer = dataset.get_vectorizer()

classifier = ReviewClassifierBiGRU(embeddings=vectorizer.review_vectors,
                                 hidden_dim=args.hidden_features,
                                 output_dim=len(vectorizer.rating_vocab),
                                 padding_idx=vectorizer.review_vocab['<pad>'],
                                 num_layers=args.num_layers,
                                 dropout=args.dropout_p)
classifier.to(args.device)
print(f'The models has {count_parameters(classifier):,} trainable parameters')

loss_func = nn.CrossEntropyLoss()
optimizer = optim.Adam(classifier.parameters(), lr=args.learning_rate)

train_size = int(0.7 * len(dataset))
val_size = len(dataset) - train_size
train_dataset, val_dataset = torch.utils.data.random_split(dataset, [train_size, val_size])

training_loss = []
training_acc = []
val_loss = []
val_acc = []
best_acc = 0.0
with tqdm(total=args.num_epochs * get_num_batches(train_dataset, args.batch_size)) as train_bar:
    for epoch_index in range(args.num_epochs):

        batch_generator = generate_rnn_batches(train_dataset, batch_size=args.batch_size, device=args.device)

        running_loss = 0.0
        running_acc = 0.0
        classifier.train()

        for batch_index, batch_dict in enumerate(batch_generator):
            # the training routine is these 5 steps:

            # --------------------------------------
            # step 1. zero the gradients
            optimizer.zero_grad()

            # step 2. compute the output
            y_pred = classifier(batch_dict['x_data'], batch_dict['x_length'])

            # step 3. compute the loss
            loss = loss_func(y_pred, batch_dict['y_target'])
            loss_t = loss.item()
            running_loss += (loss_t - running_loss) / (batch_index + 1)

            # step 4. use loss to produce gradients
            loss.backward()

            # step 5. use optimizer to take gradient step
            optimizer.step()
            # -----------------------------------------
            # compute the accuracy
            acc_t = compute_accuracy(y_pred, batch_dict['y_target'])
            running_acc += (acc_t - running_acc) / (batch_index + 1)

            # update bar
            train_bar.set_postfix(loss=running_loss, acc=running_acc, epoch=epoch_index)
            train_bar.update()

        # save training state
        training_loss.append(running_loss)
        training_acc.append(running_acc)

        # Validation
        batch_generator = generate_rnn_batches(val_dataset, batch_size=args.batch_size, device=args.device)
        running_loss = 0.
        running_acc = 0.
        classifier.eval()

        for batch_index, batch_dict in enumerate(batch_generator):
            # compute the output
            y_pred = classifier(batch_dict['x_data'], batch_dict['x_length'])

            # step 3. compute the loss
            loss = loss_func(y_pred, batch_dict['y_target'])
            loss_t = loss.item()
            running_loss += (loss_t - running_loss) / (batch_index + 1)

            # compute the accuracy
            acc_t = compute_accuracy(y_pred, batch_dict['y_target'])
            running_acc += (acc_t - running_acc) / (batch_index + 1)

        print(f'\nEpoch {epoch_index} validation loss:{running_loss}, validation acc:{running_acc}')

        # save validation state
        val_loss.append(running_loss)
        val_acc.append(running_acc)

        with open(args.training_history_filepath, 'w') as fp:
            json.dump({
                'training_loss': training_loss,
                'training_acc': training_acc,
                'val_loss': val_loss,
                'val_acc': val_acc,
            }, fp)

        # save first models or if acc is better than best acc
        if len(val_acc) == 1 or running_acc > best_acc:
            best_acc = running_acc
            torch.save(classifier.state_dict(), args.model_state_filepath)
            print(f'Validation accuracy increased. Saving models: {epoch_index}')

Yelp rewiews Multi Layer BiGRU with GloVe embeddings/train.py


import logging
from abc import abstractmethod, ABC
from dataclasses import dataclass
from typing import List, Optional

from spacepackets.cfdp import ConditionCode, FileStoreResponseTlv, MessageToUserTlv
from spacepackets.cfdp.pdu.file_data import RecordContinuationState
from spacepackets.cfdp.pdu.finished import DeliveryCode, FileDeliveryStatus
from spacepackets.util import UnsignedByteField
from tmtccmd.cfdp.defs import TransactionId
from tmtccmd.cfdp.filestore import VirtualFilestore, HostFilestore

_LOGGER = logging.getLogger(__name__)

@dataclass
class MetadataRecvParams:
    transaction_id: TransactionId
    source_id: UnsignedByteField
    file_size: Optional[int]
    source_file_name: Optional[str]
    dest_file_name: Optional[str]
    msgs_to_user: Optional[List[MessageToUserTlv]] = None

@dataclass
class TransactionFinishedParams:
    transaction_id: TransactionId
    condition_code: ConditionCode
    file_status: FileDeliveryStatus
    delivery_code: DeliveryCode
    fs_responses: Optional[List[FileStoreResponseTlv]] = None
    status_report: Optional[any] = None

@dataclass
class FileSegmentRecvdParams:
    """The length of the segment metadata is not supplied as an extra parameter as it can be
    simply queried with len(segment_metadata)
    """

    transaction_id: TransactionId
    offset: int
    length: int
    record_cont_state: Optional[RecordContinuationState]
    segment_metadata: Optional[bytes]

class CfdpUserBase(ABC):
    """This user base class provides the primary user interface to interact with CFDP handlers.
    It is also used to pass the Virtual Filestore (VFS) implementation to the CFDP handlers
    so the filestore operations can be mapped to the underlying filestore.

    This class is used by implementing it in a child class and then passing it to the CFDP
    handler objects. The base class provides default implementation for the user indication
    primitives specified in the CFDP standard. The user can override these implementations
    to provide custom indication handlers.
    """

    def __init__(self, vfs: Optional[VirtualFilestore] = None):
        if vfs is None:
            vfs = HostFilestore()
        self.vfs = vfs

    @abstractmethod
    def transaction_indication(self, transaction_id: TransactionId):
        """This indication is used to report the transaction ID to the CFDP user"""
        _LOGGER.info(f"Transaction.indication for {transaction_id}")

    @abstractmethod
    def eof_sent_indication(self, transaction_id: TransactionId):
        _LOGGER.info(f"EOF-Sent.indication for {transaction_id}")

    @abstractmethod
    def transaction_finished_indication(self, params: TransactionFinishedParams):
        _LOGGER.info(
            f"Transaction-Finished.indication for {params.transaction_id}. Parameters:"
        )
        print(params)

    @abstractmethod
    def metadata_recv_indication(self, params: MetadataRecvParams):
        _LOGGER.info(
            f"Metadata-Recv.indication for {params.transaction_id}. Parameters:"
        )
        print(params)

    @abstractmethod
    def file_segment_recv_indication(self, params: FileSegmentRecvdParams):
        _LOGGER.info(
            f"File-Segment-Recv.indication for {params.transaction_id}. Parameters:"
        )
        print(params)

    @abstractmethod
    def report_indication(self, transaction_id: TransactionId, status_report: any):
        # TODO: p.28 of the CFDP standard specifies what information the status report parameter
        #       could contain. I think it would be better to not hardcode the type of the status
        #       report here, but something like Union[any, CfdpStatusReport] with CfdpStatusReport
        #       being an implementation which supports all three information suggestions would be
        #       nice
        pass

    @abstractmethod
    def suspended_indication(
        self, transaction_id: TransactionId, cond_code: ConditionCode
    ):
        _LOGGER.info(
            f"Suspended.indication for {transaction_id} | Condition Code: {cond_code}"
        )

    @abstractmethod
    def resumed_indication(self, transaction_id: TransactionId, progress: int):
        _LOGGER.info(
            f"Resumed.indication for {transaction_id} | Progress: {progress} bytes"
        )

    @abstractmethod
    def fault_indication(
        self, transaction_id: TransactionId, cond_code: ConditionCode, progress: int
    ):
        _LOGGER.warning(
            f"Fault.indication for {transaction_id} | Condition Code: {cond_code} | "
            f"Progress: {progress} bytes"
        )

    @abstractmethod
    def abandoned_indication(
        self, transaction_id: TransactionId, cond_code: ConditionCode, progress: int
    ):
        _LOGGER.warning(
            f"Abandoned.indication for {transaction_id} | Condition Code: {cond_code} | "
            f"Progress: {progress} bytes"
        )

    @abstractmethod
    def eof_recv_indication(self, transaction_id: TransactionId):
        _LOGGER.info(f"EOF-Recv.indication for {transaction_id}")


#!/usr/bin/env python
#coding=utf-8

import web
from datetime import *
from os import *
import httplib

urls=('/(.*)','index')
app=web.application(urls,globals())

def take_photo():
	dt=datetime.now()
	s= dt.strftime('%Y%m%d%H%M%S')
	mypath=s+'.jpg'
	system('raspistill -o '+mypath+' -t 100')
	system('mv '+mypath+' /usr/share/nginx/www/'+mypath)
	return mypath

class index:
	def GET(self,path):
		web.header('Access-Control-Allow-Origin', '*')
		web.header('Access-Control-Allow-Credentials', 'true')
		mypath=take_photo()
		return  mypath

if __name__=="__main__":
	app.run()


import numpy as np
import math
import matplotlib.pyplot as plt
from combine_data import concat_data_all_temperatures
import sys
from scipy.ndimage.filters import gaussian_filter
import pdb
import sys
import seaborn as sns
from uncertainties import ufloat
from uncertainties import unumpy

sns.set(style='ticks', palette='Set2')
sns.despine()

# define temperatures of data taken
desired_temperatures = [40, 60, 70, 80, 90, 100, 110, 120]
real_temperatures = [48, 58, 68, 71, 90, 100, 110, 120]
n_files = 31

# import and concatenate all data for all temperatures
data_dir = '2016_09_RFBackgroundMeasurements/'
template = data_dir + '2016_09_23_trace3_{}K_{}.dpt'

x_all_temp, y_all_temp_smooth, xfrag_all_temp, yfrag_all_temp, cf_all_temp = concat_data_all_temperatures(
    template, desired_temperatures, n_files, linearize=True, smooth=True)

T_term = ufloat(30, 0)  # K
T_amp = ufloat(26, 0)  # K
T_circ = ufloat(30, 0)  # K
alpha_circ = ufloat(10**(-0.5 / 10), 0)

x_all_temp = x_all_temp[40]
alpha = unumpy.uarray([0] * len(x_all_temp), [0] * len(x_all_temp))
#alpha_m = np.zeros_like(x_all_temp, dtype = float)
realtemp = np.array(real_temperatures)
sys_noise_t = np.zeros((len(realtemp), len(x_all_temp)))
sig_sys_noise_t = np.zeros((len(realtemp), len(x_all_temp)))

for ind in range(0, len(x_all_temp)):  # loop through frequencies
    power_t = np.zeros_like(realtemp, dtype=float)
    for i in range(0, len(realtemp)):  # loop through temperatures
        power_t[i] = y_all_temp_smooth[desired_temperatures[i]][ind]
    z, cov = np.polyfit(realtemp, power_t, 1, cov=True)
    sig_z = np.sqrt(np.diagonal(cov))
    z = unumpy.uarray(z, sig_z)
    x_int = -z[1] / z[0]
    alpha[ind] = (-(T_amp + x_int) / (T_term - x_int))
#    alpha_m[ind] = -(T_circ*(1-alpha_circ) + T_amp +
#        x_int*alpha_circ)/(alpha_circ*(T_term*alpha_circ +
#            T_circ*(1-alpha_circ)- x_int))
for i in range(0, len(realtemp)):  # loop through temperatures
    tsys = T_term * alpha + realtemp[i] * (1 - alpha) + T_amp
    sys_noise_t[i] = unumpy.nominal_values(tsys)
    sig_sys_noise_t[i] = unumpy.std_devs(tsys)

sig_alpha = unumpy.std_devs(alpha)
alpha = unumpy.nominal_values(alpha)

plt.figure()
plt.plot(x_all_temp, alpha, rasterized=True)
plt.ylabel('Fractional Transparency')
plt.xlabel('Frequency [MHz]')
plt.tight_layout()
plt.savefig('alpha_v_freq_u.pdf')
plt.close()
#plt.plot(x_all_temp, alpha_m)

plt.figure()
plt.errorbar(x_all_temp, alpha, sig_alpha, rasterized=True)
plt.ylabel('Fractional Transparency')
plt.xlabel('Frequency [MHz]')
plt.tight_layout()
plt.savefig('alpha_v_freq_w_uncertainties_u.pdf')
plt.close()

plt.figure()
plt.plot(x_all_temp, sig_alpha / alpha, rasterized=True)
plt.xlabel('Frequency [MHz]')
plt.ylabel(r'$\sigma_{\alpha}/\alpha$')
plt.tight_layout()
plt.savefig('sig_alpha_v_freq_u.pdf')
plt.close()

plt.figure()
for i in range(0, len(realtemp)):
    plt.plot(
        x_all_temp,
        sys_noise_t[i],
        label='{} K'.format(
            realtemp[i]),
        rasterized=True)
plt.legend(loc='best')
plt.ylabel('System Noise Temperature [K]')
plt.xlabel('Frequency [MHz]')
plt.tight_layout()
plt.savefig('tsys_v_freq_u.pdf')
plt.close()

plt.figure()
for i in range(0, len(realtemp)):
    plt.errorbar(
        x_all_temp,
        sys_noise_t[i],
        sig_sys_noise_t[i],
        label='{} K'.format(
            realtemp[i]),
        rasterized=True)
plt.legend(loc='best')
plt.ylabel('System Noise Temperature [K]')
plt.xlabel('Frequency [MHz]')
plt.tight_layout()
plt.savefig('tsys_v_freq_w_uncertainties_u.pdf')
plt.close()

plt.figure()
for i in range(0, len(realtemp)):
    plt.plot(
        x_all_temp,
        sig_sys_noise_t[i] /
        sys_noise_t[i],
        label='{} K'.format(
            realtemp[i]),
        rasterized=True)
plt.legend(loc='best')
plt.ylabel(r'$\sigma_{T_{sys}}/T_{sys}$')
plt.xlabel('Frequency [MHz]')
plt.tight_layout()
plt.savefig('sig_tsys_v_freq_u.pdf')
plt.close()


import os
import glob
import shutil
import tarfile

from pathlib import Path

DESCRIPTION = """

 Prifysgol Bangor University

"""

TECHIAITH_RELEASE=os.environ["TECHIAITH_RELEASE"]

#
def copy_for_evaluation_or_publishing(source_dir, target_dir):

    Path(target_dir).mkdir(parents=True, exist_ok=True)

    # copy json files
    for file in glob.glob(os.path.join(source_dir, r"*.json")):
        print ("Copying %s" % file)
        shutil.copy(file, target_dir)

    # copy config and model binary file
    checkpoint_dir=glob.glob(os.path.join(source_dir, r"checkpoint-*"))[0]
    shutil.copy(os.path.join(checkpoint_dir, "config.json"), target_dir)
    shutil.copy(os.path.join(checkpoint_dir, "pytorch_model.bin"), target_dir)

    return target_dir

#
def make_model_tarfile(model_name, source_dir, version=TECHIAITH_RELEASE):
    output_dir = Path(source_dir).parent
    output_tar_file_path = os.path.join(output_dir, model_name.replace("/","_") + "." + version + ".tar.gz")
    with tarfile.open(output_tar_file_path, "w:gz") as tar:
        tar.add(source_dir, arcname="")

    return output_tar_file_path


import logging
import pytest

from tests.helpers import create_unique_resource_name
from ocs_ci.framework import config
from ocs_ci.framework.pytest_customization.marks import tier1

logger = logging.getLogger(__name__)

@pytest.fixture()
def create_buckets(request, noobaa_obj):
    """
    Creates multiple buckets
    """
    created_buckets = []

    def verify_bucket():
        """
        Verifies whether buckets exists after deletion

        """
        for bucket in created_buckets:
            logger.info(f"Verifying whether bucket: {bucket.name} exists"
                        f" after deletion")
            assert noobaa_obj.s3_verify_bucket_exists(bucket) is False

    request.addfinalizer(verify_bucket)

    bucket_name = create_unique_resource_name(
        resource_description='bucket', resource_type='s3'
    )
    logger.info(f'Creating bucket: {bucket_name}')
    created_buckets.append(
        noobaa_obj.s3_create_bucket(bucketname=bucket_name)
    )

    return created_buckets

@pytest.mark.skipif(condition=True, reason="NooBaa is not deployed")
@pytest.mark.filterwarnings(
    'ignore::urllib3.exceptions.InsecureRequestWarning'
)
@tier1
class TestBucketDeletion:
    """
    Test bucket Creation Deletion of buckets
    """

    @pytest.mark.skipif(
        condition=config.ENV_DATA['platform'] != 'AWS',
        reason="Tests are not running on AWS deployed cluster"
    )
    @pytest.mark.polarion_id("OCS-1299")
    def test_s3_bucket_delete(self, noobaa_obj, create_buckets):
        """
        Test deletion of bucket using the S3 SDK
        """
        for bucket in create_buckets:
            logger.info(f"Deleting bucket: {bucket.name}")
            noobaa_obj.s3_delete_bucket(bucket)


import json
from tempfile import NamedTemporaryFile

SOURCE_FORMAT_EXTENSION = {
    'CSV': '.csv',
    'TSV': '.tsv',
    'JSON_NEWLINE': '.json'
}

class FeedService:
    def __init__(self, app):
        """
        :type app: metasdk.MetaApp
        """
        self.__app = app
        self.__options = {}
        self.__data_get_cache = {}
        self.__data_get_flatten_cache = {}
        self.__metadb = app.db("meta")
        self.__media = app.MediaService
        self.__starter = app.StarterService

    def get_feed(self, datasource_id):
        """
        Получение настроек для фида
        :param datasource_id: идентификатор фида
        :return: FeedDataSource
        """
        info = self.__metadb.one(
            """
            SELECT to_json(ds) as datasource
                 , to_json(fc) as connector
                 , to_json(fct) as connector_type
                 , to_json(ctp) as connector_type_preset
                 , json_build_object('email', u.email, 'full_name', u.full_name) as author_user
              FROM meta.feed_datasource ds
              LEFT JOIN meta.feed_connector fc
                     ON fc.id=ds.connector_id
              LEFT JOIN meta.feed_connector_type fct
                     ON fct.id=fc.connector_type_id
              LEFT JOIN meta.feed_connector_type_preset ctp
                     ON ctp.id=ds.connector_type_preset_id
              LEFT JOIN meta.user_list u
                     ON u.id=ds.author_user_id
             WHERE ds.id = :datasource_id::uuid
            """,
            {"datasource_id": datasource_id}
        )
        return FeedDataSource(**info)

    def get_data(self, datasource, callback):
        """
        Сохранение медиафайла
        :param task:
        :param media_metadata:
        :param file_suffix:
        :param callback:
        :return:
        """
        task = self.__app.worker.current_task
        media_metadata = datasource.connector_type_preset['preset_data']['media_metadata']
        result_data = task['result_data']
        tmp_file = NamedTemporaryFile(delete=False, suffix=SOURCE_FORMAT_EXTENSION.get(media_metadata['sourceFormat']))
        self.__app.log.info("Открываем файл", {"filename": tmp_file.name})
        with open(tmp_file.name, 'wb') as f:
            callback(f)

        self.__app.log.info("start media upload")

        result_data['stage_id'] = "persist_media_file"
        self.__starter.update_task_result_data(task)
        result = self.__media.upload(open(tmp_file.name), {
            "ttlInSec": 60 * 60 * 24,  # 24h
            "entityId": 2770,
            "objectId": task.get('data', {}).get("ds_id"),
            "info": {"metadata": media_metadata}
        })

        result_data['stage_id'] = "generate_media_finish"
        result_data['media_id'] = result['id']
        self.__starter.update_task_result_data(task)

        return result

    def datasource_process(self, datasource_id):
        """
        deprecated
        Запускает настроенные обработки в фиде
        :param datasource_id: uuid
        """
        # TODO Выпилить потом класс используется для другого
        # TODO без applicationId не выбираются поля сущностей. Подумать на сколько это НЕ нормально
        response = self.__app.native_api_call('feed', 'datasource/' + datasource_id + '/process?applicationId=1', {},
                                              self.__options, False, None, False, http_method="POST")
        return json.loads(response.text)

class FeedDataSource:
    """
    Класс хранения данных по коннектору
    """

    def __init__(self, datasource, author_user, connector, connector_type, connector_type_preset):
        self.datasource = datasource
        self.author_user = author_user
        self.connector = connector
        self.connector_type = connector_type
        self.connector_type_preset = connector_type_preset