import pandas as pd
import scipy
import pylab
import numpy as np

from statsmodels.formula.api import OLS

from easydev import Progress, AttrDict

from gdsctools.stats import MultipleTesting
from gdsctools import readers
from gdsctools.boxplots import BoxPlots
from gdsctools.settings import ANOVASettings
from gdsctools.anova_results import ANOVAResults

__all__ = ['ANOVA']

# Not that Logging is not used: it is not pickable and prevent
# multicore analysis.
class ANOVA(object): #Logging):
    """ANOVA analysis of the IC50 vs Feature matrices

    This class is the core of the analysis. It can be used to
    compute

    #. One association between a drug and a feature
    #. The association**S** between a drug and a set of features
    #. All assocations between a set of deugs and a set of features.

    For instance here below, we read an IC50 matrix and compute the
    association for a given drug with a specific feature.

    Note that genomic features are not provided as input but a default
    file is provided with this package that contains 677 genomic
    features for 1001 cell lines. If your IC50 contains unknown cell lines,
    you can provide your own file.

    .. plot::
        :include-source:
        :width: 80%

        from gdsctools import IC50, ANOVA, ic50_test
        ic = IC50(ic50_test)
        an = ANOVA(ic)
        # This is to select a specific tissue
        an.set_cancer_type('breast')
        df = an.anova_one_drug_one_feature('Drug_1047_IC50',
            'TP53_mut', show=True)

    :Details about the anova analysis: In the example above, we perform a
        regression/anova test based on OLS regression. This is done for
        one feature one drug across all cell lines (tissue) in the method
        :meth:`anova_one_drug`. The regression
        takes into account the following factors: tissue, MSI and features.
        The order matters. If there is only one tissue, this factor is
        dropped. If the number of MSI values is less than a pre-defined
        parameter (see :class:`~gdsctools.settings.ANOVASettings`), it is
        dropped. The other
        methods :meth:`anova_one_drug` and :meth:`anova_all` are wrappers
        around :meth:`anova_one_drug_one_feature` to loop over all drugs, and
        loop over all drugs and all features, respectively.

    V17 :
        gdsc.volcano_FDR_interpolation = False
        gdsc.settings.pvalue_correction_method = 'qvalue'

    """
    def __init__(self, ic50, genomic_features=None,
            drug_decode=None, verbose=True, low_memory=True,
            set_media_factor=False):
        """.. rubric:: Constructor

        :param DataFrame IC50: a dataframe with the IC50. Rows should be
            the COSMIC identifiers and columns should be the Drug names
            (or identifiers)
        :param features: another dataframe with rows as in the IC50 matrix
            and columns as features.  The first 3 columns must be named
            specifically to hold tissues, MSI (see format).
        :param drug_decode: a 3 column CSV file with drug's name and targets
            see :mod:`readers` for more information.
        :param verbose: verbosity in "WARNING", "ERROR", "DEBUG", "INFO"

        The attribute :attr:`settings` contains specific settings related
        to the analysis or visulation.
        """
        self.verbose = verbose

        # We first need to read the IC50 using a dedicated reader
        self.ic50 = readers.IC50(ic50)

        # Create a dictionary version of the data
        # to be accessed per drug where NA have already been
        # removed. Each drug is a dictionary with 2 keys:
        # Y for the data and indices for the cosmicID where
        # there is an IC50 measured.
        ic50_parse = self.ic50.df.copy().unstack().dropna()
        self.ic50_dict = dict([(d, {'indices': ic50_parse.ix[d].index,
            'Y':ic50_parse.ix[d].values}) for d in self.ic50.drugIds])

        # Reads features if provided, otherwise use a default data set
        if genomic_features is None:
            # Reads default version provided with the package
            self.features = readers.GenomicFeatures()
        else:
            self.features = readers.GenomicFeatures(genomic_features)

        if self.features.found_media is False and \
                set_media_factor is True:
            if self.verbose:
                print('Populating MEDIA Factor in the Genomic Feature matrix')
            self.features.fill_media_factor()

        #: a CSV with 3 columns used in the report
        self.read_drug_decode(drug_decode)

        # create the multiple testing factory used in anova_all()
        self.multiple_testing = MultipleTesting()

        # We prune the genomic features by settings the cosmic ids of
        # the features to be those of the cosmic ids of the IC50. See
        # readers module. This affectation, prune the features dataframe
        # automatically. This fails if a cosmic identifier is not
        # found in the features' cosmic ids, so let us catch the error
        # before hand to give a
        unknowns = set(self.ic50.cosmicIds).difference(
                set(self.features.cosmicIds))

        if len(unknowns) > 0:
            print("WARNING: " +
                "%s cosmic identifiers in your IC50 " % len(unknowns) +
                "could not be found in the genomic feature matrix. "+
                "They will be dropped. Consider using a user-defined " +
                "genomic features matrix")

        self.ic50.drop_cosmic(list(unknowns))
        self.features.cosmicIds = self.ic50.cosmicIds
        #self.cosmicIds = self.ic50.cosmicIds

        #: an instance of :class:`~gdsctools.settings.ANOVASettings`
        self.settings = ANOVASettings()
        self.settings.low_memory = low_memory

        # alias to all column names to store results
        # cast to list (Python3).
        self.column_names = list(ANOVAResults().mapping.keys())

        # skip assoc_id for now
        self._odof_dict = dict([(name, None)
            for name in self.column_names])

        # a cache to store ANOVA results for each drug
        self.individual_anova = {}

        # must be called if ic50 or features are changed.
        self.init()

    def _autoset_msi_factor(self):
        if self.features.found_msi:
            # if the number of pos. (or neg.) factors is not large enough then
            # the MSI factor is not used
            msi_name = self.features.colnames.msi
            self.msi_factor = self.features.df[msi_name]
            total = len(self.msi_factor)
            positives = self.msi_factor.sum()
            negatives = total - positives

            # we must have at least 2 positives or 2 negative
            # This is therefore a < comparison here below. See in
            # _get_one_drug_one_feature_data that we use >= which
            # is consistent.
            if positives < self.settings.MSI_factor_threshold:
                self.settings.include_MSI_factor = False
            if negatives < self.settings.MSI_factor_threshold:
                self.settings.include_MSI_factor = False
        else:
            self.settings.include_MSI_factor = False

    def _autoset_tissue_factor(self):
        # select tissue based on the features
        tissue_name = self.features.colnames.tissue
        self.tissue_factor = self.features.df[tissue_name]
        if len(self.tissue_factor.unique()) == 1:
            # there is only one tissue
            tissue = self.tissue_factor.unique()[0]
            self.settings.analysis_type = tissue
            self.settings.directory = tissue
        else:
            # this is a PANCAN analysis
            self.settings.analysis_type = 'PANCAN'

    def _autoset_media_factor(self):

        if self.settings.analysis_type != 'PANCAN':
            self.settings.include_media_factor = False
        elif self.features.found_media is True:
            self.settings.include_media_factor = True
            colname = self.features.colnames.media
            self.media_factor = self.features.df[colname]
        else:
            self.settings.include_media_factor = False

    def set_cancer_type(self, ctype=None):
        """Select only a set of tissues.

        Input IC50 may be PANCAN (several cancer tissues).
        This  function can be used to select a subset of tissues.
        This function changes the :attr:`ic50` dataframe and possibly
        the feature as well if some are not relevant anymore (sum of the
        column is zero for instance).

        """
        if ctype is None:
            return

        if ctype == 'PANCAN':
            return

        if isinstance(ctype, str):
            ctype = [str(ctype)]

        for this in ctype:
            assert this in self.features.tissues, "%s not found" % ctype

        # keep only features that correspond to the tissue
        self.features.keep_tissue_in(ctype)

        self.ic50.df = self.ic50.df.ix[self.features.df.index]
        self.init()

    def read_settings(self, settings):
        """Read settings and update cancer type if set"""
        self.settings.from_json(settings)
        self.set_cancer_type(self.settings.analysis_type)

    def init(self):
        # Some preprocessing to speed up data access
        ic50_parse = self.ic50.df.copy().unstack().dropna()
        # for each drug, we store the IC50s (Y) and corresponding indices
        # of cosmic identifiers
        self.ic50_dict = dict([
            (d, {'indices': ic50_parse.ix[d].index,
             'Y': ic50_parse.ix[d].values}) for d in self.ic50.drugIds])

        # save the tissues
        self._autoset_tissue_factor()

        # and MSI (Microsatellite instability) status of the samples.
        self._autoset_msi_factor()

        # and (growth) media factor
        self._autoset_media_factor()

        # dictionaries to speed up code.
        self.features_dict = {}
        self.msi_dict = {}
        self.tissue_dict = {}
        self.media_dict = {}
        # fill the dictionaries for each drug once for all
        for drug_name in self.ic50.drugIds:
            indices = self.ic50_dict[drug_name]['indices']
            # if we were to store all drugs /features, this takes
            # 1Gb of memory for 265 drugs and 680 features. This is
            # therefore not scalable, especially for multiprocessing.
            if self.settings.low_memory is True:
                pass
            else:
                self.features_dict[drug_name] = self.features.df.ix[indices]

            # MSI, media and tissue are not large data files and can be store
            if self.features.found_msi:
                self.msi_dict[drug_name] = self.msi_factor.ix[indices]
            if self.features.found_media:
                self.media_dict[drug_name] = self.media_factor.ix[indices]

            self.tissue_dict[drug_name] = self.tissue_factor.ix[indices]

        # some preprocessing for the OLS computation.
        # We create the dummies for the tissue factor once for all
        # Note that to agree with R convention, we have to resort the column
        # to agree with R convention that is a<B==b<c instead of
        # where A<B<C<a<b<c (in R)
        self._tissue_dummies = pd.get_dummies(self.tissue_factor)
        columns = self._tissue_dummies.columns
        columns = sorted(columns, key=lambda s: s.lower())
        columns = ['C(tissue)[T.' + x + ']' for x in columns]
        self._tissue_dummies.columns = columns

        if self.settings.include_media_factor:
            self._media_dummies = pd.get_dummies(self.media_factor)
            columns = self._media_dummies.columns
            columns = ['C(media)[T.' + x + ']' for x in columns]
            self._media_dummies.columns = columns
            for col in columns:
                self._tissue_dummies[col] = self._media_dummies[col]

        N = len(self._tissue_dummies)
        self._tissue_dummies['C(msi)[T.1]'] = [1]*N
        self._tissue_dummies['feature'] = [1] * N
        self._tissue_dummies.insert(0, 'Intercept', [1] * N)

        # drop first feature in the tissues that seems to be used as a
        # reference in the regression
        tissues = [x for x in self._tissue_dummies.columns if 'tissue' in x]
        self._tissue_dummies.drop(tissues[0], axis=1, inplace=True)

        if self.settings.include_media_factor:
            media = [x for x in self._tissue_dummies.columns if 'media' in x]
            self._tissue_dummies.drop(media[0], axis=1, inplace=True)

        # reset the buffer.
        self.individual_anova = {}

        for this in ['tissue', 'media', 'msi', 'feature']:
            if this in self._get_analysis_mode():
                print(this.upper() + " FACTOR : included")
            else:
                print(this.upper() + " FACTOR : NOT included")

    def _get_cosmics(self):
        return self.ic50.cosmicIds
    def _set_cosmics(self, cosmics):
        self.ic50.cosmicIds = cosmics
        self.features.cosmicIds = cosmics
        self.init()
        self.individual_anova = {}
    cosmicIds = property(_get_cosmics, _set_cosmics,
        doc="get/set the cosmic identifiers in the IC50 and feature matrices")

    def _get_drug_names(self):
        return self.ic50.drugIds
    def _set_drug_names(self, drugs):
        self.ic50.drugIds = drugs
        self.init()
        # not need to init this again ? self.individual_anova = {}
    drugIds = property(_get_drug_names, _set_drug_names,
            doc="Get/Set drug identifers")

    def _get_feature_names(self):
        shift = self.features.shift
        return self.features.features[shift:]
    def _set_features_names(self, features):
        self.features.features = features
        self.init()
        self.individual_anova = {}
    feature_names = property(_get_feature_names, _set_features_names,
            doc="Get/Set feature names")

    def _get_analysis_mode(self):
        modes = []
        if self.settings.analysis_type == 'PANCAN':
            modes.append('tissue')

        if self.settings.include_MSI_factor is True:
            modes.append('msi')

        if self.settings.include_media_factor is True:
            modes.append('media')

        modes.append('feature')
        return modes

    def diagnostics(self):
        """Return dataframe with information about the analysis

        """
        n_drugs = len(self.ic50.drugIds)
        n_features = len(self.features.features) - self.features.shift
        n_combos = n_drugs * n_features
        feasible = 0
        pb = Progress(n_drugs, 1)
        counter = 0
        for drug in self.ic50.drugIds:
            for feature in self.features.features[self.features.shift:]:
                dd = self._get_one_drug_one_feature_data(drug, feature,
                        diagnostic_only=True)
                if dd.status is True:
                    feasible += 1
            counter += 1
            pb.animate(counter)

        results = {
                'n_drug': n_drugs,
                'n_combos': n_combos,
                'feasible_tests': feasible,
                'percentage_feasible_tests': float(feasible)/n_combos*100}
        return results

    def _get_one_drug_one_feature_data(self, drug_name, feature_name,
            diagnostic_only=False):
        """
        return: a dictionary with relevant information. There is also
            a test to see if the data can be analysis or not. This is
            stored ad a boolean value with key called *status*.
        """
        # dictionary  struture to hold results (can set values as attributes)
        dd = AttrDict()

        # select IC50 of a given drug
        # a fast way to select non-NA values from 1 column:
        # dropna is actually faster than a method using a mask.
        #dd.Y = self.ic50.df[drug_name].dropna()
        #indices = dd.Y.index
        #dd.masked_features = self.features.df[feature_name][indices]
        #dd.masked_tissue = self.tissue_factor[indices]
        #dd.masked_msi = self.msi_factor[indices]
        #dd.positive_feature = dd.masked_features.values.sum()
        #dd.negative_feature = len(dd.masked_features) - dd.positive_feature
        #dd.positive_msi = dd.masked_msi.values.sum()
        #dd.negative_msi = len(dd.masked_msi) - dd.positive_msi
        # using a mask instead of indices is 30% slower
        #mask = self.ic50.df[drug_name].isnull()==False
        #dd.masked_features = self.features.df[feature_name][mask]
        #dd.masked_tissue = self.tissue_factor[mask]
        #dd.masked_msi = self.msi_factor[mask]

        # Amother version using a dictionary instead of dataframer is actually
        # 2-3 times faster. It requires to transform the dataframe into a
        # dictionary once for all and dropping the NA as well.
        # Now, the next line takes no time
        dd.Y = self.ic50_dict[drug_name]['Y']

        # an alias to the indices
        indices = self.ic50_dict[drug_name]['indices']

        # select only relevant tissues/msi/features
        if self.settings.low_memory is True:
            # This line takes 50% of the time
            dd.masked_features = self.features.df.loc[indices, feature_name]
        else:
            dd.masked_features = self.features_dict[drug_name][feature_name]

        dd.masked_tissue = self.tissue_dict[drug_name]
        if self.features.found_msi:
            dd.masked_msi = self.msi_dict[drug_name]
            dd.positive_msi = dd.masked_msi.values.sum()
            dd.negative_msi = len(dd.masked_msi) - dd.positive_msi

        if self.features.found_media:
            dd.masked_media = self.media_dict[drug_name]

        # compute length of pos/neg features and MSI
        dd.positive_feature = dd.masked_features.values.sum()
        dd.negative_feature = len(dd.masked_features) - dd.positive_feature

        # Some validity tests to run the analysis or not
        feature_threshold = self.settings.feature_factor_threshold
        msi_threshold = self.settings.MSI_factor_threshold

        A = self.settings.include_MSI_factor and\
            dd.positive_feature >= feature_threshold and\
            dd.negative_feature >= feature_threshold and\
            dd.negative_msi >= msi_threshold and \
            dd.positive_msi >= msi_threshold

        B = (not self.settings.include_MSI_factor) and\
            dd.positive_feature >= feature_threshold and\
            dd.negative_feature >= feature_threshold
        # We could of course use the mean() and std() functions from pandas or
        # numpy. We could also use the glass and cohens functions from the
        # stats module but the following code is much faster because it
        # factorises the computations of mean and variance
        dd.positives = dd.Y[dd.masked_features.values == 1]
        dd.negatives = dd.Y[dd.masked_features.values == 0]
        dd.Npos = len(dd.positives)
        dd.Nneg = len(dd.negatives)

        # additional information
        dd.feature_name = feature_name
        dd.drug_name = drug_name

        # FIXME is False does not give the same results as == False
        # in the test test_anova.py !!
        if (A == False) and (B == False):
            dd.status = False
            return dd
        else:
            dd.status = True

        if diagnostic_only is True:
            return dd

        # compute mean and std of pos and neg sets;using mean() takes 15us and
        # using the already computed sum and N takes 5us
        pos_sum = dd.positives.sum()
        neg_sum = dd.negatives.sum()
        dd.pos_IC50_mean = pos_sum / dd.Npos
        dd.neg_IC50_mean = neg_sum / dd.Nneg
        dd.delta_mean_IC50 = dd.pos_IC50_mean - dd.neg_IC50_mean

        # note the ddof to agree with R convention.
        dd.pos_IC50_std = dd.positives.std(ddof=1)
        dd.neg_IC50_std = dd.negatives.std(ddof=1)

        dd.pos_IC50_std = np.sqrt(( (dd.positives**2).sum() -
            pos_sum**2/dd.Npos)/(dd.Npos-1.))
        dd.neg_IC50_std = np.sqrt(( (dd.negatives**2).sum() -
            neg_sum**2/dd.Nneg)/(dd.Nneg-1.))

        # Compute Cohens and Glass effect size. Since underlying code
        # has lots in common, we do not use the modules but add
        # the code here below
        md = np.abs(dd.pos_IC50_mean - dd.neg_IC50_mean)
        dd.pos_glass = md / dd.pos_IC50_std
        dd.neg_glass = md / dd.neg_IC50_std

        csd = (dd.Npos - 1.) * dd.pos_IC50_std**2 + \
                (dd.Nneg - 1.) * dd.neg_IC50_std**2
        csd /= dd.Npos + dd.Nneg - 2.  # make sure this is float
        dd.effectsize_ic50 = md / np.sqrt(csd)

        # Note that equal_var is a user parameter and affects
        # results. The ANOVA_results.txt obtained from SFTP
        # have different values meaning that the equal.var param
        # was set to False. Note that pvalue is stored at index 1
        dd.ttest = self._get_ttest(dd.negatives, dd.positives)
        return dd

    def _get_ttest(self, sample1, sample2):
        # this computes the ttest.
        import scipy
        return scipy.stats.ttest_ind(sample1, sample2,
                equal_var=self.settings.equal_var_ttest)[1]

    def read_drug_decode(self, filename=None):
        """Read file with the DRUG information

        .. seealso:: :class:`gdsctools.readers.DrugDecode`
        """
        # Read the DRUG decoder file into a DrugDecode/Reader instance
        self.drug_decode = readers.DrugDecode(filename)

    def anova_one_drug_one_feature(self, drug_id,
            feature_name, show=False,
            production=False, directory='.'):
        """Compute ANOVA and various tests on one drug and one feature

        :param drug_id: a valid drug identifier
        :param feature_name: a valid feature name
        :param bool show: show some plots
        :param str directory: where to save the figure.
        :param bool production: if False, returns a dataframe otherwise
            a dictionary. This is to speed up analysis when scanning
            the drug across all features.

        .. note:: **for developer** this is the core of tha analysis
            and should be kept as fast as possible. 95% of the time is spent
            here.

        .. note:: **for developer** Data used in this function comes from
            _get_one_drug_one_feature_data method, which should also be kept
            as fast as possible.
        """
        if drug_id not in self.drugIds:
            raise ValueError('Unknown drug name %s. Use e.g., %s'
                    % (drug_id, self.drugIds[0]))

        if feature_name not in self.feature_names:
            # we start index at 3 to skip tissue/name/msi
            raise ValueError('Unknown feature name %s. Use e.g. one of %s'
                    % (feature_name, self.feature_names[0:3]))

        # This extract the relevant data and some simple metrics
        # This is now pretty fast accounting for 45 seconds
        # for 265 drugs and 988 features
        odof = self._get_one_drug_one_feature_data(drug_id, feature_name)
        drug_name = self.drug_decode.get_name(drug_id)
        drug_target = self.drug_decode.get_target(drug_id)

        # if the status is False, it means the number of data points
        # in a category (e.g., positive feature) is too low.
        # If so, nothing to do, we return an 'empty' dictionary
        if odof.status is False:
            results = self._odof_dict.copy()
            results['FEATURE'] = feature_name
            results['DRUG_ID'] = drug_id
            results['DRUG_NAME'] = drug_name
            results['DRUG_TARGET'] = drug_target
            results['N_FEATURE_pos'] = odof.Npos
            results['N_FEATURE_neg'] = odof.Nneg
            if production is True:
                # return a dict
                return results
            else:
                # or a dataframe; note that index is not relevant here but
                # required.
                df = pd.DataFrame(results, index=[1])
                return df

        # with the data extract, we can now compute the regression.

        # In R or statsmodels, the regression code is simple since
        # it is based on the formula notation (Y~C(msi)+feature)
        # This is also possible in statsmodels library,  however,
        # this relies on patsy, which is very slow as compared to the
        # statsmodels without formula.
        #### self._mydata = pd.DataFrame({'Y':self.Y,
        ####    'tissue':self.masked_tissue,
        ####       'msi': self.masked_msi, 'feature':self.masked_features})
        #### self.data_lm = ols('Y ~ C(tissue) + C(msi) + feature',
        ####  data=self._mydata, missing='none').fit() #Specify C is category

        # IMPORTANT: the order of the factors in the formula
        # is important. It does not change the total sum of square errors
        # but may change individual effects of the categorical
        # components.

        # Instead of using ols function, we use the OLS one so we cannot
        # use formula. Instead, we need to create manually the input
        # data. In the case of categorical data (tissue), we need to
        # create the dummy variable, which is done in the constructor
        # once for all (slow otherwise).
        if self.settings.analysis_type == 'PANCAN':
            # IMPORTANT: tissues are sorted alphabetically in R aov
            # function. Same in statsmodels but capitalised names
            # are sorted differently. In R, a<b<B<c but in Python,
            # A<B<C<a<b<c. So, 'aero' tissue is before 'Bladder' in R,
            # not in python. Since in a linear regression
            # models, the order of the factor matters and the first
            # factor is used as a reference, we decided to use same
            # convention as in R.
            # see http://statsmodels.sourceforge.net/devel/contrasts.html
            # for a good explanation

            #self._mydata = pd.DataFrame({'Y': odof.Y.copy(),
            #    'tissue':odof.masked_tissue,
            #    'msi':  odof.masked_msi, 'feature': odof.masked_features})
            #self.data_lm2 = ols('Y ~ C(tissue) + C(msi) + feature',
            #    data=self._mydata).fit() #Specify C for Categorical

            # from statsmodels.stats.anova import anova_lm
            # import statsmodels.formula.api as smf
            # df  = pd.DataFrame({'Y': odof.Y.copy(),
            #   'tissue':odof.masked_tissue,'media'
            #    odof.masked_media, 'msi':  odof.masked_msi,
            #   'feature': odof.masked_features})
            # lm = smf.ols('Y~C(tissue)+C(media)+C(msi)+feature',
            #    data=df).fit()
            #  anova_lm(lm)
            # The code above gives same answer as the code in gdsctools
            # but is slower

            # We could use pd.get_dummies but pretty slow
            # instead we create the full matrix in init() method.
            # One issue is that some columns end up with sum == 0
            # and needs to be dropped.
            df = self._tissue_dummies.ix[odof.masked_tissue.index]
            todrop = df.columns[df.values.sum(axis=0) == 0]
            if len(todrop) > 0: # use if since drop() is slow
                df = df.drop(todrop, axis=1)

            # Here we set other variables with dataframe columns' names as
            # expected by OLS.
            if self.settings.include_media_factor == False:
                todrop = [x for x in df.columns if
                        x.startswith('C(media)')]
                df = df.drop(todrop, axis=1)

            df['C(msi)[T.1]'] = odof.masked_msi.values
            df['feature'] = odof.masked_features.values

            self.Y = odof.Y
            self.EV = df.values
            # The regression and anova summary are done here
            #
            if self.settings.regression_method == 'ElasticNet':
                self.data_lm = OLS(odof.Y, df.values).fit_regularized(
                        alpha=self.settings.regression_alpha,
                        L1_wt=self.settings.regression_L1_wt)
            elif self.settings.regression_method == 'OLS':
                self.data_lm = OLS(odof.Y, df.values).fit()
            elif self.settings.regression_method == 'Ridge':
                self.data_lm = OLS(odof.Y, df.values).fit_regularized(
                        alpha=self.settings.regression_alpha,
                        L1_wt=0)
            elif self.settings.regression_method == 'Lasso':
                self.data_lm = OLS(odof.Y, df.values).fit_regularized(
                        alpha=self.settings.regression_alpha,
                        L1_wt=1)

            # example of computing null model ?
            # Example of computing pvalues ourself
            """self.samples1 = []
            self.samples2 = []
            self.samples3 = []
            Y = odof.Y.copy()
            pb = Progress(10000,20)
            for i in range(0,10000):
                #pylab.shuffle(Y)
                #data_lm = OLS(Y, df.values).fit()
                data_lm = OLS(Y+0.3*pylab.randn(len(Y)), df.values).fit()
                anova_pvalues = self._get_anova_summary(data_lm,
                    output='dict')
                self.samples1.append(anova_pvalues['msi'])
                self.samples2.append(anova_pvalues['feature'])
                self.samples3.append(anova_pvalues['tissue'])
                pb.animate(i)
            """

        elif self.settings.include_MSI_factor is True:
            #self._mydata = pd.DataFrame({'Y': odof.Y,
            #    'msi':  odof.masked_msi, 'feature': odof.masked_features})
            #self.data_lm = ols('Y ~ C(msi) + feature',
            #    data=self._mydata).fit() #Specify C for Categorical
            df = pd.DataFrame()
            df['C(msi)[T.1]'] = odof.masked_msi.values
            df['feature'] = odof.masked_features.values
            df.insert(0, 'Intercept', [1] * (odof.Npos + odof.Nneg))
            self.data_lm = OLS(odof.Y, df.values).fit()
        else:
            df = pd.DataFrame()
            df['feature'] = odof.masked_features.values
            df.insert(0, 'Intercept', [1] * (odof.Npos + odof.Nneg))
            self.data_lm = OLS(odof.Y, df.values).fit()
            #self._mydata = pd.DataFrame({'Y': odof.Y,
            #    'feature': odof.masked_features})
            #self.data_lm = ols('Y ~ feature',
            #    data=self._mydata).fit() #Specify C for Categorical

        self.anova_pvalues = self._get_anova_summary(self.data_lm,
                 output='dict')

        # Store the pvalues. Note that some may be missing so we use try
        # except, which is faster than if/else
        try:
            tissue_PVAL = self.anova_pvalues['tissue']
        except:
            tissue_PVAL = None

        try:
            MSI_PVAL = self.anova_pvalues['msi']
        except:
            MSI_PVAL = None

        try:
            FEATURE_PVAL = self.anova_pvalues['feature']
        except:
            FEATURE_PVAL = None

        try:
            MEDIA_PVAL = self.anova_pvalues['media']
        except:
            MEDIA_PVAL = None

        if show is True:
            boxplot = BoxPlots(odof, savefig=self.settings.savefig,
                    directory=directory)
            boxplot.boxplot_association(fignum=1)

            # a boxplot to show cell lines effects. This requires
            # the settings.analyse_type to be PANCAN
            if self.settings.analysis_type == 'PANCAN':
                boxplot.boxplot_pancan(fignum=2, mode='tissue')
            if self.settings.include_MSI_factor:
                boxplot.boxplot_pancan(fignum=3, mode='msi')

        results = {'FEATURE': feature_name,
                'DRUG_ID': drug_id,
                'DRUG_NAME': drug_name,
                'DRUG_TARGET': drug_target,
                'N_FEATURE_pos': odof.Npos,
                'N_FEATURE_neg': odof.Nneg,
                'FEATURE_pos_logIC50_MEAN': odof.pos_IC50_mean,
                'FEATURE_neg_logIC50_MEAN': odof.neg_IC50_mean,
                'FEATURE_delta_MEAN_IC50': odof.delta_mean_IC50,
                'FEATURE_pos_IC50_sd': odof.pos_IC50_std,
                'FEATURE_neg_IC50_sd': odof.neg_IC50_std,
                'FEATURE_IC50_effect_size': odof.effectsize_ic50,
                'FEATURE_pos_Glass_delta': odof.pos_glass,
                'FEATURE_neg_Glass_delta': odof.neg_glass,
                'ANOVA_FEATURE_pval': FEATURE_PVAL,
                'ANOVA_TISSUE_pval': tissue_PVAL,
                'ANOVA_MSI_pval': MSI_PVAL,
                'ANOVA_MEDIA_pval': MEDIA_PVAL,
                'FEATURE_IC50_T_pval': odof.ttest # pvalues is in index 1
                }

        # 12% of the time here
        if production is True:
            return results
        else:
            df = pd.DataFrame(results, index=[1])
            return df

    def optimise_elastic_net(self, drug_name, feature_name, N=20, Nalpha=20):
        lwts = pylab.linspace(0, 1, N)
        alphas = pylab.linspace(0, 5, Nalpha)

        mses = np.zeros((N, Nalpha))

        pb = Progress(N)
        for i, lwt in enumerate(lwts):
            for j, alpha in enumerate(alphas):
                self.settings.regression_method = 'ElasticNet'
                self.settings.regression_alpha = alpha
                self.settings.regression_L1_wt = lwt
                odof = self.anova_one_drug_one_feature(drug_name,
                        feature_name)
                anova = self._get_anova_summary(self.data_lm,
                        output='dataframe')
                mses[i,j] = self.data_lm.bic
            pb.animate(i+1)
        return mses

    def optimise_ridge(self, drug_name, feature_name, alphas=None):
        return self._opt_ridge_lasso(drug_name, feature_name,
                'Ridge', alphas=alphas)

    def optimise_lasso(self, drug_name, feature_name, alphas=None):
        return self._opt_ridge_lasso(drug_name, feature_name,
                'Lasso', alphas=alphas)

    def _opt_ridge_lasso(self, drug_name, feature_name, method, alphas=None):

        if alphas is None:
            alphas = pylab.linspace(0,1, 20)

        mses = []
        params = []
        method_buf = self.settings.regression_method
        alpha_buf = self.settings.elastic_net.alpha

        pb = Progress(len(alphas))
        for j, alpha in enumerate(alphas):
            self.settings.regression_method = method
            self.settings.elastic_net.alpha = alpha
            odof = self.anova_one_drug_one_feature(drug_name,
                    feature_name)
            anova = self._get_anova_summary(self.data_lm,
                    output='dataframe')
            #mses.append(anova.ix['Residuals']['Sum Sq'])
            mses.append(anova.ix['tissue']['F value'])
            #mses.append(anova['Sum Sq'].sum())
            pb.animate(j+1)
            params.append(self.data_lm.params)
        self.settings.regression_method = method_buf
        self.settings.elastic_net.alpha = alpha_buf
        return alphas, mses, params

    # no need to optimise anymore
    def _get_anova_summary(self, data_lm, output='dict'):
        # could use this with statsmodels but somehow anova_lm with typ I
        # does not work, which is the one used in R version, so we implement
        # the anova here
        q, r = np.linalg.qr(data_lm.model.data.exog)
        effects = np.dot(q.T, data_lm.model.data.endog)

        # In the regression, the first tissue is dropped hence -1
        # The degree of freedom for tissues is N - 1
        # self.features.tissues contains all tissues even those that
        # were dropped due to lack of pos or neg features. So, we must use
        modes = self._get_analysis_mode()
        Ncolumns = data_lm.model.data.exog.shape[1]
        Ntissue = Ncolumns
        Ntissue -= 1 # remove intercept
        Ntissue -= 1 # remove feature, which are always taken into account
        if 'msi' in modes:
            Ntissue -= 1
        if 'media' in modes:
            Nmedia = len(self._media_dummies.columns)-1
        else:
            Nmedia = 0
        Ntissue -= Nmedia

        # create the W matrix using tissue and MSI if requested
        # default is that the 3 features are used
        if 'tissue' in modes and 'msi' in modes and 'media' in modes:
            dof = [Ntissue, Nmedia, 1, 1]
            indices = ['tissue', 'media', 'msi', 'feature', 'Residuals']
            # 4 stands for intercept + tissue + msi +feature
            arr = np.zeros((5, Ncolumns))
            arr[1, slice(1, Ntissue+1)] = 1
            arr[2, slice(Ntissue+1, Ntissue+Nmedia+1)] = 1
            arr[3, Ntissue + Nmedia + 1] = 1
            arr[4, Ntissue + Nmedia + 2] = 1
            self.arr = arr
        elif 'tissue' in modes and 'msi' in modes:
            dof = [Ntissue, 1, 1]
            indices = ['tissue', 'msi', 'feature', 'Residuals']
            # 4 stands for intercept + tissue + msi +feature
            arr = np.zeros((4, Ncolumns))
            arr[1, slice(1, Ntissue)] = 1
            arr[2, Ntissue + 1] = 1
            arr[3, Ntissue + 2] = 1
            self.arr = arr
        elif 'tissue' not in modes and 'msi' in modes:
            dof = [1, 1]
            indices = ['msi', 'feature', 'Residuals']
            # 3 stands for intercept + msi +feature
            arr = np.zeros((3, Ncolumns))
            arr[1, 1] = 1
            arr[2, 2] = 1
        elif 'tissue' not in modes and 'msi' not in modes:
            dof = [1]
            indices = ['feature', 'Residuals']
            # 3 stands for intercept + msi +feature
            arr = np.zeros((2, Ncolumns))
            arr[1, 1] = 1
        arr[0, 0] = 1                   # intercept

        sum_sq = np.dot(arr, effects**2)[1:] # drop the intercep
        mean_sq = sum_sq / np.array(dof)
        Fvalues = mean_sq / (data_lm.ssr / data_lm.df_resid)
        F_pvalues = scipy.stats.f.sf(Fvalues, dof, data_lm.df_resid)
        sum_sq = np.append(sum_sq, data_lm.ssr)
        mean_sq = np.append(mean_sq, data_lm.mse_resid)
        F_pvalues = np.append(F_pvalues, None)
        Fvalues = np.append(Fvalues, None)
        dof.append(data_lm.model.df_resid)
        #indices.append('Residuals')
        # dataframe is slow, return just the dict of pvalues by default
        if output == 'dataframe':
            anova = pd.DataFrame({'Sum Sq': sum_sq, 'Mean Sq': mean_sq,
                'Df': dof, 'F value': Fvalues, 'PR(>F)': F_pvalues},
                index=indices,
                columns=['Df', 'Sum Sq', 'Mean Sq', 'F value', 'PR(>F)']
                )
            return anova
        elif self.settings.analysis_type == 'PANCAN':
            if self.settings.include_media_factor:
                dd = {'tissue': F_pvalues[0],
                      'media': F_pvalues[1],
                      'msi':F_pvalues[2],
                      'feature':F_pvalues[3]}
            else:
                dd = {'tissue': F_pvalues[0],
                      'msi':F_pvalues[1],
                      'feature':F_pvalues[2]}
            return dd
        elif self.settings.include_MSI_factor is True:
            return {'msi': F_pvalues[0], 'feature':F_pvalues[1]}
        else:
            return {'feature': F_pvalues[0]}

        #return anova

    def _draft(self):
        # using sklearn
        #ols = linear_model.LinearRegression()
        #f = ols.fit(an.dff, an.Y)
        #sse = sum(np.square((f.predict(an.dff).T - an.Y))) /
        #           float(an.dff.shape[0] - an.dff.shape[1])
        # ssr = sum(np.square((f.predict(an.dff).T - an.Y.mean())))
        pass

    def _test(self):
        # for drug1047 and featuer ABCB1_mut
        print("""
        Analysis of Variance Table

        Response: Y
        Df  Sum Sq Mean Sq F value  Pr(>F)
        TISSUEpattern  26  352.35 13.5517  9.2685 < 2e-16 ***
        MSIpattern      1    5.31  5.3094  3.6313 0.05705 .
        FEATpattern     1    3.19  3.1861  2.1791 0.14028
        Residuals     817 1194.55  1.4621
        """)

    #98% of time in  method anova_one_drug_one_feature
    def anova_one_drug(self, drug_id, animate=True, output='object'):
        """Computes ANOVA for a given drug across all features

        :param str drug_id: a valid drug identifier.
        :param animate: shows the progress bar
        :return: a dataframe

        Calls :meth:`anova_one_drug_one_feature` for each feature.

        """
        # some features can be dropped ??

        # drop first and second columns that are made of strings
        # works under python2 but not python 3. Assume that the 2 first
        #columns are the sample name and tissue feature
        # Then, we keep only cases with at least 3 features.
        # MSI could be used but is not like in original R code.
        features = self.features.df.copy()
        # need to skip the FACTOR to keep only features
        shift = self.features.shift

        features = features[features.columns[shift:]]
        mask = features.sum(axis=0) >= 3

        # TODO: MSI, tissues, name must always be kept
        #
        selected_features = features[features.columns[mask]]

        # scan all features for a given drug
        assert drug_id in self.ic50.df.columns
        N = len(selected_features.columns)
        pb = Progress(N, 10)
        res = {}
        # note that we start at idnex 4 to drop sample name, tissue and MSI
        for i, feature in enumerate(selected_features.columns):
            # production True, means we do not want to create a DataFrame
            # for each call to the anova_one_drug_one_feature function
            # Instead, we require dictionaries
            this = self.anova_one_drug_one_feature(drug_id, feature,
                    production=True)
            if this['ANOVA_FEATURE_pval'] is not None:
                res[feature] = this
            if animate is True:
                pb.animate(i+1)

        # if production is False:
        # df = pid.concat(res, ignore_index=True)
        df = pd.DataFrame.from_records(res)
        df = df.T

        df = ANOVAResults().astype(df)
        if len(df) == 0:
            return df

        # append DRUG_NAME/DRUG_TARGET columns
        df = self.drug_decode.drug_annotations(df)

        # TODO: drop rows where ANOVA_FEATURE_PVAL is None
        if output != 'object':
            df = self.add_pvalues_correction(df)
            return df
        else:
            df = self.add_pvalues_correction(df)

            res = ANOVAResults(df)
            res.settings = ANOVASettings(**self.settings)
            return res

    def anova_all(self, animate=True, drugs=None):
        """Run all ANOVA tests for all drugs and all features.

        :param drugs: you may select a subset of drugs
        :param animate: shows the progress bar
        :return: an :class:`~gdsctools.anova_results.ANOVAResults`
            instance with the dataframe
            stored in an attribute called **df**

        Loops over all drugs calling :meth:`anova_one_drug` for each
        drug and concatenating all results together. Note that once all
        data are gathered, an extra column containing the FDR corrections
        is added to the dataframe using :meth:`add_pvalues_correction`
        method. An extra column  named "ASSOC_ID" is also added with
        a unique identifer sorted by ascending FDR.

        .. note:: A thorough comparison with version v17 give the same FDR
            results (difference ~1e-6); Note however that the qvalue results
            differ by about 0.3% due to different smoothing in R and Python.
        """
        # drop DRUG where number of IC50 (non-null) is below 5
        # axis=0 is default but we emphasize that sum is over
        # column (i.e. drug
        vv = (self.ic50.df.isnull() == False).sum(axis=0)
        # FIXME: should be in one_drug_one_feature ??
        drug_names = vv.index[vv >= self.settings.minimum_nonna_ic50]

        # if user provided a list of drugs, use them:
        if drugs is not None:
            # todo: check valifity of the drug names
            drug_names = drugs[:]

        pb = Progress(len(drug_names), 1)
        drug_names = list(drug_names)
        pylab.shuffle(drug_names)
        if animate is True:
            pb.animate(0)

        for i, drug_name in enumerate(drug_names):
            if drug_name in self.individual_anova.keys():
                pass
            else:
                res = self.anova_one_drug(drug_name, animate=False,
                                          output='dataframe')
                self.individual_anova[drug_name] = res
            if animate is True:
                pb.animate(i+1)
        print("\n")
        if len(self.individual_anova) == 0:
            return ANOVAResults()

        df = pd.concat(self.individual_anova, ignore_index=True)

        if len(df) == 0:
            return df
        # sort all data by ANOVA p-values
        try:
            df.sort_values('ANOVA_FEATURE_pval', inplace=True)
        except:
            df.sort('ANOVA_FEATURE_pval', inplace=True)

        # all ANOVA have been computed individually for each drug and each
        # feature. Now, we need to compute the multiple testing corrections
        if self.settings.pvalue_correction_level == 'global':
            df = self.add_pvalues_correction(df)

        # insert a unique identifier as first column
        df.insert(0, 'ASSOC_ID', range(1, len(df) + 1))

        self.df = df
        # order the column names as defined in the __init__ method
        df = df[self.column_names]
        df.reset_index(inplace=True, drop=True)

        results = ANOVAResults()
        results.df = df
        results.settings = ANOVASettings(**self.settings)
        return results

    def add_pvalues_correction(self, df, colname='ANOVA_FEATURE_pval'):
        """Add the corrected pvalues column in a dataframe based on pvalues

        The default method (FDR correction) is stored in
        :attr:`settings.pvalue_correction_method` and can be changed to other
        methods (e.g., *qvalue*)

        .. seealso:: :meth:`anova_all`,
            :class:`~gdsctools.stats.MultipleTesting`
        """
        if len(df) == 0:
            return

        # extract pvalues
        data = df[colname].values

        # set the method and compute new pvalues
        self.multiple_testing.method = self.settings.pvalue_correction_method
        new_pvalues = self.multiple_testing.get_corrected_pvalues(data)
        new_pvalues *= 100
        # insert new columns.
        colname = 'ANOVA_FEATURE_FDR'

        try:
            df.insert(len(df.columns), colname, new_pvalues)
        except:
            # replaces it otherwise
            df[colname] = new_pvalues
        return df

    def reset_buffer(self):
        self.individual_anova = {}

    def __str__(self):
        txt = self.ic50.__str__()
        txt += "\n" + self.features.__str__()
        return txt

    def __repr__(self):
        txt = self.__str__()
        return txt


from rest_framework.decorators import api_view, permission_classes
from rest_framework import generics
from lembaga.models import Lembaga, Institusi, Tema
from lembaga.serializers import LembagaSerializer, InstitusiSerializer, TemaSerializer
from rest_framework.permissions import AllowAny
from rest_framework.response import Response
from rest_framework import viewsets
from django_filters.rest_framework import DjangoFilterBackend

class LembagaView(viewsets.ModelViewSet):
    permission_classes = (AllowAny,)
    serializer_class = LembagaSerializer

    def get_queryset(self, *args, **kwargs):
        """
        Optionally restricts the returned purchases to a given user,
        by filtering against a query parameter in the URL.
        """
        try:
            queryset = Lembaga.objects.all()
            param_institusi = self.request.GET.get('institusi')
            param_tema = self.request.GET.get('tema')
            param_praktikum = self.request.GET.get('praktikum')
            param_nama = self.request.GET.get('nama')

            if param_praktikum:
                temp = param_praktikum.split(',')
                param_praktikum = []
                for i in temp:
                    param_praktikum.append(int(i[-1]))
                queryset = queryset.filter(praktikum_ke__in=param_praktikum)
            if (param_institusi and param_tema):
                param_institusi = param_institusi.split(',')
                param_tema = param_tema.split(',')
                queryset = queryset.filter(institusi__nama__in=param_institusi).filter(tema__nama__in=param_tema)
            else:
                if (param_institusi is not None):
                    param_institusi = param_institusi.split(',')
                    queryset = queryset.filter(institusi__nama__in=param_institusi)
                if (param_tema is not None):
                    param_tema = param_tema.split(',')
                    queryset = queryset.filter(tema__nama__in=param_tema)

            # Find lembaga by Name
            # by performing substring checking
            if param_nama:
                tmp_queryset = queryset
                queryset = []
                for i in tmp_queryset:
                    if param_nama.lower() in i.nama.lower():
                        queryset.append(i)

            return queryset

        # Kasus Error Jika Parameter, Variable, dan Data tidak di set saat melakukan query
        except (ValueError, RuntimeError, TypeError, NameError):
            return {"status": "Harap isi kriteria pencarian"}

class InstitusiView(viewsets.ModelViewSet):
    """
    Provides a get method handler.
    """
    permission_classes = (AllowAny, )
    queryset = Institusi.objects.all()
    serializer_class = InstitusiSerializer

class TemaView(viewsets.ModelViewSet):
    """
    Provides a get method handler.
    """
    permission_classes = (AllowAny, )
    queryset = Tema.objects.all()
    serializer_class = TemaSerializer


import numpy as np
import pandas as pd
import pandas.testing as pdt
import pytest

from cod_analytics.classes import TransformReference
from cod_analytics.math.homography import Homography, HomographyCorrection

class TestHomography:
    xy_bounds = (-1, 1)
    xy_corners = [
        [-1, -1],
        [1, -1],
        [1, 1],
        [-1, 1],
    ]
    x_bounds = (-1, 1)
    y_bounds = (-2, 2)
    xy_rect_corners = [
        [-1, -2],
        [1, -2],
        [1, 2],
        [-1, 2],
    ]
    translate_bounds = (-5, 5)
    rotate_bounds = (-np.pi, np.pi)
    scale_bounds = (-1, 1)
    center_bounds = (-1, 1)
    size = (100, 2)

    rng = np.random.RandomState(956)
    xy = rng.uniform(*xy_bounds, size=size)
    x = rng.uniform(*x_bounds, size=size[0])
    y = rng.uniform(*y_bounds, size=size[0])
    xy_rect = np.vstack([x, y]).T
    translate = rng.uniform(*translate_bounds, size=2)
    rotate = rng.uniform(*rotate_bounds)
    scale = 2 ** rng.uniform(*scale_bounds)
    center = rng.uniform(*center_bounds, size=2)

    def transform(
        self,
        points: np.ndarray,
        translate: np.ndarray,
        rotate: float,
        scale: float,
        center: np.ndarray,
    ) -> np.ndarray:
        """Transform points using a translation, rotation, and scale."""
        rotate_array = np.array(
            [
                [np.cos(rotate), -np.sin(rotate)],
                [np.sin(rotate), np.cos(rotate)],
            ]
        )
        new_points = points - center
        new_points = new_points * scale
        new_points = new_points @ rotate_array
        new_points = new_points + center
        new_points = new_points + translate
        return new_points

    def test_homography_fitted_method(self) -> None:
        """Test the fitted_method decorator."""
        homography = Homography()

        @homography.fitted_method
        def test_method(self: Homography) -> None:
            pass

        with pytest.raises(RuntimeError):
            test_method(homography)

        homography.fitted = True
        test_method(homography)

    def test_homography_fit(self) -> None:
        """Test the fit method."""
        homography = Homography()
        source = np.array([[0.0, 0.0], [1.0, 0.0], [1.0, 1.0], [0.0, 1.0]])
        target = np.array([[0.0, 0.0], [1.0, 0.0], [1.0, 1.0], [0.0, 1.0]])

        homography.fit(source, target)

        assert homography.fitted
        assert np.allclose(homography.matrix, np.eye(3))

    def test_homography_add_correction(self) -> None:
        """Test the add_correction method."""
        homography = Homography()
        correction = HomographyCorrection()
        homography.add_correction(correction)
        assert homography.correction == correction

    @pytest.mark.parametrize("scale", [True, False], ids=["S", "NS"])
    @pytest.mark.parametrize("rotate", [True, False], ids=["R", "NR"])
    @pytest.mark.parametrize("translate", [True, False], ids=["T", "NT"])
    @pytest.mark.parametrize("center", [True, False], ids=["C", "NC"])
    @pytest.mark.parametrize("square", [True, False], ids=["Sq", "Rect"])
    def test_homography_transform(
        self,
        translate: bool,
        rotate: bool,
        scale: bool,
        center: bool,
        square: bool,
    ) -> None:
        """Test the transform method."""
        translate_val = self.translate if translate else np.zeros(2)
        rotate_val = self.rotate if rotate else 0.0
        scale_val = self.scale if scale else 1.0
        center_val = self.center if center else np.zeros(2)

        # Generate XY data
        XY = (self.xy if square else self.xy_rect) + center_val

        corners = np.array(self.xy_corners if square else self.xy_rect_corners)
        transformed_corners = self.transform(
            corners, translate_val, rotate_val, scale_val, center_val
        )
        expected = self.transform(
            XY, translate_val, rotate_val, scale_val, center_val
        )
        homography = Homography()
        homography.fit(corners, transformed_corners)
        XY_transformed = homography.transform(XY)
        assert np.allclose(XY_transformed, expected)

    @pytest.mark.parametrize("scale", [True, False], ids=["S", "NS"])
    @pytest.mark.parametrize("rotate", [True, False], ids=["R", "NR"])
    @pytest.mark.parametrize("translate", [True, False], ids=["T", "NT"])
    @pytest.mark.parametrize("center", [True, False], ids=["C", "NC"])
    @pytest.mark.parametrize("square", [True, False], ids=["Sq", "Rect"])
    def test_homography_transform_reference(
        self,
        translate: bool,
        rotate: bool,
        scale: bool,
        center: bool,
        square: bool,
    ) -> None:
        translate_val = self.translate if translate else np.zeros(2)
        rotate_val = self.rotate if rotate else 0.0
        scale_val = self.scale if scale else 1.0
        center_val = self.center if center else np.zeros(2)

        # Generate XY data
        XY = (self.xy if square else self.xy_rect) + center_val
        left, right = self.xy_bounds if square else self.x_bounds
        bottom, top = self.xy_bounds if square else self.y_bounds
        source = TransformReference(
            map_left=left + center_val[0],
            map_right=right + center_val[0],
            map_bottom=bottom + center_val[1],
            map_top=top + center_val[1],
            map_rotation=0.0,
        )
        corners = np.array(self.xy_corners if square else self.xy_rect_corners)
        corners = corners + np.array([center_val[0], center_val[1]])
        transformed_corners = self.transform(
            corners,
            translate_val,
            rotate_val,
            scale_val,
            center_val,
        )
        transformed_corners_nr = self.transform(
            corners,
            translate_val,
            0.0,
            scale_val,
            center_val,
        )
        new_left, new_right = (
            transformed_corners_nr[:, 0].min(),
            transformed_corners_nr[:, 0].max(),
        )
        new_bottom, new_top = (
            transformed_corners_nr[:, 1].min(),
            transformed_corners_nr[:, 1].max(),
        )
        target = TransformReference(
            map_left=new_left,
            map_right=new_right,
            map_bottom=new_bottom,
            map_top=new_top,
            map_rotation=rotate_val * 180.0 / np.pi,
        )
        homography_expected = Homography()
        homography_expected.fit(corners, transformed_corners)
        expected = homography_expected.transform(XY)
        homography = Homography.from_transform_reference(source, target)
        XY_transformed = homography.transform(XY)
        assert np.allclose(XY_transformed, expected)

    @pytest.mark.parametrize("scale", [True, False], ids=["S", "NS"])
    @pytest.mark.parametrize("rotate", [True, False], ids=["R", "NR"])
    @pytest.mark.parametrize("translate", [True, False], ids=["T", "NT"])
    @pytest.mark.parametrize("center", [True, False], ids=["C", "NC"])
    @pytest.mark.parametrize("square", [True, False], ids=["Sq", "Rect"])
    def test_homography_with_correction(
        self,
        translate: bool,
        rotate: bool,
        scale: bool,
        center: bool,
        square: bool,
    ) -> None:
        translate_val = self.translate if translate else np.zeros(2)
        rotate_val = self.rotate if rotate else 0.0
        scale_val = self.scale if scale else 1.0
        center_val = self.center if center else np.zeros(2)

        # Generate XY data
        XY = (self.xy if square else self.xy_rect) + center_val
        corners = np.array(self.xy_corners if square else self.xy_rect_corners)
        expected = self.transform(
            XY, translate_val, rotate_val, scale_val, center_val
        )
        correction = HomographyCorrection(
            translate=translate_val,
            rotate=rotate_val,
            scale=scale_val,
            center=center_val,
        )
        homography = Homography()
        homography.fit(corners, corners)
        homography.add_correction(correction)
        XY_transformed = homography.transform(XY)
        assert np.allclose(XY_transformed, expected)

    @pytest.mark.parametrize("scale", [True, False], ids=["S", "NS"])
    @pytest.mark.parametrize("rotate", [True, False], ids=["R", "NR"])
    @pytest.mark.parametrize("translate", [True, False], ids=["T", "NT"])
    @pytest.mark.parametrize("center", [True, False], ids=["C", "NC"])
    @pytest.mark.parametrize("square", [True, False], ids=["Sq", "Rect"])
    @pytest.mark.parametrize("n", [0, 1, 2, 5, 10])
    def test_homography_transform_df(
        self,
        translate: bool,
        rotate: bool,
        scale: bool,
        center: bool,
        square: bool,
        n: int,
    ) -> None:
        """Test the transform_df method."""
        translate_val = self.translate if translate else np.zeros(2)
        rotate_val = self.rotate if rotate else 0.0
        scale_val = self.scale if scale else 1.0
        center_val = self.center if center else np.zeros(2)

        # Generate XY data
        k = n if n > 0 else 1
        if square:
            xy_data = self.rng.uniform(
                *self.xy_bounds, size=(self.size[0], self.size[1] * k)
            ) + np.tile(center_val, k)
        else:
            x_data = self.rng.uniform(*self.x_bounds, size=(self.size[0], k))
            y_data = self.rng.uniform(*self.y_bounds, size=(self.size[0], k))
            # Interleave x and y data
            xy_data = np.empty((self.size[0], self.size[1] * k))
            xy_data[:, ::2] = x_data + center_val[0]
            xy_data[:, 1::2] = y_data + center_val[1]

        corners = np.array(self.xy_corners if square else self.xy_rect_corners)
        transformed_corners = self.transform(
            corners, translate_val, rotate_val, scale_val, center_val
        )

        if n == 0:
            labels: list[str] | list[list[str]] = ["x", "y"]
            flat_labels = ["x", "y"]
        else:
            labels = [[f"x_{i}", f"y_{i}"] for i in range(n)]
            flat_labels = [item for sublist in labels for item in sublist]

        df = pd.DataFrame(xy_data, columns=flat_labels)
        homography = Homography()
        homography.fit(corners, transformed_corners)
        df_transformed = homography.transform_dataframe(df, labels)

        if n == 0:
            expected = self.transform(
                xy_data, translate_val, rotate_val, scale_val, center_val
            )
            assert np.allclose(df_transformed.values, expected)
        else:
            for i in range(n):
                k = i * 2
                expected = self.transform(
                    xy_data[:, k : k + 2],
                    translate_val,
                    rotate_val,
                    scale_val,
                    center_val,
                )
                assert np.allclose(
                    df_transformed.iloc[:, k : k + 2].values, expected
                )


from functools import partial

import chex
import jax
import numpy as np
import numpy.testing as npt
from absl.testing import parameterized

from tessellate_ipu import tile_map, tile_put_replicated
from tessellate_ipu.lax import scatter_add_p, scatter_max_p, scatter_mul_p, scatter_p

class IpuTilePrimitivesLaxScater(chex.TestCase, parameterized.TestCase):
    def setUp(self):
        super().setUp()
        self.device = jax.devices("ipu")[0]
        self.num_tiles = self.device.num_tiles
        # Not very clean, but for better reproducibility.
        np.random.seed(123)

    @parameterized.parameters(
        {"num_elements": 8, "num_indices": 3, "scatter_prim": scatter_p},
        {"num_elements": 8, "num_indices": 16, "scatter_prim": scatter_add_p},
        {"num_elements": 8, "num_indices": 16, "scatter_prim": scatter_max_p},
        {"num_elements": 8, "num_indices": 16, "scatter_prim": scatter_mul_p},
        {"num_elements": 8, "num_indices": 3, "scatter_prim": scatter_add_p},
        {"num_elements": 8, "num_indices": 12, "scatter_prim": scatter_add_p},
        {"num_elements": 256, "num_indices": 512, "scatter_prim": scatter_add_p},
    )
    def test__tile_map__scatter__jitting__multi_sizes__proper_result(self, num_elements, num_indices, scatter_prim):
        tiles = (0,)
        data = np.random.randn(num_elements).astype(np.float32)
        indices = np.random.randint(low=0, high=num_elements, size=num_indices)
        indices = indices.reshape(-1, 1).astype(np.uint32)
        updates = np.random.randn(indices.size).astype(np.float32)

        # Only supported configuration!
        scatter_dnums = jax.lax.ScatterDimensionNumbers(
            update_window_dims=(), inserted_window_dims=(0,), scatter_dims_to_operand_dims=(0,)
        )

        def scatter_add_fn(data, indices, updates):
            data = tile_put_replicated(data, tiles)
            indices = tile_put_replicated(indices, tiles)
            updates = tile_put_replicated(updates, tiles)
            return tile_map(
                scatter_prim,
                data,
                indices,
                updates,
                dimension_numbers=scatter_dnums,
                indices_are_sorted=False,
                unique_indices=False,
                mode=jax.lax.GatherScatterMode.PROMISE_IN_BOUNDS,
                update_jaxpr=None,
                update_consts=None,
            )

        cpu_scatter_add_fn = partial(jax.jit, backend="cpu")(scatter_add_fn)
        ipu_scatter_add_fn = partial(jax.jit, backend="ipu")(scatter_add_fn)

        cpu_output = cpu_scatter_add_fn(data, indices, updates)
        ipu_output = ipu_scatter_add_fn(data, indices, updates)

        assert ipu_output.tiles == tiles
        assert ipu_output.dtype == data.dtype
        npt.assert_array_almost_equal(ipu_output, cpu_output)


import json
import os
import pickle
import numpy as np
from pprint import pprint

winningRuns, losingRuns, cards, relics, X, Y = None, None, None, None, None, None
cwd = os.path.dirname(__file__)
pathToData = os.path.join(cwd, '..', 'Data')

#Parses the wins and loses from the dump file
def loadFromFile(winsPath, losePath):
    #runsPath = os.path.join(pathToData, 'data_2018-10-24_0-5000.json')
    global winningRuns, losingRuns
    runs = []
    count = 0
    winningRuns = []
    losingRuns = []
    winningCount = 0
    lossCount = 0

    for f in os.listdir(os.path.join(pathToData, 'runs')):
        runsPath = os.path.join(pathToData, 'runs', f, f)
        runs = json.load(open(runsPath))
        count += len(runs)

        # If floors reached = 50 then died to final boss, 51 means victory
        for run in runs:
            floor = run['event']['floor_reached']
            if floor == 50:
                losingRuns.append(run)
                lossCount+=1
            elif floor == 51:
                winningRuns.append(run)
                winningCount += 1
        print("Reading: ", f, "Cumulative size: ", count, "NumWins: ", winningCount, "NumLosses", lossCount)

        if len(runs) > 200000:
            break

    #runs = json.load(open(runsPath))

    # global winningRuns, losingRuns
    # winningRuns = []
    # losingRuns = []
    #
    # #If floors reached = 50 then died to final boss, 51 means victory
    # for run in runs:
    #     floor = run['event']['floor_reached']
    #     if floor == 50:
    #         losingRuns.append(run)
    #     elif floor == 51:
    #         winningRuns.append(run)

    pickle.dump(winningRuns, open(winsPath, 'w'))
    pickle.dump(losingRuns, open(losePath, 'w'))

#Load all the variables from
def loadVars():
    varFileNames = ['cards.json', 'relics.json']
    varFilePaths = [None, None]
    varList = [None, None]

    for file in range(len(varFileNames)):
        varFilePaths[file] = os.path.join(pathToData, varFileNames[file])
        varList[file] = json.load(open(varFilePaths[file]))

    #Set Variables to global scope and write cards and relics
    global cards, relics, winningRuns, losingRuns
    cards = varList[0]
    relics = varList[1]

    #Try to load up the wins and loses, if unable to then parse them from dump file
    winsPath = os.path.join(pathToData, 'wins.pkl')
    losePath = os.path.join(pathToData, 'loses.pkl')

    try:
        pickle.load(open("break")) #This is for debugging
        winningRuns = pickle.load(open(winsPath, 'r+'))
        losingRuns = pickle.load(open(losePath, 'r+'))
        print("Run data files found and loaded")
    except IOError:
        print("No run files found, loading from bulk data")
        loadFromFile(winsPath, losePath)

def loadArrays():
    global X, Y
    xPath = os.path.join(pathToData, 'X.npy')
    yPath = os.path.join(pathToData, 'Y.npy')
    try:
        #np.load("break") #For debugging
        # np.load("Break")
        X = np.load(xPath)
        Y = np.load(yPath)
        print("Loaded Arrays from file")
    except IOError:
        print("No array files found, parsing from data files")
        loadVars()
        index = list(cards.keys())
        numCards = len(index)
        runs = winningRuns + losingRuns
        X = np.zeros((len(runs), numCards*2))
        Y = np.concatenate((np.ones(len(winningRuns)), np.zeros(len(losingRuns))))
        np.save(yPath, Y)
        rowCount = 0
        for run in runs:
            #print ('row: ', rowCount)
            for card in run[u'event'][u'master_deck']:
                cardName = str(card)
                #print ('index: ', str(index[collumnCount]))
                #print ('card: ', str(card))
                for i in range(len(index)):
                    indexCardName = str(index[i])
                    upgradedIndexCardName = str(indexCardName + '+1')
                    if cardName == indexCardName:
                        #print cardName, i
                        X[rowCount][i] += 1
                    elif cardName == upgradedIndexCardName:
                         X[rowCount][i+numCards] += 1
            rowCount += 1
        np.save(xPath, X)

def main():
    loadArrays()

if __name__=='__main__':
    loadArrays()


# implementing pymoo multi-objective optimization with FEniCS FEA objectives

import numpy as np
import matplotlib.pyplot as plt
from matplotlib.offsetbox import TextArea, DrawingArea, OffsetImage, AnnotationBbox
import pathlib
from pathlib import Path

from pymoo.core.problem import ElementwiseProblem
from pymoo.algorithms.moo.nsga2 import NSGA2
from pymoo.termination import get_termination
from pymoo.optimize import minimize

from pymoo.operators.sampling.rnd import BinaryRandomSampling
from pymoo.operators.crossover.hux import HalfUniformCrossover
from pymoo.operators.crossover.ux import UniformCrossover
from pymoo.operators.crossover.pntx import TwoPointCrossover
from pymoo.operators.mutation.bitflip import BitflipMutation
from pymoo.decomposition.asf import ASF

# importing required modules (have to be in the same directory!)
from core_functions import *
from boundary_conditions import *

# defining the multiobjective problem for square FEA problem: Weight/vm_stress obj
class MyProblem(ElementwiseProblem):

    def __init__(self, conn, coords, num_el, meshpath):
        super().__init__(n_var=num_el,
                         n_obj=2,
                         n_ieq_constr=0,
                         xl=np.zeros(num_el),
                         xu=np.ones(num_el)
                         )
        self.conn = conn
        self.coords = coords
        self.num_el = num_el
        self.meshpath = meshpath  # temp mesh for evaluation

    def _evaluate(self, x, out, *args, **kwargs):

        # generate the infill file
        generate_infill_file(self.meshpath, x, self.conn, self.coords)

        # Loading the created infill mesh to FEniCS
        mesh, _, _, _ = extract_mesh_data_fenics(meshfile)

        # perform FEA and obtain displacements and von mises stress
        d_max, d_tot, d_avg, max_vm = square_FEA(mesh)

        # objective values: we want to minimize *weight* and *maximal stress*
        f1 = np.sum(x)  # objective 1 - "weight"
        f2 = d_max     # objective 2 - "maxStress"

        # constraints
        #g1 = np.sum(x) - 200   # dummy constraint: can be adjusted to limit the infill rate

        out["F"] = [f1, f2]  # dictionary key for objectives
        #out["G"] = [g1]      # dictionary key for constraints

# Filepath management
cwd = Path.cwd().parent
mesh_dir = cwd.joinpath("MOEA_meshes", "smallSquare7x7")  # adjust the dir here
inf_file = str(mesh_dir.stem) + '-infill.msh'
mesh_inp = mesh_dir / inf_file  # mesh input file

# Extracting geometry from the infill mesh -> num_el is the design dimension
conn, coords, num_el = extract_mesh_data(mesh_inp)   # extracting the geometry

# init the MOP
meshfile = mesh_dir / 'infill_gen.msh'
problem = MyProblem(conn, coords, num_el, meshfile)

# initialize the algorithm object
algorithm = NSGA2(
    pop_size=100,
    n_offsprings=40,
    sampling=BinaryRandomSampling(),
    crossover=TwoPointCrossover(prob=1.0),
    #crossover=UniformCrossover(),
    mutation=BitflipMutation(prob=0.5, prob_var= 2 * 1/num_el),  # mutation scaled by the num of elements
    eliminate_duplicates=True
)

# define the termination criterion
termination = get_termination("n_gen", 10)

# solve the MOP
res = minimize(problem,
               algorithm,
               termination,
               seed=1,
               save_history=True,
               verbose=True)

# n_gen  - generation counter,
# n_eval - number of evaluations
# cv_min - minimum constraint violation
# cv_avg - average constraint violation in the current population.
# n_nds  - number of non-dominated solutions
# eps/ind- running performance metrics

X = res.X  # solutions in the decision space
F = res.F  # solutions in the objective space
hist = res.history  # getting the info for each iteration

#print(X)
#print(F)

# save solutions
out_dir = mesh_dir / "infill_gen"
img_dir = out_dir / "png"
out_dir.mkdir(exist_ok=True)
img_dir.mkdir(exist_ok=True)
for i, x in enumerate(X):
    ifile = out_dir / f"sol{i:02d}.msh"  # generate mesh
    generate_infill_file(ifile, x, conn, coords)
    imfile = img_dir / f"sol_{i}.png"    # generate png
    generate_img_file(imfile, x, conn, coords)
    print(f"File {ifile.name} saved.")###

# writing to a file
with open(out_dir / "sq_solutions.txt", "w+") as file:
    alg = str(type(algorithm))[8:-2]
    header = f"Algorithm:{alg}, pop_size: {algorithm.pop_size},\
     n_offsp: {algorithm.n_offsprings}, n_gen: {termination.n_max_gen},\
     t_tot: {res.exec_time:10.1f}, t_gen: {res.exec_time/termination.n_max_gen:10.2f}\n"
    file.writelines(header)
    file.writelines(f"{str(list(sol))} \t w: {F[i][0]:10.1f} \t vm: {F[i][1]:10.06f} \n" for i, sol in enumerate(1*X))

# plotting the decision space
xl, xu = problem.bounds()
plt.figure(figsize=(7, 5))
plt.imshow(X)
plt.title("Design Space")
plt.show()

# plotting the objective space
plt.figure(figsize=(5, 3))
plt.scatter(F[:, 0], F[:, 1], s=20, facecolors='black', edgecolors='black')
plt.title("Objective Space")
plt.savefig(out_dir / 'ObjectiveSpace.png')
plt.show()

# Multi-Criteria Decision Making: subset selection
n_keep = 9  # how many solutions should we keep
if len(X) > n_keep:
    approx_ideal = F.min(axis=0)  # ideal point
    approx_nadir = F.max(axis=0)  # nadir point

    # normalizing with respect to both objectives
    nF = (F - approx_ideal) / (approx_nadir - approx_ideal)  # normalized objectives

    # Decision making: Using Augmented Scalarization Function (ASF)
    decomp = ASF()                  # init the ASF metric to be minimized
    weights = [np.array([(k + 1)/(n_keep+1), (n_keep - k)/(n_keep+1)]) for k in range(n_keep)]
    best_ASF = []  # indexes of chosen solutions
    for weight in weights:
        best_ASF.append(decomp.do(nF, 1/weight).argmin())  # index of the best solution regarding ASF
    best_ASF = list(set(best_ASF))  # remove duplicates
    F_ASF = F[best_ASF, :]          # objectives
    X_ASF = X[best_ASF, :]          # solutions
    n_kept = len(best_ASF)          # number of kept solutions

    # plotting the objective space with mesh png annotations
    fig, ax = plt.subplots(figsize=(13, 8))
    plt.scatter(F_ASF[:, 0], F_ASF[:, 1], s=20, facecolors='black', edgecolors='black')

    for i, (x, y) in enumerate(F_ASF):
        ind = best_ASF[i]
        img_lab = plt.imread(img_dir / f"sol_{ind}.png")
        imagebox = OffsetImage(img_lab, zoom=0.045)
        imagebox.image.axes = ax

        fl_offset = (-1)**i * 20  # spreading fluctuating offset
        ab = AnnotationBbox(imagebox, (x, y),
                            xybox=(32 - 0.6 * fl_offset, 38 + fl_offset),
                            xycoords='data',
                            boxcoords=("offset points", "offset points"),
                            pad=-1,
                            arrowprops=dict(
                                arrowstyle="->",
                                connectionstyle="angle,angleA=0,angleB=90,rad=3",
                                linewidth=0.5)
                            )
        ax.add_artist(ab)
        ax.text(x, y, f"  {str(ind)}", va='top', ha='left', zorder=4)

    xrang = np.abs(np.max(F_ASF[:,0]) - np.min(F_ASF[:,0]))
    yrang = np.abs(np.max(F_ASF[:,1]) - np.min(F_ASF[:,1]))
    ax.set_xlim((np.min(F_ASF[:,0]) - 0.05*xrang, np.max(F_ASF[:,0]) + 0.15*xrang ))
    ax.set_ylim((np.min(F_ASF[:,1]) - 0.1*yrang, np.max(F_ASF[:,1]) + 0.25*yrang ))
    plt.title(f"Objective Space of {n_kept} ASF-selected solutions")
    plt.savefig(out_dir / 'ObjectiveSpace-ASF.png')
    plt.show()


# -*- coding: utf-8 -*-
"""
Created on Tue Aug 24 19:55:28 2021

@author: jon-f
"""

import numpy as np
import matplotlib.pyplot as plt
import os
from skimage.transform import resize

def binary_array_to_hex(arr):

	bit_string = ''.join(str(int(b)) for b in arr.flatten())
	width = int(np.ceil(len(bit_string)/4))
	return '{:0>{width}x}'.format(int(bit_string, 2), width=width)

def image_to_edgehex(numpy_image):
    numpy_array = resize(numpy_image, (64, 64))

    size = numpy_array.shape
    grayscale = np.zeros(size[:-1])
    grayscale1der = np.zeros(size[:-1])

    for i in range(size[0]):
        for j in range(size[1]):

            if size[2] == 4:
                val = numpy_array[i, j]
                n_val = (0.299*val[0]+0.587*val[1]+0.114*val[2])*val[3]
            elif size[2] == 3:
                val = numpy_array[i, j]
                n_val = 0.299*val[0]+0.587*val[1]+0.114*val[2]
            else:
                n_val = numpy_array[i, j]

            grayscale[i, j] = n_val

            if i == 0 or j == 0:
                continue

            sub = grayscale[i-1:i+1, j-1:j+1]

            dIdx = 1/2 * ((sub[0, 1] - sub[0, 0]) + (sub[1, 1] - sub[1, 0]))
            dIdy = 1/2 * ((sub[1, 0] - sub[0, 0]) + (sub[1, 1] - sub[0, 1]))

            dIdp = abs(dIdx/2 + dIdy/2)
            if dIdp < 0.03:
                continue

            grayscale1der[i, j] = 1

    return binary_array_to_hex(grayscale1der)

def dilate(image_file):
    numpy_array = resize(image_file, (64, 64))

    size = numpy_array.shape[:-1]
    dilated = np.zeros(size)

    for i in range(size[0]):
        for j in range(size[1]):

            sub = image_file[i-1:i+1, j-1:j+1]

            if (sub == 1).any():
                dilated[i, j] = 1
    return dilated

def hamming_distance(hash1, hash2):
    dist = sum(n1 != n2 for n1, n2 in zip(hash1, hash2))
    return dist

def hash_match(hash1, hash2, percent_match=60):
    # returns True if at least percent_match %
    return hamming_distance(hash1, hash2)/len(hash1) < percent_match/100


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import datetime as dt
from datetime import datetime

#%config InlineBackend.figure_format = 'retina'
#%matplotlib inline
#%%
path='C:/Users/Mr.Goldss/Desktop/health care digital ticket EDA/'
filename='7_23_2020 Facility Itemized Tracker (1)'
df=pd.read_excel(path+filename+'.xlsx',sheet_name='New ALL')

df.shape

#%%                     fix basic formatting: preprocessing
'''data preprocessing'''

def formatting_preprocessing(df):
    want_col=['Date', 'Facility Name',
              'Facility Type', 'Item', 'Product Source', 'Type', 'UOM',
              'Qty', 'Total']
    df=df[want_col]
    df.columns=['Delivered Date', 'Facility Name',
       'Facility Type', 'Item Description', 'Product Source', 'Type', 'Units',
       'QTY Per UOM', 'Total Amount']
    df=df[df['Delivered Date'].notna()]
    df=df[df['Item Description'].notna()]
    df=df[(df['Units']>0) & (df['Units'].notna())]
    df['Product Source']=df['Product Source'].fillna('SNS')
    df_clean=df
    return df_clean
df_clean=formatting_preprocessing(df)

def del_wrong_ems(df_clean):

    del_cond1=(df_clean['Delivered Date']>='2020-04-01') & (df_clean['Delivered Date']<='2020-04-30')
    del_cond2=(df_clean['Facility Type'] =='EMS') & (df_clean['Facility Name'] !='Phoenix Fire Resource Management')
    to_be_drop_EMS=df_clean.loc[del_cond1 & del_cond2]
    Ems_drop_list=to_be_drop_EMS.index
    df_clean_copy=df_clean
    df_clean_EMS=df_clean_copy.drop(Ems_drop_list)
    return df_clean_EMS
df_clean_EMS=del_wrong_ems(df_clean)

def deep_clean_str_manipulate(df_clean_EMS):

    df_clean_EMS['QTY Per UOM']=df_clean_EMS['QTY Per UOM'].fillna(1)
    df_clean_EMS['Type']=df_clean_EMS['Type'].str.replace('Surgical Masks','Surgical Mask')
    df_clean_EMS['Type']=df_clean_EMS['Type'].str.lower()
    df_clean_EMS.groupby('Type')['Total Amount'].sum()

    df_clean_EMS['Facility Type']=df_clean_EMS['Facility Type'].str.lower()
    df_clean_EMS['Facility Type']=df_clean_EMS['Facility Type'].str.strip()
    df_clean_EMS['Facility Type']=df_clean_EMS['Facility Type'].str.capitalize()

    to_replace={'Lctf':'Ltcf','Ltfc':'Ltcf'}
    for key in to_replace.keys():
        df_clean_EMS['Facility Type']=df_clean_EMS['Facility Type'].str.replace(key,to_replace[key])

    df_clean_EMS['Facility Type']=df_clean_EMS['Facility Type'].str.replace('Acute care','Acute')

    df_clean_EMS['Item Description']=df_clean_EMS['Item Description'].str.lower()

    s=[]
    m=[]
    l=[]
    other=[]

    for i in df_clean_EMS['Item Description']:

        if 'small' in i:
            s.append(i)
        elif 'medium' in i:
            m.append(i)
        elif 'large' in i:
            l.append(i)
        else:
            other.append(i)
    df_clean_EMS['size']=df_clean_EMS['Item Description'].apply(
        lambda x:x.replace(x,'small') if x in s
        else(x.replace(x,'medium') if x in m else( x.replace(x,'large') if x in l else x.replace(x,'universal')) ))

    df_clean_EMS['week_of_year']=df_clean_EMS['Delivered Date'].dt.strftime('%U')
    df_clean_EMS['Month']=df_clean_EMS['Delivered Date'].dt.month
    df_clean_EMS['Total Amount']=df_clean_EMS['Total Amount'].astype(int)
    return df_clean_EMS
deep_clean_str_manipulate(df_clean_EMS)
#%%
'''countplot'''
fig, ax1, = plt.subplots()
fig.set_size_inches(25, 12)
sns.countplot(x='Facility Type',data=df_clean_EMS,ax=ax1)
plt.xticks(fontsize=18,rotation=90)
plt.xlabel('Facility', fontsize=15)
plt.ylabel('count', fontsize=20)
plt.title('Beltmann Inventory Distribution', fontsize=30)

plt.tight_layout()
plt.show()
#%%
#,barplot with plt.text to label all columns'''
'''
for index, value in enumerate(df_clean_EMS['Total Amount']):
    plt.text(index,value+400, str(value),fontsize=12)
plt.tight_layout()
'''

# %%
'''sns.lineplot---->date as x axis'''
#we only care about 5 product type
type_needed=['n95','surgical mask','gowns','gloves']
fig, ax1, = plt.subplots()
fig.set_size_inches(25, 12)
sns.lineplot(x='Month',y='Total Amount',hue='Type',\
    data=df_clean_EMS[df_clean_EMS['Type'].isin(type_needed)],ax=ax1,\
        ci=False, markers=True, style='Type')
plt.xticks(fontsize=18,rotation=90)
plt.xlabel('Month', fontsize=20)
plt.ylabel('Amount', fontsize=25)
plt.title('day', fontsize=30)

plt.tight_layout()
plt.show()
#%%
type_needed=['n95','surgical mask','gowns','gloves']
fig, ax1, = plt.subplots()
fig.set_size_inches(25, 12)
sns.relplot(x='Month',y='Total Amount',hue='Type',\
    data=df_clean_EMS[df_clean_EMS['Type'].isin(type_needed)],ax=ax1,\
        ci=False, markers=True, style='Type',kind='line')
plt.xticks(fontsize=18,rotation=90)
plt.xlabel('Month', fontsize=20)
plt.ylabel('Amount', fontsize=25)
plt.title('day', fontsize=30)

plt.tight_layout()
plt.show()

# %%
'''sns.barplot()'''
#one thing need to realize is that the bar is not the sum but the mean

type_needed=['n95','surgical mask','gowns','gloves']
fig, ax1, = plt.subplots()
fig.set_size_inches(25, 12)
sns.barplot(x='Month',y='Total Amount',hue='Type',\
    data=df_clean_EMS[df_clean_EMS['Type'].isin(type_needed)],ax=ax1,\
        ci=False)
plt.yticks(fontsize=18)
plt.xticks(fontsize=20)
plt.xlabel('Month of the year', fontsize=20)
plt.ylabel('mean Amount', fontsize=25)
plt.title('day', fontsize=30)

plt.tight_layout()
plt.show()
# %%
'''switch x and y and use orient=h to transpose '''
type_needed=['n95','surgical mask','gowns','gloves']
fig, ax1, = plt.subplots()
fig.set_size_inches(25, 12)
sns.barplot(y='Month',x='Total Amount',hue='Type',\
    data=df_clean_EMS[df_clean_EMS['Type'].isin(type_needed)],ax=ax1,\
        ci=False,orient='h')
plt.yticks(fontsize=18)
plt.xticks(fontsize=20)
plt.xlabel('Month of the year', fontsize=20)
plt.ylabel('mean Amount', fontsize=25)
plt.title('day', fontsize=30)

plt.tight_layout()
plt.show()

# %%
'''how can i change the color?'''
#go to seaborn official web to find color option
# search for html color code--> find the (hex) code like this :#E315C7
type_needed=['n95','surgical mask','gowns','gloves']
fig, ax1, = plt.subplots()
fig.set_size_inches(25, 12)
sns.barplot(y='Month',x='Total Amount',hue='Type',\
    data=df_clean_EMS[df_clean_EMS['Type'].isin(type_needed)],ax=ax1,\
        ci=False,orient='h',color='#E315C7')
plt.yticks(fontsize=18)
plt.xticks(fontsize=20)
plt.xlabel('Month of the year', fontsize=20)
plt.ylabel('mean Amount', fontsize=25)
plt.title('day', fontsize=30)

plt.tight_layout()
plt.show()

# %%
'''hist is a good way to analysis the distribution of the data'''

fig, ax1, = plt.subplots()
fig.set_size_inches(25, 12)
sns.distplot(df_clean_EMS[(df_clean_EMS['Type']=='gloves')\
    &(df_clean_EMS['Facility Type']=='Ltcf')\
        &(df_clean_EMS['Total Amount']<=500)]['Total Amount'],ax=ax1)
plt.yticks(fontsize=18)
plt.xticks(fontsize=20)
plt.xlabel('give amount', fontsize=20)
plt.ylabel('count', fontsize=25)
plt.title('give out distribution', fontsize=30)

mean=df_clean_EMS[(df_clean_EMS['Type']=='gloves')\
    &(df_clean_EMS['Facility Type']=='Ltcf')\
        &(df_clean_EMS['Total Amount']<=500)]['Total Amount'].mean()
plt.axvline(mean, color='red')

plt.tight_layout()
plt.show()

# %%
def draw_type_facility_type_amount_dist(Type,FacilityType,upper_level=50000):

    fig, ax1, = plt.subplots()
    fig.set_size_inches(25, 12)
    sns.distplot(df_clean_EMS[(df_clean_EMS['Type']==Type)\
        &(df_clean_EMS['Facility Type']==FacilityType)\
            &(df_clean_EMS['Total Amount']<=upper_level)]['Total Amount'],ax=ax1)
    plt.yticks(fontsize=18)
    plt.xticks(fontsize=20)
    plt.xlabel('give amount', fontsize=20)
    plt.ylabel('count', fontsize=25)
    plt.title('give out distribution', fontsize=30)

    mean=df_clean_EMS[(df_clean_EMS['Type']==Type)\
        &(df_clean_EMS['Facility Type']==FacilityType)\
            &(df_clean_EMS['Total Amount']<=upper_level)]['Total Amount'].mean()
    plt.axvline(mean, 0,1,color='red')

    plt.tight_layout()
    plt.show()

# %%
draw_type_facility_type_amount_dist('n95','Acute',2000)

# %%
'''box plot is another good way to show distribution'''
fig, ax1, = plt.subplots()
fig.set_size_inches(25, 12)
sns.boxplot(df_clean_EMS[(df_clean_EMS['Type']=='gloves')\
    &(df_clean_EMS['Facility Type']=='Ltcf')\
        &(df_clean_EMS['Total Amount']<=5000)]['Total Amount'],ax=ax1)
plt.yticks(fontsize=18)
plt.xticks(fontsize=20)
plt.xlabel('give amount', fontsize=20)
plt.ylabel('count', fontsize=25)
plt.title('give out distribution', fontsize=30)

mean=df_clean_EMS[(df_clean_EMS['Type']=='gloves')\
    &(df_clean_EMS['Facility Type']=='Ltcf')\
        &(df_clean_EMS['Total Amount']<=500)]['Total Amount'].mean()
plt.axvline(mean, color='red')

plt.tight_layout()
plt.show()

# %%
fig, ax1, = plt.subplots()
#fig.set_size_inches(25, 12)
sns.set(rc={'figure.figsize':(12,10)})
sns.boxplot(y='Total Amount',
            x='Month',data=df_clean_EMS,ax=ax1)
plt.tight_layout()
plt.show()

# %%
'''sns.swarmplot(), which can show every data point with a distribution,
 is a good helper for sns.boxplot, '''

fig, ax1, = plt.subplots()
#fig.set_size_inches(25, 12)
sns.set(rc={'figure.figsize':(12,10)})
sns.boxplot(x='Total Amount',
            y='Month',data=df_clean_EMS[df_clean_EMS['Total Amount']<=5000],ax=ax1,orient='h',color='#E55C94')

sns.swarmplot(x='Total Amount',
            y='Month',data=df_clean_EMS[df_clean_EMS['Total Amount']<=5000],ax=ax1,orient='h',color='green')
plt.tight_layout()
plt.show()

# %%
'''same as sns.violinplot'''
fig, ax1, = plt.subplots()
#fig.set_size_inches(25, 12)
#sns.set(rc={'figure.figsize':(12,10)})
sns.boxplot(x='Total Amount',
            y='Month',data=df_clean_EMS[df_clean_EMS['Total Amount']<=5000],ax=ax1,orient='h',color='#E55C94')

sns.violinplot(x='Total Amount',
            y='Month',data=df_clean_EMS[df_clean_EMS['Total Amount']<=5000],ax=ax1,orient='h',color='green')
plt.tight_layout()
plt.show()

# %%
'''let's try sns.scatterplot()'''
fig, ax1, = plt.subplots()
#fig.set_size_inches(25, 12)
#sns.set(rc={'figure.figsize':(12,10)})
sns.scatterplot(y='Total Amount',
            x='week_of_year',data=df_clean_EMS[df_clean_EMS['Total Amount']<=5000],\
                ax=ax1,color='#E55C94')
plt.tight_layout()
plt.show()

# %%
'''what is lmplot()==> basically: scattern plot with regression line'''
#this is not a good example, basically you need to find 2 continue variable

sns.lmplot(y='Total Amount',
            x='Month',data=df_clean_EMS[df_clean_EMS['Total Amount']<=5000],height=10)

plt.tight_layout()
plt.show()

health care digital ticket EDA/python models/seaborn viz center.py


import pygame as pg
import ChessEngine
import os
from itertools import cycle
from copy import deepcopy
pg.init()
pg.mixer.init()

ROOT_DIR = os.path.dirname(__file__)
IMAGE_DIR = os.path.join(ROOT_DIR, 'images')
WIDTH = HEIGHT = 512
DIMENSION = 8
SQ_SIZE = HEIGHT // 8
MAX_FPS = 15
IMAGES = {}
WHITE = (215,215,215)
BLACK = (155,155,155)

def load_images():
    for file in os.listdir(IMAGE_DIR):
        IMAGES[file[:2]] = pg.transform.scale(pg.image.load(os.path.join(IMAGE_DIR, file)), (SQ_SIZE, SQ_SIZE) )

def main():
    screen = pg.display.set_mode((WIDTH,HEIGHT))
    clock = pg.time.Clock()
    game = ChessEngine.GameState()
    load_images()
    running = True
    slctSq = ()
    playerClicks = []
    moves =  []
    while running:
        clock.tick(MAX_FPS)
        for e in pg.event.get():
            if e.type == pg.QUIT:
                running = False

            elif e.type == pg.KEYDOWN:
                if e.key == pg.K_LEFT:
                    game.undo_move()

                if e.key == pg.K_RIGHT:
                    game.redo_move()

                if e.key == pg.K_BACKSPACE:
                    game.unmake_move()

            elif e.type == pg.MOUSEBUTTONDOWN:
                if game.viewmode:
                    game.viewboard = deepcopy(game.board)
                    game.viewmove = len(game.moveLog)
                    game.viewmode = False
                else:
                    location = pg.mouse.get_pos()
                    col = location[0]//SQ_SIZE
                    row = location[1]//SQ_SIZE
                    if slctSq == (row, col):
                        slctSq = ()
                        playerClicks = []
                        moves = []
                    elif slctSq and (row, col) not in moves:
                        slctSq = ()
                        playerClicks = []
                        moves = []
                    elif len(slctSq) == 0 and game.board[row][col] == "--":
                        pass
                    else:
                        if slctSq:
                            slctSq  = (row, col)
                            playerClicks.append(slctSq)
                            pass
                        else:
                            if (game.board[row][col][0] == 'w' and game.whiteToMove) or (game.board[row][col][0] == 'b' and not game.whiteToMove):
                                moves = game.legal_moves(col, row)
                                slctSq  = (row, col)
                                playerClicks.append(slctSq)
                            else:
                                pass

                    if len(playerClicks) == 2:
                        if game.board[playerClicks[0][0]][playerClicks[0][1]] != '--':
                            move = ChessEngine.Move(playerClicks[0], playerClicks[1], game.board)
                            print(move.get_chess_notation())
                            game.make_move(move)
                            slctSq = ()
                            playerClicks = []
                            moves = []
                        else:
                            slctSq = ()
                            playerClicks.pop(0)

        screen.fill(WHITE)
        draw_game(screen, game, moves)
        if playerClicks:
            if game.board[playerClicks[0][0]][playerClicks[0][1]] != '--':
                highlight(screen, playerClicks[0][0],playerClicks[0][1])
        pg.display.flip()

def highlight(screen, row, col):
    s = pg.Surface((SQ_SIZE,SQ_SIZE))  # the size of your rect
    s.set_alpha(128)                # alpha level
    s.fill((0,128,128))           # this fills the entire surface
    screen.blit(s, (col*SQ_SIZE,row*SQ_SIZE))

def draw_game(screen, game, moves):
    draw_board(screen)
    draw_pieces(screen, game)
    draw_moves(screen, moves)

def draw_board(screen):
    tile_color = cycle([WHITE, BLACK])
    for row in range(DIMENSION):
        for col in range(DIMENSION):
            pg.draw.rect(screen, next(tile_color), (col*SQ_SIZE, row*SQ_SIZE, SQ_SIZE,SQ_SIZE))
        next(tile_color)

def draw_pieces(screen, game):
    for row in range(DIMENSION):
        for col in range(DIMENSION):
            if game.viewboard[row][col] != '--':
                screen.blit(IMAGES[game.viewboard[row][col]], (col*64,row*64))

def draw_moves(screen, moves):
    for i in moves:
        pg.draw.circle(screen, (168, 123, 116), (32+i[1]*64, 32+i[0]*64), 5)

if __name__ == '__main__':
    main()


from django.shortcuts import render, get_object_or_404
from .models import Post, Category, User
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from taggit.models import Tag
from django.db.models import Count
from django.utils import timezone

def post_list(request, tag_slug=None):
    posts = Post.objects.filter(status="published")
    latest_posts = Post.objects.filter(published_date__lte=timezone.now()).reverse()[:3]

    # post tag
    tag = None
    if tag_slug:
        tag = get_object_or_404(Tag, slug=tag_slug)
        posts = posts.filter(tags__in=[tag])

    paginator = Paginator(posts, 5)  # change this to accommodate number of posts per page
    page = request.GET.get('page', 1)
    try:
        posts = paginator.page(page)
    except PageNotAnInteger:
        posts = paginator.page(1)
    except EmptyPage:
        posts = paginator.page(paginator.num_pages)

    context = {
        'posts': posts,
        'page': page,
        'tag': tag,
        'latest_posts': latest_posts,
    }

    return render(request, 'blog/post_list.html', context, )

def author_post_list(request, author):
    author = User.objects.get(username=author)

    posts_list = author.posts.filter(status="published")

    paginator = Paginator(posts_list, 3)  # change this to accommodate number of posts per page
    page = request.GET.get('page', 1)

    try:
        posts = paginator.page(page)
    except PageNotAnInteger:
        posts = paginator.page(1)
    except EmptyPage:
        posts = paginator.page(paginator.num_pages)

    context = {
        'author': author,
        'posts': posts,
        'page': page
    }

    return render(request, 'blog/author_post_list.html', context)

def category_post_list(request, slug):
    category = Category.objects.get(slug=slug)

    posts_list = category.posts.filter(status="published")
    paginator = Paginator(posts_list, 3)  # change this to accommodate number of posts per page

    page = request.GET.get('page', 1)

    try:
        posts = paginator.page(page)
    except PageNotAnInteger:
        posts = paginator.page(1)
    except EmptyPage:
        posts = paginator.page(paginator.num_pages)

    context = {
        'category': category,
        'posts': posts,
        'page': page
    }

    return render(request, 'blog/category.html', context)

def blog_post(request, slug):
    # post = Post.objects.get(slug=slug)
    post = get_object_or_404(Post, slug=slug, status='published')

    # List of similar posts
    post_tags_ids = post.tags.values_list('id', flat=True)
    similar_posts = Post.objects.filter(active=True, tags__in=post_tags_ids).exclude(id=post.id)
    similar_posts = similar_posts.annotate(same_tags=Count('tags')).order_by('-same_tags', '-active')[
                    :3]  # adjust slice here for amount of posts to show

    context = {
        'post': post,
        'similar_posts': similar_posts,
    }

    return render(request, 'blog/blog_post.html', context)