#!/usr/bin/env python

# Copyright INRA (Institut National de la Recherche Agronomique)
# http://www.inra.fr
# http://urgi.versailles.inra.fr
#
# This software is governed by the CeCILL license under French law and
# abiding by the rules of distribution of free software.  You can  use,
# modify and/ or redistribute the software under the terms of the CeCILL
# license as circulated by CEA, CNRS and INRIA at the following URL
# "http://www.cecill.info".
#
# As a counterpart to the access to the source code and  rights to copy,
# modify and redistribute granted by the license, users are provided only
# with a limited warranty  and the software's author,  the holder of the
# economic rights,  and the successive licensors  have only  limited
# liability.
#
# In this respect, the user's attention is drawn to the risks associated
# with loading,  using,  modifying and/or developing or reproducing the
# software by the user in light of its specific status of free software,
# that may mean  that it is complicated to manipulate,  and  that  also
# therefore means  that it is reserved for developers  and  experienced
# professionals having in-depth computer knowledge. Users are therefore
# encouraged to load and test the software's suitability as regards their
# requirements in conditions enabling the security of their systems and/or
# data to be ensured and,  more generally, to use and operate it in the
# same conditions as regards security.
#
# The fact that you are presently reading this means that you have had
# knowledge of the CeCILL license and that you accept its terms.

##@file
# Give summary information on a TE annotation table.
# options:
#     -h: this help
#     -t: analysis type (default = 1, 1: per transposable element (TE), 2: per cluster, 3: per classification, 4: with map input file)
#     -p: name of the table (_path) or file (.path) with the annotated TE copies
#     -s: name of the table (_seq) or file (.fasta or .fa) with the TE reference sequences
#     -g: length of the genome (in bp)
#     -m: name of the file with the group and the corresponding TE names (format = 'map')
#     -o: name of the output file (default = pathTableName + '_stats.txt')
#     -C: name of the configuration file to access MySQL (e.g. 'TEannot.cfg')
#     -c: remove map files and blastclust file (if analysis type is 2 or 3)
#     -I: identity coverage threshold (default = 0)
#     -L: length coverage threshold (default=0.8)
#     -v: verbosity level (default = 0)

from commons.core.LoggerFactory import LoggerFactory
from commons.core.stat.Stat import Stat
from commons.core.sql.DbFactory import DbFactory
from commons.core.sql.TablePathAdaptator import TablePathAdaptator
from commons.core.sql.TableSeqAdaptator import TableSeqAdaptator
from commons.tools.getCumulLengthFromTEannot import getCumulLengthFromTEannot

LOG_DEPTH = "repet.tools"

# TODO: use templating engine instead of raw strings for AnnotationStatsWriter
class AnnotationStats(object):

    def __init__(self, analysisName = "TE", clusterFileName = "", seqTableName = "", pathTableName = "", genomeLength = 0, statsFileName = "", globalStatsFileName = "", verbosity = 3):
        self._analysisName = analysisName
        self._clusterFileName = clusterFileName
        self._seqTableName = seqTableName
        self._pathTableName = pathTableName
        self._genomeLength = genomeLength
        self._statsFileName = statsFileName
        self._globalStatsFileName = globalStatsFileName
        self._iDb = None
        self._iTablePathAdaptator = None
        self._iTableSeqAdaptator = None
        self._save = False
        self._clean = False
        self._verbosity = verbosity
        self._log = LoggerFactory.createLogger("%s.%s" % (LOG_DEPTH, self.__class__.__name__), self._verbosity)

    def _logAndRaise(self, errorMsg):
        self._log.error(errorMsg)
        raise Exception(errorMsg)

    def setCoverageThreshold(self, lengthThresh):
        self._coverageThreshold = float(lengthThresh)

    def setIdentityThreshold(self, identityThresh):
        self._identityThreshold = int(identityThresh)

    def setAnalyseType(self, analyseType):
        self._analyseType = str(analyseType)

    def setPathTableName(self, pathTableName):
        self._pathTableName = pathTableName

    def setDBInstance(self, iDb):
        self._iDb = iDb

    def setTablePathAdaptator(self, iTablePathAdaptator):
        self._iTablePathAdaptator = iTablePathAdaptator

    def setTableSeqAdaptator(self, iTableSeqAdaptator):
        self._iTableSeqAdaptator = iTableSeqAdaptator

    ## Get the coverage of TE copies for a given family (using 'mapOp')
    #
    # @param consensus string name of a TE family ('subject_name' in the 'path' table)
    # @return cumulCoverage integer cumulative coverage
    #
    def getCumulCoverage(self, consensus = ""):
        gclft = getCumulLengthFromTEannot()
        gclft.setInputTable(self._pathTableName)
        gclft.setTErefseq(consensus)
        gclft.setClean()
        gclft._db = self._iDb
        gclft._tpA = self._iTablePathAdaptator
        mapFileName = gclft.getAllSubjectsAsMapOfQueries()
        mergeFileName = gclft.mergeRanges(mapFileName)
        cumulCoverage = gclft.getCumulLength(mergeFileName)
        # self._iTablePathAdaptator.getCumulPathLength_from_subject( consensus )
        return cumulCoverage

    ## Get the number of full-lengths (95% <= L =< 105%)
    #
    # @param consensusLength integer
    # @param lLengths list of integers
    # @return fullLengthConsensusNb integer
    #
    def getNbFullLengths(self, consensusLength, lLengths):
        fullLengthConsensusNb = 0
        for i in lLengths:
            if i / float(consensusLength) >= 0.95 and i / float(consensusLength) <= 1.05:
                fullLengthConsensusNb += 1
        return fullLengthConsensusNb

# TODO: copies flagged as full-length fragment CAN'T belong to multi-fragments copies (fragments that have been joined). Use the beginning of implementation commented below.
    def getStatPerTE(self, consensusName):
        dConsensusStats = {}
        lLengthPerFragment = self._iTablePathAdaptator.getPathLengthListFromSubject(consensusName)
#         lLengthPerNotJoinedFragment = self._iTablePathAdaptator.getPathLengthListFromNotJoinedSubject(consensusName) # Method name and location is for information only. You may implement it elsewhere.
        lLengthPerCopy = self._iTablePathAdaptator.getChainLengthListFromSubject(consensusName)
        lIdentityPerCopy = self._iTablePathAdaptator.getChainIdentityListFromSubject(consensusName)
        dConsensusStats["length"] = self._iTableSeqAdaptator.getSeqLengthFromAccession(consensusName)
        dConsensusStats["cumulCoverage"] = self.getCumulCoverage(consensusName)
        dConsensusStats["nbFragments"] = len(lLengthPerFragment)
#         dConsensusStats["nbFullLengthFragments"] = self.getNbFullLengths(dConsensusStats["length"], lLengthPerNotJoinedFragment)
        dConsensusStats["nbFullLengthFragments"] = self.getNbFullLengths(dConsensusStats["length"], lLengthPerFragment)
        dConsensusStats["nbCopies"] = len(lLengthPerCopy)
        dConsensusStats["nbFullLengthCopies"] = self.getNbFullLengths(dConsensusStats["length"], lLengthPerCopy)
        dConsensusStats["statsIdentityPerChain"] = Stat()
        dConsensusStats["statsLengthPerChain"] = Stat()
        dConsensusStats["statsLengthPerChainPerc"] = Stat()
        self._statsForIdentityAndLength(dConsensusStats, lLengthPerCopy, lIdentityPerCopy)
        return dConsensusStats

    def getStatPerCluster(self, lConsensusNames):
        dConsensusClusterStats = {}
        lLengthPerFragment = []
        lLengthPerCopy = []
        cumulCoverageLength = 0
        for consensusName in lConsensusNames:
            cumulCoverageLength += self.getCumulCoverage(consensusName)
            lLengthPerFragment.extend(self._iTablePathAdaptator.getPathLengthListFromSubject(consensusName))
            lLengthPerCopy.extend(self._iTablePathAdaptator.getChainLengthListFromSubject(consensusName))
        dConsensusClusterStats["cumulCoverage"] = cumulCoverageLength
        dConsensusClusterStats["nbFragments"] = len(lLengthPerFragment)
        dConsensusClusterStats["nbCopies"] = len(lLengthPerCopy)
        return dConsensusClusterStats

    def getClusterListFromFile(self):
        lClusters = []
        with open(self._clusterFileName) as fCluster:
            for line in fCluster:
                lConsensusNames = line.rstrip().split("\t")
                lClusters.append(lConsensusNames)
        return lClusters

    def run(self):
        LoggerFactory.setLevel(self._log, self._verbosity)
        self._log.info("START")
        self._iDb = DbFactory.createInstance()
        self._iTablePathAdaptator = TablePathAdaptator(self._iDb, self._pathTableName)
        self._iTableSeqAdaptator = TableSeqAdaptator(self._iDb, self._seqTableName)

        iASW = AnnotationStatsWriter()
        if self._analysisName == "TE":

            with open(self._statsFileName, "w") as fStats:
                string = "%s\tlength\tcovg\tfrags\tfullLgthFrags\tcopies\tfullLgthCopies\t" % self._analysisName
                string += "meanId\tsdId\tminId\tq25Id\tmedId\tq75Id\tmaxId\t" # 7 items
                string += "meanLgth\tsdLgth\tminLgth\tq25Lgth\tmedLgth\tq75Lgth\tmaxLgth\t" # 7 items
                string += "meanLgthPerc\tsdLgthPerc\tminLgthPerc\tq25LgthPerc\tmedLgthPerc\tq75LgthPerc\tmaxLgthPerc\n" # 7 items
                fStats.write(string)

                lNamesTErefseq = self._iTableSeqAdaptator.getAccessionsList()
                lDistinctSubjects = self._iTablePathAdaptator.getSubjectList()
                totalCumulCoverage = self.getCumulCoverage()

                with open(self._globalStatsFileName, "w") as fG:
                    fG.write("%s\n" % iASW.printResume(lNamesTErefseq, lDistinctSubjects, totalCumulCoverage, self._genomeLength))
                    for consensusName in lNamesTErefseq:
                        self._log.debug("processing '%s'..." % consensusName)
                        dStatForOneConsensus = self.getStatPerTE(consensusName)
                        iASW.addCalculsOfOneTE(dStatForOneConsensus)
                        fStats.write("%s\n" % iASW.getStatAsStringForTE(consensusName, dStatForOneConsensus))
                    fG.write(iASW.printStatsForAllTEs(len(lNamesTErefseq)))

        elif self._analysisName == "Cluster":
            lClusters = self.getClusterListFromFile()
            lClusters.sort(key = lambda k: len(k), reverse = True)
            with open(self._statsFileName, "w") as fStats:
                string = "%s\tcovg\tfrags\tcopies\n" % self._analysisName
                # TODO: add fullLgthFrags and fullLgthCopies ? Is addition of previous results significant ?
                fStats.write(string)
                for index, lConsensus in enumerate(lClusters):
                    self._log.debug("processing '%s'..." % lConsensus)
                    dStatForOneCluster = self.getStatPerCluster(lConsensus)
                    fStats.write("%s\n" % iASW.getStatAsStringForCluster(str(index + 1), dStatForOneCluster))

        if self._save:
            outTableName = "%s_statsPer%s" % (self._pathTableName, self._analysisName)
            self._iDb.createTable(outTableName, "pathstat", self._statsFileName)

        self._iDb.close()
        self._log.info("END")

    def _statsForIdentityAndLength(self, dStat, lLengthPerCopy, lIdentityPerCopy):
        for i in lIdentityPerCopy:
            dStat["statsIdentityPerChain"].add(i)
        lLengthPercPerCopy = []
        for i in lLengthPerCopy:
            dStat["statsLengthPerChain"].add(i)
            lperc = 100 * i / float(dStat["length"])
            lLengthPercPerCopy.append(lperc)
            dStat["statsLengthPerChainPerc"].add(lperc)

class AnnotationStatsWriter(object):

    def __init__(self):
        self._dAllTErefseqs = { "sumCumulCoverage": 0,
                         "totalNbFragments": 0,
                         "totalNbFullLengthFragments": 0,
                         "totalNbCopies": 0,
                         "totalNbFullLengthCopies": 0,
                         "nbFamWithFullLengthFragments": 0,
                         "nbFamWithOneFullLengthFragment": 0,
                         "nbFamWithTwoFullLengthFragments": 0,
                         "nbFamWithThreeFullLengthFragments": 0,
                         "nbFamWithMoreThanThreeFullLengthFragments": 0,
                         "nbFamWithFullLengthCopies": 0,
                         "nbFamWithOneFullLengthCopy": 0,
                         "nbFamWithTwoFullLengthCopies": 0,
                         "nbFamWithThreeFullLengthCopies": 0,
                         "nbFamWithMoreThanThreeFullLengthCopies": 0,
                         "statsAllCopiesMedIdentity": Stat(),
                         "statsAllCopiesMedLengthPerc": Stat()
                         }

    def getAllTEsRefSeqDict(self):
        return self._dAllTErefseqs

    def getStatAsStringForTE(self, name, d):
        """
        Return a string with all data properly formatted.
        """

        string = ""
        string += "%s" % name
        string += "\t%i" % d["length"]
        string += "\t%i" % d["cumulCoverage"]
        string += "\t%i" % d["nbFragments"]
        string += "\t%i" % d["nbFullLengthFragments"]
        string += "\t%i" % d["nbCopies"]
        string += "\t%i" % d["nbFullLengthCopies"]

        # string += "\t%i" % d["maxLength"]
        # string += "\t%i" % d["meanLength"]

        if d["statsIdentityPerChain"].getValuesNumber() != 0:
            string += "\t%.2f" % d["statsIdentityPerChain"].mean()

            string += "\t%.2f" % d["statsIdentityPerChain"].sd()
            string += "\t%.2f" % d["statsIdentityPerChain"].getMin()
            string += "\t%.2f" % d["statsIdentityPerChain"].quantile(0.25)
            string += "\t%.2f" % d["statsIdentityPerChain"].median()
            string += "\t%.2f" % d["statsIdentityPerChain"].quantile(0.75)
            string += "\t%.2f" % d["statsIdentityPerChain"].getMax()

        else:
            for i in range(0, 7):
                string += "\tNA"

        if d["statsLengthPerChain"].getValuesNumber() != 0:
            string += "\t%.2f" % d["statsLengthPerChain"].mean()

            string += "\t%.2f" % d["statsLengthPerChain"].sd()
            string += "\t%i" % d["statsLengthPerChain"].getMin()
            string += "\t%.2f" % d["statsLengthPerChain"].quantile(0.25)
            string += "\t%.2f" % d["statsLengthPerChain"].median()
            string += "\t%.2f" % d["statsLengthPerChain"].quantile(0.75)
            string += "\t%i" % d["statsLengthPerChain"].getMax()
        else:
            for i in range(0, 7):
                string += "\tNA"

        if d["statsLengthPerChainPerc"].getValuesNumber() != 0:
            string += "\t%.2f" % d["statsLengthPerChainPerc"].mean()

            string += "\t%.2f" % d["statsLengthPerChainPerc"].sd()
            string += "\t%.2f" % d["statsLengthPerChainPerc"].getMin()
            string += "\t%.2f" % d["statsLengthPerChainPerc"].quantile(0.25)
            string += "\t%.2f" % d["statsLengthPerChainPerc"].median()
            string += "\t%.2f" % d["statsLengthPerChainPerc"].quantile(0.75)
            string += "\t%.2f" % d["statsLengthPerChainPerc"].getMax()
        else:
            for i in range(0, 7):
                string += "\tNA"

        return string

    def getStatAsStringForCluster(self, name, d):
        """
        Return a string with all data properly formatted.
        """
        string = ""
        string += "%s" % name
        string += "\t%i" % d["cumulCoverage"]
        string += "\t%i" % d["nbFragments"]
        string += "\t%i" % d["nbCopies"]

        return string

    def addCalculsOfOneTE(self, dOneTErefseq):
        self._dAllTErefseqs[ "sumCumulCoverage" ] += dOneTErefseq[ "cumulCoverage" ]

        self._dAllTErefseqs[ "totalNbFragments" ] += dOneTErefseq[ "nbFragments" ]
        self._dAllTErefseqs[ "totalNbFullLengthFragments" ] += dOneTErefseq[ "nbFullLengthFragments" ]
        if dOneTErefseq[ "nbFullLengthFragments" ] > 0:
            self._dAllTErefseqs[ "nbFamWithFullLengthFragments" ] += 1
        if dOneTErefseq[ "nbFullLengthFragments" ] == 1:
            self._dAllTErefseqs[ "nbFamWithOneFullLengthFragment" ] += 1
        elif dOneTErefseq[ "nbFullLengthFragments" ] == 2:
            self._dAllTErefseqs[ "nbFamWithTwoFullLengthFragments" ] += 1
        elif dOneTErefseq[ "nbFullLengthFragments" ] == 3:
            self._dAllTErefseqs[ "nbFamWithThreeFullLengthFragments" ] += 1
        elif dOneTErefseq[ "nbFullLengthFragments" ] > 3:
            self._dAllTErefseqs[ "nbFamWithMoreThanThreeFullLengthFragments" ] += 1

        self._dAllTErefseqs[ "totalNbCopies" ] += dOneTErefseq[ "nbCopies" ]
        self._dAllTErefseqs[ "totalNbFullLengthCopies" ] += dOneTErefseq[ "nbFullLengthCopies" ]
        if dOneTErefseq[ "nbFullLengthCopies" ] > 0:
            self._dAllTErefseqs[ "nbFamWithFullLengthCopies" ] += 1
        if dOneTErefseq[ "nbFullLengthCopies" ] == 1:
            self._dAllTErefseqs[ "nbFamWithOneFullLengthCopy" ] += 1
        elif dOneTErefseq[ "nbFullLengthCopies" ] == 2:
            self._dAllTErefseqs[ "nbFamWithTwoFullLengthCopies" ] += 1
        elif dOneTErefseq[ "nbFullLengthCopies" ] == 3:
            self._dAllTErefseqs[ "nbFamWithThreeFullLengthCopies" ] += 1
        elif dOneTErefseq[ "nbFullLengthCopies" ] > 3:
            self._dAllTErefseqs[ "nbFamWithMoreThanThreeFullLengthCopies" ] += 1

        if dOneTErefseq[ "statsIdentityPerChain" ].getValuesNumber() != 0:
            self._dAllTErefseqs[ "statsAllCopiesMedIdentity" ].add(dOneTErefseq[ "statsIdentityPerChain" ].median())

        if dOneTErefseq[ "statsLengthPerChainPerc" ].getValuesNumber() != 0:
            self._dAllTErefseqs[ "statsAllCopiesMedLengthPerc" ].add(dOneTErefseq[ "statsLengthPerChainPerc" ].median())

    def printStatsForAllTEs(self, TEnb):
#        statString += "(sum of cumulative coverages: %i bp)" % ( self._dAllTErefseqs[ "sumCumulCoverage" ] )
        statString = "total nb of TE fragments: %i\n" % (self._dAllTErefseqs[ "totalNbFragments" ])

        if self._dAllTErefseqs[ "totalNbFragments" ] != 0:

            statString += "total nb full-length fragments: %i (%.2f%%)\n" % \
            (self._dAllTErefseqs[ "totalNbFullLengthFragments" ], \
              100 * self._dAllTErefseqs[ "totalNbFullLengthFragments" ] / float(self._dAllTErefseqs[ "totalNbFragments" ]))

            statString += "total nb of TE copies: %i\n" % (self._dAllTErefseqs[ "totalNbCopies" ])

            statString += "total nb full-length copies: %i (%.2f%%)\n" % \
            (self._dAllTErefseqs[ "totalNbFullLengthCopies" ], \
              100 * self._dAllTErefseqs[ "totalNbFullLengthCopies" ] / float(self._dAllTErefseqs[ "totalNbCopies" ]))

            statString += "families with full-length fragments: %i (%.2f%%)\n" % \
            (self._dAllTErefseqs[ "nbFamWithFullLengthFragments" ], \
              100 * self._dAllTErefseqs[ "nbFamWithFullLengthFragments" ] / float(TEnb))
            statString += " with only one full-length fragment: %i\n" % (self._dAllTErefseqs[ "nbFamWithOneFullLengthFragment" ])
            statString += " with only two full-length fragments: %i\n" % (self._dAllTErefseqs[ "nbFamWithTwoFullLengthFragments" ])
            statString += " with only three full-length fragments: %i\n" % (self._dAllTErefseqs[ "nbFamWithThreeFullLengthFragments" ])
            statString += " with more than three full-length fragments: %i\n" % (self._dAllTErefseqs[ "nbFamWithMoreThanThreeFullLengthFragments" ])

            statString += "families with full-length copies: %i (%.2f%%)\n" % \
            (self._dAllTErefseqs[ "nbFamWithFullLengthCopies" ], \
              100 * self._dAllTErefseqs[ "nbFamWithFullLengthCopies" ] / float(TEnb))
            statString += " with only one full-length copy: %i\n" % (self._dAllTErefseqs[ "nbFamWithOneFullLengthCopy" ])
            statString += " with only two full-length copies: %i\n" % (self._dAllTErefseqs[ "nbFamWithTwoFullLengthCopies" ])
            statString += " with only three full-length copies: %i\n" % (self._dAllTErefseqs[ "nbFamWithThreeFullLengthCopies" ])
            statString += " with more than three full-length copies: %i\n" % (self._dAllTErefseqs[ "nbFamWithMoreThanThreeFullLengthCopies" ])

            statString += "mean of median identity of all families: %.2f +- %.2f\n" % \
            (self._dAllTErefseqs[ "statsAllCopiesMedIdentity" ].mean(), \
              self._dAllTErefseqs[ "statsAllCopiesMedIdentity" ].sd())

            statString += "mean of median length percentage of all families: %.2f +- %.2f\n" % \
            (self._dAllTErefseqs[ "statsAllCopiesMedLengthPerc" ].mean(), \
              self._dAllTErefseqs[ "statsAllCopiesMedLengthPerc" ].sd())
        return statString

    def printResume(self, lNamesTErefseq, lDistinctSubjects, totalCumulCoverage, genomeLength):
        statString = "nb of sequences: %i\n" % len(lNamesTErefseq)
        statString += "nb of matched sequences: %i\n" % len(lDistinctSubjects)
        statString += "cumulative coverage: %i bp\n" % totalCumulCoverage
        statString += "coverage percentage: %.2f%%\n" % (100 * totalCumulCoverage / float(genomeLength))
#            statString += "processing the %i TE families..." % len(lNamesTErefseq)
        return statString


#!/usr/bin/python3
"""THis module makes a pascal's triangle
"""

def pascal_triangle(n):
    """PAscal solution
    Args:
        n (lenght): lenght of pascal
    Returns:
        list: list of lists of integers
    """
    if n <= 0:
        return []
    list = []
    for y in range(n):
        list.append([1])
        if y > 1:
            for x in range(y - 1):
                list[y].append(list[y - 1][x] + list[y - 1][x + 1])
        if y is not 0:
            list[y].append(1)
    return list


from hyperapp.common.htypes import bundle_t
from hyperapp.common.htypes.packet_coders import packet_coders
from hyperapp.common.module import Module

class ThisModule(Module):

    def __init__(self, module_name, services, config):
        super().__init__(module_name, services, config)

        master_peer_bundle = packet_coders.decode('cdr', config['master_peer_bundle_cdr'], bundle_t)
        services.unbundler.register_bundle(master_peer_bundle)
        master_peer_ref = master_peer_bundle.roots[0]
        master_peer = services.peer_registry.invite(master_peer_ref)
        my_identity = services.generate_rsa_identity(fast=True)

        services.transport.send(master_peer, my_identity, [])


# Copyright (C) 2018 Atsushi Togo
# All rights reserved.
#
# This file is part of phonopy.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# * Redistributions of source code must retain the above copyright
#   notice, this list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright
#   notice, this list of conditions and the following disclaimer in
#   the documentation and/or other materials provided with the
#   distribution.
#
# * Neither the name of the phonopy project nor the names of its
#   contributors may be used to endorse or promote products derived
#   from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

import os
import numpy as np
from phonopy.api_phonopy import Phonopy
from phonopy.structure.cells import (guess_primitive_matrix,
                                     get_primitive_matrix_by_centring)
from phonopy.interface import (read_crystal_structure,
                               get_default_physical_units)
from phonopy.file_IO import (parse_BORN, parse_FORCE_SETS,
                             read_force_constants_hdf5,
                             parse_FORCE_CONSTANTS)

def load(supercell_matrix=None,
         primitive_matrix=None,
         is_nac=False,
         calculator="vasp",
         unitcell=None,
         supercell=None,
         nac_params=None,
         unitcell_filename=None,
         supercell_filename=None,
         born_filename=None,
         force_sets_filename=None,
         force_constants_filename=None,
         use_alm=False,
         factor=None,
         frequency_scale_factor=None,
         symprec=1e-5,
         is_symmetry=True,
         log_level=0):
    """Create Phonopy instance from parameters and/or input files.

    When unitcell and unitcell_filename are not given, file name that is
    default for the chosen calculator is looked for in the current directory
    as the default behaviour.

    When force_sets_filename and force_constants_filename are not given,
    'FORCE_SETS' is looked for in the current directory as the default
    behaviour.

    Parameters
    ----------
    supercell_matrix : array_like, optional
        Supercell matrix multiplied to input cell basis vectors.
        shape=(3, ) or (3, 3), where the former is considered a diagonal
        matrix. Default is the unit matrix.
        dtype=int
    primitive_matrix : array_like or str, optional
        Primitive matrix multiplied to input cell basis vectors. Default is
        the identity matrix.
        shape=(3, 3)
        dtype=float
        When 'F', 'I', 'A', 'C', or 'R' is given instead of a 3x3 matrix,
        the primitive matrix defined at
        https://atztogo.github.io/spglib/definition.html
        is used.
    is_nac : bool, optional
        If True, look for 'BORN' file.
        The priority for NAC is nac_params > born_filename > is_nac ('BORN')
    calculator : str, optional.
        Calculator used for computing forces. This is used to switch the set
        of physical units. Default is 'vasp'.
    unitcell : PhonopyAtoms, optional
        Input unit cell. Default is None. The priority for cell is
        unitcell_filename > supercell_filename > unitcell > supercell.
    supercell : PhonopyAtoms, optional
        Input supercell cell. Default value of primitive_matrix is set to
        'auto' (can be overwitten). supercell_matrix is ignored. Default is
        None. The priority for cell is
        unitcell_filename > supercell_filename > unitcell > supercell.
    nac_params : dict, optional
        Parameters required for non-analytical term correction. Default is
        None. The priority for NAC is nac_params > born_filename > is_nac.
        {'born': Born effective charges
                 (array_like, shape=(primitive cell atoms, 3, 3), dtype=float),
         'dielectric': Dielectric constant matrix
                       (array_like, shape=(3, 3), dtype=float),
         'factor': unit conversion facotr (float)}
    unitcell_filename : str, optional
        Input unit cell filename. Default is None. The priority for cell is
        unitcell_filename > supercell_filename > unitcell > supercell.
    supercell_filename : str, optional
        Input supercell filename. Default value of primitive_matrix is set to
        'auto' (can be overwitten). supercell_matrix is ignored. Default is
        None. The priority for cell is
        unitcell_filename > supercell_filename > unitcell > supercell.
    born_filename : str, optional
        Filename corresponding to 'BORN', a file contains non-analytical term
        correction parameters.
        The priority for NAC is nac_params > born_filename > is_nac ('BORN').
    force_sets_filename : str, optional
        Filename of a file corresponding to 'FORCE_SETS', a file contains sets
        of forces and displacements. Default is None.
        The priority for force constants is
        force_constants_filename > force_sets_filename > 'FORCE_SETS'.
    force_constants_filename : str, optional
        Filename of a file corresponding to 'FORCE_CONSTANTS' or
        'force_constants.hdf5', a file contains force constants.
        Default is None.
        The priority for force constants is
        force_constants_filename > force_sets_filename > 'FORCE_SETS'.
    use_alm : bool, optional
        Default is False.
    factor : float, optional
        Phonon frequency unit conversion factor. Unless specified, default
        unit conversion factor for each calculator is used.
    frequency_scale_factor : float, optional
        Factor multiplied to calculated phonon frequency. Default is None,
        i.e., effectively 1.
    symprec : float, optional
        Tolerance used to find crystal symmetry. Default is 1e-5.
    is_symmetry : bool, optional
        Setting False, crystal symmetry except for lattice translation is not
        considered. Default is True.
    log_level : int, optional
        Verbosity control. Default is 0.

    """

    cell, smat, pmat = _get_cell_settings(unitcell_filename,
                                          supercell_filename,
                                          unitcell,
                                          supercell,
                                          calculator,
                                          primitive_matrix,
                                          supercell_matrix,
                                          symprec)

    # units keywords: factor, nac_factor, distance_to_A
    units = get_default_physical_units(calculator)
    if factor is None:
        _factor = units['factor']
    else:
        _factor = factor
    phonon = Phonopy(cell,
                     smat,
                     primitive_matrix=pmat,
                     factor=_factor,
                     frequency_scale_factor=frequency_scale_factor,
                     symprec=symprec,
                     is_symmetry=is_symmetry,
                     log_level=log_level)
    _set_nac_params(phonon,
                    nac_params,
                    born_filename,
                    is_nac,
                    units['nac_factor'])
    _set_force_constants(phonon,
                         force_constants_filename,
                         force_sets_filename,
                         use_alm)
    return phonon

def _get_cell_settings(unitcell_filename,
                       supercell_filename,
                       unitcell,
                       supercell,
                       calculator,
                       pmat,
                       smat,
                       symprec):
    if unitcell_filename is not None:
        cell, filename = read_crystal_structure(
            filename=unitcell_filename, interface_mode=calculator)
        _smat = _get_supercell_matrix(smat)
        _pmat = pmat
    elif supercell_filename is not None:
        cell, filename = read_crystal_structure(
            filename=supercell_filename, interface_mode=calculator)
        _smat = np.eye(3, dtype='intc', order='C')
        if pmat is None:
            _pmat = 'auto'
    elif unitcell is not None:
        cell = unitcell
    elif supercell is not None:
        cell = supercell
        _smat = np.eye(3, dtype='intc', order='C')
        if pmat is None:
            _pmat = 'auto'
    else:
        raise RuntimeError("Cell has to be specified.")

    if cell is None:
        msg = "'%s' could not be found." % filename
        raise FileNotFoundError(msg)

    _pmat = _get_primitive_matrix(_pmat, cell, symprec)

    return cell, _smat, _pmat

def _get_supercell_matrix(smat):
    if smat is None:
        _smat = np.eye(3, dtype='intc', order='C')
    elif len(np.ravel(smat)) == 3:
        _smat = np.diag(smat)
    elif len(np.ravel(smat)) == 9:
        _smat = np.reshape(smat, (3, 3))
    else:
        msg = "supercell_matrix shape has to be (3,) or (3, 3)"
        raise RuntimeError(msg)
    return _smat

def _get_primitive_matrix(pmat, unitcell, symprec):
    if type(pmat) is str and pmat in ('F', 'I', 'A', 'C', 'R', 'auto'):
        if pmat == 'auto':
            _pmat = guess_primitive_matrix(unitcell, symprec=symprec)
        else:
            _pmat = get_primitive_matrix_by_centring(pmat)
    elif pmat is None:
        _pmat = None
    elif len(np.ravel(pmat)) == 9:
        matrix = np.reshape(pmat, (3, 3))
        if matrix.dtype.kind in ('i', 'u', 'f'):
            det = np.linalg.det(matrix)
            if symprec < det and det < 1 + symprec:
                _pmat = matrix
            else:
                msg = ("Determinant of primitive_matrix has to be larger "
                       "than 0")
                raise RuntimeError(msg)
    else:
        msg = ("primitive_matrix has to be a 3x3 matrix, None, 'auto', "
               "'F', 'I', 'A', 'C', or 'R'")
        raise RuntimeError(msg)

    return _pmat

def _set_nac_params(phonon, nac_params, born_filename, is_nac, nac_factor):
    if nac_params is not None:
        _nac_params = nac_params
    elif born_filename is not None:
        _nac_params = parse_BORN(phonon.primitive, filename=born_filename)
    elif is_nac is True:
        if os.path.isfile("BORN"):
            _nac_params = parse_BORN(phonon.primitive, filename="BORN")
        else:
            raise RuntimeError("BORN file doesn't exist at current directory.")
    else:
        _nac_params = None

    if _nac_params is not None:
        if _nac_params['factor'] is None:
            _nac_params['factor'] = nac_factor
        phonon.set_nac_params(_nac_params)

def _set_force_constants(phonon,
                         force_constants_filename,
                         force_sets_filename,
                         use_alm):
    natom = phonon.supercell.get_number_of_atoms()

    if force_constants_filename is not None:
        dot_split = force_constants_filename.split('.')
        p2s_map = phonon.primitive.get_primitive_to_supercell_map()
        if len(dot_split) > 1 and dot_split[-1] == 'hdf5':
            fc = read_force_constants_hdf5(filename=force_constants_filename,
                                           p2s_map=p2s_map)
        else:
            fc = parse_FORCE_CONSTANTS(filename=force_constants_filename,
                                       p2s_map=p2s_map)
        phonon.set_force_constants(fc)
    elif force_sets_filename is not None:
        force_sets = parse_FORCE_SETS(natom=natom,
                                      filename=force_sets_filename)
        if force_sets:
            phonon.set_displacement_dataset(force_sets)
            phonon.produce_force_constants(
                calculate_full_force_constants=False,
                use_alm=use_alm)
    elif os.path.isfile("FORCE_SETS"):
        force_sets = parse_FORCE_SETS(natom=natom)
        if force_sets:
            phonon.set_displacement_dataset(force_sets)
            phonon.produce_force_constants(
                calculate_full_force_constants=False,
                use_alm=use_alm)


#coding:utf-8

import requests
import re
from bs4 import BeautifulSoup
from urllib import quote
from time import sleep
import threading
from time import sleep
#from urllib3.exceptions import NewConnectionError
TIME_SPLIT2=1

class User():
    def __init__(self,name):
        self.TIME_SPLIT=0.2
        self.name=name
        self.base_url = "http://300report.jumpw.com/list.html?name=%s"%name
        self.match_urls=[]
        self.match_info=[]    #保存每场比赛的信息,每个元素都是一个字典
        self.USER_VALID=False      #标记用户名是否存在

        html=requests.get(self.base_url)
        #print html.text
        soup=BeautifulSoup(html.text,'lxml')
        #print soup.tr
        tr=soup.find_all("tr")
        if len(tr)!=0:
            self.USER_VALID=True
            self.set_user_info(tr)

    def set_user_info(self,tr):
        self.user_info={ 'username':tr[0].td.next_sibling.string,\
                         'userlevel':int(tr[1].td.next_sibling.string),\
                         'jiecao':int(tr[2].td.next_sibling.string),
                         'totalwins':int(tr[3].td.next_sibling.string),\
                         'totalplays':int(tr[4].td.next_sibling.string)}

        #获取每个比赛详情页面的地址
        self.match_urls=self.find_match_url(self.base_url)

        #循环访问比赛详情页面,获取每局比赛的信息
        threads=[threading.Thread(target=self.find_match_content,args=(match_urlsi,)) \
                 for match_urlsi in self.match_urls]
        for t in threads:
            try:
                sleep(self.TIME_SPLIT)
                t.start()
            except:
                sleep(1)
                pass
        #match_urls = re.findall(re_match_url, html.text)

    def find_match_url(self,url):
        re_match_url=r"match\.html\?id\=\d+"
        re_match_url=re.compile(re_match_url)
        match_urls=[]
        pages=int((self.user_info['totalplays']-1)/10)  #每页10条信息,计算一共有多少页
        for i in range(1,pages+2):
            try:
                sleep(TIME_SPLIT2)
                content=requests.get(self.base_url+"&index=%s"%str((i-1)*10)).text
                tmplist=re.findall(re_match_url,content)        #抓取比赛详情页面的url
                if len(tmplist)!=0:
                    for l in tmplist:
                        match_urls.append("http://300report.jumpw.com/"+l)
            except:
                sleep(TIME_SPLIT2)
                continue
        return match_urls

    #获取每场比赛的详细数据
    def find_match_content(self,match_url):
        content=requests.get(match_url).text
        match_id=int( match_url.split('=')[1] )
        soup=BeautifulSoup(content,'lxml')
        tr=soup.find_all('tr')
        for t in tr:
            if re.search(self.name,t.text):
                tds=[td for td in t.children]
                this_match_info={"id":match_id,\
                                 "hero":tds[3].text.split(")")[1].split("(")[0],\
                                 'kills':tds[5].string,\
                                 'wins':True if (tds[7].parent.parent.previous_element.previous_element==u"胜利") else False,\
                                 'buildings':int(tds[9].string),\
                                 'soldiers':int(tds[11].string),\
                                 'golds':int(tds[13].string),\
                                 'score':tds[15].string,\
                                 'exps':tds[21].string,\
                                 'jiecao':int(tds[23].string),\
                                 'wins_p':tds[25].string,}
        self.match_info.append(this_match_info)
        return this_match_info


import torch as pt

def get_num_splits(size: int, split_size: int) -> int:
    if size % split_size == 0:
        return size // split_size

    return (size + split_size - 1) // split_size

def get_tf_device(device: pt.device):
    if device.type == "cuda":
        name = "GPU"
        index = device.index
        if index is None:
            index = 0
    elif device.type == "cpu":
        name = "CPU"
        index = 0
    else:
        raise RuntimeError(f"Unknown device type: {device.type}!")

    return f"{name}:{index}"


import time,datetime
#from svgcandle import *
from math import ceil
from mongo_settings import *

class Stage:
    def get_result(self):

        c = self.cache

        pos = self.todo[0]
        _Max = 20
        _result = list(self.db[pos].find({},sort=[('_id',desc)],limit=_Max))

        ma10 = sum([one['c'] for one in _result[:10]])/10.0
        ma20 = sum([one['c'] for one in _result[:20]])/20.0

        for i in self.todo:
            c[i][0]['ma10'] = ma10
            c[i][0]['ma20'] = ma20
            self.cache[i][0] = c[i][0]
            self.save(i,c[i][0])

        if ma10>ma20:
            return 1
        else:
            return -1
#=====================================================================
    def __init__(self,symbol,time=None,plus=""):
        self.plus = plus
        self.db = {}
        self.data={}
        self.symbol = symbol+plus
        self.result = {}
        self.todo = [3600]
        if time:
            self.time = time
        else:
            self.time = time.time()
        for i in self.todo:
            self.db[i] = conn[self.symbol][str(i)]

    def price(self,price):
        for i in self.todo:
            self.new_price(price,i)

    def new_price(self,price,pos):
        _result = list(self.db[pos].find({},sort=[('_id',desc)],limit=1))

        if len(_result)>0:
            now = _result[0]
            now['c'] = price
            now['h'] = max(now['c'],now['h'])
            now['l'] = min(now['c'],now['l'])
            now['time'] = self.time
            now = self.check_k_hour(now,pos)
        else:
            last = None
            now = {'_id':0,'o':price,'h':price,'l':price,'c':price,'begin':self.time}
            self.save(pos,now)
    def check_k_hour(self,now,pos):
        if int(now.get('begin',-1)/pos)!=int(now.get('time',-1)/pos):
            p = now['c']
            new = {'o':p,'h':p,'l':p,'c':p,'do':0,'begin':self.time}
            new['_id'] = int(self.time/pos)
            self.save(pos,now)
            self.save(pos,new)
            return new
        else:
            self.save(pos,now)
            return now

    def data_out(self,pos):
        _result = self.db[pos].find({},sort=[('_id',desc)],limit=2)
        return jsondump(list(_result))
#=====================================================================
    def save(self,Pos,Dict):
        self.db[Pos].save(Dict)
#=====================================================================
'''
    def get_image(self,pos,lens,group,offset=0):
        data = self.db[int(pos)]
        result = list(data.find(sort=[('_id',desc)],limit=int(lens),skip=int(offset)*int(lens)))
        _l = self.state.get('his',['none'])[::-1]
        out = SVG(group,result[::-1],_l,data).to_html()
        return out
    def only_image(self,pos,lens,group,offset=0):
        data = self.db[int(pos)]
        result = list(data.find(sort=[('_id',desc)],limit=int(lens),skip=int(offset)*int(lens)))
        out = SVG(group,result[::-1],[self.symbol+" "+str(datetime.datetime.now())[:19]],data).to_html()
        return out'''
############################################################################################################
'''
#        end
'''
############################################################################################################


from mapswipe_workers.ProjectTypes.BuildArea.BuildAreaTask import *
from mapswipe_workers.basic.BaseGroup import *

class BuildAreaGroup(BaseGroup):
    """
        The subclass of BaseGroup to specify groups of the build area
        project type.

        Attributes
        ----------
        zoomLevel: int
            The zoom level of the defined tiled imagery server used for the project
        xMax: int
            The maximum x coordinate of the extent of the group
        xMin: int
            The minimum x coordinate of the extent of the group
        yMax: int
            The maximum y coordinate of the extent of the group
        yMin: int
            The minimum y coordinate of the extent of the group
    """
    type = 1

    def __init__(self, imp, project_id, group_id, slice: dict):
        """
            The constructor method for a group instance of the build area project type

        Parameters
        ----------
        project: BuildAreaProject object
            The project the group is associated with
        group_id: int
            The id of the group
        slice: dict
            The dictionary containing the spatial representation of the group as extent
            consisting of 4 values
        """
        # super() executes fine now
        super(BuildAreaGroup, self).__init__(imp, project_id, group_id)

        # add the type specific attributes
        self.zoomLevel = imp.info['zoomLevel']
        self.xMax = slice['xMax']
        self.xMin = slice['xMin']
        self.yMax = slice['yMax']
        self.yMin = slice['yMin']

        # we need to add the tasks then, is this happening during init?
        self.create_tasks(imp)

    def create_tasks(self, imp: object):
        """
        The Function to create tasks for the group of the build area project type

        Parameters
        ----------
        project: BuildAreaProject object
            The project the group is associated with
        """
        tasks = {}
        for TileX in range(int(self.xMin), int(self.xMax) + 1):
            for TileY in range(int(self.yMin), int(self.yMax) + 1):

                task = BuildAreaTask(self, imp, TileX, TileY)
                tasks[task.id] = task

        self.tasks = tasks
        self.count = len(tasks)


import os
import configparser
import requests
import sqlalchemy as sa
from requests_cache import CachedSession
from requests_futures.sessions import FuturesSession
from concurrent.futures import as_completed

from tqdm import tqdm
import sys

def getcorps(corplist):
    corpfuture = []
    print("get corps")
    for corpid in corplist:
        if isinstance(corpid, str) and corpid.startswith("https"):
            corpfuture.append(session.get(str(corpid)))
        else:
            corpfuture.append(session.get(corplookupurl.format(corpid)))
    badlist = []
    pbar = tqdm(total=len(corplist))
    for corpdata in as_completed(corpfuture):
        if corpdata.result().status_code == 200:
            corpid = int(str(corpdata.result().url).split('/')[5])
            corpjson = corpdata.result().json()
            connection.execute(invNames.insert(),
                               itemID=corpid,
                               itemName=corpjson.get('name', None)
                               )

            connection.execute(crpNPCCorporations.insert(),
                               corporationID=corpid,
                               description=corpjson.get('description', None),
                               )
        else:
            badlist.append(corpdata.result().url)
            print(corpdata.result().url)
        pbar.update(1)
    return badlist

if len(sys.argv) < 2:
    print("Load.py destination")
    exit()

database = sys.argv[1]

if len(sys.argv) == 3:
    language = sys.argv[2]
else:
    language = 'en'

fileLocation = os.path.dirname(os.path.realpath(__file__))
inifile = fileLocation + '/sdeloader.cfg'
config = configparser.ConfigParser()
config.read(inifile)
destination = config.get('Database', database)
sourcePath = config.get('Files', 'sourcePath')
redis_server = config.get('Redis', 'server')
redis_db = config.get('Redis', 'db')

schema = None
if database == "postgresschema":
    schema = "evesde"

engine = sa.create_engine(destination, echo=False)
metadata = sa.MetaData(schema=schema)
connection = engine.connect()
trans = connection.begin()

crpNPCCorporations = sa.Table('crpNPCCorporations', metadata,
                              sa.Column('corporationID', sa.INTEGER(), primary_key=True, autoincrement=False, nullable=False),
                              sa.Column('size', sa.CHAR(length=1)),
                              sa.Column('extent', sa.CHAR(length=1)),
                              sa.Column('solarSystemID', sa.INTEGER()),
                              sa.Column('investorID1', sa.INTEGER()),
                              sa.Column('investorShares1', sa.INTEGER()),
                              sa.Column('investorID2', sa.INTEGER()),
                              sa.Column('investorShares2', sa.INTEGER()),
                              sa.Column('investorID3', sa.INTEGER()),
                              sa.Column('investorShares3', sa.INTEGER()),
                              sa.Column('investorID4', sa.INTEGER()),
                              sa.Column('investorShares4', sa.INTEGER()),
                              sa.Column('friendID', sa.INTEGER()),
                              sa.Column('enemyID', sa.INTEGER()),
                              sa.Column('publicShares', sa.INTEGER()),
                              sa.Column('initialPrice', sa.INTEGER()),
                              sa.Column('minSecurity', sa.FLOAT()),
                              sa.Column('scattered', sa.Boolean(name='cnpcc_scatt')),
                              sa.Column('fringe', sa.INTEGER()),
                              sa.Column('corridor', sa.INTEGER()),
                              sa.Column('hub', sa.INTEGER()),
                              sa.Column('border', sa.INTEGER()),
                              sa.Column('factionID', sa.INTEGER()),
                              sa.Column('sizeFactor', sa.FLOAT()),
                              sa.Column('stationCount', sa.INTEGER()),
                              sa.Column('stationSystemCount', sa.INTEGER()),
                              sa.Column('description', sa.VARCHAR(length=4000)),
                              sa.Column('iconID', sa.INTEGER()),
                              schema=schema
                              )

invNames = sa.Table('invNames', metadata,
                    sa.Column('itemID', sa.INTEGER(), primary_key=True, autoincrement=False, nullable=False),
                    sa.Column('itemName', sa.VARCHAR(length=200), nullable=False),
                    schema=schema
                    )

maincorplist = []

corpurl = "https://esi.evetech.net/latest/corporations/npccorps/?datasource=tranquility"

corplookupurl = 'https://esi.evetech.net/latest/corporations/{}/?datasource=tranquility'

errorcount = 0

base_session = CachedSession("evesde", backend="redis")

reqs_num_workers = 50

session = FuturesSession(max_workers=reqs_num_workers, session=base_session)

corps = requests.get(corpurl).json()

firstbadlist = getcorps(corps)
print("Getting badlist")
secondbadlist = getcorps(firstbadlist)

trans.commit()


# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

from setuptools import setup

install_requires = [
    "flask==0.10.1",
    "requests==2.0.0",
    "mongoengine==0.8.4",
    "flask_mongoengine==0.7.0",
]

setup(name="laughing-happiness",
    version="0.0",
    description="",
    install_requires=install_requires,
)


import numpy as np

def split_data(x, y, ratio, myseed=1):
    """Function to split training data and test data based on given splitting ratio.
    Args:
        x      (numpy array): Original matrix input (features) of size N x D.
        y      (numpy array): Original matrix output of size N x 1.
        ratio  (float)      : Ratio of data splitting for training data with value of {0,1} with the rest (1-ratio) goes to test.
        myseed (int)        : Integer value to seed the random generator.
    Returns:
        x_train (numpy array) : Matrix input (features) for training set with size of (ratio*N) x D.
        y_train (numpy array) : Matrix output for training set with size of (ratio*N) x D.
        x_test  (numpy array) : Matrix input (features) for testing set with size of ((1-ratio)*N) x D.
        y_test  (numpy array) : Matrix output for testing set with size of ((1-ratio)*N) x D.
    """

    # set seed
    np.random.seed(myseed)

    # generate random indices
    num_row          = len(y)
    indices          = np.random.permutation(num_row)
    index_split      = int(np.floor(ratio * num_row))
    training_indices = indices[:index_split]
    test_indices     = indices[index_split:]

    # splitting the data
    x_train = x[training_indices]
    y_train = y[training_indices]
    x_test  = x[test_indices]
    y_test  = y[test_indices]

    return x_train, y_train, x_test, y_test


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('my_blog', '0001_initial'),
    ]

    operations = [
        migrations.AlterModelOptions(
            name='user',
            options={'ordering': ['-id'], 'verbose_name': '\u7528\u6237', 'verbose_name_plural': '\u7528\u6237'},
        ),
        migrations.RemoveField(
            model_name='comment',
            name='user',
        ),
        migrations.AddField(
            model_name='comment',
            name='artical',
            field=models.ForeignKey(verbose_name=b'\xe6\x96\x87\xe7\xab\xa0', blank=True, to='my_blog.Artical', null=True),
        ),
        migrations.AlterField(
            model_name='artical',
            name='tag',
            field=models.ManyToManyField(to='my_blog.Tag', verbose_name='\u6807\u7b7e'),
        ),
    ]


#!venv/bin/python
from flask import Flask, request, render_template, jsonify
from strava_monitoring.fetcher import get_distances

app = Flask(
    __name__, template_folder='strava_monitoring\\templates')

@app.route('/')
@app.route('/index')
def index():
    distances = get_distances()
    return render_template('index.html', distances=distances)

if __name__ == '__main__':
    app.run(debug=True, host='0.0.0.0')


import json
import logging
# import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import sys

from collections import Counter
from sklearn.cluster import KMeans
from sklearn.metrics import silhouette_score
from tqdm import tqdm

log_filename = None

if len(sys.argv) == 2:
    log_filename = './logs/entropies_clustering_results.log'
elif len(sys.argv) == 3:
    log_filename = './logs/entropies_clustering_grid_search.log'
elif len(sys.argv) == 4:
    log_filename = './logs/entropies_deeper_clustering_results.log'
else:
    log_filename = './logs/entropies_deeper_clustering_grid_search.log'

logging.basicConfig(filename=log_filename, filemode='w', level=logging.INFO,
                    format='%(message)s')

clustering_df, valid_clustering_df, valid_MAG_ids = None, None, None

if len(sys.argv) > 3:
    clustering_df = pd.read_csv(sys.argv[2],
                                dtype={'MAG_id': str, 'cluster': int})
    valid_clustering_df = \
        clustering_df[clustering_df.cluster != int(sys.argv[3])]

    valid_MAG_ids = sorted(valid_clustering_df['MAG_id'].values.tolist())

entropies_dict, total_years = dict(), set()

with open(sys.argv[1], 'r') as entropies_file:
    for row in tqdm(entropies_file, desc='READING ENTROPIES FILE'):
        creator = json.loads(row)

        entropies_dict.update(creator)

        for MAG_id in creator:
            for year in creator[MAG_id]:
                total_years.add(year)

entropies_matrix, total_years, local_years = list(), sorted(total_years), set()

if len(sys.argv) > 3:
    for MAG_id in valid_MAG_ids:
        for year in entropies_dict[MAG_id]:
            local_years.add(year)

    local_years = sorted(local_years)
else:
    local_years = total_years

for MAG_id in tqdm(valid_MAG_ids if len(sys.argv) > 3
                   else sorted(entropies_dict),
                   desc='BUILDING ENTROPIES MATRIX'):
    row = list()

    for year in local_years:
        if year in entropies_dict[MAG_id]:
            row.append(entropies_dict[MAG_id][year]['entropy'])
        else:
            row.append(np.nan)

    entropies_matrix.append(row)

entropies_matrix = np.array(entropies_matrix)

means = [np.nanmean(entropies_matrix[:, idx])
         for idx in np.arange(0, entropies_matrix.shape[1])]

for i, row in tqdm(enumerate(entropies_matrix), desc='PREPROCESSING',
                   total=entropies_matrix.shape[0]):
    for j, val in enumerate(np.isnan(row)):
        if val == 1:
            entropies_matrix[i, j] = means[j]

if len(sys.argv) == 3 or len(sys.argv) == 5:
    iterations = int(sys.argv[2]) if len(sys.argv) == 3 else int(sys.argv[4])

    for iteration in tqdm(np.arange(0, iterations),
                          desc='GRID SEARCH ITERATION'):
        for clusters_number in tqdm(np.arange(2, 11), desc='GRID SEARCH'):
            classifier = KMeans(n_clusters=clusters_number, max_iter=100)
            labels = classifier.fit_predict(entropies_matrix)

            silhouette_avg = \
                silhouette_score(entropies_matrix, labels,
                                 sample_size=100000 if len(sys.argv) == 3
                                 else None)

            logging.info("KMEANS' N_CLUSTERS GRID SEARCH -- N_CLUSTERS: {}, "
                         "AVERAGE SILHOUETTE SCORE: {}".format(clusters_number,
                                                               silhouette_avg))
else:
    classifier = KMeans(n_clusters=5 if len(sys.argv) == 2 else 2,
                        random_state=42)
    labels = classifier.fit_predict(entropies_matrix).tolist()

    if len(sys.argv) > 3:
        for idx, label in enumerate(labels):
            if label == 0:
                labels[idx] = 1
            else:
                labels[idx] = 2

    centroids = classifier.cluster_centers_

    clusters_infos, dataframe_infos = dict(), [[], [], []]
    clusters_records, clusters_records_without_mean = None, None

    if len(sys.argv) == 2:
        clusters_records, clusters_records_without_mean = \
            [[], [], [], [], []], [[], [], [], [], []]
    else:
        clusters_records, clusters_records_without_mean = [[], []], [[], []]

    total = len(entropies_dict) if len(sys.argv) == 2 else len(valid_MAG_ids)

    for idx, MAG_id in tqdm(enumerate(sorted(entropies_dict))
                            if len(sys.argv) == 2 else
                            enumerate(valid_MAG_ids),
                            desc='ASSIGNING CLUSTERS', total=total):
        if labels[idx] not in clusters_infos:
            clusters_infos[labels[idx]] = \
                {'years': set(list(entropies_dict[MAG_id])),
                 'countries': set([entropies_dict[MAG_id][y]['affiliation']
                                  for y in entropies_dict[MAG_id]])}
        else:
            for year in entropies_dict[MAG_id]:
                clusters_infos[labels[idx]]['years'].add(year)
                clusters_infos[labels[idx]]['countries']\
                    .add(entropies_dict[MAG_id][year]['affiliation'])

        c_idx = labels[idx] if len(sys.argv) == 2 else labels[idx] - 1

        dataframe_infos[0].append(MAG_id)
        dataframe_infos[1].append(labels[idx])
        dataframe_infos[2].append(
            np.linalg.norm(entropies_matrix[labels[idx]] - centroids[c_idx]))

        clusters_records[c_idx].append(np.mean(entropies_matrix[idx]))

    new_clustering_df = pd.DataFrame({'MAG_id': dataframe_infos[0],
                                      'cluster': dataframe_infos[1],
                                      'DFC': dataframe_infos[2]})

    representative_records = list()

    for cluster in sorted(new_clustering_df.cluster.unique()):
        cluster_dataframe = \
            new_clustering_df[new_clustering_df.cluster == cluster]

        records = new_clustering_df\
            .query('cluster == {} and DFC == {}'
                   .format(cluster, min(cluster_dataframe['DFC']))).iloc[0:5]

        for record in records.values:
            representative_records\
                .append({record[0]: entropies_dict[record[0]]})

        for MAG_id in cluster_dataframe['MAG_id']:
            entropies = list()

            for year in entropies_dict[MAG_id]:
                entropies.append(entropies_dict[MAG_id][year]['entropy'])

            c_idx = cluster if len(sys.argv) == 2 else cluster - 1

            clusters_records_without_mean[c_idx].append(np.mean(entropies))

    creators_per_cluster = Counter(labels)

    for cluster in sorted(creators_per_cluster):
        denominator = len(entropies_dict) if len(sys.argv) == 2 else \
            len(valid_MAG_ids)
        percentage = \
            np.round((creators_per_cluster[cluster] / denominator) * 100, 2)

        logging.info('CLUSTER {}:\n\t{} RESEARCHERS, {}% OF THE TOTAL\n'
                     .format(cluster, creators_per_cluster[cluster],
                             percentage))
        logging.info('\tYEARS: {}\n'
                     .format(sorted(clusters_infos[cluster]['years'])))
        logging.info('\tCOUNTRIES: {}\n'
                     .format(sorted(clusters_infos[cluster]['countries'])))
        logging.info('\tREPRESENTATIVE RECORDS:\n')

        step = int(cluster) if len(sys.argv) == 2 else int(cluster) - 1

        for record in representative_records[step * 5:(step * 5) + 5]:
            logging.info('\t\t{}'.format(record))

    if len(sys.argv) == 2:
        new_clustering_df[['MAG_id', 'cluster']].to_csv(
            '~/mydata/zenodo/clustering/clustering_data.csv', index=False)
    else:
        clustering_df.loc[
            clustering_df.query('MAG_id in {}'.format(valid_MAG_ids)).index,
            'cluster'] = labels
        clustering_df.to_csv(
            '~/mydata/zenodo/clustering/deeper_clustering.csv', index=False)


import numpy as np
import pyaudio
from scipy.fftpack import fft, fftfreq
import scipy
import struct
from random import randint

class g_soundrandom():
    def __init__(self):
        self.amount = 1
        # sound2light stuff
        self.sample_rate = 44100
        self.buffer_size = 2**11
        self.thres_factor = 0
        self.channel = 1

        pa = pyaudio.PyAudio()
        chosen_device_index = -1
        for x in range(0,pa.get_device_count()):
            info = pa.get_device_info_by_index(x)
            if info["name"] == "pulse":
                chosen_device_index = info["index"]

        p = pyaudio.PyAudio()

        self.stream = p.open(
            format = pyaudio.paInt16,	# paFloat32 vs paInt16 ?
            channels = 1,
            rate = self.sample_rate,
            input = True,
            output = False,
            frames_per_buffer = self.buffer_size
            )

        self.threshold = 0.5

    def control(self, amount, threshold, channel):
        self.amount = amount*6
        self.threshold = threshold
        self.channel = int(channel*4)

#        self.thres_list = self.thres_list*self.thres_factor
        # new_list = [x+1 for x in my_list]
#        self.thres_list = [x+self.thres_factor for x in self.thres_list]

    def label(self):
        return ['amount',round(self.amount,2),
                'threshold',round(self.threshold,2),
                'channel',round(self.channel)]

    def generate(self, step, world):
        tempworld = np.zeros([10, 10, 10])

        number = int(self.amount*(self.update_line()[self.channel]-self.threshold))

        for i in range(number):
            world[:, randint(0, 9), randint(0, 9), randint(0, 9)] = 1.0

        return np.clip(world, 0, 1)

    def get_fft(self, data):
        FFT = fft(data)                                # Returns an array of complex numbers
        freqs = fftfreq(self.buffer_size, 1.0/self.sample_rate)  # Returns an array containing the frequency values

        y = abs(FFT[0:int(len(FFT)/2)])/1000                # Get amplitude and scale
        y = scipy.log(y) - 2                           # Subtract noise floor (empirically determined)
        return (freqs, y)

    def control_threshold(self, dat, thres):
        if dat < thres:
            return 0.0
        else:
            return dat

    def update_line(self):
        buf = self.stream.read(self.buffer_size)
        data = scipy.array(struct.unpack("%dh"%(self.buffer_size), buf))

        freqs, y = self.get_fft(data)

        # Normalize
        y = y / 5.0

        # Average into chunks of N
        N = 100

        yy = [scipy.average(y[n:n+N]) for n in range(0, len(y), N)]

        yy = yy[:int(len(yy)/2)] # Discard half of the samples, as they are mirrored

        # now do some threshold detection
        # for i in range(len(yy)):
        #    yy[i] = self.control_threshold(yy[i], self.threshold)

        return np.round(np.clip(yy,0,1),2)


import time
from selenium import webdriver

# 使用chrome的驱动器
browser = webdriver.Chrome()
# 设置超时阈值（最好设置长一点）
browser.set_page_load_timeout(120)
# 打开页面，由于页面动态加载，需要模拟滚动页面，让其加载完成
browser.get('http://www.acfun.cn/search/?#query=南方公园')
browser.execute_script("window.scrollTo(0, document.body.scrollHeight);")
# 根据目测知道有多少页，右键检查，复制为selector
page_info = browser.find_element_by_css_selector('#list-pager > div > span.hint')
# 页号信息文本格式为：当前页: 1/39页
# 先用/分开取第二个，并去除最后一个字
pages = int(page_info.text.split('/')[1][:-1])
print('总页数: %d' % (pages))
# 接着去遍历页数
for page in range(1, pages + 1):
    if page > 3:
        break
    print('当前页面: %d' % (page))
    # 遍历页面
    url = 'http://www.acfun.cn/search/?#page=' + str(page) + ';query=南方公园;type=video'
    browser.get(url)
    browser.execute_script("window.scrollTo(0, document.body.scrollHeight);")
    time.sleep(3)   # 等待滚动的时间
    # 逐层取selector，第一层找到视频列表（层级关系可以简化)
    # 层级关系为list-wrap -> video-cell clearfix -> div ->[div title, video info, description, video-tags-and-channel](更深略)
    vlist = browser.find_element_by_css_selector('#main > section > div.column-left > div.video-list.list-warp.active > div.video-list-wrap')
    # 列表下取所有video-cell clearfix标签，注意element和elements，这里使用 xpath 形式取标签
    velems = vlist.find_elements_by_xpath("//*[@class='video-cell clearfix']")
    for velem in velems:
        # 往下有一层div
        vitem = velem.find_element_by_tag_name('div')
        # 取得播放地址，title和播放信息
        vtitle = vitem.find_element_by_css_selector('div.title').find_element_by_tag_name('a')
        # 取得链接
        vlink = vtitle.get_attribute('href')
        # 取得信息
        vinfo = vitem.find_element_by_css_selector('div.video-info.clearfix').find_element_by_tag_name('div')
        print(vtitle.text, vinfo.text, vlink)


# coding=utf-8
import tensorflow as tf
import numpy as np

# # Saver
# # Declaration
# ## Remember to define the same dtype & shape when restore
# weights = tf.Variable([[1,2,3],[3,4,5]], dtype=tf.float32, name='weights')
# biases = tf.Variable([[1,2,3]], dtype=tf.float32, name='biases')

# init = tf.global_variables_initializer()

# saver = tf.train.Saver()

# with tf.Session() as sess:
#     sess.run(init)
#     save_path = saver.save(sess, "my_net/save_net.ckpt")
#     print("Save to: ", save_path)

# Restore variables
# Redefine the same shape and same type for your varibales
weights = tf.Variable(np.arange(6).reshape((2,3)), dtype=tf.float32, name="weights")
biases = tf.Variable(np.arange(3).reshape((1,3)), dtype=tf.float32, name="biases")

# Doesn't require initialize here

saver = tf.train.Saver()
with tf.Session() as sess:
    # Extract variables
    saver.restore(sess, "my_net/save_net.ckpt")
    print("Weights:", sess.run(weights))
    print("Biases:", sess.run(biases))


#1. 빈 딕셔너리를 생성
dic = {}
#2. 사용자로부터 key와 value를 입력받아 dictionary에 추가
while(True):
    print("===insert===")
    Key = str(input("Key >> "))
    value = str(input("value >> "))
    dic[Key] = value
    if (Key=="cake"):
        break
print("===result===")
#3. key값이 cake가 아니라면 2번으로 돌아감

#4. dictionary의 value값을 모두 출력
for i in dic.values():
    print(i)


import web

from web import form
from globals import *
from lesson import *
from utils import *

class display:
    def GET(self):
        data = web.input()
        #session = web.config.get('_session')
        session = web.config._session
        if 'page' in data:
            lesson_data = readLessonData(convertToRealPath(data['page']))
            storeVisit(lesson_data['index'], session.user)

            if lesson_data['type'] == 'lesson':
                raise web.seeother('/lesson?page=%s' % (data['page']))
            elif lesson_data['type'] == 'quiz':
                raise web.seeother('/quiz?page=%s' % (data['page']))

        return render.lesson(getLesson(session.user, ''))

    def POST(self):
        return self.GET()


#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of this
# software and associated documentation files (the "Software"), to deal in the Software
# without restriction, including without limitation the rights to use, copy, modify,
# merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
# INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
# PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

from __future__ import print_function

import os
import sys

# Required to load modules from vendored subfolder (for clean development env)
sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), "./vendored"))

import re
import json
from collections import defaultdict
import boto3
import datetime
import logging
import pandas as pd

# For date
from dateutil.relativedelta import relativedelta

session = boto3.Session()

# For email
from email.mime.application import MIMEApplication
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email.utils import COMMASPACE, formatdate

# GLOBALS
SES_REGION = os.environ.get("SES_REGION")
if not SES_REGION:
    SES_REGION = "us-east-1"
ACCOUNT_LABEL = os.environ.get("ACCOUNT_LABEL")
if not ACCOUNT_LABEL:
    ACCOUNT_LABEL = "Email"

CURRENT_MONTH = os.environ.get("CURRENT_MONTH")
if CURRENT_MONTH == "true":
    CURRENT_MONTH = True
else:
    CURRENT_MONTH = False

LAST_MONTH_ONLY = os.environ.get("LAST_MONTH_ONLY")

# Default exclude support, as for Enterprise Support
# as support billing is finalised later in month so skews trends
INC_SUPPORT = os.environ.get("INC_SUPPORT")
if INC_SUPPORT == "true":
    INC_SUPPORT = True
else:
    INC_SUPPORT = False

TAG_VALUE_FILTER = os.environ.get("TAG_VALUE_FILTER") or "*"
TAG_KEY = os.environ.get("TAG_KEY")

class CostExplorer:
    """Retrieves BillingInfo checks from CostExplorer API
    >>> costexplorer = CostExplorer()
    >>> costexplorer.addReport(GroupBy=[{"Type": "DIMENSION","Key": "SERVICE"}])
    >>> costexplorer.generateExcel()
    """

    def __init__(self, CurrentMonth=False):
        # Array of reports ready to be output to Excel.
        self.reports = []
        self.client = session.client("ce", region_name="us-east-1")
        self.end = datetime.date.today().replace(day=1)
        self.riend = datetime.date.today()
        if CurrentMonth or CURRENT_MONTH:
            self.end = self.riend

        if LAST_MONTH_ONLY:
            self.start = (datetime.date.today() - relativedelta(months=+1)).replace(
                day=1
            )  # 1st day of month a month ago
        else:
            # Default is last 12 months
            self.start = (datetime.date.today() - relativedelta(months=+12)).replace(
                day=1
            )  # 1st day of month 12 months ago

        self.ristart = (datetime.date.today() - relativedelta(months=+11)).replace(
            day=1
        )  # 1st day of month 11 months ago
        self.sixmonth = (datetime.date.today() - relativedelta(months=+6)).replace(
            day=1
        )  # 1st day of month 6 months ago, so RI util has savings values
        try:
            self.accounts = self.getAccounts()
        except:
            logging.exception("Getting Account names failed")
            self.accounts = {}

    def getAccounts(self):
        accounts = {}
        client = session.client("organizations", region_name="us-east-1")
        paginator = client.get_paginator("list_accounts")
        for response in paginator.paginate():
            for acc in response["Accounts"]:
                accounts[acc["Id"]] = acc
        return accounts

    def addReport(
        self,
        Name="Default",
        GroupBy=[{"Type": "DIMENSION", "Key": "SERVICE"}],
        Style="Total",
        Region=None,
        NoCredits=True,
        CreditsOnly=False,
        RefundOnly=False,
        UpfrontOnly=False,
        IncSupport=False,
    ):
        type = "chart"  # other option table
        results = []
        if not NoCredits:
            response = self.client.get_cost_and_usage(
                TimePeriod={
                    "Start": self.start.isoformat(),
                    "End": self.end.isoformat(),
                },
                Granularity="MONTHLY",
                Metrics=["UnblendedCost"],
                GroupBy=GroupBy,
            )
        else:
            Filter = {"And": []}

            Dimensions = {
                "Not": {
                    "Dimensions": {
                        "Key": "RECORD_TYPE",
                        "Values": ["Credit", "Refund", "Upfront", "Support"],
                    }
                }
            }
            if (
                INC_SUPPORT or IncSupport
            ):  # If global set for including support, we dont exclude it
                Dimensions = {
                    "Not": {
                        "Dimensions": {
                            "Key": "RECORD_TYPE",
                            "Values": ["Credit", "Refund", "Upfront"],
                        }
                    }
                }
            if CreditsOnly:
                Dimensions = {
                    "Dimensions": {"Key": "RECORD_TYPE", "Values": ["Credit"]}
                }
            if RefundOnly:
                Dimensions = {
                    "Dimensions": {"Key": "RECORD_TYPE", "Values": ["Refund"]}
                }
            if UpfrontOnly:
                Dimensions = {
                    "Dimensions": {"Key": "RECORD_TYPE", "Values": ["Upfront"]}
                }

            tagValues = None
            if TAG_KEY:
                tagValues = self.client.get_tags(
                    SearchString=TAG_VALUE_FILTER,
                    TimePeriod={
                        "Start": self.start.isoformat(),
                        "End": datetime.date.today().isoformat(),
                    },
                    TagKey=TAG_KEY,
                )

            if tagValues:
                Filter["And"].append(Dimensions)
                if len(tagValues["Tags"]) > 0:
                    Tags = {"Tags": {"Key": TAG_KEY, "Values": tagValues["Tags"]}}
                    Filter["And"].append(Tags)
            else:
                Filter = Dimensions.copy()
            if Region:
                Filter= {'And': [{'Dimensions': {'Key': 'REGION', 'Values': [Region]}}, Filter]}

            response = self.client.get_cost_and_usage(
                TimePeriod={
                    "Start": self.start.isoformat(),
                    "End": self.end.isoformat(),
                },
                Granularity="MONTHLY",
                Metrics=["UnblendedCost"],
                GroupBy=GroupBy,
                Filter=Filter,
            )

        if response:
            results.extend(response["ResultsByTime"])

            while "nextToken" in response:
                nextToken = response["nextToken"]
                response = self.client.get_cost_and_usage(
                    TimePeriod={
                        "Start": self.start.isoformat(),
                        "End": self.end.isoformat(),
                    },
                    Granularity="MONTHLY",
                    Metrics=["UnblendedCost"],
                    GroupBy=GroupBy,
                    NextPageToken=nextToken,
                )

                results.extend(response["ResultsByTime"])
                if "nextToken" in response:
                    nextToken = response["nextToken"]
                else:
                    nextToken = False
        rows = []
        sort = ""
        for v in results:
            row = {"date": v["TimePeriod"]["Start"]}
            sort = v["TimePeriod"]["Start"]
            for i in v["Groups"]:
                key = i["Keys"][0]
                if key in self.accounts:
                    key = self.accounts[key][ACCOUNT_LABEL]
                row.update({key: float(i["Metrics"]["UnblendedCost"]["Amount"])})
            if not v["Groups"]:
                row.update({"Total": float(v["Total"]["UnblendedCost"]["Amount"])})
            rows.append(row)

        df = pd.DataFrame(rows)
        df.set_index("date", inplace=True)
        df = df.fillna(0.0)

        if Style == "Change":
            dfc = df.copy()
            lastindex = None
            for index, row in df.iterrows():
                if lastindex:
                    for i in row.index:
                        try:
                            df.at[index, i] = dfc.at[index, i] - dfc.at[lastindex, i]
                        except:
                            logging.exception("Error")
                            df.at[index, i] = 0
                lastindex = index
        df = df.T
        df = df.sort_values(sort, ascending=False)
        self.reports.append({"Name": Name, "Data": df, "Type": type})
        return df

    def generateExcel(self):
        # Create a Pandas Excel writer using XlsxWriter as the engine.\
        os.chdir("/tmp")
        writer = pd.ExcelWriter("cost_explorer_report.xlsx", engine="xlsxwriter")
        workbook = writer.book
        for report in self.reports:
            print(report["Name"], report["Type"])
            report["Data"].to_excel(writer, sheet_name=report["Name"])
            worksheet = writer.sheets[report["Name"]]
        writer.save()

        # Time to deliver the file to S3
        if os.environ.get("S3_BUCKET"):
            s3 = session.client("s3")
            s3.upload_file(
                "cost_explorer_report.xlsx",
                os.environ.get("S3_BUCKET"),
                "cost_explorer_report.xlsx",
            )
        if os.environ.get("SES_SEND"):
            # Email logic
            msg = MIMEMultipart()
            msg["From"] = os.environ.get("SES_FROM")
            msg["To"] = COMMASPACE.join(os.environ.get("SES_SEND").split(","))
            msg["Date"] = formatdate(localtime=True)
            msg["Subject"] = "Cost Explorer Report"
            text = "Find your Cost Explorer report attached\n\n"
            msg.attach(MIMEText(text))
            with open("cost_explorer_report.xlsx", "rb") as fil:
                part = MIMEApplication(fil.read(), Name="cost_explorer_report.xlsx")
            part["Content-Disposition"] = (
                'attachment; filename="%s"' % "cost_explorer_report.xlsx"
            )
            msg.attach(part)
            # SES Sending
            ses = session.client("ses", region_name=SES_REGION)
            result = ses.send_raw_email(
                Source=msg["From"],
                Destinations=os.environ.get("SES_SEND").split(","),
                RawMessage={"Data": msg.as_string()},
            )

def main_handler(event=None, context=None):
    costexplorer = CostExplorer(CurrentMonth=False)
    # Default addReport has filter to remove Support / Credits / Refunds / UpfrontRI
    # Overall Billing Reports
    # costexplorer.addReport(Name="Total", GroupBy=[],Style='Total',IncSupport=True)
    # costexplorer.addReport(Name="TotalChange", GroupBy=[],Style='Change')
    # costexplorer.addReport(Name="TotalInclCredits", GroupBy=[],Style='Total',NoCredits=False,IncSupport=True)
    # costexplorer.addReport(Name="TotalInclCreditsChange", GroupBy=[],Style='Change',NoCredits=False)
    # costexplorer.addReport(Name="Credits", GroupBy=[],Style='Total',CreditsOnly=True)
    # costexplorer.addReport(Name="Refunds", GroupBy=[],Style='Total',RefundOnly=True)
    # costexplorer.addReport(Name="RIUpfront", GroupBy=[],Style='Total',UpfrontOnly=True)
    # GroupBy Reports
    usage_type = costexplorer.addReport(
        Name="Usage",
        GroupBy=[{"Type": "DIMENSION", "Key": "USAGE_TYPE"}],
        Style="Total",
    )
    costexplorer.addReport(
        Name="Instances",
        GroupBy=[{"Type": "DIMENSION", "Key": "INSTANCE_TYPE"}],
        Style="Total",
    )
    costexplorer.addReport(
        Name="Services",
        GroupBy=[{"Type": "DIMENSION", "Key": "SERVICE"}],
        Style="Total",
        IncSupport=True,
    )
    # costexplorer.addReport(Name="ServicesChange", GroupBy=[{"Type": "DIMENSION","Key": "SERVICE"}],Style='Change')
    #costexplorer.addReport(
    #    Name="Accounts",
    #    GroupBy=[{"Type": "DIMENSION", "Key": "LINKED_ACCOUNT"}],
    #    Style="Total",
    #)
    # costexplorer.addReport(Name="AccountsChange", GroupBy=[{"Type": "DIMENSION","Key": "LINKED_ACCOUNT"}],Style='Change')
    regions = costexplorer.addReport(
        Name="Regions", GroupBy=[{"Type": "DIMENSION", "Key": "REGION"}], Style="Total"
    )
    # costexplorer.addReport(Name="RegionsChange", GroupBy=[{"Type": "DIMENSION","Key": "REGION"}],Style='Change')

    all_used_regions = list(next(regions.items())[1].to_dict().keys())
    print(f"Using regions {all_used_regions}")
    regional_reports = {}
    #for region in all_used_regions:
    for region in ['us-east-2']:
        if region in ['NoRegion', 'global']:
            continue
        regional_reports[region] = costexplorer.addReport(
            Name=region,
            GroupBy=[{"Type": "DIMENSION", "Key": "USAGE_TYPE"}],
            Region=region,
            Style="Total",
        )
        calculate_carbon(regional_reports[region])

    costexplorer.generateExcel()

    #from ptpython.repl import embed
    #embed(globals(), locals(), vi_mode=True)

    #for month, report in usage_type.items():
    #    categorize_month(report.to_dict())

    return "Report Generated"

def lambda_handler(event, context):
    global session
    session = boto3.Session()
    print(json.dumps(event, sort_keys=True))
    main_handler()
    s3 = boto3.resource('s3')
    bucket = s3.Bucket(os.getenv("REPORT_STORAGE"))

    bucket.upload_file(
        '/tmp/cost_explorer_report.xlsx',
        boto3.session('sts').get_caller_identity()['Account'] + '.xlsx'
    )

def calculate_carbon(regional_report):
    for month, usage in regional_report.items():
        for usage_type, amount in usage.to_dict().items():
            if amount < 1:
                continue
            if usage_type.endswith('Bytes') and amount < 10**9:
                continue
            print(month, usage_type, amount)
    from ptpython.repl import embed
    embed(globals(), locals(), vi_mode=True)

def divine_region(usage_type_name):
    mapping = dict(
        USW1='us-west-1',
        USW2='us-west-2',
        USE1='us-east-1',
        USE2='us-east-2',
        EUN1="eu-north-1",
        EU="eu-north-1",
        APS3="ap-south-1",
        EUW3="eu-west-3",
        EUW2="eu-west-2",
        EUW1="eu-west-1",
        APN2="ap-northeast-2",
        APN1="ap-northeast-1",
        SAE1="sa-east-1",
        CAN1="ca-central-1",
        APS1="ap-southeast-1",
        APS2="ap-southeast-2",
        EUC1="eu-central-1",
    )

    for k, v in mapping.items():
        if usage_type_name.startswith(k):
            return v
        if usage_type_name.startswith(v):
            return v
    if re.match('^(Box|Instance|Node|Heavy)Usage', usage_type_name):
        return 'us-east-1'
    if re.match('^(RDS:|)(Multi-AZ|)(Usage:db|-?GP2|-?PIOPS|StorageUsag)', usage_type_name):
        return 'us-east-1'
    if re.match('^EBS(Optimized|):', usage_type_name):
        return 'us-east-1'
    if re.match('^ets-(hd|sd|audio)-', usage_type_name):
        return 'us-east-1'
    if usage_type_name.startswith('agent-assessment'):
        return 'us-east-1'
    if usage_type_name.startswith('Storage-ShardHour'):
        return 'us-east-1'
    if usage_type_name.startswith('Lambda-GB-Second'):
        return 'us-east-1'
    print(f"WARN: Could not regionalize {usage_type_name}")
    return None

def categorize_month(month):
    regional = defaultdict(list)
    for k, v in month.items():
        if v < 1:
            continue
        region = divine_region(k)
    pass

if __name__ == "__main__":
    main_handler()


from pandas import json

import sklearn
import json

#
# We implement cosine similarity here.
#
def calculateSim(path, mat, user_id):
	with open('%s_user_id.json' % path, 'r') as f:
		train_user_id = json.load(f)

	num = train_user_id[user_id]

	temDict = sklearn.metrics.pairwise.cosine_similarity(mat[num].reshape(1, -1), mat)
	result = {}
	i = 0
	for key,value in train_user_id.items():
		result[key] = temDict[0][i]
		i+=1
	# result = sorted(result.items(), key=lambda d: d[1], reverse=True)
	print("Similarity calculation done! Start writing.\n")

	# with open("%s_sim_result.json" % path,"w") as fp:
	# 	json.dump(result,fp, sort_keys=False)

	with open('%s_simi_result.json' % path,'w') as fp:
		json.dump(result,fp)


from __future__ import print_function, absolute_import, division  # makes KratosMultiphysics backward compatible with python 2.6 and 2.7

# Importing the Kratos Library
import KratosMultiphysics

# Import KratosUnittest
import KratosMultiphysics.KratosUnittest as UnitTest

# Other imports
from KratosMultiphysics.CompressiblePotentialFlowApplication.potential_flow_analysis import PotentialFlowAnalysis
import KratosMultiphysics.kratos_utilities as kratos_utilities
import os

class WorkFolderScope:
    def __init__(self, work_folder):
        self.currentPath = os.getcwd()
        self.scope = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)),work_folder))

    def __enter__(self):
        os.chdir(self.scope)

    def __exit__(self, exc_type, exc_value, traceback):
        os.chdir(self.currentPath)

class PotentialFlowTestFactory(UnitTest.TestCase):

    def setUp(self):
        # Set to true to get post-process files for the test
        self.print_output = False

    def test_execution(self):

        with WorkFolderScope(self.work_folder):
            self._run_test()

            kratos_utilities.DeleteFileIfExisting("naca0012_Case_5.time")

    def _run_test(self):
        model = KratosMultiphysics.Model()
        with open(self.file_name + "_parameters.json",'r') as parameter_file:
            ProjectParameters = KratosMultiphysics.Parameters(parameter_file.read())

        if self.print_output:
            ProjectParameters.AddValue("output_processes", KratosMultiphysics.Parameters(r'''{
                "gid_output" : [{
                    "python_module" : "gid_output_process",
                    "kratos_module" : "KratosMultiphysics",
                    "process_name"  : "GiDOutputProcess",
                    "help"          : "This process writes postprocessing files for GiD",
                    "Parameters"    : {
                        "model_part_name"        : "MainModelPart",
                        "output_name"            : "naca0012",
                        "postprocess_parameters" : {
                            "result_file_configuration" : {
                                "gidpost_flags"       : {
                                    "GiDPostMode"           : "GiD_PostBinary",
                                    "WriteDeformedMeshFlag" : "WriteDeformed",
                                    "WriteConditionsFlag"   : "WriteConditions",
                                    "MultiFileFlag"         : "SingleFile"
                                },
                                "file_label"          : "step",
                                "output_control_type" : "step",
                                "output_frequency"    : 1,
                                "body_output"         : true,
                                "node_output"         : false,
                                "skin_output"         : false,
                                "plane_output"        : [],
                                "nodal_results"       : ["VELOCITY_POTENTIAL","AUXILIARY_VELOCITY_POTENTIAL","DISTANCE","UPPER_WAKE","LOWER_WAKE","POTENTIAL_JUMP","AIRFOIL","TRAILING_EDGE","KUTTA"],
                                "gauss_point_results" : ["PRESSURE","VELOCITY","VELOCITY_LOWER","PRESSURE_LOWER","WAKE","TRAILING_EDGE","ELEMENTAL_DISTANCES","KUTTA","DISTANCE"]
                            },
                            "point_data_configuration"  : []
                        }
                    }
                }]
            }'''))

        potential_flow_analysis = PotentialFlowAnalysis(model,ProjectParameters)
        potential_flow_analysis.Run()

class Naca0012SmallTest(PotentialFlowTestFactory):
    file_name = "naca0012_small"
    work_folder = "naca0012_small_test"

if __name__ == '__main__':
    UnitTest.main()