import pygame
from pygame.math import Vector2 as Vector

class Ball(pygame.sprite.Sprite):
    # This class represents a Disc. It inherits the Sprite class of pygame

    def __init__(self, position, gravity=.20, friction=-.025):
        # Call parent class constructor
        super().__init__()

        self.radius = self.get_radius()

        self.on_ground = False
        self.gravity = gravity
        self.friction = friction

        self.position = position
        self.x, self.y = self.position
        self.velocity = Vector(0, 0)
        self.acceleration = Vector(0, self.gravity)

    def collision(self, rect):
        return self.get_rect().colliderect(rect) == 1

    def dist(self, player):
        """
        Check if ball is close to player to allow more forgiving ball movement
        :param player: Player()
        :return: boolean
        """
        dist_x = abs((self.position.x + self.radius) - (player.position.x + player.width/2))
        dist_y = abs((self.position.y + self.radius) - (player.position.y + player.height / 2))
        return dist_x < 60 and dist_y < 40

    def horizontal_movement(self, player, dt):
        """
        Handles the players horizontal movement using
        Newton's Equations of Motions:
        new velocity = previous velocity + acceleration * time
        new position = previous position + (velocity*time) + (.5*acceleration*(time^2))
        :param dt: float - Multiplier that makes up for loss of frames
        :return: None
        """
        # if keys not pressed than the default acceleration is 0
        self.acceleration.x = 0

        if self.collision(player.get_rect()):
            if player.LEFT_KEY or player.RIGHT_KEY:
                self.acceleration.x += player.acceleration.x / 2
                self.velocity.x += player.velocity.x

            else:
                self.acceleration.x = -self.acceleration.x / 2
                self.velocity.x = -self.velocity.x

        """
        since self.friction has a negative multiplier,
        if a player is moving to a certain direction, the following line of code will simulate
        a force going to the opposite direction
        """
        if self.on_ground:
            self.acceleration.x += self.velocity.x * self.friction
        self.velocity.x += self.acceleration.x * dt
        self.position.x += self.velocity.x * dt + (self.acceleration.x * .5) * (dt * dt)
        self.x = int(self.position.x)

    def vertical_movement(self, player, bottom, dt):
        """
        Handles the players vertical movement using
        Newton's Equations of Motions:
        new velocity = previous velocity + acceleration * time
        new position = previous position + (velocity*time) + (.5*acceleration*(time^2))
        :param bottom: int - bottom boundary
        :param dt: float - Multiplier that makes up for loss of frames
        :return: None
        """

        if self.dist(player):
            if player.is_jumping:
                self.velocity.y = player.velocity.y * 1.2

            else:
                self.velocity.y = -self.velocity.y

        self.velocity.y += self.acceleration.y * dt

        # Limit Velocity
        if self.velocity.y > 15:
            self.velocity.y = 15

        self.position.y += self.velocity.y * dt + (self.acceleration.y * .5) * (dt * dt)

        # Bound Y Position
        if self.position.y > bottom:
            self.on_ground = True
            self.velocity.y = 0
            self.position.y = bottom
        else:
            self.on_ground = False
        self.y = self.position.y - self.radius

    def update(self, player, boundaries, dt):
        pass
        """
        updates players movement
        :param boundaries: tuple(left, right, top, bottom)
        :param dt: float - Multiplier that makes up for loss of frames
        :return: None
        """
        left, right, top, bottom = boundaries
        self.horizontal_movement(player, dt)
        self.vertical_movement(player, bottom, dt)

    def get_radius(self):
        image = pygame.image.load(
            "Images/ball.png")
        return image.get_width()/2

    def get_rect(self):
        radius = self.get_radius()
        return pygame.Rect(self.x, self.y, radius, radius)

    def draw(self):
        image = pygame.image.load(
            "Images/ball.png")

        radius = image.get_width()/2
        return image, (int(self.x), int(self.y), int(radius), int(radius))


from __future__ import print_function

import sys
import pickle

import numpy as np
import lasagne
import theano
import theano.tensor as T

import hyperopt
from hyperopt import fmin, tpe, hp, Trials, rand
from hyperopt.mongoexp import MongoTrials

sys.path.append("./utils")
import otto_train as otto_tr
import otto_load_data as otto_dl

def objective(space, num_epochs=100):
    """
    Defines the objective function used by hyperopt. The objective function
    is the best validation score of a trained neural network.
    """

    #Some weirdness with the scope of hyperopt.
    import pickle
    import sys

    import numpy as np

    import lasagne
    import theano
    import theano.tensor as T

    sys.path.append("./utils")
    import otto_train as otto_tr
    import otto_load_data as otto_dl
    import model_building as mb

    print(space)

    #Train code
    dataset=otto_dl.load_data(transform='sqrt',
                              whiten=True,
                              test_size=0.05)

    output_layer = mb.build_model(space,
                               num_features=dataset['input_dim'],
                               output_dim=dataset['output_dim'])

    valid_loss, train_loss = otto_tr.run_training_hyperopt(
                                 output_layer=output_layer,
                                 dataset=dataset,
                                 batch_size=334,
                                 num_epochs=num_epochs,
                                 learning_rate=space['learning_rate'])

    #Monitoring
    best_valid_loss = valid_loss.min()
    #Haha this looks fun
    best_valid_epoch = np.argmin(valid_loss)+1 #correct of 0-index
    corresponding_train_loss = train_loss[best_valid_epoch-1]
    print("Best valid loss: ")
    print(best_valid_loss)
    print("Corresponding train loss: ")
    print(corresponding_train_loss)
    print("At epoch: ")
    print(best_valid_epoch)

    return {
            'loss': str(best_valid_loss),
            'attachments':
            {'valid_loss': pickle.dumps(valid_loss),
             'train_loss': pickle.dumps(train_loss),
             },
           }

def main():
    """
    Performs hyperparameter optimization with hyperopt. It consists of three
    steps. First define a trials object connected to a mongo database where all
    the results will be stored. Secondly define a stochastic search space from
    which hyperopt will sample hyperparameter configurations. Thirdly define
    the define the objective function and run the minimization function.
    """

    trials = MongoTrials('mongo://localhost:1234/otto-sqrt-pca-95-5/jobs',
            exp_key='15-11-03')

    #Search space
    space={

    'dense_part':  {'num_units' : hp.quniform('DL1', 512, 2048, 512),
                    'more_layers' :
                        {'num_units' : hp.quniform('DL2', 512, 2048, 512),
                         'more_layers' :
                          hp.choice('MD2', [0,
                            {'num_units' : hp.quniform('DL3', 512, 2048, 512),
                             'more_layers' : 0,} #DL3
                                        ])},#DL2
                                                },#DL1

    'leakiness' : hp.choice('leak', [0, 0.01, 0.15] ),

    'weight_init' : hp.choice('weight',['orto','uni']),
    'input_dropout' : hp.quniform('p_in', 0.1, 0.4, 0.1),

    'learning_rate': hp.choice('lr',[0.001,0.01,0.025,0.05,0.1]),
    }

    #Optimize
    best = fmin(objective, space=space, algo=tpe.suggest, max_evals=100,
               trials=trials)

    print(trials.losses())
    print(best)

if __name__ == '__main__':
    main()


#!/bin/env python
"""
Program to manage and run ATS regression tests.

With inspiration, and in some places just stolen, from Ben Andre's
PFloTran regression test suite.

Author: Ethan Coon (ecoon@lanl.gov)
"""

from __future__ import print_function
from __future__ import division

import sys

if sys.hexversion < 0x02070000:
    print(70*"*")
    print("ERROR: The regression test manager requires python >= 2.7.x. ")
    print("It appears that you are running python {0}.{1}.{2}".format(
        sys.version_info[0], sys.version_info[1], sys.version_info[2]))
    print(70*"*")
    sys.exit(1)

import argparse
import datetime
import os
#import pprint
import shutil
import subprocess
import textwrap
import time
import traceback
import distutils.spawn
import numpy

import ats_h5

class NoCatchException(Exception):
    def __init__(self,*args,**kwargs):
        Exception.__init__(self,*args,**kwargs)

if sys.version_info[0] == 2:
    from ConfigParser import SafeConfigParser as config_parser
else:
    from configparser import ConfigParser as config_parser

class TestStatus(object):
    """
    Simple class to hold status info.
    """
    def __init__(self):
        self.fail = 0
        self.warning = 0
        self.error = 0
        self.skipped = 0
        self.test_count = 0

    def __str__(self):
        message = "fail = {0}\n".format(self.fail)
        message += "warning = {0}\n".format(self.warning)
        message += "error = {0}\n".format(self.error)
        message += "skipped = {0}\n".format(self.skipped)
        message += "test_count = {0}\n".format(self.test_count)
        return message

class RegressionTest(object):
    """
    Class to collect data about a test problem, run the problem, and
    compare the results to a known result.
    """
    # things to compare
    _TIME = "time"
    _TIMESTEPS = "timesteps"
    _DEFAULT = "default" # compare a variable

    # ways to compare it
    _ABSOLUTE = "absolute"
    _RELATIVE = "relative"
    _PERCENT = "percent"
    _DISCRETE = "discrete"

    # ways to measure what to compare
    _NORM = numpy.inf

    def __init__(self):
        self._EXECUTABLE = "ats"
        self._SUCCESS = 0
        self._RESERVED = [self._TIME, self._TIMESTEPS]

        # misc test parameters
        #        self._pprint = pprint.PrettyPrinter(indent=2)
        self._txtwrap = textwrap.TextWrapper(width=78, subsequent_indent=4*" ")
        self._executable = None
        self._input_arg = "--xml_file="
        self._input_suffix = "xml"
        self._np = None
        self._timeout = 60.0
        self._skip_check_gold = False
        self._check_performance = False
        self._num_failed = 0
        self._test_name = None

        # assign default tolerances for different classes of variables
        # absolute min and max thresholds for determining whether to
        # compare to baseline, i.e. if (min_threshold <= abs(value) <=
        # max_threshold) then compare values. By default we use the
        # python definitions for this platform
        self._default_tolerance = {}
        self._default_tolerance[self._TIME] = [5.0, self._PERCENT, \
                                               0.0, sys.float_info.max]
        self._default_tolerance[self._TIMESTEPS] = [5, self._RELATIVE, \
                                                    0, sys.maxsize]
        self._default_tolerance[self._DISCRETE] = [0, self._ABSOLUTE, 0, sys.maxsize]

        self._default_tolerance[self._DEFAULT] = [1.0e-12, self._RELATIVE, \
                                                  0.0, sys.float_info.max]
        self._eps = 1.e-14

        self._checkpoint = None

        # dictionary indexed by domain, each of which is a dictionary of
        # (key, tolerance) pairs
        self._criteria = {}
        self._file_format_with_domain = "visdump_{0}_data.h5"
        self._file_format_no_domain = "visdump_data.h5"

    def __str__(self):
        message = "  {0} :\n".format(self.name())
        message += "    timeout = {0}\n".format(self._timeout)
        message += "    np = {0}\n".format(self._np)
        message += "    ATS args :\n"
        message += "        exec  : {0}\n".format(self._executable)
        message += "        input : {0}../{1}.{2}\n".format(
            self._input_arg,self._test_name,self._input_suffix)
        for domain,criteria in self._criteria.iteritems():
            message += "    test criteria on \"{0}\" :\n".format(domain)
            for key,tol in criteria.iteritems():
                message += "        {0} : {1} [{2}] : {3} <= abs(value) <= {4}\n".format(
                    key, tol[0], tol[1], tol[2], tol[3])
        return message

    def setup(self, cfg_criteria, test_data,
              timeout, check_performance, testlog):
        """
        Setup the test object

        cfg_criteria - dict from cfg file, all tests in file
        test_data - dict from cfg file, test specific
        timeout - list(?) from command line option
        check_performance - bool from command line option
        """
        self._test_name = test_data.pop('name')
        self._np = test_data.pop('np', None)
        self._skip_check_gold = test_data.pop('skip_check_gold', None)
        self._check_performance = check_performance

        # timeout : preference (1) command line (2) test data (3) class default
        self._timeout = float(cfg_criteria.pop('timeout', self._timeout))
        self._timeout = float(test_data.pop('timeout', self._timeout))
        if timeout is not None:
            self._timeout = float(timeout[0])

        # check whether this is a checkpoint
        checkpoint = test_data.pop('checkpoint',
                                   cfg_criteria.pop('checkpoint', None))
        if checkpoint is not None:
            self._checkpoint = checkpoint

        # pop a default and/or default discrete
        default = test_data.pop(self._DEFAULT,
                                cfg_criteria.pop(self._DEFAULT, None))
        if default is not None:
            tol = self._validate_tolerance(self._DEFAULT, self._DEFAULT, default)
            self._default_tolerance[self._DEFAULT] = tol

        discrete = test_data.pop(self._DISCRETE,
                                cfg_criteria.pop(self._DISCRETE, None))
        if discrete is not None:
            tol = self._validate_tolerance(self._DISCRETE, self._DISCRETE, discrete)
            self._default_tolerance[self._DISCRETE] = tol

        # requested criteria, skipping time and timesteps
        for key in set(cfg_criteria.keys() + test_data.keys()):
            if key != self._TIME and key != self._TIMESTEPS:
                self._set_criteria(key, cfg_criteria, test_data)

        # criteria defaults
        # - time exists on all domains, but must make sure it is on a valid one
        if self._check_performance:
            self._set_criteria("({0})".format(self._criteria.keys()[0])+self._TIME,
                               cfg_criteria, test_data)

        # - timestep exists on all domains, but must make sure it is on a valid one
        if self._checkpoint is None:
            self._set_criteria("({0})".format(self._criteria.keys()[0])+self._TIMESTEPS,
                               cfg_criteria, test_data)

    def name(self):
        return self._test_name

    def dirname(self, gold=False):
        suffix = ".regression"
        if gold:
            suffix += ".gold"
        return self.name() + suffix

    def filename(self, domain, gold=False):
        if self._checkpoint is not None:
            return os.path.join(self.dirname(gold),
                            self._checkpoint)
        else:
            if domain == "":
                return os.path.join(self.dirname(gold),
                                    self._file_format_no_domain)
            else:
                return os.path.join(self.dirname(gold),
                                    self._file_format_with_domain.format(domain))

    def run(self, mpiexec, executable, dry_run, status, testlog):
        """
        Run the test.
        """
        self._cleanup_generated_files()
        self._run_test(mpiexec, executable, self.name(), dry_run, status,
                       testlog)

    def _run_test(self, mpiexec, executable, test_name, dry_run, status, testlog):
        """
        Build up the run command, including mpiexec, np, ats,
        input file, output file. Then run the job as a subprocess.

        * NOTE(bja) - starting in python 3.3, we can use:

          subprocess.Popen(...).wait(timeout)

          to catch hanging jobs, but for python < 3.3 we have to
          manually manage the timeout...?
        """
        command = []
        if self._np is not None:
            if mpiexec:
                command.append(mpiexec)
                command.append("-np")
                command.append(self._np)
            else:
                # parallel test, but don't have mpiexec, we mark the
                # test as skipped and bail....
                message = self._txtwrap.fill(
                    "WARNING : mpiexec was not provided for a parallel test '{0}'.\n"
                    "This test was skipped!".format(self.name()))
                print(message, file=testlog)
                status.skipped = 1
                return None

        command.append(executable)
        command.append("{0}../{1}.{2}".format(self._input_arg,test_name,self._input_suffix))

        test_directory = os.getcwd()
        run_directory = os.path.join(test_directory, self.dirname())
        os.mkdir(run_directory)
        os.chdir(run_directory)

        print("    cd {0}".format(run_directory), file=testlog)
        print("    {0}".format(" ".join(command)), file=testlog)

        if not dry_run:
            run_stdout = open(test_name + ".stdout", 'w')
            start = time.time()
            proc = subprocess.Popen(command,
                                    shell=False,
                                    stdout=run_stdout,
                                    stderr=subprocess.STDOUT)
            while proc.poll() is None:
                time.sleep(0.1)
                if time.time() - start > self._timeout:
                    proc.kill()
                    time.sleep(0.1)
                    message = self._txtwrap.fill(
                        "ERROR: job '{0}' has exceeded timeout limit of "
                        "{1} seconds.".format(test_name, self._timeout))
                    print(''.join(['\n', message, '\n']), file=testlog)
            finish = time.time()
            print("    # {0} : run time : {1:.2f} seconds".format(test_name, finish - start), file=testlog)
            ierr_status = abs(proc.returncode)
            run_stdout.close()

            if ierr_status != self._SUCCESS:
                status.fail = 1
                message = self._txtwrap.fill(
                    "FAIL : {name} : {execute} return an error "
                    "code ({status}) indicating the simulation may have "
                    "failed. Please check '{name}.stdout' "
                    "for error messages (included below).".format(
                        execute=self._EXECUTABLE, name=test_name, status=ierr_status))

                print("".join(['\n', message, '\n']), file=testlog)
                print("~~~~~ {0}.stdout ~~~~~".format(test_name), file=testlog)
                try:
                    with open("{0}.stdout".format(test_name), 'r') as tempfile:
                        shutil.copyfileobj(tempfile, testlog)
                except Exception as e:
                    print("   Error opening file: {0}.stdout\n    {1}".format(test_name, e))
                print("~~~~~~~~~~", file=testlog)

        os.chdir(test_directory)

    def _cleanup_generated_files(self):
        """Cleanup old generated files that may be hanging around from a
        previous run.
        """
        run_dir = self.dirname(False)
        shutil.rmtree(run_dir, ignore_errors=True)

    def check(self, status, testlog):
        """
        Check the test results against the gold standard
        """

        for domain in self._criteria.keys():
            self._check_gold(status, domain, testlog)

    def update(self, status, testlog):
        """
        Update the gold standard test results to the current
        output. Both the current regression output and a gold file
        must exist.
        """
        gold_dir = self.dirname(True)
        run_dir = self.dirname(False)
        old_gold_dir = gold_dir+".old"

        # verify that the gold file exists
        if not os.path.isdir(gold_dir):
            print("ERROR: test '{0}' results cannot be updated "
                  "because a gold directory does not "
                  "exist!".format(self.name()), file=testlog)
            status.error = 1

        # verify that the regression directory exists
        if not os.path.isdir(run_dir):
            print("ERROR: test '{0}' results cannot be updated "
                  "because no regression run directory "
                  "exists!".format(self.name()), file=testlog)
            status.error = 1

        # check that the regression file was created in the regression directory
        if (not status.error and not status.fail):
            for domain in self._criteria.keys():
                reg_name = self.filename(domain, False)

                if not os.path.isfile(reg_name):
                    print("ERROR: run for "
                          "test '{0}' did not create the needed regression file "
                          "'{1}', not updating!".format(self.name(),
                                                        reg_name), file=testlog)
                    status.fail = 1

        if not status.error and not status.fail:
            # remove old-old gold
            if os.path.isdir(old_gold_dir):
                try:
                    shutil.rmtree(old_gold_dir)
                except Exception as error:
                    message = str(error)
                    message += "\nFAIL : Could not remove old gold '{0}'. ".format(old_gold_dir)
                    message += "Please remove the directory manually!"
                    message += "    rm -rf {0}".format(old_gold_dir)
                    print(message, file=testlog)
                    status.fail = 1

            # move gold -> old_gold
            if not status.fail:
                try:
                    os.rename(gold_dir, old_gold_dir)
                except Exception as error:
                    message = str(error)
                    message += "\nFAIL : Could not back up gold '{0}' as '{1}'. ".format(
                        gold_dir, old_gold_dir)
                    message += "Please rename or remove the gold directory manually!"
                    message += "    mv {0} {1}".format(gold_dir, old_gold_dir)
                    print(message, file=testlog)
                    status.fail = 1

            if not status.fail:
                # move run -> gold
                try:
                    os.rename(run_dir, gold_dir)
                except Exception as error:
                    message = str(error)
                    message += "\nFAIL : Could not move '{0}' to '{1}'. ".format(
                        run_dir, gold_dir)
                    message += "Please rename the directory manually!"
                    message += "    mv {0} {1}".format(run_dir, gold_dir)
                    print(message, file=testlog)
                    status.fail = 1

        print("done", file=testlog)

    def new_test(self, status, testlog):
        """
        A new test does not have a gold standard regression test. We
        will check to see if a gold standard file exists (an error),
        then create the gold file by copying the current regression
        file to gold.
        """
        gold_dir = self.dirname(True)
        run_dir = self.dirname(False)

        # verify that the gold file does not exisit
        if os.path.isdir(gold_dir) or os.path.isfile(gold_dir):
            print("ERROR: test '{0}' was classified as new, "
                               "but a gold file already "
                               "exists!".format(self.name()), file=testlog)
            status.error = 1

        # verify that the run directory exists
        if not os.path.isdir(run_dir):
            print("ERROR: test '{0}' results cannot be new "
                  "because no run directory "
                  "exists!".format(self.name()), file=testlog)
            status.error = 1

        # verify that the run directory created good files
        if (not status.error and not status.fail):
            for domain in self._criteria.keys():
                reg_name = self.filename(domain, False)

                if not os.path.isfile(reg_name):
                    print("ERROR: run for "
                          "test '{0}' did not create the needed regression file "
                          "'{1}', not making new gold!".format(self.name(),
                                                    reg_name), file=testlog)
                    status.fail = 1

        # move the run to gold
        if not status.error and not status.fail:
            print("  creating gold directory '{0}'... ".format(self.name()),
                  file=testlog)
            try:
                os.rename(run_dir, gold_dir)
            except Exception as error:
                message = str(error)
                message += "\nFAIL : Could not move '{0}' to '{1}'. ".format(
                    run_dir, gold_dir)
                message += "Please rename the directory manually!"
                message += "    mv {0} {1}".format(run_dir, gold_dir)
                print(message, file=testlog)
                status.fail = 1

        print("done", file=testlog)

    def _check_gold(self, status, domain, testlog):
        """
        Test the output from the run against the known "gold standard"
        output and determine if the test succeeded or failed.

        We return zero on success, one on failure so that the test
        manager can track how many tests succeeded and failed.
        """
        if self._skip_check_gold:
            message = "    Skipping comparison to regression gold file (only test if model runs to completion)."
            print("".join(['\n', message, '\n']), file=testlog)
            return

        gold_filename = self.filename(domain, True)
        if not os.path.isfile(gold_filename):
            message = self._txtwrap.fill(
                "FAIL: could not find regression test gold file "
                "'{0}'. If this is a new test, please create "
                "it with '--new-test'.".format(gold_filename))
            print("".join(['\n', message, '\n']), file=testlog)
            status.fail = 1
            return

        current_filename = self.filename(domain, False)
        if not os.path.isfile(current_filename):
            message = self._txtwrap.fill(
                "FAIL: could not find regression test file '{0}'."
                " Please check the standard output file for "
                "errors.".format(current_filename))
            print("".join(['\n', message, '\n']), file=testlog)
            status.fail = 1
            return

        try:
            h5_current = ats_h5.File(current_filename)
        except Exception as e:
            print("    FAIL: Could not open file: '{0}'".format(current_filename), file=testlog)
            status.fail = 1
            h5_current = None

        try:
            h5_gold = ats_h5.File(gold_filename)
        except Exception as e:
            print("    FAIL: Could not open file: '{0}'".format(gold_filename), file=testlog)
            status.fail = 1
            h5_gold = None

        if h5_gold is not None and h5_current is not None:
            self._compare(h5_current, h5_gold, domain, status, testlog)

        h5_gold.close()
        h5_current.close()

    def _compare(self, h5_current, h5_gold, domain, status, testlog):
        """Check that output hdf5 file has not changed from the baseline.
        """
        for key, tolerance in self._criteria[domain].iteritems():
            if key == self._TIME:
                self._compare_time(h5_current, h5_gold, tolerance,
                                   status, testlog)
            elif key == self._TIMESTEPS:
                self._compare_timesteps(h5_current, h5_gold, tolerance,
                                        status, testlog)
            else:
                self._compare_values(h5_current, h5_gold, key, tolerance,
                                     status, testlog)
        if status.fail == 0:
            print("    Passed tests.", file=testlog)

    def _compare_time(self, h5_current, h5_gold, tolerance, status, testlog):
        self._check_tolerance(h5_current.simulationTime(), h5_gold.simulationTime(),
                              self._TIME, tolerance, status, testlog)

    def _compare_timesteps(self, h5_current, h5_gold, tolerance, status, testlog):
        self._check_tolerance(h5_current.numSteps(), h5_gold.numSteps(),
                              self._TIMESTEPS, tolerance, status, testlog)

    def _compare_values(self, h5_current, h5_gold, key, tolerance, status, testlog):
        if set(h5_current.matches(key)) != set(h5_gold.matches(key)):
            status.fail = 1
            print("    FAIL: {0} : {1} :".format(self.name(), key), file=testlog)
            print("        gold matches: {1}".format(";".join(h5_gold.matches(key))),
                  file=testlog)
            print("        current matches: {1}".format(";".join(h5_current.matches(key))),
                  file=testlog)

        else:
            if self._checkpoint is not None:
                for k in h5_gold.matches(key):
                    self._check_tolerance(h5_current[k][:], h5_gold[k][:],
                                          k, tolerance, status, testlog)
            else:
                for k in h5_gold.matches(key):
                    for i_current, i_gold in zip(h5_current.steps(), h5_gold.steps()):
                        self._check_tolerance(h5_current[k][i_current][:],
                                              h5_gold[k][i_gold][:],
                                              key, tolerance,
                                              status, testlog)

    def _norm(self, diff):
        """
        Determine the difference between two values
        """
        if type(diff) is numpy.ndarray:
            delta = numpy.linalg.norm(diff.flatten(), self._NORM)
        else:
            delta = abs(diff)
        return delta

    def _check_tolerance(self, current, gold, key, tolerance, status, testlog):
        """
        Compare the values using the appropriate tolerance and criteria.
        """
        my_status = 0

        # unpack the tolerance
        tol, tol_type, min_threshold, max_threshold = tuple(tolerance)

        if tol_type == self._ABSOLUTE:
            delta = self._norm(current-gold)

        elif (tol_type == self._RELATIVE or
              tol_type == self._PERCENT):

            if type(gold) is numpy.ndarray:
                rel_to = numpy.where(gold > self._eps, gold, current)
                filter = numpy.where(rel_to > self._eps)[0]
                if filter.shape[0] == 0:
                    delta = 0
                else:
                    delta = self._norm((gold[filter] - current[filter]) / rel_to[filter])

            else:
                if gold > self._eps:
                    delta = self._norm((gold - current) / gold)
                elif current > self._eps:
                    delta = self._norm((gold - current) / current)
                else:
                    delta = 0

            if tol_type == self._PERCENT:
                delta *= 100.0
        else:
            # should never get here....
            raise RuntimeError("ERROR: unknown test tolerance_type '{0}' for "
                               "variable '{1}, {2}.'".format(tol_type,
                                                          self.name(), key))

        if delta > tol:
            status.fail = 1
            print("    FAIL: {0} : {1} : {2} > {3} [{4}]".format(
                    self.name(), key, delta, tol, tol_type), file=testlog)
        else:
            print("    PASS: {0} : {1} <= {2} [{3}]".format(
                self.name(), delta, tol, tol_type), file=testlog)
        return

    def _set_criteria(self, key, cfg_criteria, test_data):
        """
        Our preferred order for selecting test criteria is:
        (1) test data section of the config file
        (2) config-file wide defaults
        (3) hard coded class default
        """
        # strip the domain prefix
        if key.startswith("("):
            domain,varname = key[1:].split(")")
        elif "-" in key:
            domain = key.split("-")[0]
            varname = key
        else:
            domain = ""
            varname = key

        # parse the criteria
        if key in test_data:
            criteria = domain, varname, self._validate_tolerance(key, varname, test_data[key])
        elif key in cfg_criteria:
            criteria = domain, varname, self._validate_tolerance(key, varname, cfg_criteria[key])
        elif varname == self._TIME:
            criteria = domain, self._TIME, self._default_tolerance[self._TIME]
        elif varname == self._TIMESTEPS:
            criteria = domain, self._TIMESTEPS, self._default_tolerance[self._TIMESTEPS]
        else:
            criteria = domain, varname, self._default_tolerance[self._DEFAULT]

        if criteria is not None:
            domain,varname,tol = criteria
            if tol is not None:
                domain_dict = self._criteria.setdefault(domain,dict())
                domain_dict[varname] = tol

    def _validate_tolerance(self, key, varname, test_data):
        """
        Validate the tolerance string from a config file.

        Valid input configurations are:

        * (domain)varname =
        * (domain)varname = default
        * (domain)varname = no
        * (domain)varname = tolerance type
        * (domain)varname = tolerance type [, min_threshold value] [, max_threshold value]

        where the first two use defaults, the third turns the test
        off, and the last two specify the tolerance explicitly, where
        min_threshold and max_threshold are optional

        """
        # deal with defaults first
        if test_data.lower() == "none" or test_data.lower() == "no" or test_data.lower() == "n":
            return None
        if test_data == "" or test_data.lower() == self._DEFAULT:
            return self._default_tolerance[self._DEFAULT]
        if test_data == self._DISCRETE:
            return self._default_tolerance[self._DISCRETE]

        # if we get here, parse the string
        criteria = 4*[None]
        test_data = test_data.split(",")
        test_criteria = test_data[0]
        value = test_criteria.split()[0]

        try:
            value = float(value)
        except Exception:
            raise RuntimeError("ERROR : Could not convert '{0}' test criteria "
                               "value '{1}' into a float!".format(key, value))
        criteria[0] = value

        criteria_type = test_criteria.split()[1]
        if (criteria_type.lower() != self._PERCENT and
            criteria_type.lower() != self._ABSOLUTE and
                criteria_type.lower() != self._RELATIVE):
            raise RuntimeError("ERROR : invalid test criteria string '{0}' "
                               "for '{1}'".format(criteria_type, key))
        criteria[1] = criteria_type

        thresholds = {}
        for t in range(1, len(test_data)):
            name = test_data[t].split()[0].strip()
            value = test_data[t].split()[1].strip()
            try:
                value = float(value)
            except Exception:
                raise RuntimeError(
                    "ERROR : Could not convert '{0}' test threshold '{1}'"
                    "value '{2}' into a float!".format(key, name, value))
            thresholds[name] = value
        value = thresholds.pop("min_threshold", None)
        criteria[2] = value
        value = thresholds.pop("max_threshold", None)
        criteria[3] = value
        if len(thresholds) > 0:
            raise RuntimeError("ERROR: test {0} : unknown criteria threshold: {1}",
                               key, thresholds)

        return criteria

class RegressionTestManager(object):
    """
    Class to open a configuration file, process it into a group of
    tests, and manage running the tests.

    Notes:

        * The ConfigParser class converts all section names and key
          names into lower case. This means we need to preprocess user
          input names to lower case.

    """

    def __init__(self):
        self._debug = False
        self._file_status = TestStatus()
        self._config_filename = None
        self._default_test_criteria = {}
        self._available_tests = {}
        self._available_suites = {}
        self._tests = []
        self._txtwrap = textwrap.TextWrapper(width=78, subsequent_indent=4*" ")
        #        self._pprint = pprint.PrettyPrinter(indent=2)

    def __str__(self):
        data = "Regression Test Manager :\n"
        data += "    configuration file : {0}\n".format(self._config_filename)
        data += "    default test criteria :\n"
        data += self._dict_to_string(self._default_test_criteria)
        data += "    suites :\n"
        data += self._dict_to_string(self._available_suites)
        data += "    available tests :\n"
        data += self._dict_to_string(self._available_tests)

        data += "Tests :\n"
        for test in self._tests:
            data += test.__str__()

        return data

    def debug(self, debug):
        self._debug = debug

    def num_tests(self):
        return len(self._tests)

    def generate_tests(self, config_file, user_suites, user_tests,
                       timeout, check_performance, testlog):
        """
        Read the config file, validate the input and generate the test objects.
        """
        self._read_config_file(config_file)
        self._validate_suites()
        user_suites, user_tests = self._validate_user_lists(user_suites,
                                                            user_tests, testlog)
        self._create_tests(user_suites, user_tests, timeout, check_performance,
                           testlog)

    def run_tests(self, mpiexec, executable,
                  dry_run, update, new_test, check_only, testlog):
        """
        Run the tests specified in the config file.

        * dry_run - flag indicates that the test is setup then print
          the command that would be used, but don't actually run
          anything or compare results.

        * new_test - flag indicates that the test is a new test, and
          there should not be a gold standard regression file
          present. Run the executable and create the gold file.

        * update - flag indicates that the output from ats has
          changed, and we want to update the gold standard regression
          file to reflect this. Run the executable and replace the
          gold file.

        * check_only - flag to indicate just diffing the existing
          regression files without rerunning ats.
        """
        if self.num_tests() > 0:
            if new_test:
                self._run_new(mpiexec, executable, dry_run, testlog)
            elif update:
                self._run_update(mpiexec, executable, dry_run, testlog)
            elif check_only:
                self._check_only(dry_run, testlog)
            else:
                self._run_check(mpiexec, executable, dry_run, testlog)
        else:
            self._file_status.test_count = 0

    def _run_check(self, mpiexec, executable, dry_run, testlog):
        """
        Run the test and check the results.
        """
        if dry_run:
            print("Dry run:")
        print("Running tests from '{0}':".format(self._config_filename), file=testlog)
        print(50 * '-', file=testlog)

        for test in self._tests:
            status = TestStatus()
            self._test_header(test.name(), testlog)

            test.run(mpiexec, executable, dry_run, status, testlog)

            if not dry_run and status.skipped == 0:
                test.check(status, testlog)

            self._add_to_file_status(status)

            self._test_summary(test.name(), status, dry_run,
                               "passed", "failed", testlog)

        self._print_file_summary(dry_run, "passed", "failed", testlog)

    def _check_only(self, dry_run, testlog):
        """
        Recheck the regression files from a previous run.
        """
        if dry_run:
            print("Dry run:")
        print("Checking existing test results from '{0}':".format(
            self._config_filename), file=testlog)
        print(50 * '-', file=testlog)

        for test in self._tests:
            status = TestStatus()
            self._test_header(test.name(), testlog)

            if not dry_run and status.skipped == 0:
                test.check(status, testlog)

            self._add_to_file_status(status)

            self._test_summary(test.name(), status, dry_run,
                               "passed", "failed", testlog)

        self._print_file_summary(dry_run, "passed", "failed", testlog)

    def _run_new(self, mpiexec, executable, dry_run, testlog):
        """
        Run the tests and create new gold files.
        """
        if dry_run:
            print("Dry run:")

        print("New tests from '{0}':".format(self._config_filename), file=testlog)
        print(50 * '-', file=testlog)

        for test in self._tests:
            status = TestStatus()
            self._test_header(test.name(), testlog)

            test.run(mpiexec, executable, dry_run, status, testlog)

            if not dry_run and status.skipped == 0:
                test.new_test(status, testlog)
            self._add_to_file_status(status)
            self._test_summary(test.name(), status, dry_run,
                               "created", "error creating new test files.", testlog)

        self._print_file_summary(dry_run, "created", "could not be created", testlog)

    def _run_update(self, mpiexec, executable, dry_run, testlog):
        """
        Run the tests and update the gold file with the current output
        """
        if dry_run:
            print("Dry run:")
        print("Updating tests from '{0}':".format(self._config_filename), file=testlog)
        print(50 * '-', file=testlog)

        for test in self._tests:
            status = TestStatus()
            self._test_header(test.name(), testlog)
            test.run(mpiexec, executable, dry_run, status, testlog)

            if not dry_run and status.skipped == 0:
                test.update(status, testlog)
            self._add_to_file_status(status)
            self._test_summary(test.name(), status, dry_run,
                               "updated", "error updating test.", testlog)

        self._print_file_summary(dry_run, "updated", "could not be updated", testlog)

    def _test_header(self, name, testlog):
        """
        Write a header to the log file to separate tests.
        """
        print(40 * '-', file=testlog)
        print("{0}... ".format(name), file=testlog)

    def _test_summary(self, name, status, dry_run,
                      success_message, fail_message, testlog):
        """
        Write the test status information to stdout and the test log.
        """
        if dry_run:
            print("D", end='', file=sys.stdout)
            print(" dry run.", file=testlog)
        else:
            if (status.fail == 0 and
                    status.warning == 0 and
                    status.error == 0 and
                    status.skipped == 0):
                print(".", end='', file=sys.stdout)
                print("{0}... {1}.".format(name, success_message), file=testlog)
            elif status.fail != 0:
                print("F", end='', file=sys.stdout)
                print("{0}... {1}.".format(name, fail_message), file=testlog)
            elif status.warning != 0:
                print("W", end='', file=sys.stdout)
            elif status.error != 0:
                print("E", end='', file=sys.stdout)
            elif status.skipped != 0:
                print("S", end='', file=sys.stdout)
                print("{0}... skipped.".format(name), file=testlog)
            else:
                print("?", end='', file=sys.stdout)

        sys.stdout.flush()

    def _print_file_summary(self, dry_run, success_message, fail_message, testlog):
        """
        Print a summary of the results for this config file
        """
        print("", file=sys.stdout)
        print(50 * '-', file=testlog)
        if dry_run:
            print("{0} : dry run.".format(self._config_filename), file=testlog)
        elif self._file_status.test_count == 0:
            print("{0} : no tests run.".format(self._config_filename), file=testlog)
        else:
            line = "{0} : {1} tests : ".format(self._config_filename,
                                               self._file_status.test_count)
            if self._file_status.fail > 0:
                line = "{0} {1} tests {2}, ".format(
                    line, self._file_status.fail, fail_message)
            if self._file_status.skipped > 0:
                line = "{0} {1} tests {2}, ".format(
                    line, self._file_status.skipped, "skipped")
            num_passed = (self._file_status.test_count -
                          self._file_status.fail - self._file_status.skipped)
            line = "{0} {1} tests {2}".format(line, num_passed, success_message)
            print(line, file=testlog)

    def _add_to_file_status(self, status):
        """
        Add the current test status to the overall status for the file.
        """
        self._file_status.fail += status.fail
        self._file_status.warning += status.warning
        self._file_status.error += status.error
        self._file_status.skipped += status.skipped
        self._file_status.test_count += 1

    def run_status(self):
        return self._file_status

    def display_available_tests(self):
        print("Available tests: ")
        for test in sorted(self._available_tests.keys()):
            print("    {0}".format(test))

    def display_available_suites(self):
        print("Available test suites: ")
        for suite in self._available_suites:
            print("    {0} :".format(suite))
            for test in self._available_suites[suite].split():
                print("        {0}".format(test))

    def _read_config_file(self, config_file):
        """
        Read the configuration file.

        Sections : The config file will have known sections:
        "suites", "default-test-criteria".

        All other sections are assumed to be test names.
        """
        if config_file is None:
            raise RuntimeError("Error, must provide a config filename")
        self._config_filename = config_file
        config = config_parser()
        config.read(self._config_filename)

        if config.has_section("default-test-criteria"):
            self._default_test_criteria = \
                self._list_to_dict(config.items("default-test-criteria"))

        if config.has_section("suites"):
            self._available_suites = \
                self._list_to_dict(config.items("suites"))

        self._identify_tests(config)

    def _identify_tests(self, config):
        """
        Create a list of all tests in a config file.

        Assumes every section is a test except for some fixed section
        names

        """
        # section names are test names
        test_names = config.sections()

        # remove the fixed section names
        if config.has_section("default-test-criteria"):
            test_names.remove("default-test-criteria")
        if config.has_section("suites"):
            test_names.remove("suites")

        # all remaining sections should be individual tests
        for test in test_names:
            self._available_tests[test] = self._list_to_dict(config.items(test))
            self._available_tests[test]['name'] = test

    def _dict_to_string(self, data):
        """
        Format dictionary key-value pairs in a string
        """
        temp = ""
        for key, value in data.items():
            temp += "        {0} : {1}\n".format(key, value)
        return temp

    def _list_to_dict(self, input_list):
        """
        Convert a list of key-value pairs into a dictionary.
        """
        output_dict = {}
        for item in input_list:
            output_dict[item[0]] = item[1]
        return output_dict

    def _validate_suites(self):
        """
        Validates the suites defined in configuration file by
        checking that each test in a suite is one of the available
        tests.

        If the config file has an empty suite, we report that to the
        user then remove it from the list.
        """
        invalid_tests = []
        empty_suites = []
        for suite in self._available_suites:
            suite_tests = self._available_suites[suite].split()
            if len(suite_tests) == 0:
                empty_suites.append(suite)
            else:
                # validate the list
                for test in suite_tests:
                    if test not in self._available_tests:
                        name = "suite : '{0}' --> test : '{1}'".format(
                            suite, test)
                        invalid_tests.append(name)

        for suite in empty_suites:
            # empty suite, warn the user and remove it from the list
            del self._available_suites[suite]
            print("DEV WARNING : {0} : cfg validation : empty suite "
                  ": '{1}'".format(self._config_filename, suite))

        if len(invalid_tests) != 0:
            raise RuntimeError("ERROR : suites contain unknown tests in "
                               "configuration file '{0}' : {1}".format(
                                   self._config_filename, invalid_tests))

    def _validate_user_lists(self, user_suites, user_tests, testlog):
        """
        Check that the list of suites or tests passed from the command
        line are valid.
        """
        # if no suites or tests is specified, use all available tests
        if len(user_suites) == 0 and len(user_tests) == 0:
            u_suites = []
            u_tests = self._available_tests
        else:
            # check that the processed user supplied names are valid
            # convert user supplied names to lower case
            u_suites = []
            for suite in user_suites:
                if suite.lower() in self._available_suites:
                    u_suites.append(suite.lower())
                else:
                    message = self._txtwrap.fill(
                        "WARNING : {0} : Skipping requested suite '{1}' (not "
                        "present, misspelled or empty).".format(
                            self._config_filename, suite))
                    print(message, file=testlog)

            u_tests = []
            for test in user_tests:
                if test in self._available_tests:
                    u_tests.append(test.lower())
                else:
                    message = self._txtwrap.fill(
                        "WARNING : {0} : Skipping test '{1}' (not present or "
                        "misspelled).".format(self._config_filename, test))
                    print(message, file=testlog)

        return u_suites, u_tests

    def _create_tests(self, user_suites, user_tests, timeout, check_performance,
                      testlog):
        """
        Create the test objects for all user specified suites and tests.
        """
        all_tests = user_tests
        for suite in user_suites:
            for test in self._available_suites[suite].split():
                all_tests.append(test)

        for test in all_tests:
            try:
                new_test = RegressionTest()
                criteria = self._default_test_criteria.copy()
                new_test.setup(criteria,
                               self._available_tests[test], timeout,
                               check_performance, testlog)
                self._tests.append(new_test)
            except Exception as error:
                raise RuntimeError("ERROR : could not create test '{0}' from "
                                   "config file '{1}'. {2}".format(
                                       test, self._config_filename, str(error)))

def commandline_options():
    """
    Process the command line arguments and return them as a dict.
    """
    parser = argparse.ArgumentParser(description='Run an ATS regression '
                                     'tests or suite of tests.')

    parser.add_argument('--backtrace', action='store_true',
                        help='show exception backtraces as extra debugging '
                        'output')

    parser.add_argument('--check-only', action='store_true', default=False,
                        help="diff the existing regression files without "
                        "running ATS again.")

    parser.add_argument('--check-performance', action='store_true', default=False,
                        help="include the performance metrics ('SOLUTION' blocks) "
                        "in regression checks.")

    parser.add_argument('--debug', action='store_true',
                        help='extra debugging output')

    parser.add_argument('-d', '--dry-run',
                        default=False, action='store_true',
                        help='perform a dry run, setup the test commands but '
                        'don\'t run them')

    parser.add_argument('-e', '--executable', nargs=1, default=None,
                        help='path to executable to use for testing')

    parser.add_argument('--list-suites', default=False, action='store_true',
                        help='print the list of test suites from the config '
                        'file and exit')

    parser.add_argument('--list-tests', default=False, action='store_true',
                        help='print the list of tests from the config file '
                        'and exit')

    parser.add_argument('-m', '--mpiexec', nargs=1, default=None,
                        help="path to the executable for mpiexec (mpirun, etc)"
                        "on the current machine.")

    parser.add_argument('-n', '--new-tests',
                        action="store_true", default=False,
                        help="indicate that there are new tests being run. "
                        "Skips the output check and creates a new gold file.")

    parser.add_argument('-s', '--suites', nargs="+", default=[],
                        help='space separated list of test suite names')

    parser.add_argument('-t', '--tests', nargs="+", default=[],
                        help='space separated list of test names')

    parser.add_argument('--timeout', nargs=1, default=None,
                        help="test timeout (for assuming a job has hung and "
                        "needs to be killed)")

    parser.add_argument('-u', '--update',
                        action="store_true", default=False,
                        help='update the tests listed by the "--tests" '
                        'option, with the current output becoming the new '
                        'gold standard')

    parser.add_argument('configs', metavar='CONFIG_LOCATION', type=str,
                        nargs='+', help='list of directories and/or configuration '
                        'files to parse for suites and tests')

    options = parser.parse_args()
    return options

def config_list_includes_search(options):
    """
    Check if there are any directories in the config list.
    """
    for f in options.configs:
        if (os.path.isdir(f)):
            return True
    return False

def generate_config_file_list(options):
    """
    Try to generate a list of configuration files from the commandline
    options.
    """
    config_file_list = []
    # loop through the list, adding files and searching through directories
    for f in options.configs:
        if not os.path.isabs(f):
            f = os.path.abspath(f)

        if os.path.isfile(f):
            config_file_list.append(f)
        elif os.path.isdir(f):
            search_for_config_files(f, config_file_list)
        else:
            raise RuntimeError("ERROR: specified config file/search "
                               "directory '{0}' does not "
                               "exist!".format(f))

    if options.debug:
        print("\nFound config files:")
        for config_file in config_file_list:
            print("    {0}".format(config_file))

    if len(config_file_list) == 0:
        raise RuntimeError("ERROR: no config files were found. Please specify a "
                           "config file or search directory containing config files.")

    return config_file_list

def search_for_config_files(base_dir, config_file_list):
    """
    recursively search the directory tree, creating a list of all config files
    """
    for root, dirnames, filenames in os.walk(base_dir):
        for filename in filenames:
            if filename.endswith(".cfg") and os.path.isfile(os.path.join(root, filename)):
                config_file_list.append(os.path.join(root, filename))

def check_options(options):
    """
    Run some sanity checks on the commandline options.
    """
    # prevent the user from updating regression output during a
    # recursive search for config files
    if options.update and config_list_includes_search(options):
        raise RuntimeError("ERROR: cannot update gold regression files "
                           "during a recursive search for config files.")

    if options.update and options.new_tests:
        raise RuntimeError("ERROR: cannot create new tests and update gold "
                           "regression files at the same time.")

def check_for_executable(options, testlog):
    """
    Try to verify that we have something reasonable for the executable
    """
    # check the executable
    if options.executable is None:
        # try to detect from env
        try:
            executable = distutils.spawn.find_executable("ats")
        except Exception:
            executable = None
        finally:
            if executable is None:
                options.dry_run = True

    else:
        # absolute path to the executable
        executable = os.path.abspath(options.executable[0])
        # is it a valid file?
        if not os.path.isfile(executable):
            raise RuntimeError("ERROR: executable is not a valid file: "
                               "'{0}'".format(executable))

    if executable is None:
        message = ("\n** WARNING ** : ATS executable was not provided on the command line\n"
                   "               and was not found in the PATH.  Will run as 'dry run.'\n")
        print(message, file=sys.stdout)
        print(message, file=testlog)
        executable = "/usr/bin/false"

    return executable

def check_for_mpiexec(options, testlog):
    """
    Try to verify that we have something reasonable for the mpiexec executable

    Notes:

    geh: need to add code to determine full path of mpiexec if not specified

    bja: the problem is that we don't know how to get the correct
    mpiexec. On the mac, there is a system mpiexe that shows up in the
    path, but this is provided by apple and it is not the correct one
    to use because it doesn't include a fortran compiler. We need the
    exact mpiexec/mpirun that was used to compile petsc, which may
    come from a system installed package in /usr/bin or /opt/local/bin
    or maybe petsc compiled it. This is best handled outside the test
    manager, e.g. use make to identify mpiexec from the petsc
    variables
    """

    # check for mpiexec
    mpiexec = None
    if options.mpiexec is not None:
        # mpiexec = os.path.abspath(options.mpiexec[0])
        mpiexec = options.mpiexec[0]
        # try to log some info about mpiexec
        print("MPI information :", file=testlog)
        print("-----------------", file=testlog)
        tempfile = "{0}/tmp-ats-regression-test-mpi-info.txt".format(os.getcwd())
        command = [mpiexec, "--version"]
        append_command_to_log(command, testlog, tempfile)
        print("\n\n", file=testlog)
    else:
        message = ("\n** WARNING ** : mpiexec was not provided on the command line.\n"
                   "                All parallel tests will be skipped!\n")
        print(message, file=sys.stdout)
        print(message, file=testlog)

    return mpiexec

def summary_report_by_file(report, outfile):
    """
    Summarize the results for each config file.
    """
    print(70 * '-', file=outfile)
    print("Regression test file summary:", file=outfile)
    for filename in report:
        line = "    {0}... {1} tests : ".format(filename, report[filename].test_count)
        if report[filename].warning > 0:
            line = "{0} {1} test warnings, ".format(line, report[filename].warning)
        if report[filename].error > 0:
            line = "{0} {1} test errors, ".format(line, report[filename].error)

        if report[filename].test_count == 0:
            line = "{0}... no tests were run.".format(line)
        else:
            if report[filename].fail > 0:
                line = "{0} {1} tests failed, ".format(line, report[filename].fail)
            if report[filename].skipped > 0:
                line = "{0} {1} tests skipped, ".format(line, report[filename].skipped)
            if report[filename].fail == 0 and report[filename].skipped == 0:
                line = "{0} all tests passed".format(line)
            else:
                num_passed = (report[filename].test_count - report[filename].fail -
                              report[filename].skipped)
                line = "{0} {1} passed.".format(line, num_passed)

        print("{0}".format(line), file=outfile)

    print("\n", file=outfile)

def summary_report(run_time, report, outfile):
    """
    Overall summary of test results
    """
    print(70 * '-', file=outfile)
    print("Regression test summary:", file=outfile)
    print("    Total run time: {0:4g} [s]".format(run_time), file=outfile)
    test_count = 0
    num_failures = 0
    num_errors = 0
    num_warnings = 0
    num_skipped = 0
    for filename in report:
        test_count += report[filename].test_count
        num_failures += report[filename].fail
        num_errors += report[filename].error
        num_warnings += report[filename].warning
        num_skipped += report[filename].skipped

    print("    Total tests : {0}".format(test_count), file=outfile)

    if num_skipped > 0:
        print("    Skipped : {0}".format(num_skipped), file=outfile)
        success = False

    print("    Tests run : {0}".format(test_count - num_skipped), file=outfile)

    success = True
    if num_failures > 0:
        print("    Failed : {0}".format(num_failures), file=outfile)
        success = False

    if num_errors > 0:
        print("    Errors : {0}".format(num_errors), file=outfile)
        success = False

    if num_warnings > 0:
        print("    Warnings : {0}".format(num_warnings), file=outfile)
        success = False

    if success:
        print("    All tests passed.", file=outfile)

    print("\n", file=outfile)
    return num_failures

def append_command_to_log(command, testlog, tempfile):
    """
    Append the results of a shell command to the test log
    """
    print("$ {0}".format(" ".join(command)), file=testlog)
    testlog.flush()
    with open(tempfile, "w") as tempinfo:
        subprocess.call(command, shell=False,
                        stdout=tempinfo,
                        stderr=subprocess.STDOUT)
        # NOTE(bja) 2013-06 : need a short sleep to ensure the
        # contents get written...?
        time.sleep(0.01)
    with open(tempfile, 'r') as tempinfo:
        shutil.copyfileobj(tempinfo, testlog)
    os.remove(tempfile)

def setup_testlog(txtwrap):
    """
    Create the test log and try to add some useful information about
    the environment.
    """
    now = datetime.datetime.today().strftime("%Y-%m-%d_%H-%M-%S")
    filename = os.path.join("LOGS", "ats-tests-{0}.testlog".format(now))
    testlog = open(filename, 'w')
    print("  Test log file : {0}".format(filename))

    print("ATS Regression Test Log", file=testlog)
    print("Date : {0}".format(now), file=testlog)
    print("System Info :", file=testlog)
    print("    platform : {0}".format(sys.platform), file=testlog)
    test_dir = os.getcwd()
    print("Test directory : ", file=testlog)
    print("    {0}".format(test_dir), file=testlog)

    if os.environ.has_key("ATS_SRC_DIR"):
        tempfile = "{0}/tmp-ats-regression-test-hg-info.txt".format(test_dir)

        print("\nATS repository status :", file=testlog)
        print("----------------------------", file=testlog)
        if os.path.isdir("{0}/.hg".format(os.environ["ATS_SRC_DIR"])):
            cmd = ["hg", "parent"]
            append_command_to_log(cmd, testlog, tempfile)
            cmd = ["hg", "status", "-q"]
            append_command_to_log(cmd, testlog, tempfile)
            print("\n\n", file=testlog)
        else:
            print("    unknown", file=testlog)

    return testlog

def main(options):
    txtwrap = textwrap.TextWrapper(width=78, subsequent_indent=4*" ")
    testlog = setup_testlog(txtwrap)

    root_dir = os.getcwd()

    check_options(options)
    executable = check_for_executable(options, testlog)
    mpiexec = check_for_mpiexec(options, testlog)
    config_file_list = generate_config_file_list(options)

    print("Running ATS regression tests :")

    # loop through config files, cd into the appropriate directory,
    # read the appropriate config file and run the various tests.
    start = time.time()
    report = {}
    for config_file in config_file_list:
        try:
            # NOTE(bja): the try block is inside this loop so that if
            # a single test throws an exception in a large batch of
            # tests, we can recover and at least try running the other
            # config files.
            print(80 * '=', file=testlog)

            # get the absolute path of the directory
            test_dir = os.path.dirname(config_file)
            # cd into the test directory so that the relative paths in
            # test files are correct
            os.chdir(test_dir)
            if options.debug:
                print("Changed to working directory: {0}".format(test_dir))

            test_manager = RegressionTestManager()

            if options.debug:
                test_manager.debug(True)

            # get the relative file name
            filename = os.path.basename(config_file)

            test_manager.generate_tests(filename,
                                        options.suites,
                                        options.tests,
                                        options.timeout,
                                        options.check_performance,
                                        testlog)

            if options.debug:
                print(70 * '-')
                print(test_manager)

            if options.list_suites:
                test_manager.display_available_suites()

            if options.list_tests:
                test_manager.display_available_tests()

            test_manager.run_tests(mpiexec,
                                   executable,
                                   options.dry_run,
                                   options.update,
                                   options.new_tests,
                                   options.check_only,
                                   testlog)

            report[filename] = test_manager.run_status()
            os.chdir(root_dir)
        except Exception as error:
            message = txtwrap.fill(
                "ERROR: a problem occured in file '{0}'.  This is "
                "probably an error with commandline options, the "
                "configuration file, or an internal error.  The "
                "error is:\n{1}".format(config_file, str(error)))
            print(''.join(['\n', message, '\n']), file=testlog)
            if options.backtrace:
                traceback.print_exc()
            print('F', end='', file=sys.stdout)
            report[filename] = TestStatus()
            report[filename].fail = 1

    stop = time.time()
    status = 0
    if not options.dry_run and not options.update:
        print("")
        run_time = stop - start
        summary_report_by_file(report, testlog)
        summary_report(run_time, report, testlog)
        status = summary_report(run_time, report, sys.stdout)

    if options.update:
        message = txtwrap.fill(
            "Test results were updated! Please document why you modified the "
            "gold standard test results in your revision control commit message!\n")
        print(''.join(['\n', message, '\n']))

    testlog.close()

    return status

if __name__ == "__main__":
    cmdl_options = commandline_options()
    try:
        suite_status = main(cmdl_options)
        sys.exit(suite_status)
    except Exception as error:
        print(str(error))
        if cmdl_options.backtrace:
            traceback.print_exc()
        sys.exit(1)


import os
import json
import datetime
from openpyxl import Workbook
from openpyxl.styles import Font
from openpyxl.utils import get_column_letter

def adjust_column_widths(sheet):
    for column in sheet.columns:
        max_length = max(len(str(cell.value)) for cell in column)
        column_letter = get_column_letter(column[0].column)
        adjusted_width = (max_length + 2) * 1.2
        sheet.column_dimensions[column_letter].width = adjusted_width

def format_datetime(datetime_str):
    datetime_obj = datetime.datetime.strptime(datetime_str, "%Y-%m-%dT%H:%M:%S")
    formatted_datetime = datetime_obj.strftime("%d.%m.%y-%H:%M")
    return formatted_datetime

def replace_shop_name(shop_name):
    replace_file_path = os.path.join("sys", "shopnamereplace.json")

    # Load data from shopnamereplace.json for shop name replacements
    with open(replace_file_path, "r", encoding="utf-8") as replace_file:
        replace_data = json.load(replace_file)

    # Replace shop name if there is a matching entry in shopnamereplace.json
    for entry in replace_data:
        if entry.get("original_name") == shop_name:
            return entry.get("short_name")

    return shop_name

def create_export_file(data):
    # Path to the output directory
    output_dir = "output/"

    # Load excluded shop names
    excluded_file_path = os.path.join("sys", "excluded.json")
    excluded_shops = []
    if os.path.exists(excluded_file_path):
        with open(excluded_file_path, "r", encoding="utf-8") as excluded_file:
            excluded_shops = json.load(excluded_file)

    # Create a new Excel workbook for export
    export_workbook = Workbook()

    # Remove the default "Sheet" that is automatically created
    default_sheet = export_workbook["Sheet"]
    export_workbook.remove(default_sheet)

    # Check if there is any data available
    if not data:
        # Create a single sheet named "No data" if there is no data
        export_sheet = export_workbook.active
        export_sheet.title = "No data"
    else:
        # Group the data by month and day
        data_by_month = {}
        for item in data:
            receipt = item.get("ticket", {}).get("document", {}).get("receipt", {})
            datetime_str = receipt.get("dateTime")
            shop_name = replace_shop_name(receipt.get("user"))
            if datetime_str and shop_name not in excluded_shops:
                datetime_obj = datetime.datetime.strptime(datetime_str, "%Y-%m-%dT%H:%M:%S")
                month = datetime_obj.strftime("%Y-%m")
                day = datetime_obj.date()
                if month not in data_by_month:
                    data_by_month[month] = {}
                if day not in data_by_month[month]:
                    data_by_month[month][day] = []
                data_by_month[month][day].append(item)

        # Create separate sheets for each month and add data for each day
        for month, month_data in data_by_month.items():
            month_sheet = export_workbook.create_sheet(title=month)

            for day, day_data in month_data.items():
                export_sheet = month_sheet

                # Add a separator row if it's not the first day
                if day != min(month_data.keys()):
                    export_sheet.append([])

                # Add a header row
                export_sheet.append(["Date/Time", "Shop", "Product", "Price", "Quantity", "Sum"])
                export_sheet.freeze_panes = "A2"

                # Apply bold font to the header row
                export_bold_font = Font(bold=True)
                for cell in export_sheet[1]:
                    cell.font = export_bold_font

                # Export data to the export sheet
                sum_total = 0
                for item in day_data:
                    receipt = item.get("ticket", {}).get("document", {}).get("receipt", {})
                    shop_name = replace_shop_name(receipt.get("user"))
                    for product in receipt.get("items", []):
                        row = [
                            format_datetime(receipt.get("dateTime")), shop_name, product.get("name"),
                            product.get("price") / 100, product.get("quantity"), product.get("sum") / 100
                        ]
                        export_sheet.append(row)
                        sum_total += product.get("sum") / 100

                # Add a row with the total sum for the day
                export_sheet.append([])  # Add an empty row
                total_row = ["Total", "", "", "", "", sum_total]
                export_sheet.append(total_row)

                # Apply bold font to the total row
                for row in export_sheet.iter_rows(max_row=export_sheet.max_row, max_col=export_sheet.max_column):
                    for cell in row:
                        if cell.row == export_sheet.max_row:
                            cell.font = export_bold_font

            # Adjust column widths for the month sheet
            adjust_column_widths(month_sheet)

    # Save the export workbook as export.xlsx
    export_file = os.path.join(output_dir, "export.xlsx")
    export_workbook.save(export_file)

    print(f"Export file {export_file} created successfully.")

# Path to the input directory
input_dir = "input/"

# Get a list of JSON files in the directory
json_files = [file for file in os.listdir(input_dir) if file.endswith(".json")]

data = []  # Store the data from all JSON files

for json_file in json_files:
    # Load data from the JSON file
    with open(os.path.join(input_dir, json_file), "r", encoding="utf-8") as file:
        data.extend(json.load(file))

# Create the export file with separate sheets for each month and add separators between days
create_export_file(data)


from PyQt5.QtCore import pyqtSignal, Qt
from PyQt5.QtWidgets import QWidget, QLabel, QGridLayout, QTextBrowser,\
        QFrame, QPushButton, QSizePolicy
from PyQt5.QtGui import QPixmap, QIcon, QImage, QPainter, QDesktopServices
from gui.media_display import MediaWidget

class StatusWidget(QWidget):
    delete_tweets = pyqtSignal(str)

    def __init__(self, tweet, cache):
        super().__init__()

        self.tid = tweet.tid
        self.tweet = tweet
        self.cache = cache

        self.initUI(tweet)

    def getPix(self, url1, url2=None):
        img1 = self.cache.get_resource(url1)
        i1 = QImage.fromData(img1)
        i1 = i1.scaledToHeight(60, Qt.SmoothTransformation)
        if url2 is not None:
            img2 = self.cache.get_resource(url2)
            i2 = QImage.fromData(img2)
            i2 = i2.scaledToHeight(30, Qt.SmoothTransformation)
            p = QPainter()
            p.begin(i1)
            p.drawImage(0, 30, i2)
            p.end()

        ret = QLabel()
        ret.setPixmap(QPixmap.fromImage(i1))
        return ret

    def choose_video(self, vids):
        chosen = vids['expanded_url']
        max_br = -1
        for i in vids['video_info']['variants']:
            try:
                if i['bitrate'] > max_br:
                    max_br = i['bitrate']
                    chosen = i['url']
            except:
                pass
        return(chosen)

    def process_text(self):
        html_text = self.tweet.text
        pretty_text = self.tweet.text

        html_text = html_text.replace("\n", "
")
        html_text = html_text.replace("

", "
")

        for i, j in self.tweet.ent['pic']:
            html_text = html_text.replace(i, "")
            html_text += " Pic".format(j)
            pretty_text = pretty_text.replace(i, '')
            pretty_text += ' Pic'
        for i, j in self.tweet.ent['gif']:
            html_text = html_text.replace(i, "")
            html_text += " Gif".format(j)
            pretty_text = pretty_text.replace(i, '')
            pretty_text += ' Gif'
        for i, j in self.tweet.ent['vid']:
            html_text = html_text.replace(i, '')
            html_text += " Vid".format(j)
            pretty_text = pretty_text.replace(i, "")
            pretty_text += ' Vid'
        for i, j, k in self.tweet.ent['url']:
            to_rep = self.tweet.text[i[0]:i[1]]
            html_text = html_text.replace(to_rep, "\
                    {1}".format(k, j))
            pretty_text = pretty_text.replace(to_rep, j)

        self.text = html_text

        splitted = pretty_text.split("\n")
        nb = len(splitted)
        for i in splitted:
            nb += int(len(i)/72)
            if len(i) == 0:
                nb -= 1

        return nb

    def add_pic(self):
            if not self.tweet.rt:
                pic = self.getPix(self.tweet.ent['profile'][0])
            else:
                pic = self.getPix(self.tweet.ent['profile'][0],
                                  self.tweet.ent['profile'][1])

            self.lay.addWidget(pic, 0, 0, 2, 1)

    def add_time(self):
        time = self.tweet.time
        string = time.strftime("%d/%m - %H:%M")

        self.lay.addWidget(QLabel(string), 0, 3)

    def add_text(self):
        self.process_text()
        text = QTextBrowser()
        text.setHtml(self.text)
        text.setOpenLinks(False)
        # text.moveCursor(QTextCursor.End)
        text.setMinimumWidth(500)
        text.setMaximumWidth(500)
        text.document().setTextWidth(500)
        text.setMinimumHeight((text.document().size().height())+5)
        text.setSizePolicy(QSizePolicy.Preferred, QSizePolicy.Expanding)

        text.anchorClicked.connect(self.link_clicked)

        self.lay.addWidget(text, 1, 1, 1, 3)

    def add_username(self):
        name = "" + self.tweet.username + " @" +\
                self.tweet.screen_name + ""

        self.lay.addWidget(QLabel(name), 0, 1)

    def add_button(self):
        icon = QIcon.fromTheme("edit-delete")
        self.button = QPushButton(icon, "")
        self.button.pressed.connect(self.send_delete)

        self.lay.addWidget(self.button, 1, 4)

    def send_delete(self):
        self.delete_tweets.emit(str(self.tweet.tid))

    def link_clicked(self, url):
        sc = url.scheme()
        if sc == "pic" or sc == "gif" or sc == "vid":
            self.media = MediaWidget(url.query(), sc, self.cache)
        else:
            QDesktopServices.openUrl(url)

    def initUI(self, tweet):
        layout = QGridLayout()
        self.setLayout(layout)
        self.lay = layout

        layout.setColumnStretch(10, 100)

        self.add_pic()
        self.add_time()
        self.add_text()
        self.add_username()
        self.add_button()

        line = QFrame()
        line.setFrameShape(QFrame.HLine)
        line.setFrameShadow(QFrame.Sunken)
        layout.addWidget(line, 4, 0, 1, -1)

        self.setLayout(layout)

        layout.setSpacing(5)
        layout.setContentsMargins(0, 0, 0, 0)


from scipy import optimize
import pulp
import numpy as np
#简单的题目一维变量
#法一:调用scipy库中optimize模块的linprog函数
#（约束条件无等号）函数默认取最小值
c = np.array([4, -1])  #目标函数
A = np.array([[-1, 1], [-1, -1]])#限制条件1/2 且符号为<=
b = np.array([5, 0])#<=右侧条件
x = (None, 3) #范围
y = (None, None)
res1 = optimize.linprog(c, A, b, bounds=(x, y))

#约束条件带等号
c = np.array([2,3,-5])
A = np.array([[-2,5,-1],[1,3,1]])
b = np.array([-10,12])
Aeq = np.array([[1,1,1]])
beq = np.array([7])
x1 = (0,None)
x2 = (0,None)
x3 = (0,None)
res2 = optimize.linprog(-c,A,b,Aeq,beq,bounds=(x1,x2,x3))#-c最大值

#法二调用pulp库
def LPproblem():
    Myproblem = pulp.LpProblem(sense=pulp.LpMaximize)
    #定义变量
    x1 = pulp.LpVariable('x1', lowBound=0, upBound=None, cat='Continuous')
    x2 = pulp.LpVariable('x2', lowBound=0, upBound=None, cat='Continuous')#带绝对值的话就改参数范围https://blog.csdn.net/HsinglukLiu/article/details/123123292
    x3 = pulp.LpVariable('x3', lowBound=0, upBound=None, cat='Continuous')
    #目标函数及约束条件
    Myproblem += 2 * x1 + 3 * x2 - 5 * x3
    Myproblem += (x1 + x2 + x3 == 7)
    Myproblem += (2 * x1 - 5 * x2 + x3 >= 10)
    Myproblem += (x1 + 3 * x2 + x3 <= 12)
    Myproblem.solve()
    for i in Myproblem.variables():
        print(i.name, '=', i.varValue)  #Xi的值
    Fx = pulp.value(Myproblem.objective) #函数表达式
    return Fx,x1,x2,x3

#二维变量
#产销问题，运费最少的线性规划问题
import gurobipy as gb
production, a = gb.multidict({'A1': 7, 'A2': 4, 'A3': 9})  # 产地
sales, b = gb.multidict({'B1': 3, 'B2': 6, 'B3': 5, 'B4': 6})  # 销地
route, cost = gb.multidict({
    ('A1', 'B1'): 3,
    ('A1', 'B2'): 11,
    ('A1', 'B3'): 3,
    ('A1', 'B4'): 10,
    ('A2', 'B1'): 1,
    ('A2', 'B2'): 9,
    ('A2', 'B3'): 2,
    ('A2', 'B4'): 8,
    ('A3', 'B1'): 7,
    ('A3', 'B2'): 4,
    ('A3', 'B3'): 10,
    ('A3', 'B4'): 5})  # 单位运价

m = gb.Model("lp4")  # 构建模型
x = {}
for i, j in route:
    x[i, j] = m.addVar(vtype=gb.GRB.INTEGER)  # 构建决策变量x为整数

obj = gb.quicksum(x[i, j] * cost[i, j] for i, j in route)  # 目标函数
m.setObjective(obj)  # 默认最小化. 最大化则添加参数 gb.GRB.MAXIMIZE

x = gb.tupledict(x)
m.addConstrs((x.sum("*", j) == b[j] for j in sales), name="con1")  # 添加约束条件1
m.addConstrs((x.sum(i, "*") == a[i]for i in production), name="con2")  # 添加约束条件2
m.write("lp4.lp")
m.optimize()  # 求解
for v in m.getVars(): #输出
    print("%s %g" % (v.varName, v.x))
print("Obj: %g" % m.objVal)

#匈牙利算法，指派问题（0，1分配）
#高阶玩法：考虑多个代价矩阵，考虑相关性https://blog.csdn.net/weixin_42353399/article/details/103450457
from scipy.optimize import linear_sum_assignment

cost = np.array([[4, 1, 3], [2, 0, 5], [3, 2, 2]])
row_ind, col_ind = linear_sum_assignment(cost)
print(row_ind)  # 开销矩阵对应的行索引
print(col_ind)  # 对应行索引的最优指派的列索引
print(cost[row_ind, col_ind])  # 提取每个行索引的最优指派列索引所在的元素，形成数组
print(cost[row_ind, col_ind].sum())  # 数组求和
#分段函数的整数规划??


import re

from sqlalchemy import Column, String, Float, PrimaryKeyConstraint, ForeignKeyConstraint
from sqlalchemy.orm import relationship

from fr.tagc.rainet.core.util.sql.Base import Base

from fr.tagc.rainet.core.util.sql.SQLManager import SQLManager
from fr.tagc.rainet.core.util.exception.RainetException import RainetException
from fr.tagc.rainet.core.data import DataConstants
from fr.tagc.rainet.core.util.pattern.PatternUtil import PatternUtil
from fr.tagc.rainet.core.util.sql.SQLUtil import SQLUtil
from fr.tagc.rainet.core.util.log.Logger import Logger
from fr.tagc.rainet.core.util.exception.NotRequiredInstantiationException import NotRequiredInstantiationException
from fr.tagc.rainet.core.data.ProteinCrossReference import ProteinCrossReference

# #
# This class describes a Protein to Protein interaction
#
class ProteinInteraction( Base ):
    __tablename__ = 'ProteinInteraction'

    # The first interactor
    interactorA_id = Column( String )
    interactorA = relationship( 'Protein', foreign_keys = interactorA_id )
    # The first interactor
    interactorB_id = Column( String )
    interactorB = relationship( 'Protein', foreign_keys = interactorB_id )
    # The method used to detect the interaction
    detectionMethod = Column( String )
    # The pubmedID of the publication reporting the interaction
    pubmedID = Column( String )
    # The type of the interaction
    interactionType = Column( String )
    # The database reporting where the interaction was found
    sourceDB = Column( String )
    # The confidence score associated to the interaction
    confidenceScore = Column( Float )
    # Define the Composite PrimaryKey and ForeignKeys
    __table_args__ = (
        PrimaryKeyConstraint( 'interactorA_id', 'interactorB_id', 'detectionMethod', 'pubmedID' ),
        ForeignKeyConstraint( ['interactorA_id', 'interactorB_id'],
                            ['Protein.uniprotAC', 'Protein.uniprotAC'] )
                            , )

    # #
    # The constructor
    #
    def __init__( self, id_A, id_B, alt_id_A, alt_id_B, method, pubmed_string, interaction_type, source_db, score_string ):

        Logger.get_instance().debug( "\nSearching proteins" )
        protein_A = self.find_protein( [id_A, alt_id_A] )
        Logger.get_instance().debug( "|--proteinA found = " + str( protein_A) )

        protein_B = self.find_protein( [id_B, alt_id_B] )
        Logger.get_instance().debug( "|--proteinB found = " + str( protein_B ) )

        # If both interacting proteins have been found
        if protein_A != None and protein_B != None:
            self.pubmedID = PatternUtil.find_single_group_in_string( DataConstants.INTERACTOME_PUBMED_REGEX, pubmed_string.lower(), False )
#             self.detectionMethod = PatternUtil.find_single_group_in_string( DataConstants.INTERACTOME_METHOD_REGEX, method, False )
#             self.interactionType = PatternUtil.find_single_group_in_string( DataConstants.INTERACTOME_TYPE_REGEX, interaction_type, False )
#             self.sourceDB = PatternUtil.find_single_group_in_string( DataConstants.INTERACTOME_SOURCEDB_REGEX, source_db, False )
            self.detectionMethod= method
            self.interactionType = interaction_type
            self.sourceDB = source_db
            self.confidenceScore = self.get_score( score_string )
            Logger.get_instance().debug( "--->score is " + str( self.confidenceScore))

            # Test if a similar interaction already exists (may be possible due to isoform of proteins)
            sql_session = SQLManager.get_instance().get_session()
            if self.search_similar_interaction( protein_A, protein_B, sql_session ) != None:
                raise NotRequiredInstantiationException( "ProteinInteraction: A similar interaction exists in database." )

            # Add interaction to interacting proteins
            self.interactorA = protein_A
            self.interactorB = protein_B
            sql_session.add( protein_A )
            sql_session.add( protein_B )
        else:
            raise NotRequiredInstantiationException( "ProteinInteraction: At least one interacting protein was not found in database." )

    # #
    # Search for a protein ID (uniprotAC or other) in the ids of the provided id_list
    #
    def find_protein( self, id_list ):

        #=======================================================================
        # Search for a UniprotAC in the provided IDs
        #=======================================================================
        uniprotkb_pattern = re.compile( DataConstants.INTERACTOME_ID_UNIPROTKB_REGEX )
        uniprot_ac_list = PatternUtil.find_groups_in_list( uniprotkb_pattern, id_list )
        # -- If the uniprotAC contains a "-", it is a isoform uniprotAC so only the base
        # of the AC must be kept

        # Search for other ID type using also cross references if no uniprot AC was found before
        if uniprot_ac_list == None or len ( uniprot_ac_list ) == 0:

            Logger.get_instance().debug( "|--Find_proteins :Trying crossrefences ---" )
            # Initialize uniprot_ac_list
            uniprot_ac_list = []

            # Get a SQLalchemy session
            sql_session = SQLManager.get_instance().get_session()

            # Parse the cross reference regular expression to find matching ID that will be
            # converted to uniprotAC thanks to cross references present in database
            for crossref_source, crossref_pattern in DataConstants.INTERACTOME_ID_CROSSREF_REGEX_DICT.items():
                Logger.get_instance().debug( "|--|--Cross reference = " + crossref_source )
                # Compile the regular expression
                crossref_compiled_pattern = re.compile( crossref_pattern )
                # Search matching string (if any) in the provided strings
                matched_crossref_list = PatternUtil.find_groups_in_list( crossref_compiled_pattern, id_list )
                Logger.get_instance().debug( "|--|--|--Cross reference list found = " + str( matched_crossref_list ) )
                # If matching string exists, look in database if corresponding crossreference is found
                # If so, add the equivalent uniprotAC to the final list.
                if matched_crossref_list != None and len( matched_crossref_list ) > 0:
                    for matched_crossref in matched_crossref_list:
                        db_regex = DataConstants.INTERACTOME_ID_CROSSREF_DB_REGEX_DICT[ crossref_source].replace( "VALUE", matched_crossref )
                        uniprot_ac = sql_session.query( ProteinCrossReference.protein_id ).filter( ProteinCrossReference.sourceDB == crossref_source, ProteinCrossReference.crossReferenceID.like( db_regex ) ).all()
                        if uniprot_ac != None and len( uniprot_ac) >= 1:
                            uniprot_ac_list.extend( uniprot_ac[0] )

        Logger.get_instance().debug( "|--Found unitproAC= " + str( uniprot_ac_list ) )

        #=======================================================================
        # Return the protein with the found uniprot_ac
        #=======================================================================
        try:
            return SQLUtil.find_unique_protein_from_uniprotAC_list( uniprot_ac_list )
        except RainetException as e:
            Logger.get_instance().debug( e.to_string() )
            return None

    # #
    # Retrieve the interaction score in the given string
    #
    # @param score_string : string - The string containing the score like "intact-miscore:0.96"
    #
    # @return float - The interaction score or -1 if an issue occurred
    #
    def get_score( self, score_string):

        compiled_pattern = re.compile( DataConstants.INTERACTOME_SCORE_REGEX )
        score = PatternUtil.find_groups_in_string( compiled_pattern, score_string )

        Logger.get_instance().debug( "|--|--|-score groups = " + str( score ) )
        if score == None:
            return -1
        elif len( score) >= 1:
            score_number = score[0]
        else:
            return -1

        try:
            return float( score_number )
        except ValueError:
            Logger.get_instance().warning( "ProteinInteraction.get_score : score is not a number : " + score_number )
            return -1

    ##
    # Search for a ProteinInteraction between protein_a and protein_b with detectionMethod and PubmedID
    # equals to those inserted in the current ProteinInteraction. The search is done for protein in
    # both directions.
    #
    # @param protein_a : Protein - The first protein
    # @param protein_b : Protein - The second protein
    # @param sql_session : SQLSession - The current SQLAlchemy session
    def search_similar_interaction( self, protein_a, protein_b, sql_session ):

        # Look at an interaction from A to B
        a_to_b_list = sql_session.query( ProteinInteraction ).filter( ProteinInteraction.interactorA == protein_a,
                                                       ProteinInteraction.interactorB == protein_b,
                                                       ProteinInteraction.detectionMethod == self.detectionMethod,
                                                       ProteinInteraction.pubmedID == self.pubmedID ).all()

        # Check the result: if a single interaction is found, return it. If several are found, raise an exception
        if a_to_b_list != None and len( a_to_b_list ) > 0:
            if len( a_to_b_list ) == 1:
                return a_to_b_list[0]
            else:
                raise RainetException( "ProteinInteraction.__init__ : Found several protein interaction for interactor A = " + protein_a.uniprotAC +
                                      " /interactor B = " + protein_b.uniprotAC + " /detection method = " + self.detectionMethod + " /pubmedID = " + self.pubmedID )

        # Look at an interaction from B to A
        b_to_a_list = sql_session.query( ProteinInteraction ).filter( ProteinInteraction.interactorA == protein_b,
                                                       ProteinInteraction.interactorB == protein_a,
                                                       ProteinInteraction.detectionMethod == self.detectionMethod,
                                                       ProteinInteraction.pubmedID == self.pubmedID ).all()

        # Check the result: if a single interaction is found, return it. If several are found, raise an exception
        if b_to_a_list != None and len( b_to_a_list ) > 0:
            if len( b_to_a_list ) == 1:
                return b_to_a_list[0]
            else:
                raise RainetException( "ProteinInteraction.__init__ : Found several protein interaction for interactor A = " + protein_b.uniprotAC +
                                      " /interactor B = " + protein_a.uniprotAC + " /detection method = " + self.detectionMethod + " /pubmedID = " + self.pubmedID )

        return None

    # #
    # Add the object to SQLAlchemy session
    def add_to_session( self ):

        sql_session = SQLManager.get_instance().get_session()
        sql_session.add( self )


import os.path
import shutil
import sysconfig
from pathlib import Path
import time

from setuptools import setup
from setuptools.command.install import install
from setuptools.command.sdist import sdist
from wheel.bdist_wheel import bdist_wheel

home_dir = Path.home()
install_dir = sysconfig.get_paths()['purelib']
print(home_dir)
print(install_dir)

current = Path(__file__).parent.absolute()

VERSION = "0.0.5"

def generate_sumake(home_dir, current):
    sumake_dir = home_dir / ".sumake"
    bin_dir = current / "bin"

    utils_mk = sumake_dir / "utils.mk"

    if not os.path.exists(sumake_dir):
        os.makedirs(sumake_dir)
    if not os.path.exists(bin_dir):
        os.makedirs(bin_dir)
    print("current", current)

    shutil.copy(current / "utils.mk", utils_mk)
    # open("./sumake")
    with open(current / "./sumake", "r") as template:
        with open(bin_dir / "sumake", "w") as target:
            res = template.read().format(
                UTILS_MK=utils_mk,
                VERSION=VERSION, )
            target.write(res)

def generate_version():
    with open(current / "version.txt", "w") as f:
        f.write(VERSION)
        # f.write(f"""VERSION = "{VERSION}" """)

class PreBdistWheel(bdist_wheel):
    def run(self):
        generate_sumake(home_dir, current)
        bdist_wheel.run(self)

class PreSDist(sdist):
    def run(self):
        generate_sumake(home_dir, current)
        sdist.run(self)

class PreInstall(install):
    def run(self):
        generate_sumake(home_dir, current)
        install.run(self)

if __name__ == '__main__':
    generate_version()
    setup(
        author="SuCicada",
        author_email="pengyifu@gmail.com",
        classifiers=[],
        description="A generic makefile for projects.",
        scripts=["bin/sumake"],
        cmdclass={
            "install": PreInstall,
            "bdist_wheel": PreBdistWheel,
            "sdist": PreSDist,
        },
        install_requires=["setuptools"],

        include_package_data=True,
        # package_data={
        #     '': ['sumake', 'utils.mk'],
        # },
        # install_requires=[],
        # keywords="",
        # license="",
        long_description=open("README.md").read(),
        name="sumake",
        # namespace_packages=[],
        # packages=find_packages(),
        # py_modules=["."],
        # test_suite="",
        url="https://github.com/SuCicada/sumake",
        version=VERSION,
        zip_safe=False,
    )