"""Create a file containing xdmf metadata for results stored in HDF5."""

import KratosMultiphysics
import KratosMultiphysics.HDF5Application as KratosHDF5
import os, sys, xdmf
import warnings
with warnings.catch_warnings():
    # suppressing an import-related warningfrom h5py
    # problem appears when using it in a test with python >=3.6
    warnings.simplefilter('ignore', category=ImportWarning)
    import h5py

def GenerateXdmfConnectivities(file_name):
    with h5py.File(file_name, "r") as h5py_file:
        has_xdmf = ("Xdmf" in h5py_file.get('/ModelData').keys())
    if not has_xdmf:
        KratosHDF5.HDF5XdmfConnectivitiesWriterProcess(file_name, "/ModelData").Execute()

def GetSpatialGrid(h5py_file):
    elements_path = "/ModelData/Xdmf/Elements/"
    coordinates_path = "/ModelData/Nodes/Local/Coordinates"
    spatial_grid = xdmf.SpatialGrid()
    coords_data = xdmf.HDF5UniformDataItem(h5py_file.get(coordinates_path))
    geom = xdmf.Geometry(coords_data)
    elems_group = h5py_file.get(elements_path)
    for name in elems_group.keys():
        if isinstance(elems_group[name], h5py.Group):
            single_elem_group = elems_group.get(name)
            dim = single_elem_group.attrs["Dimension"]
            num_points = single_elem_group.attrs["NumberOfNodes"]
            cell_type = xdmf.TopologyCellType(dim, num_points)
            connectivity_data = xdmf.HDF5UniformDataItem(h5py_file.get(elements_path + '/' + name + '/Connectivities'))
            topology = xdmf.UniformMeshTopology(cell_type, connectivity_data)
            spatial_grid.add_grid(xdmf.UniformGrid(name, geom, topology))
    return spatial_grid

def GetNodalResults(h5py_file):
    results = {}
    if "/ResultsData/NodalSolutionStepData" in h5py_file:
        AddNodalData(h5py_file.get("/ResultsData/NodalSolutionStepData"), results)
    if "/ResultsData/NodalDataValues" in h5py_file:
        AddNodalData(h5py_file.get("/ResultsData/NodalDataValues"), results)

    return list(results.values())

def AddNodalData(results_group, results):
    for variable_name in results_group.keys():
        if isinstance(results_group[variable_name], h5py.Dataset):
            if variable_name in results:
                raise ValueError('Nodal result "' + variable_name + '" is already defined.')
            data = xdmf.HDF5UniformDataItem(results_group.get(variable_name))
            results[variable_name] = xdmf.NodalData(variable_name, data)

def GetElementResults(h5py_file):
    element_results_path = "/ResultsData/ElementDataValues"
    results = []
    if not element_results_path in h5py_file:
        return results
    results_group = h5py_file.get(element_results_path)
    for variable_name in results_group.keys():
        if isinstance(results_group[variable_name], h5py.Dataset):
            data = xdmf.HDF5UniformDataItem(results_group.get(variable_name))
            results.append(xdmf.ElementSolutionStepData(variable_name, data))
    return results

def GetListOfTimeLabels(file_name):
    list_of_file_names = []
    path, file_name = os.path.split(file_name)
    if path == "": path = "." # os.listdir fails with empty path
    time_prefix = file_name.replace(".h5", "-")
    for name in os.listdir(path):
        if name.find(time_prefix) == 0:
            list_of_file_names.append(name)
    list_of_time_labels = []
    for name in list_of_file_names:
        list_of_time_labels.append(name.replace(".h5", "")[len(time_prefix):])
    list_of_time_labels.sort(key=float)
    return list_of_time_labels

def WriteXdmfFile(file_name):
    #todo(msandre): generalize to WriteXdmfFile(xdmf_file_name, list_of_h5_file_paths):
    temporal_grid = xdmf.TemporalGrid()
    GenerateXdmfConnectivities(file_name)
    # Get the initial spatial grid from the base file.
    with h5py.File(file_name, "r") as h5py_file:
        current_spatial_grid = GetSpatialGrid(h5py_file)
    for current_time in GetListOfTimeLabels(file_name):
        current_file_name = file_name.replace(".h5", "-" + current_time + ".h5")
        try:
            # Check if the current file has mesh information.
            with h5py.File(current_file_name, "r") as h5py_file:
                has_mesh = ("ModelData" in h5py_file.keys())
                has_data = ("/ResultsData" in h5py_file.keys())
        except OSError:
            # in case this file cannot be opened skip it
            # this can be the case if the file is already opened
            warn_msg  = 'No xdmf-data was written for file:\n"'
            warn_msg += current_file_name + '"'
            KratosMultiphysics.Logger.PrintWarning("XDMF-Writing", warn_msg)
            continue
        if not has_data:
            continue
        if has_mesh:
            GenerateXdmfConnectivities(current_file_name)
        with h5py.File(current_file_name, "r") as h5py_file:
            if has_mesh:
                # Update current spatial grid
                current_spatial_grid = GetSpatialGrid(h5py_file)
            # Initialize the current grid with the spatial grid.
            current_grid = xdmf.SpatialGrid()
            for grid in current_spatial_grid.grids:
                current_grid.add_grid(xdmf.UniformGrid(grid.name, grid.geometry, grid.topology))
            # Add the (time-dependent) results.
            for nodal_result in GetNodalResults(h5py_file):
                current_grid.add_attribute(nodal_result)
            for element_result in GetElementResults(h5py_file):
                current_grid.add_attribute(element_result)
        # Add the current grid to the temporal grid.
        temporal_grid.add_grid(xdmf.Time(current_time), current_grid)
    # Create the domain.
    domain = xdmf.Domain(temporal_grid)
    # Write.
    raw_file_name = os.path.split(file_name)[1]
    xdmf_file_name = raw_file_name.replace(".h5", ".xdmf")
    xdmf.ET.ElementTree(xdmf.Xdmf(domain).create_xml_element()).write(xdmf_file_name)

if __name__ == '__main__':
    file_name = sys.argv[1]
    WriteXdmfFile(file_name)


# -*- coding: utf-8 -*-

from config import config
from service_base import ServiceBase
from tariffs.decorator import memoize
from urllib import unquote
import json
from pyquery import PyQuery
from itertools import izip
import logging

logger = logging.getLogger('tariffs.services.o2')
logging.basicConfig()

class o2(ServiceBase):
    def __init__(self, *countries, **kwargs):
        """This class takes care of querying the O2 service for international calling tariffs
        :param countries: one or more country names
        :param kwargs: not used
        """
        if not countries:
            logger.error('Provide at least one country')
            raise ValueError('Please provide at least one country')

        super(o2, self).__init__()

        self._config = config['o2']

        self.country_data = {}
        self.country_list = countries
        self.tariffs = {}

    def _get_country_data(self):
        """Gets URL encoded list of country data, extracted from a elements in countryLinkContainer div

        :return: List of URL encoded country data
        """
        logger.debug('Getting main page')
        self.driver.get(self._config['url'])

        self.driver.find_element_by_id("showAllCountries").click()
        countries_encoded = self.driver.find_elements_by_xpath('//div[@id="countryLinkContainer"]//a')

        return countries_encoded

    def _process_country_data(self):
        """Converts the URL encoded list of country data into a Python dictionary indexed by country
        :return: {country_name: {countryId: COUNTRY_CODE}, ...}
        """
        logger.debug('Processing country data, extracting from <a> elements')
        countries_encoded = self._get_country_data()

        countries = [json.loads(unquote(elem.get_attribute('data-country'))) for elem in countries_encoded]
        self.country_data = {country['name']: country for country in countries}

    @memoize
    def _get_country_page(self, country):
        """Gets HTML page for a country

        :param country: country name
        :return: PyQuery object containing the page
        """
        logger.debug('Getting html for %s', country)

        if not self.country_data or country not in self.country_list or country not in self.country_data.keys():
            logger.error('Invalid country name')
            raise ValueError("Invalid or empty country name or function ran out of sequence, use get()")

        country_code = self.country_data[country]['countryId']

        pq = PyQuery(self._config['tariff_details_url'],
                     {self._config['tariff_details_arg']: country_code},
                     method='get')
        return pq

    def _get_rates(self, country, period):
        """Gets tariffs for a country
        :param country: Country name
        :return: dictionary of {landline: amount, mobile: amount, text: amount}
        """
        if not country:
            logger.error('Country not provided')
            raise ValueError('Country not provided')

        if not self.country_data:
            self._get_country_data()
            self._process_country_data()

        logger.debug('Getting page for %s', country)
        page = self._get_country_page(country)

        func = {'monthly': self._get_monthly_rates(page),
                'paygo': self._get_paygo_rates(page)}

        return func[period]

    def _get_paygo_rates(self, page):
        rates = {}
        for rate_type, rate_id in self._config['rates'].iteritems():
            rates[rate_type] = page.find(rate_id).text()

        return rates
    def _get_monthly_rates(self, page):
        monthly_table = page.find(self._config['montlhly_elem'])

        rates = dict(izip(('landline', 'mobile', 'text'), map(lambda elem: elem.text, monthly_table('td')[1::2])))

        return rates

    def close(self):
        """Closes the Selenium driver
        """
        self.driver.close()

    def get(self, *args, **kwargs):
        """Gets all the country data in country_list provided at object init

        :return: dictionary of {country1: {landline: amount, ...}, country2: {}...}
        """
        tariffs = {}

        if not args or len(args) != 1 or args[0] not in ('monthly', 'paygo'):
            raise ValueError("Provide period")

        period = args[0]

        for country in self.country_list:
            logger.info("Getting tariff for %s", country)
            tariffs[country] = self._get_rates(country, period)

        self.tariffs = tariffs

    def __str__(self):
        all_data = []
        for country, data in self.tariffs.iteritems():
            tariffs = []
            for tariff_type, amount in data.iteritems():
                tariffs.append(u'\t{}: {}'.format(tariff_type, amount))
            country_data = u'{}:\n{}'.format(country, u'\n'.join(tariffs))
            all_data.append(country_data)

        return u'\n'.join(all_data).encode('utf-8')


"""
# Plotting Benchmark Results

## Usage

    usage: benchmark_plot.py [-h] [--test TEST] [--kind KIND] [--show] [--save]
                             filename name

    Plotting

    positional arguments:
      filename     path to file with benchmark data
      name         type name

    optional arguments:
      -h, --help   show this help message and exit
      --test TEST
      --kind KIND
      --show
      --save

## Compare Implementations

    python -m tests.benchmark_sortedlist --bare > tests/results_sortedlist.txt
    python -m tests.benchmark_plot tests/results_sortedlist.txt SortedList --save

    python -m tests.benchmark_sorteddict --bare > tests/results_sorteddict.txt
    python -m tests.benchmark_plot tests/results_sorteddict.txt SortedDict --save

    python -m tests.benchmark_sortedset --bare > tests/results_sortedset.txt
    python -m tests.benchmark_plot tests/results_sortedset.txt SortedSet --save

## Compare Python Versions

    rm tests/results_runtime_sortedlist.txt
    echo ". env27/bin/activate && python -m tests.benchmark_sortedlist --bare --kind SortedList --suffix _Py27 >> tests/results_runtime_sortedlist.txt" | bash
    echo ". env34/bin/activate && python -m tests.benchmark_sortedlist --bare --kind SortedList --suffix _Py34 >> tests/results_runtime_sortedlist.txt" | bash
    echo ". env27/bin/activate && pypy -m tests.benchmark_sortedlist --bare --kind SortedList --suffix _PyPy >> tests/results_runtime_sortedlist.txt" | bash
    python -m tests.benchmark_plot tests/results_runtime_sortedlist.txt SortedList --suffix _runtime --save

    rm tests/results_runtime_sorteddict.txt
    echo ". env27/bin/activate && python -m tests.benchmark_sorteddict --bare --kind SortedDict --suffix _Py27 >> tests/results_runtime_sorteddict.txt" | bash
    echo ". env34/bin/activate && python -m tests.benchmark_sorteddict --bare --kind SortedDict --suffix _Py34 >> tests/results_runtime_sorteddict.txt" | bash
    echo ". env27/bin/activate && pypy -m tests.benchmark_sorteddict --bare --kind SortedDict --suffix _PyPy >> tests/results_runtime_sorteddict.txt" | bash
    python -m tests.benchmark_plot tests/results_runtime_sorteddict.txt SortedDict --suffix _runtime --save

    rm tests/results_runtime_sortedset.txt
    echo ". env27/bin/activate && python -m tests.benchmark_sortedset --bare --kind SortedSet --suffix _Py27 >> tests/results_runtime_sortedset.txt" | bash
    echo ". env34/bin/activate && python -m tests.benchmark_sortedset --bare --kind SortedSet --suffix _Py34 >> tests/results_runtime_sortedset.txt" | bash
    echo ". env27/bin/activate && pypy -m tests.benchmark_sortedset --bare --kind SortedSet --suffix _PyPy >> tests/results_runtime_sortedset.txt" | bash
    python -m tests.benchmark_plot tests/results_runtime_sortedset.txt SortedSet --suffix _runtime --save

## Compare Loads

    rm tests/results_load_sortedlist.txt
    python -m tests.benchmark_sortedlist --bare --kind SortedList --suffix _100 --load 100 --no-limit >> tests/results_load_sortedlist.txt
    python -m tests.benchmark_sortedlist --bare --kind SortedList --suffix _1000 --load 1000 --no-limit >> tests/results_load_sortedlist.txt
    python -m tests.benchmark_sortedlist --bare --kind SortedList --suffix _10000 --load 10000 --no-limit >> tests/results_load_sortedlist.txt
    python -m tests.benchmark_plot tests/results_load_sortedlist.txt SortedList --suffix _load --save

    rm tests/results_load_sorteddict.txt
    python -m tests.benchmark_sorteddict --bare --kind SortedDict --suffix _100 --load 100 --no-limit >> tests/results_load_sorteddict.txt
    python -m tests.benchmark_sorteddict --bare --kind SortedDict --suffix _1000 --load 1000 --no-limit >> tests/results_load_sorteddict.txt
    python -m tests.benchmark_sorteddict --bare --kind SortedDict --suffix _10000 --load 10000 --no-limit >> tests/results_load_sorteddict.txt
    python -m tests.benchmark_plot tests/results_load_sorteddict.txt SortedDict --suffix _load --save

    rm tests/results_load_sortedset.txt
    python -m tests.benchmark_sortedset --bare --kind SortedSet --suffix _100 --load 100 --no-limit >> tests/results_load_sortedset.txt
    python -m tests.benchmark_sortedset --bare --kind SortedSet --suffix _1000 --load 1000 --no-limit >> tests/results_load_sortedset.txt
    python -m tests.benchmark_sortedset --bare --kind SortedSet --suffix _10000 --load 10000 --no-limit >> tests/results_load_sortedset.txt
    python -m tests.benchmark_plot tests/results_load_sortedset.txt SortedSet --suffix _load --save
"""

from __future__ import print_function

import argparse
import matplotlib.pyplot as plt
from collections import defaultdict

def tree():
    return defaultdict(tree)

parser = argparse.ArgumentParser(description='Plotting')
parser.add_argument('filename', help='path to file with benchmark data')
parser.add_argument('name', help='type name')
parser.add_argument('--test', action='append', help='filter tests by name')
parser.add_argument('--kind', action='append', help='filter types by name')
parser.add_argument('--suffix', default='', help='suffix for output')
parser.add_argument('--show', action='store_true', help='show results')
parser.add_argument('--save', action='store_true', help='save results')

args = parser.parse_args()

text = open(args.filename).read()

lines = text.splitlines()
lines = [line.split() for line in lines]
for line in lines:
    line[2] = int(line[2])
    line[3:] = map(float, line[3:])
data = tree()
for line in lines:
    data[line[0]][line[1]][line[2]] = line

def order_kinds(kinds):
    for idx, kind in enumerate(kinds):
        if kind.startswith('Sorted'):
            del kinds[idx]
            kinds.insert(0, kind)
            break
    return kinds

def test_plot(test):
    ax = plt.gca()
    ax.set_color_cycle(['b', 'g', 'r', 'c', 'm', 'y', 'k', '0.8'])
    kinds = order_kinds(sorted(args.kind or list(data[test])))
    for kind in kinds:
        kind_plot(test, kind)
    plt.ylim(ymin=9e-7)
    plt.loglog()
    plt.title(args.name + ' Performance: ' + test)
    plt.ylabel('Seconds')
    plt.xlabel('List Size')
    plt.legend(kinds, loc=2)

def kind_plot(test, kind):
    sizes = sorted(data[test][kind].keys())
    yrange = [[(data[test][kind][size][5] - data[test][kind][size][3])
               for size in sizes],
              [(data[test][kind][size][4] - data[test][kind][size][5])
               for size in sizes]]
    # Timer isn't any better than micro-second resolution.
    yvalues = [max(1e-6, data[test][kind][size][5]) for size in sizes]
    plt.errorbar(sizes, yvalues, yerr=yrange)

tests = args.test or list(data)

for test in tests:
    test_plot(test)

    if args.show:
        plt.show()

    if args.save:
        plt.savefig('{0}{1}-{2}.png'.format(args.name, args.suffix, test))

    plt.close()


from typing import List

def MCM_aux(P,m,s,i,j):
    m[i][j] = None

    for k in range(i, j):
        q = m[i][k] + m[k+1][j] + P[i]*P[k+1]*P[j+1]
        if m[i][j] is None or q < m[i][j]:
            m[i][j] = q
            s[i][j] = k + 1 # python starts counting from zero not 1, that makes us add it by hand

def matrix_chain_mult(P: List[int]):
    n = len(P)
    m = [[0 for j in range(n-1)] for i in range(n-1)]
    s = [[0 for j in range(n-1)] for i in range(n-1)]

    for diag in range(1,n):
        for i in range(n-diag-1):
            j = i + diag    # until now we have selected the cell of the diagonal

            # select minumum sum

            MCM_aux(P,m,s,i,j)

    return m, s


import numpy as np
import pickle
import os
import tensorflow as tf
import argparse

os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'

parser = argparse.ArgumentParser()
parser.add_argument('--n_hidden', type=int,
                    help='Enter num of hidden LSTM cells', default = 512)
parser.add_argument('--time_step', type=int,
                    help='Enter Time Step', default = 10)
parser.add_argument('--lr', type=float,
                    help='Enter Learning Rate', default = 0.001)
parser.add_argument('--nb_epoch', type=int,
                    help='Enter Number of Epochs', default = 100)
parser.add_argument('--d_step', type=int,
                    help='Enter Display Step', default = 1)

args = parser.parse_args()

with open('./Training_Data/train_data.pkl', 'rb') as f:
    train_data = pickle.load(f)

num_notes = 128
num_vel = 128
n_hidden = args.n_hidden
num_time = 1
time_steps = args.time_step
lr = args.lr
num_epoch = args.nb_epoch
num_total = num_notes + num_vel + num_time
display_step = args.d_step
hidden_layer = 512

print('[i] Number of Epochs:          ', num_epoch)
print('[i] Learning Rate:          ', lr)
print('[i] Number of hidden units:          ', n_hidden)
print('[i] Time Step:          ', time_steps)
print('[i] Display Step:          ', display_step)

with tf.variable_scope('input'):

    X = tf.placeholder(dtype=tf.float32, shape=[None, time_steps, num_total])
    Y = tf.placeholder(dtype=tf.float32, shape=[None, num_total])

with tf.variable_scope('rnn'):
    weights = {
        'hidden_layer': tf.Variable(tf.random_normal( [n_hidden, hidden_layer] )),
        'out': tf.Variable(tf.random_normal( [hidden_layer, num_total] ))
    }
    biases = {
        'hidden_layer': tf.Variable(tf.random_normal( [hidden_layer] )),
        'out': tf.Variable(tf.random_normal([num_total])),
    }

    input = tf.unstack(X, time_steps, 1)

    rnn_cell = tf.contrib.rnn.BasicLSTMCell(n_hidden)
    outputs_rnn, _ = tf.contrib.rnn.static_rnn(rnn_cell, input, dtype=tf.float32)

    outputs_rnn = tf.nn.dropout(outputs_rnn, keep_prob=0.6)

    outputs = tf.matmul( outputs_rnn[-1], weights['hidden_layer'] ) + biases['hidden_layer']
    outputs = tf.nn.relu( outputs )

    outputs = tf.nn.dropout( outputs, keep_prob=0.5 )

    pred = tf.matmul(outputs, weights['out']) + biases['out']

    pred_note = pred[:,:num_notes]
    pred_vel = pred[:,num_notes: num_notes + num_vel]
    pred_time = pred[:,-1]

with tf.variable_scope('costs'):
    note_cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=Y[:,:num_notes], logits=pred_note))
    vel_cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=Y[:,num_notes: num_notes + num_vel], logits=pred_vel))
    time_cost = tf.reduce_mean(tf.squared_difference(pred_time, Y[:,-1]))

with tf.variable_scope('train'):
    opt_note = tf.train.AdamOptimizer(learning_rate=lr).minimize(note_cost)
    opt_vel = tf.train.AdamOptimizer(learning_rate=lr).minimize(vel_cost)
    opt_time = tf.train.AdamOptimizer(learning_rate=lr).minimize(time_cost)

with tf.variable_scope('logging'):
    tf.summary.scalar('note_cost', note_cost)
    tf.summary.scalar('velocity_cost', vel_cost)
    tf.summary.scalar('time_cost', time_cost)
    summary = tf.summary.merge_all()

init = tf.global_variables_initializer()
saver = tf.train.Saver()

def Get_Data(path):
    with open(path, 'rb') as f:
        data = pickle.load(f)

    data['time_train'] = train_data['scaler'].transform(data['time_train'])
    data['time_test'] = train_data['scaler'].transform(data['time_test'])

    return data

def Get_Next_X(note, vel, time, curr):
    n = np.array( note[curr: curr + time_steps] )
    v = np.array( vel[curr: curr + time_steps] )
    t = np.array( time[curr: curr + time_steps] )
    return n, v, t

def Get_Next_Y(note, vel, time, curr):
    n = np.array( [ note[curr + time_steps] ] )
    v = np.array( [ vel[curr + time_steps] ] )
    t = np.array( [ time[curr + time_steps] ] )
    return n, v, t

def Get_Processed_Data(note, vel, time, n_samples):
    curr = 0
    X = []
    Y = []
    while curr + time_steps < n_samples:
        n_x, v_x, t_x = Get_Next_X(note, vel, time, curr)
        n_y, v_y, t_y = Get_Next_Y(note, vel, time, curr)

        final_x = np.concatenate( (n_x, v_x, t_x), axis=1 )
        final_y = np.concatenate( (n_y, v_y, t_y), axis=1 )

        X.append( final_x )
        Y.append( final_y )

        curr += 1

    X = np.array( X )
    Y = np.array( Y )

    return X,Y

train_dir = './Training_Data/'
pickle_files = os.listdir(train_dir)

with tf.Session() as sess:
    sess.run(init)

    print("Training has Started")
    print ("--------------------")

    for epoch in range(num_epoch):

        print ("Epoch {}:".format(epoch + 1))

        train_note_loss = 0.0
        train_vel_loss = 0.0
        train_time_loss = 0.0
        tot_samples_train = 0

        test_note_loss = 0.0
        test_vel_loss = 0.0
        test_time_loss = 0.0
        tot_samples_test = 0

        train_writer = tf.summary.FileWriter('./logs/training', sess.graph)
        test_writer = tf.summary.FileWriter('./logs/testing', sess.graph)

        for pkl in pickle_files:

            if pkl == 'train_data.pkl':
                continue
            if pkl[-3:] != 'pkl':
                continue

            data = Get_Data(train_dir + pkl)

            n_samples = data['note_train'].shape[0]

            inp, out = Get_Processed_Data( data['note_train'], data['vel_train'], data['time_train'], n_samples )

            for i in range(inp.shape[0]):
                sess.run([opt_note, opt_vel, opt_time], feed_dict={
                    X: np.array( [ inp[i] ] ),
                    Y: np.array( out[i]   )
                })

            tot_samples_train += inp.shape[0]

            for i in range( inp.shape[0] ):
                n_loss, v_loss, t_loss, train_summary = sess.run([note_cost, vel_cost, time_cost,summary], feed_dict={
                    X: np.array( [ inp[i] ] ),
                    Y: np.array( out[i]   )
                })

                train_note_loss += n_loss
                train_vel_loss += v_loss
                train_time_loss += t_loss

            n_samples = data['note_test'].shape[0]

            inp_test, out_test = Get_Processed_Data(data['note_test'], data['vel_test'], data['time_test'], n_samples)

            for i in range( inp_test.shape[0] ):
                n_loss, v_loss, t_loss, test_summary = sess.run([note_cost, vel_cost, time_cost, summary], feed_dict={
                    X: np.array( [ inp_test[i] ] ),
                    Y: np.array( out_test[i]  )
                })

                test_note_loss += n_loss
                test_vel_loss += v_loss
                test_time_loss += t_loss

            tot_samples_test += inp_test.shape[0]

        train_note_loss /= tot_samples_train
        train_vel_loss /= tot_samples_train
        train_time_loss /= tot_samples_train

        test_note_loss /= tot_samples_test
        test_vel_loss /= tot_samples_test
        test_time_loss /= tot_samples_test

        if (epoch+1) % display_step == 0:
            train_writer.add_summary(train_summary, epoch)
            test_writer.add_summary(test_summary, epoch)
            saver.save(sess, './logs/model_' + str(epoch+1) + '.ckpt')

        print ("Training Note Loss is {:,.6f} & Testing Note Loss is {:,.6f}".format(train_note_loss, test_note_loss))
        print ("Training Velocity Loss is {:,.6f} & Testing Velocity Loss is {:,.6f}".format(train_vel_loss, test_vel_loss))
        print ("Training Time Loss is {:,.6f} & Testing Time Loss is {:,.6f}".format(train_time_loss, test_time_loss))

        print ("-------------------")

    print("Training is Complete")
    save_path = saver.save(sess, './logs/final.ckpt')
    print ("Model saved: {}".format(save_path))


from tools.constants import SQLITE_FULL_PATH
from tools.constants import TABLE_NAME_ALL_GAMES
from tools.utils import df_to_sqlite_table
from tools.utils import drop_table_if_exists_from_sqlite
from tools.utils import sqlite_table_to_df

import logging
import os
import pandas as pd
import sqlite3

log = logging.getLogger(__name__)

class MergeCsvToSqlite:
    def __init__(self, clean_csv_info):
        log.info(f"start {self.__class__.__name__}")
        self.clean_csv_info = clean_csv_info

    def merge_game_csv_to_sqlite(self, csv_file_path, table_name=None):
        assert table_name in ["league", "cup"]
        log.info(f"merging {csv_file_path}")
        df = pd.read_csv(csv_file_path, sep="\t")
        df_to_sqlite_table(df, table_name=table_name, if_exists="append")

    def merge_two_sqlite_tables(self):
        df_cup = sqlite_table_to_df(table_name="cup")
        df_league = sqlite_table_to_df(table_name="league")
        all_games = pd.concat(
            [df_cup, df_league], ignore_index=False, axis=0, sort=False
        )
        all_games.set_index("date")
        all_games.sort_values(by="date", ascending=True)
        df_to_sqlite_table(
            all_games, table_name=TABLE_NAME_ALL_GAMES, if_exists="replace"
        )

    def drop_both_seperate_tables(self):
        drop_table_if_exists_from_sqlite(table_name="cup")
        drop_table_if_exists_from_sqlite(table_name="league")

    def run(self):
        if os.path.isfile(SQLITE_FULL_PATH):
            log.debug(f"deleting {SQLITE_FULL_PATH}")
            os.remove(SQLITE_FULL_PATH)
        """ for now, we always create new sqlite with tables: cup, league, player """
        # Opens file if exists, else creates file
        log.debug(f"connect to {SQLITE_FULL_PATH}")
        self.connex = sqlite3.connect(SQLITE_FULL_PATH)
        for csv_type, csv_file_path in self.clean_csv_info.csv_info:
            if not csv_file_path.endswith("_invalid.csv"):
                self.merge_game_csv_to_sqlite(csv_file_path, table_name=csv_type)
        self.merge_two_sqlite_tables()
        self.drop_both_seperate_tables()


#!/usr/bin/env python

import os
import xmltodict
from sysutils.utils.file_utils import read_file

from abstractPackagePerformer import AbstractPerformer

class PomxmlPerformer(AbstractPerformer):
    FILE_NAME = "pom.xml"

    def __init__(self, root_folder='../data', delimiter=AbstractPerformer.DELIMITER):
        super().__init__(root_folder, delimiter)
        self._data = {}
        self._properties = {}

    @property
    def data(self):
        return self._data

    @property
    def properties(self):
        return self._properties

    def get(self, keys='project'):
        content = self.data
        for key in keys.split('.'):
            content = dict(content).get(key)
        return content

    def as_version(self, value):
        if not value or value is None:
            return ''
        if value.startswith('${'):
            key = value[2:-1]
            return self.properties.get(key) or ''
        return ''

    def process(self, keys='project.dependency'):
        content = self.get(keys)
        return [{self.PACKAGE_NAME: "{}.{}".format(d.get('groupId', ''), d.get('artifactId', '')),
                 self.PACKAGE_VERSION: "{}".format(self.as_version(d.get('version')))} for d in content]

    def load(self):
        try:
            self._data = dict(xmltodict.parse(read_file(self.FILE_NAME, self.root_folder)))
            self._properties = self.get('project.properties')
            return "OK"
        except Exception as ex:
            return "ERROR. Cannot process file '{}'; Reason: {}".format(self.FILE_NAME, ex)

    def extract(self):
        status = self.load()
        if status != 'OK':
            return status

        for key in (
                {'key': 'project.dependencies.dependency', 'name': 'dependencies'},
                {'key': 'project.build.plugins.plugin', 'name': 'plugins'},
                {'key': 'project.build.plugins.plugin', 'name': 'plugins'},
                {'key': 'project.build.pluginManagement.plugins.plugin', 'name': 'management'},
            ):
            self.save(self.process(key['key']), os.path.join(self.root_folder, "{}-java.csv".format(key['name'])))
        return "OK"

def extract(performer):
    status = performer.extract()
    return "Command {} completed. Status: {}".format('extract', status)

def main(command):
    print("PackageJson performer started. command: {}".format(command))

    performer = PomxmlPerformer(root_folder='../data')

    commands = {
        "extract": extract
    }
    return commands.get(command)(performer) if commands.get(command) else "invalid command {}".format(command)

if __name__ == "__main__":
    command = os.environ.get('COMMAND') or "extract"
    print(main(command))


#-*- coding: utf-8 -*-
import numpy as np
import copy
import gym
from population import Individual,Population,RouletteWheelSelection,Crossover,Mutation

from util import dump_model,load_model

from config import Config
conf = Config()

class GA:
    def __init__(self, population, selection, crossover, mutation):
        self.population = population
        self.selection = selection
        self.crossover = crossover
        self.mutation = mutation

    def run(self, env,num_games=1, visualization=True):
        self.population.initialize()
        gen = conf.train_generations
        get_max_num = 0
        for n in range(1, gen+1):
            print("="*20,"generations: ",n,"="*20)
            fitness_list, best_individual = self.population.fitness(env, num_games, visualization)
            best_individual = copy.deepcopy(best_individual)
            print("initial P.individuals.shape, best_individual.fitness",self.population.individuals.shape, best_individual.fitness)

            self.selection.select(self.population, conf.select_rate, conf.lucky_select_rate)
            print("selected P.individuals.shape",self.population.individuals.shape)

            self.crossover.cross(self.population)
            print("cross P.individuals.shape",self.population.individuals.shape)

            self.mutation.mutate(self.population, conf.mutation_alpha)
            print("mutate P.individuals.shape",self.population.individuals.shape)

            # 精英策略，保留最好的进入下一代
            if conf.elitism:
                pos = np.random.randint(self.population.size)
                self.population.individuals[pos] = best_individual

            if best_individual.fitness >= conf.max_fitness:
                get_max_num+=1
            if get_max_num >= 10:
                break

        return best_individual

def run_train():
    env = gym.make("Pong-ram-v0").env
    observation_space = env.observation_space.shape[0] #Box(128,)
    action_space = env.action_space.n  # Discrete(6)
    conf.set_params(observation_space=observation_space, action_space=action_space)

    I = Individual(conf)
    P = Population(I,conf)
    S = RouletteWheelSelection()
    C = Crossover(conf)
    M = Mutation(conf.mutation_rate)
    g = GA(P, S, C, M)

    best_individual = g.run(env,num_games=conf.num_games_per_individual, visualization=True)
    dump_model(best_individual, conf.save_path)
    env.close()

def test_best_ind():
    env = gym.make("Pong-ram-v0").env
    best_individual = load_model(conf.save_path)
    fitness = best_individual.play_and_evaluation(env, num_games=3, visualization=False)
    print("fitness", fitness)
    env.close()

if __name__ == "__main__":
    run_train()
    # test_best_ind()


import multiprocessing
from pathlib import Path

if Path('/vagrant').exists():
    chdir = '/vagrant'
else:
    chdir = str(Path.home())

workers = max(multiprocessing.cpu_count(), 2)

bind = '0.0.0.0:80'
keepalive = 120
errorlog = 'gunicorn.log'
pidfile = 'gunicorn.pid'
timeout = 5

worker_class = 'aiohttp.worker.GunicornUVLoopWebWorker'


import tensorflow as tf
from tensorflow import keras
import time

import gpflow as gp
from tensorflow.keras.regularizers import l2

from tensorflow.keras import Sequential
from tensorflow.keras.layers import Conv2D
from tensorflow.keras.layers import BatchNormalization
from tensorflow.keras.layers import MaxPooling2D
from tensorflow.keras.layers import Dropout
from tensorflow.keras.layers import Dense
from tensorflow.keras.layers import Flatten
from tensorflow.keras.optimizers import Adam
# from sklearn.preprocessing import to_categorical
from tensorflow.keras.utils import to_categorical
from gpflow.utilities import print_summary
import tensorflow_probability as tfp
import tensorflow as tf
from matplotlib import pyplot
from tensorflow.keras.preprocessing.image import load_img
import numpy as np

def prep_pixels(train, test):
	# convert from integers to floats
	train_norm = train.astype('float32')
	test_norm = test.astype('float32')
	# normalize to range 0-1
	train_norm = train_norm / 255.0
	test_norm = test_norm / 255.0
	# return normalized images
	return train_norm, test_norm

def load_cifar():
	# load dataset
	(trainX, trainY), (testX, testY) = keras.datasets.cifar10.load_data()
	# one hot encode target values
	trainY = to_categorical(trainY)
	testY = to_categorical(testY)
	return trainX, trainY, testX, testY

class Keras_model_fashion():

    def __init__(self):
        (train_images, train_labels), (test_images, test_labels) = keras.datasets.fashion_mnist.load_data()
        self.train_images= train_images
        self.train_labels= train_labels
        self.test_images= test_images
        self.test_labels= test_labels

    def evaluate_error_and_cost(self, layer_sizes, alpha, l2_regul, num_epoch):
        layer_sizes = np.ndarray.astype(np.rint(np.power(2, layer_sizes)), dtype= int)
        alpha= np.power(10.0, alpha)
        l2_regul= np.power(10.0, l2_regul)
        num_epoch= np.int(np.rint(np.power(10, num_epoch)))

        xt_eval= np.empty([1, len(layer_sizes)+ 2])

        xt_eval[0, 0:len(layer_sizes)]= np.log2(layer_sizes[:])
        # xt_eval[0,len(filter_sizes):len(filter_sizes)+ len(dense_sizes)]= np.log2(dense_sizes[:])
        xt_eval[0,len(layer_sizes)]= np.log10(alpha)
        xt_eval[0, len(layer_sizes)+1]= np.log10(l2_regul)

        print('layer_sizes:{}\nalpha:{}\nl2_regul:{}\nnum_epoch:{}'.format(layer_sizes, alpha, l2_regul, num_epoch))
        '''define model'''
        t1= time.clock()
        model = keras.Sequential([
            keras.layers.Flatten(input_shape=(28, 28)),
            keras.layers.Dense(layer_sizes[0], activation='relu',  kernel_initializer='he_uniform', kernel_regularizer= l2(l2_regul)),
            keras.layers.Dense(layer_sizes[1], activation='relu', kernel_initializer='he_uniform', kernel_regularizer= l2(l2_regul)),
            keras.layers.Dense(layer_sizes[2], activation='relu', kernel_initializer='he_uniform', kernel_regularizer= l2(l2_regul)),
            keras.layers.Dense(layer_sizes[3], activation='relu', kernel_initializer='he_uniform', kernel_regularizer= l2(l2_regul)),
            keras.layers.Dense(10)
        ])
        opt = keras.optimizers.Adam(learning_rate= alpha)
        '''compile model'''
        model.compile(optimizer= opt,  loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
                      metrics=['accuracy'])

        '''fit'''
        model.fit(self.train_images, self.train_labels, epochs= num_epoch)

        '''test set accuracy'''
        test_loss, test_acc = model.evaluate(self.test_images,  self.test_labels, verbose=2)
        print('\nTest accuracy:', test_acc)
        t2= time.clock()

        return np.atleast_2d(-test_acc), np.atleast_2d(t2- t1), xt_eval

class Keras_model_fashion2():

    def __init__(self):
        (train_images, train_labels), (test_images, test_labels) = keras.datasets.fashion_mnist.load_data()
        self.train_images= train_images
        self.train_labels= train_labels
        self.test_images= test_images
        self.test_labels= test_labels

    def evaluate_error_and_cost(self, layer_size, num_layers, alpha, num_epoch):
        layer_size= np.int(np.power(2.0, layer_size))
        alpha= np.power(10.0, alpha)
        # l2_regul= np.power(10.0, l2_regul)
        num_epoch= np.int(np.rint(np.power(10, num_epoch)))
        num_layers= np.int(num_layers)

        xt_eval= np.empty([1, 3])

        xt_eval[0, 0]= np.log2(layer_size)
        xt_eval[0, 1]= num_layers
        # xt_eval[0,len(filter_sizes):len(filter_sizes)+ len(dense_sizes)]= np.log2(dense_sizes[:])
        xt_eval[0,2]= np.log10(alpha)
        # xt_eval[0,2]= np.log10(l2_regul)

        print('layer_sizes:{}\nnum_layers:{}\nalpha:{}\nnum_epoch:{}'.format(layer_size, num_layers, alpha,
                                                                            num_epoch))
        '''define model'''
        t1= time.clock()
        model = Sequential()
        model.add(keras.layers.Flatten(input_shape=(28, 28)))
        for i in range(num_layers):
            model.add(keras.layers.Dense(layer_size, activation='relu',
                                         kernel_initializer='he_uniform'))
        model.add(keras.layers.Dense(10))
        # model = keras.Sequential([
        #     keras.layers.Flatten(input_shape=(28, 28)),
        #     keras.layers.Dense(layer_sizes[0], activation='relu',  kernel_initializer='he_uniform', kernel_regularizer= l2(l2_regul)),
        #     keras.layers.Dense(layer_sizes[1], activation='relu', kernel_initializer='he_uniform', kernel_regularizer= l2(l2_regul)),
        #     keras.layers.Dense(layer_sizes[2], activation='relu', kernel_initializer='he_uniform', kernel_regularizer= l2(l2_regul)),
        #     keras.layers.Dense(layer_sizes[3], activation='relu', kernel_initializer='he_uniform', kernel_regularizer= l2(l2_regul)),
        #     keras.layers.Dense(10)
        # ])
        opt = keras.optimizers.Adam(learning_rate= alpha)
        '''compile model'''
        model.compile(optimizer= opt,  loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
                      metrics=['accuracy'])

        '''fit'''
        model.fit(self.train_images, self.train_labels, epochs= num_epoch)

        '''test set accuracy'''
        test_loss, test_acc = model.evaluate(self.test_images,  self.test_labels, verbose=2)
        print('\nTest accuracy:', test_acc)
        t2= time.clock()

        return np.atleast_2d(-test_acc), np.atleast_2d(t2- t1), xt_eval

class Keras_model_cifar():

    def __init__(self):

        train_images, train_labels, test_images, test_labels= load_cifar()
        train_images,  test_images = prep_pixels( train_images, test_images)

        self.train_images= train_images
        self.train_labels= train_labels
        self.test_images= test_images
        self.test_labels= test_labels

    def evaluate_error_and_cost(self, filter_sizes, dense_sizes, alpha, l2_regul, dropout, num_epoch=0):

        filter_sizes= np.ndarray.astype(np.rint(np.power(2, filter_sizes)), dtype= int)
        dense_sizes= np.ndarray.astype(np.rint(np.power(2, dense_sizes)), dtype= int)
        alpha= np.power(10, alpha)
        l2_regul= np.power(10, l2_regul)
        # dropout= np.power(10, dropout)
        num_epoch= np.int(np.rint(np.power(10, num_epoch)))

        xt_eval= np.empty([1, len(filter_sizes)+ len(dense_sizes)+3])

        xt_eval[0, 0:len(filter_sizes)]= np.log2(filter_sizes[:])
        xt_eval[0,len(filter_sizes):len(filter_sizes)+ len(dense_sizes)]= np.log2(dense_sizes[:])
        xt_eval[0, len(filter_sizes)+ len(dense_sizes)]= np.log10(alpha)
        xt_eval[0, len(filter_sizes)+ len(dense_sizes)+1]= np.log10(l2_regul)
        xt_eval[0, len(filter_sizes)+ len(dense_sizes)+2]= dropout

        t1 = time.clock()
        model = Sequential()

        model.add(
            Conv2D(filter_sizes[0], (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same',
                   input_shape=(32, 32, 3), kernel_regularizer=l2(l2_regul)))
        model.add(BatchNormalization())

        model.add(Conv2D(filter_sizes[0], (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same',
                         kernel_regularizer=l2(l2_regul)))
        model.add(BatchNormalization())

        model.add(MaxPooling2D((2, 2)))
        model.add(Dropout(dropout))

        # model.add(Conv2D(filter_sizes[1], (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same',
        #                  kernel_regularizer=l2(l2_regul)))
        # model.add(BatchNormalization())
        #
        # model.add(Conv2D(filter_sizes[1], (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same',
        #                  kernel_regularizer=l2(l2_regul)))
        # model.add(BatchNormalization())
        #
        # model.add(MaxPooling2D((2, 2)))
        # model.add(Dropout(0.3))

        # model.add(Conv2D(filter_sizes[2], (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same',
        #                  kernel_regularizer=l2(l2_regul)))
        # model.add(BatchNormalization())
        #
        # model.add(Conv2D(filter_sizes[2], (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same',
        #                  kernel_regularizer=l2(l2_regul)))
        # model.add(BatchNormalization())
        #
        # model.add(MaxPooling2D((2, 2)))
        # model.add(Dropout(0.4))

        model.add(Flatten())
        model.add(Dense(dense_sizes[0], activation='relu', kernel_initializer='he_uniform', kernel_regularizer=l2(l2_regul)))
        model.add(Dropout(dropout))
        model.add(BatchNormalization())
        model.add(Dense(10, activation='softmax'))

        # compile model
        opt = Adam(lr=alpha)
        model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])

        '''fit'''
        model.fit(self.train_images, self.train_labels, epochs= num_epoch)

        '''test set accuracy'''
        test_loss, test_acc = model.evaluate(self.test_images,  self.test_labels, verbose=2)
        print('\nTest accuracy:', test_acc)
        t2= time.clock()

        return np.atleast_2d(test_loss), np.atleast_2d(t2- t1), xt_eval

class Keras_model_boston():

    def __init__(self):
        (X_train, Y_train), (X_test, Y_test) = keras.datasets.boston_housing.load_data()
        self.X_train= X_train
        self.Y_train= Y_train
        self.X_test= X_test
        self.Y_test= Y_test

    def evaluate_error_and_cost(self, layer_sizes, alpha, l2_regul, num_epoch):

        layer_sizes = np.ndarray.astype(np.rint(np.power(2, layer_sizes)), dtype= int)
        alpha= np.power(10.0, alpha)
        l2_regul= np.power(10.0, l2_regul)
        num_epoch= np.int(np.rint(np.power(10, num_epoch)))
        print('layer_sizes:{}\nalpha:{}\nl2_regul:{}\nnum_epoch:{}'.format(layer_sizes, alpha, l2_regul, num_epoch))
        '''define model'''
        t1= time.clock()
        model = keras.Sequential([
            keras.layers.Flatten(input_shape=(13, )),
            keras.layers.Dense(layer_sizes[0], activation='relu',  kernel_initializer='he_uniform', kernel_regularizer= l2(l2_regul)),
            keras.layers.Dense(layer_sizes[1], activation='relu', kernel_initializer='he_uniform', kernel_regularizer= l2(l2_regul)),
            keras.layers.Dense(layer_sizes[2], activation='relu', kernel_initializer='he_uniform', kernel_regularizer= l2(l2_regul)),
            keras.layers.Dense(layer_sizes[3], activation='relu', kernel_initializer='he_uniform', kernel_regularizer= l2(l2_regul)),
            keras.layers.Dense(1)
        ])
        opt = keras.optimizers.Adam(learning_rate= alpha)
        '''compile model'''
        model.compile(optimizer= opt,  loss=tf.keras.losses.mean_squared_error,
                      metrics=['MeanSquaredError'])

        '''fit'''
        model.fit(self.X_train, self.Y_train, epochs= num_epoch)

        '''test set accuracy'''
        test_loss, test_mse = model.evaluate(self.X_test,  self.Y_test, verbose=2)
        print('\nTest mse:', test_mse)
        t2= time.clock()

        return np.atleast_2d(test_mse), np.atleast_2d(t2- t1), model

def uniformly_choose_from_domain(domain, num_samples):
    X= np.empty([num_samples, len(domain)])
    # Y= np.empty([num_samples, 1])

    for i in range(len(domain)):
        xi= np.random.uniform(domain[i][0], domain[i][1], (num_samples,1))
        X[:, i]= xi[:,0]
    return X

def initial_training_cifar(domain, num_samples, num_layer, num_dense, num_epoch= 1.7):

    X= uniformly_choose_from_domain(domain, num_samples)

    keras_model = Keras_model_cifar()

    Y= np.empty([X.shape[0], 1])
    Y_cost= np.empty([X.shape[0], 1])

    for i in range(X.shape[0]):
        xi= X[i, :].reshape(1,-1)
        z= num_layer+num_dense
        filter_sizes =xi[0, 0:num_layer]; dense_sizes = xi[0, num_layer:z];
        alpha = xi[0, z]; l2_regul = xi[0, z+1]; dropout = xi[0, z+2]

        Y[i, 0], Y_cost[i, 0], X[i, :]= keras_model.evaluate_error_and_cost(filter_sizes, dense_sizes, alpha, l2_regul, dropout,
                                                                  num_epoch= num_epoch)

    return X, Y, Y_cost

def initial_training_fashion(domain, num_samples, num_layer, num_epoch= 1.7):

    X= uniformly_choose_from_domain(domain, num_samples)

    keras_model= Keras_model_fashion()

    Y= np.empty([X.shape[0], 1])
    Y_cost= np.empty([X.shape[0], 1])

    for i in range(X.shape[0]):
        xi= X[i, :].reshape(1,-1)
        layer_sizes =xi[0, 0:num_layer]; alpha = xi[0, num_layer]; l2_regul = xi[0, num_layer+1];
        #num_epoch= xi[0, num_layer+2]

        Y[i, 0], Y_cost[i, 0], X[i, :]= keras_model.evaluate_error_and_cost(layer_sizes, alpha, l2_regul, num_epoch)

    return X, Y, Y_cost

def initial_training_fashion2(domain, num_samples, num_epoch= 1.7):

    X= uniformly_choose_from_domain(domain, num_samples)

    keras_model= Keras_model_fashion2()

    Y= np.empty([X.shape[0], 1])
    Y_cost= np.empty([X.shape[0], 1])

    for i in range(X.shape[0]):
        xi= X[i, :].reshape(1,-1)
        layer_size = xi[0, 0]; num_layers = xi[0, 1];alpha = xi[0, 2];
        #num_epoch= xi[0, num_layer+2]

        Y[i, 0], Y_cost[i, 0], X[i, :]= keras_model.evaluate_error_and_cost(layer_size, num_layers,  alpha, num_epoch)

    return X, Y, Y_cost

def get_cifar_domain(num_layer, num_dense):

    domain_filter_size_log2= [0.0, 8.0]; domain_dense_size_log2= [0.0, 8.0]; domain_alpha_log10= [-6.0, 0.0];
    domain_l2_regul = [0.0, 0.1]; domain_dropout= [0.0, 0.8]; domain_num_epoch_log10 = [0, 2]

    domain= []

    for i in range(num_layer):
        domain.append(domain_filter_size_log2)
    for k in range(num_dense):
        domain.append(domain_dense_size_log2)

    domain.append(domain_alpha_log10)
    domain.append(domain_l2_regul)
    domain.append(domain_dropout)
    # domain.append(domain_num_epoch_log10)
    return domain

def get_fashion_domain(num_layer):

    domain_layer_size_log2= [0.0, 8.0]; domain_alpha_log10= [-6.0, 0.0];
    domain_l2_regul = [-8.0, -1.0];  #domain_num_epoch_log10 = [0, 1.3]

    domain= []

    for i in range(num_layer):
        domain.append(domain_layer_size_log2)

    domain.append(domain_alpha_log10)
    domain.append(domain_l2_regul)
    #domain.append(domain_num_epoch_log10)

    return domain

def get_fashion2_domain():

    domain_layer_size_log2= [0.0, 8.0]; domain_alpha_log10= [-6.0, 0.0];
    domain_num_layers = [1, 8]

    domain= [domain_layer_size_log2, domain_num_layers, domain_alpha_log10]

    return domain

def logistic_bjt(lower, upper):
    '''define logistic bijector'''
    low= tf.constant(lower, dtype= tf.float64); high= tf.constant(upper, dtype= tf.float64)
    affine = tfp.bijectors.AffineScalar(shift=low.numpy(), scale=(high.numpy() - low.numpy()))
    sigmoid = tfp.bijectors.Sigmoid()
    logistic = tfp.bijectors.Chain([affine, sigmoid])
    return logistic

def find_best_suited_gp_kernels(X, Y, Y_cost, noise, noise_cost):

    '''constraint values'''
    lower= 10**(-3); upper= 10**(6); #lengtscale and variance constarint
    lower_noise= 10**(-3); upper_noise= 10**(6); #noise constarint

    latent_kernel= gp.kernels.RBF()
    latent_cost_model= gp.models.GPR((X, np.log(Y_cost)), latent_kernel)
    '''limit latent cost kernel hyperparameters'''
    latent_cost_model.kernel.lengthscales= gp.Parameter(latent_cost_model.kernel.lengthscales.numpy(),transform=logistic_bjt(lower, upper))
    latent_cost_model.kernel.variance= gp.Parameter(latent_cost_model.kernel.variance.numpy(), transform= logistic_bjt(lower, upper))
    latent_cost_model.likelihood.variance= gp.Parameter(latent_cost_model.likelihood.variance.numpy(), transform= logistic_bjt(lower_noise, upper_noise))

    # latent_cost_model.likelihood.variance.assign(noise_cost)
    '''optimize cost model hyperparamters'''
    # gp.set_trainable(latent_cost_model.likelihood, False)
    opt_cost = gp.optimizers.Scipy()
    opt_cost.minimize(latent_cost_model.training_loss, latent_cost_model.trainable_variables, options=dict(maxiter=100))

    kernel= gp.kernels.RBF()
    model= gp.models.GPR((X, Y), kernel)
    '''limit objective kernel hyperparameters'''
    model.kernel.lengthscales= gp.Parameter(model.kernel.lengthscales.numpy(), transform= logistic_bjt(lower, upper))
    model.kernel.variance= gp.Parameter(model.kernel.variance.numpy(), transform= logistic_bjt(lower, upper))
    model.likelihood.variance= gp.Parameter(model.likelihood.variance.numpy(), transform= logistic_bjt(lower_noise, upper_noise))

    # model.likelihood.variance.assign(noise)

    '''optimize objective model hyperparamters'''
    opt_obj = gp.optimizers.Scipy()
    opt_obj.minimize(model.training_loss, model.trainable_variables, options=dict(maxiter=100))

    print_summary(model)
    print_summary(latent_cost_model)

    return kernel, latent_kernel

import sys
sys.path.append('../bo_cost_budget_cont_domain')
from util import *
import h5py
def grid_search_fashion():
    keras_model = Keras_model_fashion2()
    disc= 21
    domain= get_fashion2_domain()

    '''create grid'''
    x_list= []
    x_layer_size= np.linspace(1, 8, 8)
    x_width = np.linspace(0, 8.0, 21)
    x_list.append(x_layer_size); x_list.append(x_width)
    X_temp = np.meshgrid(*x_list)
    X_grid = np.empty([8*21,2])
    for i in range(2):
        X_grid[:, i] = X_temp[i].flatten()

    '''evaluate'''
    loss_list= np.empty([8*21, 1])
    X_eval=  np.empty([8*21, 3])
    cost_list=  np.empty([8*21, 1])

    for i in range(X_grid.shape[0]):
        xi = X_grid[i, :].reshape(1,-1)
        layer_size= xi[0, 0]; num_layers= xi[0, 1]
        alpha= -4
        test_lossi, time_costi, x_evali= keras_model.evaluate_error_and_cost(layer_size, num_layers, alpha, 1.7)
        loss_list[i, 0]= test_lossi
        cost_list[i, 0]= time_costi
        X_eval[i, :]= x_evali[0, :]
    with h5py.File('./keras_results/'+'fashion_grid.h5', 'a') as hf:

        hf.create_dataset('X_eval', data= X_eval)
        hf.create_dataset('loss_list', data= loss_list)
        hf.create_dataset('cost list', data= cost_list)

def test_model_fashion():
    fashion_mnist = keras.datasets.fashion_mnist
    (train_images, train_labels), (test_images, test_labels) = fashion_mnist.load_data()

    keras_model= Keras_model_fashion()
    domain_alpha= [1e-6, 1.0]
    domain_l2= [0, 0.1]
    domain_epochs= [1,100]
    layer_sizes= [32, 32, 32, 32, 32]; alpha_log= -4; l2_regul_log= -2; num_epoch_log= 1
    l2_regul= 10**(l2_regul_log); alpha= 10**(alpha_log); num_epoch= 10**(num_epoch_log)
    test_loss, time_cost, x_eval= keras_model.evaluate_error_and_cost(layer_sizes, alpha, l2_regul, num_epoch)
    print('test_loss:{}, time_cost:{}'.format(test_loss, time_cost))

def test_model_fashion2():
    fashion_mnist = keras.datasets.fashion_mnist
    (train_images, train_labels), (test_images, test_labels) = fashion_mnist.load_data()
    domain= get_fashion2_domain()
    keras_model= Keras_model_fashion2()

    layer_size= 5; num_layers= 4; alpha= -4
    test_loss, time_cost, x_eval= keras_model.evaluate_error_and_cost(layer_size, num_layers, alpha, 1.7)
    print('test_loss:{}, time_cost:{}'.format(test_loss, time_cost))

def test_model_cifar():
    #batch size
    # (train_images, train_labels), (test_images, test_labels) = keras.datasets.cifar10.load_data()

    filter_sizes, dense_sizes, alpha, l2_regul, num_epoch, dropout= np.array([32, 64, 128]), np.array([128]), 10**(-4), 0.01, 1, 0.1
    model_cifar= Keras_model_cifar()
    test_loss, time_cost= model_cifar.evaluate_error_and_cost(filter_sizes, dense_sizes, alpha, l2_regul, dropout)

def test_boston_model():

    layer_sizes= [5, 5, 5, 5]; alpha_log= -4; l2_regul_log= -2; num_epoch_log= 1
    boston_model= Keras_model_boston()
    test_loss, time_cost, model= boston_model.evaluate_error_and_cost(layer_sizes, alpha_log, l2_regul_log, 2)
    index= 2
    x_test= (boston_model.X_test[index, :]).reshape(1,-1)
    y_test = (boston_model.Y_test[index])
    print(model.predict(x_test))
    print('mean squared error:{}, time_cost:{}'.format(test_loss, time_cost))


'''
This python script wraps some R packages such as annotables and qqman, or some R codes in python environment using rpy2
Please see more documentation at http://rpy2.readthedocs.org/en/version_2.7.x/
'''

'''
################################
#### Software prerequistes #####
################################
'''
# python packages:
# rpy2:       pip install rpy2

# R packages:
# annotables:     install.packages("devtools")
#                 devtools::install_github("stephenturner/annotables")
# dplyr:          install.packages('dplyr')
# ggplot2:        install.packages("tidyverse")
# qqman:          install.packages("qqman")

'''
###################
#### Modules  #####
###################
'''
from rpy2.robjects import r
from rpy2.robjects.packages import importr
from rpy2.robjects import pandas2ri
from rpy2 import robjects
from helpers import *
from datetime import datetime
import os, pandas, glob, sqlite3, shutil, subprocess
import uuid as myuuid

pandas2ri.activate()

projectID = str(myuuid.uuid4())

'''
######################
#### Annotations #####
######################
'''
def annotate_metaxcan_result(projectName):

    global projectID

    # load R packages
    importr('annotables')
    importr('dplyr',  on_conflict="warn")

    # current path
    currentPath = os.getcwd()

    # root path
    os.chdir('../../../')
    root_path = os.getcwd()

    # input path
    input_path = root_path + '/input/'

    if not os.path.exists(input_path):
        warning = "Please make sure that you have created an input folder with input files"
        add_log(warning)
        add_log('Input path should be: %s' %input_path)
    os.chdir(input_path)

    # annoated_files output file path
    annoated_files_path = root_path + '/output/' + projectName + "_" + CURRENT_TIME + '/annotated_metaxcan_output_files/'
    if not os.path.exists(annoated_files_path):
        os.makedirs(annoated_files_path)

    # get input file lists (raw metaxcan output files *.csv)
    inputFileList = glob.glob("*.csv")

    # loop through file lists
    for inputFilename in inputFileList:
        msg = "ANNOTATING GENES: " + inputFilename
        add_log(msg)

        # read data
        r("data <- read.csv('%s')" %inputFilename)
        data = r('na.omit(data)')
        robjects.globalenv['dataframe'] = data

        # annotation library
        grch37 = r('grch37')
        robjects.globalenv['dataframe'] = grch37

        # annotating
        # if inputFilename == 'xxxxxDGN-WB-unscaled.csv':
        #     annotatedData = r("inner_join(data[, c('gene', 'gene_name', 'zscore', 'effect_size', 'pvalue', 'n_snps_in_model', 'pred_perf_p', 'pred_perf_R2')], grch37[, c('symbol', 'chr', 'start', 'end')], by=c('gene'='symbol'))")
        # else:
        annotatedData = r("inner_join(data[, c('gene', 'gene_name', 'zscore', 'effect_size', 'pvalue', 'n_snps_in_model', 'pred_perf_p', 'pred_perf_R2')], grch37[, c('ensgene', 'chr', 'start', 'end')], by =c('gene'='ensgene'))")
        annotatedData = annotatedData.drop_duplicates()

        # ouput annotated data
        os.chdir(annoated_files_path)
        annotatedData.to_csv(annoated_files_path + inputFilename[:-4] + '_annotated.csv', index=None)

        os.chdir(input_path)

    # merge all annotated metaxcan files
    # get annotated file lists
    os.chdir(annoated_files_path)
    outputFileListWithoutSNPs = glob.glob("*.csv")

    # merged annotated file path
    merged_annotated_file_path = annoated_files_path + 'merged/'
    if not os.path.exists(merged_annotated_file_path):
        os.makedirs(merged_annotated_file_path)

    dfListWithoutSNPs = []
    for outputFileName in outputFileListWithoutSNPs:
        # read data
        msg = "MERGING ALL ANNOTATED FILES ..."
        add_log(msg)
        msg = "CONCATING FILE: " + outputFileName
        add_log(msg)

        r("data <- read.csv('%s')" %outputFileName)
        data = r('data <- na.omit(data)')
        robjects.globalenv['dataframe'] = data

        if 'CrossTissue_elasticNet' in outputFileName:
            outputFileName = outputFileName[:-25]
        elif 'DGN-WB-unscaled' in outputFileName:
            outputFileName = outputFileName [:-23]
        else:
            outputFileName = outputFileName[:-25]
            outputFileName = outputFileName[3:]

        data.insert(2, 'tissue', outputFileName)
        dfListWithoutSNPs.append(data)
    concatDfWithoutSNPs = pandas.concat(dfListWithoutSNPs, axis = 0)

    #output data
    os.chdir(merged_annotated_file_path)
    concatDfWithoutSNPs.to_csv(merged_annotated_file_path + 'merged_annotated_metaxcan_output.csv', index = None)

    add_log(datetime.now().strftime('%Y.%m.%d.%H:%M:%S ') + "Done!")

'''
####################
####  QQ-Plot  #####
####################
'''
def qqplot(projectName):

    global projectID

    # load R package
    qqman = importr('qqman')

    # current path
    currentPath = os.getcwd()

    # back to root path
    os.chdir('../../../')
    root_path = os.getcwd()

    # annotated metaxcan file path
    annoated_files_path = root_path + '/output/' + projectName + "_" + CURRENT_TIME + '/annotated_metaxcan_output_files/'
    os.chdir(annoated_files_path)

    qqplot_output_path = root_path + '/output/' + projectName + "_" + CURRENT_TIME +'/QQ-plot/'
    if not os.path.exists(qqplot_output_path):
         os.makedirs(qqplot_output_path)

    outputFileListWithoutSNPs = glob.glob("*.csv")

    # loop through file lists
    for outputFileName in outputFileListWithoutSNPs:
        msg = "QQ-PLOT(no snps): " + outputFileName
        add_log(msg)

        # get GWAS output data
        r("data <- read.csv('%s')" %outputFileName)
        data = r('data <- na.omit(data)')
        robjects.globalenv['dataframe'] = data
        os.chdir(qqplot_output_path)

        # draw qq-plot and save them to files
        r.png('%s%s%s'%('QQ-Plot_', outputFileName[:-4],'.png'))
        qqman.qq(data['pvalue'],  main = "MetaXcan genes")
        r['dev.off']()
        os.chdir(annoated_files_path)

    msg = "QQ-PLOT(no snps): " + "merged_annotated_metaxcan_output"
    add_log(msg)

    # set up path and read merged annotated metaxcan output files
    os.chdir(annoated_files_path + 'merged/')
    r("data <- read.csv('merged_annotated_metaxcan_output.csv')")
    data = r('data <- na.omit(data)')
    robjects.globalenv['dataframe'] = data
    os.chdir(qqplot_output_path)

    # draw qq-plot and save them to files
    r.png('%s%s%s'%('QQ-Plot_', 'metaxcan_output', '.png'))
    qqman.qq(data['pvalue'],  main = "QQ plot")
    r['dev.off']()

    add_log(datetime.now().strftime('%Y.%m.%d.%H:%M:%S ') + "Done!")

'''
###########################
####  Manhattan-Plot  #####
###########################
'''
def manhattan(projectName):

    global projectID

    # load module
    qqman = importr('qqman')

    # current path
    currentPath = os.getcwd()

    # back to root path
    os.chdir('../../../')
    root_path = os.getcwd()

    # annotated metaxcan file path
    annoated_files_path = root_path + '/output/' + projectName + "_" + CURRENT_TIME +'/annotated_metaxcan_output_files/'
    os.chdir(annoated_files_path)

    manhattanplot_output_path = root_path + '/output/' + projectName + "_" + CURRENT_TIME + '/Manhattan-Plot/'
    if not os.path.exists(manhattanplot_output_path):
         os.makedirs(manhattanplot_output_path)

    # get file lists
    outputFileListWithoutSNPs = glob.glob("*.csv")

    # loop through file lists
    for outputFileName in outputFileListWithoutSNPs:
        msg = "MANHATTAN-PLOT(no snps): " + outputFileName
        add_log(msg)

        # read data
        r("data <- read.csv('%s')" %outputFileName)
        r('data$chr <- as.numeric(as.character(data$chr))')
        data = r('data <- na.omit(data)')
        robjects.globalenv['dataframe'] = data

        os.chdir(manhattanplot_output_path)

        # draw manhattan and save them to files
        r.png('%s%s%s'%('Manhattan-Plot_', outputFileName[:-4], '.png'))
        qqman.manhattan(data, chr = 'chr', bp='start', p='pvalue', snp='gene_name', main = 'MetaXcan genes')
        r['dev.off']()
        os.chdir(annoated_files_path)

    msg = "MANHATTAN-PLOT(no snps): " + "merged_annotated_metaxcan_output"
    add_log(msg)

    # read data
    os.chdir(annoated_files_path + 'merged/')
    r("data <- read.csv('merged_annotated_metaxcan_output.csv')")
    r('data$chr <- as.numeric(as.character(data$chr))')
    data = r('data <- na.omit(data)')
    robjects.globalenv['dataframe'] = data

    os.chdir(manhattanplot_output_path)

    # draw manhattan and save them to files
    r.png('%s%s%s'%('Manhattan-Plot_', 'merged_annotated_metaxcan_output.csv', '.png'))
    qqman.manhattan(data, chr = 'chr', bp='start', p='pvalue', snp='gene_name', main = 'Manhattan plot')
    r['dev.off']()

    add_log(datetime.now().strftime('%Y.%m.%d.%H:%M:%S ') + "Done!")

'''
######################################
####   Top Gene List without SNPs ####
######################################
'''
def sortTopGeneList(projectName):

    global projectID

    # current path
    currentPath = os.getcwd()

    # back to root path
    os.chdir('../../../')
    root_path = os.getcwd()

    # annotated metaxcan file path
    annoated_files_path = root_path + '/output/' + projectName + "_" + CURRENT_TIME +'/annotated_metaxcan_output_files/'
    os.chdir(annoated_files_path)

    top_genes_output_path = root_path + '/output/' + projectName + "_" + CURRENT_TIME +'/top_genes/'
    if not os.path.exists(top_genes_output_path):
         os.makedirs(top_genes_output_path)

    # get file lists
    fileLists = glob.glob("*.csv")

    # dfList = []
    # total_rows_in_total = 0
    for filename in fileLists:
        msg = "TOP GENE LIST: " + filename
        add_log(msg)

        df = pandas.read_csv(filename)

        if 'CrossTissue_elasticNet' in filename:
            tissue_name = filename[:-25]
        elif 'DGN-WB-unscaled' in filename:
            tissue_name = filename [:-23]
        else:
            tissue_name = filename[:-25]
            tissue_name = tissue_name[3:]
        df.insert(2, 'tissue', tissue_name)

        # sort data by defined column
        df.sort_values(['pvalue', 'n_snps_in_model'], ascending=[1, 0], inplace=True)
        total_rows = df.shape[0] # number of row count shape[1] is number of col count
        msg = 'total_row within tissue: ' + str(total_rows)
        add_log(msg)
        # total_rows_in_total += total_rows
        cut_off = 0.05/total_rows

        # cut_off_list.append(cut_off)
        top_gene_list = df[df['pvalue'] < cut_off]
        # top_gene_list = top_gene_list[top_gene_list['pred_perf_R2'] > 0.01]
        # top_gene_list.drop('gene', axis=1, inplace=True)
        msg = 'CALCULATING TISSUE-WIDE P-VALUE: ' + str(cut_off)
        add_log(msg)
        add_log(datetime.now().strftime('%Y.%m.%d.%H:%M:%S ') + "Done!")

        #output data
        top_gene_list.to_csv(top_genes_output_path + "sorted_top_genes_%s"%filename, index = None)

        # dfList.append(top_gene_list)

    os.chdir(annoated_files_path + 'merged/')

    msg = "TOP GENES: merged_annotated_metaxcan_output.csv"
    add_log(msg)

    df = pandas.read_csv('merged_annotated_metaxcan_output.csv')

    total_rows_in_total = df.shape[0]
    msg = 'total_row_in_total: ' + str(total_rows_in_total)
    add_log(msg)

    total_cut_off = 0.05/total_rows_in_total

    msg = 'CALCULATING GENOME-WIDE P-VALUE: ' + str(total_cut_off)
    add_log(msg)

    df = df[df['pvalue'] < total_cut_off]

    # sort data by defined column
    df.sort_values(['pvalue', 'n_snps_in_model'], ascending=[1, 0], inplace=True)

    #output data
    df.to_csv(top_genes_output_path+ 'sorted_top_genes.csv', index = None)

    add_log(datetime.now().strftime('%Y.%m.%d.%H:%M:%S ') + "Done!")

'''
####################################
####   Top Gene List with SNPs #####
####################################
'''
def sortTopGeneListWithSNPs(projectName):

    global projectID

    # current path
    currentPath = os.getcwd()

    # back to root path
    os.chdir('../../../')
    root_path = os.getcwd()

    # set up path
    top_genes_file_path = root_path + '/output/' + projectName + "_" + CURRENT_TIME + '/top_genes/'
    os.chdir(top_genes_file_path)

    # read data
    top_genes = pandas.read_csv('sorted_top_genes.csv')

    # add into files
    gene_lists = top_genes['gene_name']
    tissue_lists = top_genes['tissue']
    pvalue_lists = top_genes['pvalue']
    zscore_lists = top_genes['zscore']
    model_n_lists = top_genes['n_snps_in_model']
    pred_perf_R2_lists = top_genes['pred_perf_R2']
    chr_lists = top_genes['chr']
    start_lists = top_genes['start']
    end_lists = top_genes['end']

    # get db lists
    input_path = root_path + '/input/'

    if not os.path.exists(input_path):
        warning = "Please make sure that you have created an input folder with input files"
        add_log(warning)
        add_log('Input path should be: %s' %input_path)
    os.chdir(input_path)

    dbFileList = glob.glob("*.db")

    database_names = []
    for dbFilename in dbFileList:
       database_names.append(dbFilename)

    # Loop through databases
    query_output_list = []
    for i in range(len(database_names)):
        for k in range(len(tissue_lists)):
            if tissue_lists[k] in database_names[i]:
                # Connect databases
                conn = sqlite3.connect(database_names[i])

                full_query_name = None
                if database_names[i] == 'DGN-WB-unscaled_0.5.db':
                    full_query_name = SQL_QUERY_PREFIX_DNG + gene_lists[k] + "'"
                else:
                    full_query_name = SQL_QUERY_PREFIX + gene_lists[k] + "'"

                query_output = pandas.read_sql(full_query_name, conn, index_col=None)

                if database_names[i] == 'DGN-WB-unscaled_0.5.db':
                    query_output.rename(columns={'gene':'genename'}, inplace=True)

                # # Add correspinding parameters to the new output file
                query_output['tissue'] = tissue_lists[k]
                query_output['pvalue'] =  pvalue_lists[k]
                query_output['zscore'] =  zscore_lists[k]
                query_output['model_n'] = model_n_lists[k]
                query_output['pred_perf_R2'] = pred_perf_R2_lists[k]
                query_output['chr'] =  chr_lists[k]
                query_output['start'] = start_lists[k]
                query_output['end'] = end_lists[k]
                query_output_list.append(query_output)

                msg = 'FETCH SNPs for GENE %s ' % gene_lists[k] +  'FROM DATABASE: %s' % database_names[i]
                add_log(msg)

                # Close database
                conn.close()

    # Merge output data
    query_output_of = pandas.concat(query_output_list, axis = 0)
    # gwas_snp = pandas.read_csv('gwas_snp.txt', sep="\s+")
    # gwas_snp.rename(columns={'MarkerName':'rsid', 'P-value':'gwas_pvalue'}, inplace=True)

    # query_output_of.merge(gwas_snp, on='rsid', how='inner')

    # get tissue abbr name
    # load module
    ggplot2 = importr('ggplot2')
    dplyr = importr('dplyr',  on_conflict="warn")

    r("""

    tissue_abbr <- read.delim('gtex_tissue_abbr.txt', sep='\t') %>%
    select(tissue_site_detail_abbr,tissue_site_detail_id)
    colnames(tissue_abbr)[colnames(tissue_abbr) == 'tissue_site_detail_id'] = "tissue"

    """)

    top_genes_snps_output_path = root_path + '/output/' + projectName + "_" + CURRENT_TIME + '/top_genes_snps/'

    if not os.path.exists(top_genes_snps_output_path):
         os.makedirs(top_genes_snps_output_path)
    os.chdir(top_genes_snps_output_path)

    # Output merged data
    query_output_of.to_csv("top_genes_snps.csv", index=None)

    add_log(datetime.now().strftime('%Y.%m.%d.%H:%M:%S ') + "Done!")

'''
######################
###  Bubble Plot #####
######################
'''
def bubbleplot(projectName):

    global projectID

    # load module
    ggplot2 = importr('ggplot2')
    dplyr = importr('dplyr',  on_conflict="warn")

    # current path
    currentPath = os.getcwd()

    # back to root path
    os.chdir('../../../')
    root_path = os.getcwd()

    # get gwas_lead_snp.txt file
    input_path = root_path + '/output/' + projectName + "_" + CURRENT_TIME + '/top_genes/'
    if not os.path.exists(input_path):
        warning = "Please make sure that you have created an input folder with input files"
        add_log(warning)
        add_log('Input path should be: %s' %input_path)
    os.chdir(input_path)

    gwas_lead_snp = r("""
        gwas_lead_snp <- read.csv('sorted_top_genes.csv') %>%
          select(gene_name, chr, start) %>%
          distinct()
        colnames(gwas_lead_snp) = c('snpsNames', 'chrosome', 'startSites')

        """)
    robjects.globalenv['dataframe'] = gwas_lead_snp

    # annotated metaxcan file path
    merged_annoated_files_path = root_path + '/output/' + projectName + "_" + CURRENT_TIME + '/annotated_metaxcan_output_files/merged/'
    os.chdir(merged_annoated_files_path)

    data = r("data <- read.csv('merged_annotated_metaxcan_output.csv')")
    robjects.globalenv['dataframe'] = data

    bubble_plot_output_path = root_path + '/output/' + projectName + "_" + CURRENT_TIME + '/bubble_plot/'
    if not os.path.exists(bubble_plot_output_path):
         os.makedirs(bubble_plot_output_path)
    os.chdir(bubble_plot_output_path)

    # for loop through each snps site +/- 1000000 bp
    r("""
        # read dataframe
        startSites = as.numeric(as.character(gwas_lead_snp$startSites))
        snpsNames = as.character(gwas_lead_snp$snpsNames)
        chrosome = as.numeric(as.character(gwas_lead_snp$chrosome))

        # for loop through each snps site +/- 1000000 bp
        for (i in 1:length(startSites))
        {
            print(paste("BUBBLE PLOTING", ": ", snpsNames[i]))

            # subset data for each snps
            subData <- subset(data, data$start > startSites[i] - 1000000  &
                data$start < startSites[i] + 1000000 & data$chr==chrosome[i])
            subData <- subData[order(subData$start),]
            subData <- mutate(subData, z_score=ifelse(subData$zscore > 0, '   +   ', '   -   '))
            # Labels = subData$gene_name
            subData$gene_name <- factor(subData$gene_name, levels=subData$gene_name)
            # subData$tissue <- factor(subData$tissue, levels=subData$tissue)
            subData <- subData %>%
            inner_join(tissue_abbr, by = "tissue")

            # draw plot
            p <- ggplot(subData, aes(x=subData$gene_name, y=subData$tissue_site_detail_abbr, size=abs(subData$zscore)))
            p +
            geom_point(aes(colour=z_score)) +
            scale_color_manual(values=c('blue', 'brown')) +
            scale_size_continuous(guide=FALSE, range=c(0,max(abs(subData$zscore))))+
            # ggtitle(paste('locus: ', snpsNames[i], '(chromosome', chrosome[i], ')')) +
            labs(x='Gene', y='Tissue') +
            ggtitle(snpsNames[i])+
            # scale_x_discrete(breaks = subData$gene_name, labels=Labels) +
            theme(axis.text.x = element_text(size=16, face='bold', angle = 90, hjust = 1)) +
            theme(axis.text.y = element_text(size=16, face='bold')) +
            theme(plot.title = element_text(size=18, face='bold')) +
            # theme(axis.title= element_text(size=18, face='bold')) +
            theme(axis.title.x = element_blank(), axis.title.y = element_blank()) +
            theme(legend.position = "none") +
            theme(panel.grid.major = element_blank(), panel.grid.minor = element_blank())

            # save plot
            ggsave(paste(snpsNames[i], '_bubble_plot', '.png', sep=''), width=12, height=12)
        }
    """)

    add_log(datetime.now().strftime('%Y.%m.%d.%H:%M:%S ') + "Done!")

'''
######################
###  Region Plot #####
######################
'''
def regionplot(projectName):

    global projectID

    # load module
    ggplot2 = importr('ggplot2')
    dplyr = importr('dplyr',  on_conflict="warn")

    # current path
    currentPath = os.getcwd()

    # back to root path
    os.chdir('../../../')
    root_path = os.getcwd()

    # get gwas_lead_snp.txt file
    input_path = root_path + '/output/' + projectName + "_" + CURRENT_TIME + '/top_genes/'
    if not os.path.exists(input_path):
        warning = "Please make sure that you have created an input folder with input files"
        add_log(warning)
        add_log('Input path should be: %s' %input_path)
    os.chdir(input_path)

    gwas_lead_snp = r("""
        gwas_lead_snp <- read.csv('sorted_top_genes.csv') %>%
          select(gene_name, chr, start) %>%
          distinct()
        colnames(gwas_lead_snp) = c('snpsNames', 'chrosome', 'startSites')

        """)
    robjects.globalenv['dataframe'] = gwas_lead_snp

    # annotated metaxcan file path
    merged_annoated_files_path = root_path + '/output/' + projectName +  "_" + CURRENT_TIME + '/annotated_metaxcan_output_files/merged/'
    os.chdir(merged_annoated_files_path)

    data = r("data <- read.csv('merged_annotated_metaxcan_output.csv')")
    robjects.globalenv['dataframe'] = data

    bubble_plot_output_path = root_path + '/output/' + projectName + "_" + CURRENT_TIME + '/region_plot/'
    if not os.path.exists(bubble_plot_output_path):
         os.makedirs(bubble_plot_output_path)
    os.chdir(bubble_plot_output_path)

    r("""
        # read dataframe
        startSites = as.numeric(as.character(gwas_lead_snp$startSites))
        snpsNames = as.character(gwas_lead_snp$snpsNames)
        chrosome = as.numeric(as.character(gwas_lead_snp$chrosome))

        # loop through snps site +/- 1000,000 bps
        for (i in 1:length(startSites))
        {
        print(paste("REGION PLOTING", ": ", snpsNames[i]))

        # subset data
        subData <- subset(data, data$start > startSites[i] - 1000000  & data$start < startSites[i]
            + 1000000 & data$chr==chrosome[i])
        subData$logp <- -log10(subData$pvalue)
        subData <- subData[order(subData$start),]

        subData <- mutate(subData, sig=ifelse(subData$logp > -log10(0.05/nrow(data)), 'Most Sig',
        ifelse(subData$logp > 5.30103 & subData$logp <= -log10(0.05/nrow(data)), 'Sig',
        ifelse(subData$logp > -log10(0.05/nrow(subData))
            & subData$logp <= 5.30103, 'Less Sig','Not Sig'))))
        subData$gene_name <- factor(subData$gene_name, levels=subData$gene_name)

        p <- ggplot(subData, aes(x=gene_name, y=logp))
        p + geom_point(aes(colour = sig)) +
        scale_color_manual(guide=FALSE, values=c('black', 'black', 'black', 'black')) +
            # ggtitle(paste("locus: ", snpsNames[i], '(chromosome', chrosome[i], ')')) +
        labs(x='Gene', y='-log10(p-value)') +
        ggtitle(snpsNames[i])+
        geom_hline(yintercept = 5.30103, linetype='dashed', color='black') +
        geom_hline(yintercept = -log10(0.05/nrow(data)), color='black') +
        geom_hline(yintercept = -log10(0.05/nrow(subData)), linetype='dashed', color='black') +
        theme(axis.text.x = element_text(size=16, face='bold', angle = 90, hjust = 1)) +
        theme(axis.text.y = element_text(size=16, face='bold')) +
        theme(plot.title = element_text(size=18, face='bold')) +
        theme(axis.title.x = element_blank(), axis.title.y=element_text(size=18, face='bold')) +
        theme(legend.position = "none")+
        theme(panel.grid.major = element_blank(), panel.grid.minor = element_blank())

        # output plot
        ggsave(paste(snpsNames[i], '_region_plot', '.png',sep=''), width=12, height=12)
        }
    """)

    add_log(datetime.now().strftime('%Y.%m.%d.%H:%M:%S ') + "Done!")

'''
#########################
###  locuszoom Plot #####
#########################
'''
def locuszoom_plot(projectName):

    dplyr = importr('dplyr',  on_conflict="warn")

    global projectID

    # current path
    currentPath = os.getcwd()

    # back to root path
    os.chdir('../../../')
    root_path = os.getcwd()

    # make batch_locuszoom.txt file
    input_path = root_path + '/output/' + projectName + "_" + CURRENT_TIME + '/top_genes/'
    if not os.path.exists(input_path):
        warning = "Please make sure that you have created an input folder with input files"
        add_log(warning)
        add_log('Input path should be: %s' %input_path)
    os.chdir(input_path)

    gwas_lead_snp = r("""
        gwas_lead_snp <- read.csv('sorted_top_genes.csv') %>%
          select(gene_name, chr, start, end) %>%
          distinct() %>%
          mutate(flank='1.0MB', run = 'yes', m2zargs= "showAnnot=F")
          colnames(gwas_lead_snp) = c('snp', 'chr', 'start', 'stop', 'flank', 'run', 'm2zargs')
          write.table(gwas_lead_snp, file="batch_locuszoom.txt", sep=" ", quote=FALSE, row.names=F)

        """)

    # write gwas_lead_snp into txt file
    # gwas_lead_snp.to_csv("batch_locuszoom.txt", sep='\t')

    # create a folder to hold locuszoom results
    locuszoom_plot_path = root_path + '/locuszoom/locuszoom_plots/'
    if not os.path.exists(locuszoom_plot_path):
        os.makedirs(locuszoom_plot_path)

    # source files from input path
    # copy the files including plink, run_locuszoom.py and two other .txt from input path into locuszoom program
    destination = locuszoom_plot_path     # locuszoom plot destination path
    plink_destination = root_path + '/locuszoom'

    source = os.listdir(input_path)

    for file in source:
        if file.endswith(".txt"):
            shutil.copy(file, destination)
            msg = "COPYING: the file '%s' into the folder '%s'" % (file, destination)
            add_log(msg)
    os.system('rm batch_locuszoom.txt')

    # input path
    input_path = root_path + '/input/'
    if not os.path.exists(input_path):
        warning = "Please make sure that you have created an input folder with input files"
        add_log(warning)
        add_log('Input path should be: %s' % input_path)
    os.chdir(input_path)

    source = os.listdir(input_path)

    plink_file = 'plink'
    shutil.copy(plink_file, plink_destination)

    msg = "COPYING: the file '%s' into the folder '%s'" % (plink_file, plink_destination)
    add_log(msg)

    for file in source:
        if file.endswith(".txt"):
            shutil.copy(file, destination)
            msg = "COPYING: the file '%s' into the folder '%s'" % (file, destination)
            add_log(msg)

    locuszoom_path = root_path + '/MetaXcan-Postprocess/scripts/Python_scripts'
    os.chdir(locuszoom_path)

    locuszoom_scrip_cmd = 'run_locuszoom.py'
    shutil.copy(locuszoom_scrip_cmd, destination)
    msg = "COPYING: the file '%s' into the folder '%s'" % (locuszoom_scrip_cmd, destination)
    add_log(msg)

    # run locuszoom program
    os.chdir(destination)
    os.system('./run_locuszoom.py')
    add_log('RUNNING: run_locuszoom.py...')

    # setup locuszoom plot output file path and move all files to this new output folder
    locuszoom_plot_files_path = root_path + '/output/' + projectName + "_" + CURRENT_TIME + '/looszoom_plot/'
    if not os.path.exists(locuszoom_plot_files_path):
        os.makedirs(locuszoom_plot_files_path)
    os.chdir(destination)

    src = destination
    dst = locuszoom_plot_files_path
    os.system('mv %s %s' % (src, dst))

    msg = "MOVING: all files from the folder '%s' into the folder '%s'" % (src, locuszoom_plot_files_path)
    add_log(msg)

    add_log(datetime.now().strftime('%Y.%m.%d.%H:%M:%S ') + "Done!")


from cherrypy.process import plugins
from sqlalchemy import create_engine
from sqlalchemy.orm import scoped_session, sessionmaker

__all__ = ['SaPlugin']

class SaPlugin(plugins.SimplePlugin):
    def __init__(self, bus, connection_string=None):
        """
        The plugin is registered to the CherryPy engine and therefore
        is part of the bus (the engine *is* a bus) registery.

        We use this plugin to create the SA engine. At the same time,
        when the plugin starts we create the tables into the database
        using the mapped class of the global metadata.
        """
        plugins.SimplePlugin.__init__(self, bus)
        self.sa_engine = None
        self.connection_string = connection_string
        self.session = scoped_session(sessionmaker(autoflush=True, autocommit=False))

    def start(self):
        self.sa_engine = create_engine(self.connection_string, echo=False)
        self.bus.subscribe("bind-session", self.bind)
        self.bus.subscribe("commit-session", self.commit)
        self.bus.log("SA plugin started")

    def stop(self):
        self.bus.unsubscribe("bind-session", self.bind)
        self.bus.unsubscribe("commit-session", self.commit)
        if self.sa_engine:
            self.sa_engine.dispose()
            self.sa_engine = None
        self.bus.log("SA plugin stopped.")

    def bind(self):
        self.session.configure(bind=self.sa_engine)
        return self.session

    def commit(self):
        try:
            self.session.commit()
        except:
            self.session.rollback()
            raise
        finally:
            self.session.remove()