#!/usr/bin/env python
# -*- coding: utf-8 -*-

################################################################################
#
# 将mongoDB中的赛程数据导出为XML模板表格式
#
# usage:
#   python export_fixtures.py --help
#
#   导出英超2016/2017赛季赛程数据到8_2017.xml
#   python export_fixtures.py -h 10.1.0.6 -p 27017 -d football -c fixtures -m 8 -s 2016 -o 8_2016.xml
#
#   OR
#   python export_fixtures.py --config config.ini
#
#
################################################################################
from ConfigParser import ConfigParser
import pymongo
import codecs
import datetime
import argparse
import sys
import enum
import common

# 比赛类型
SPORT_TYPE = 1
FINAL_TIME = 60 * 30 # 秒

DOCUMENT = """<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<ns1:Document xmlns:ns1="http://viivgame.cn/schema/db_match_schedule.xsd">
%s
</ns1:Document>
"""
ROW_TEMPLATE = """<ns1:db_match_schedule N_match_id="%d" N_match_type="%d" N_sport_type="%s" N_match_round="%s" N_match_start_time="%s" N_match_valide_time="%s" N_home_id="%d" N_away_id="%d" N_home_score="%d" N_away_score="%d"/>"""

class ConfigFile:
    """docstring for ConfigFile:"""
    def __init__(self, cfgfile="config.ini"):
        fd = codecs.open(cfgfile, 'r', encoding='utf-8')
        self.cfg = ConfigParser()
        self.cfg.readfp(fd)
    def __getattr__(self, name):
        try:
            return self.cfg.get('main', name)
        except Exception as e:
            return None

def export_fixtures_to_template(args):

    condition = dict()
    try:
        if args.match:
            condition['compid'] = str(args.match)
    except Exception as e:
        pass
    try:
        if args.season:
            condition['season'] = str(args.season)
    except Exception as e:
        pass

    sport_type = enum.ST_FOOTBALL
    try:
        sport_type = int(args.type)
    except Exception as e:
        pass

    if not(args.host and args.port and args.db and args.collection):
        assert False
        print("导出赛程失败！")
        return

    client = pymongo.MongoClient(args.host, int(args.port))
    database = client.get_database(args.db)
    collection = database.get_collection(args.collection)

    content = []
    content_classify = dict()
    for record in collection.find(condition):
        start_time = datetime.datetime.strptime("%s %s" % (record['date'], record['time']), "%Y-%m-%d %H:%M").strftime('%s')

        if record['date'] in content_classify:
            content_classify[record['date']][0].append(record)
            if int(start_time) < content_classify[record['date']][1]:
                content_classify[record['date']][1] = int(start_time)
        else:
            content_classify[record['date']] = [[record], int(start_time)]

    for k, v in content_classify.iteritems():
        # 最早开赛前30分钟
        valide_time = str(v[1] - FINAL_TIME)
        for record in v[0]:

            now = datetime.datetime.now().strftime('%s')
            start_time = datetime.datetime.strptime("%s %s" % (record['date'], record['time']), "%Y-%m-%d %H:%M").strftime('%s')
            homescore = awayscore = -1
            if int(now) > int(start_time):
                try:
                    homescore = int(record['homescore'])
                    awayscore = int(record['awayscore'])
                except Exception as e:
                    print("%s 比赛有结果有误!")
                    pass

            mid = int(record['id'])
            mid = common.generate_uid(mid, sport_type)
            homeid = common.generate_uid(int(record['homeid'][1:]), sport_type)
            awayid = common.generate_uid(int(record['awayid'][1:]), sport_type)

            row = ROW_TEMPLATE % (
                        mid,
                        int(record['compid']),
                        str(sport_type),
                        record['round'],
                        start_time,
                        #record['date'],
                        valide_time,
                        homeid,
                        awayid,
                        homescore,
                        awayscore
                    )
            content.append(row)

    open(args.output,'w').write(DOCUMENT % '\n'.join(content))

def parse_cmd_options():
    parser = argparse.ArgumentParser(
            description="将爬虫抓取的赛程数据输出为赛程模板表格式",
            conflict_handler='resolve'
            )

    #parser.add_argument('-?', '--help', action='store_true')
    parser.add_argument(
            '-h', '--host', default='10.1.0.6',
            help='存放爬虫数据的MongoDB数据库ID default: %(default)s'
            )
    parser.add_argument(
                '-p', '--port', type=int, default=27017,
                help='MongoDB使用的端口。default: %(default)s'
            )
    parser.add_argument(
                '-d', '--db', default='football', help='db名'
            )
    parser.add_argument(
                '-c', '--collection', default='fixture', help='collection名'
            )
    parser.add_argument(
                '--config', help='从文件中读取配置信息。'
            )
    parser.add_argument(
                '-t', '--type', default=1,
                help='体育类型。1:足球. default: %(default)s'
            )
    parser.add_argument(
                '-m', '--match', help='联赛类型。208:中超'
            )
    parser.add_argument(
                '-s', '--season', help='赛季'
            )
    parser.add_argument(
                '-o', '--output', default="db_match_schedules.xml",
                help='输出文件名.'
            )

    args = parser.parse_args()

    if not(args.config or args.db or args.collection):
        sys.exit('must set -d, -c or --config')

    return args

def main():

    args = parse_cmd_options()

    if args.config:
        # 如果有指定配置文件，则优先使用配置文件
        args = ConfigFile(args.config)

    export_fixtures_to_template(args)

if '__main__' == __name__:
    main()


from .base import FunctionalTest
import time
from selenium.common.exceptions import WebDriverException
from datetime import datetime, date
from django.contrib.auth.models import User
# from selenium.webdriver.firefox.webdriver import WebDriver
# from selenium.webdriver.common.keys import Keys
# import unittest
# import os

# StaticLiveServerTestCase -> serve static files during test, spin up development server for test, creates and deletes test database without interfering with production database
class TestNewObjectSubmission(FunctionalTest):

	# User makes a submission and observes on objectList
	def test_1_new_userToDo_UserNote__NoteDescription_submission(self):
		User.objects.create_user('admin', 'admin@testuser.com', 'adminTest')

		# Webpage startup
		self.selenium.get( ('%s%s' % (self.live_server_url, '/')) )
		self.wait_for_(lambda: self.find_object_by_id("loginHeader", "Welcome to Marindec."))
		self.insert_Login(
			"admin",
			"adminTest",
			)

		self.wait_for_(lambda: self.find_object_by_id("updateItem", "ajax loaded."))

		self.insert_UserToDo(
			"Test New User To Do",
			"In Progress",
			failure=False,
			)

		self.selenium.find_element_by_id('refreshObjectList').click()
		self.wait_for_(lambda: self.find_object_by_id("UserToDo_subject", "Test New User To Do"))

		time.sleep(5)

		self.selenium.execute_script("window.scrollTo(0, document.body.scrollHeight);")
		# User inserts a new usernote
		self.selenium.find_element_by_id('UserToDo_subject').click()
		self.wait_for_(lambda: self.find_object_by_id("addUserNote_Toggle", "Insert a task"))
		self.selenium.find_element_by_id('addUserNote_Toggle').click()

		self.insert_UserNote(
			"Test New User Note",
			"In Progress",
			# datetime.strftime(date.today(), '%Y-%m-%d'),
			failure=False,
			)
		time.sleep(5)
		self.selenium.execute_script("window.scrollTo(0, document.body.scrollHeight);")

		self.selenium.find_element_by_id('refreshObjectList').click()
		time.sleep(5)
		self.selenium.find_element_by_id('UserToDo_subject').click()
		time.sleep(1)	# Allow for animation
		self.wait_for_(lambda: self.find_object_by_id("userNote_subject", "Test New User Note"))

		# User inserts a new note description
		self.selenium.find_element_by_id('userNote_subject').click()
		time.sleep(1)	# Allow for animation
		self.selenium.find_element_by_id('addNoteDescription_Toggle').click()
		self.insert_NoteDescription(
			"Test New Note Description",
			"In Progress",
			)
		time.sleep(5)
		self.selenium.find_element_by_id('refreshObjectList').click()
		time.sleep(5)
		self.selenium.find_element_by_id('UserToDo_subject').click()
		time.sleep(1)	# Allow for animation
		self.selenium.find_element_by_id('userNote_subject').click()
		time.sleep(1)	# Allow for animation
		self.wait_for_(lambda: self.find_object_by_id("noteDescription_1", "Test New Note Description"))

		# user clicks on edit objective to being making changes to UserToDO
		self.selenium.find_element_by_id('editUserToDo_Toggle').click()

		# Edit objective.
		self.elementID_wait("editUserToDo_heading")

		self.edit_UserToDo(
			"Edited New User To Do",
			"Completed",
			)
		self.selenium.find_element_by_id('refreshObjectList').click()
		self.wait_for_(lambda: self.find_object_by_id("UserToDo_subject", "Edited New User To Do"))
		print("TEST: test_new_userToDo_submission -> complete")

		# # User decides to make changes to userNote
		# self.selenium.find_element_by_id('UserToDo_subject').click()
		# time.sleep(1)	# Allow for animation
		# self.wait_for_(lambda: self.find_object_by_id("addUserNote_Toggle", "Insert a task"))
		# self.selenium.find_element_by_id('editUserNote_Toggle').click()
		# self.wait_for_(lambda: self.find_object_by_id("editUserNote_heading", "Edit this secondary objective"))
		# self.wait_for_(lambda: self.find_object_by_id("editUserNote_taskNote", "Test New User Note"))
		# self.selenium.execute_script("window.scrollTo(0, 1200)")
		# time.sleep(1)
		# self.edit_UserNote(
		# 	"Edited New User To Do",
		# 	"Completed",
		# 	)

		# # User decides to change a note description
		# self.selenium.find_element_by_id('refreshObjectList').click()
		# time.sleep(1)	# Allow for animation
		# self.selenium.find_element_by_id('UserToDo_subject').click()
		# time.sleep(1)	# Allow for animation
		# self.selenium.find_element_by_id('userNote_subject').click()
		# time.sleep(1)	# Allow for animation
		# self.selenium.find_element_by_id('editNoteDescription_Toggle').click()
		# self.edit_NoteDescription(
		# 	"Test New Note Description",
		# 	"In Progress",
		# 	)
		# self.selenium.execute_script("window.scrollTo(0, 1200)")
		# time.sleep(1)

		print("\ndef:test_1_new_userToDo_UserNote__NoteDescription_submission -> complete\n")

		return True


#!/usr/bin/env python3
#importo librerías
import pyspark
from pyspark import SparkContext, SparkConf, SQLContext
from pyspark.sql.functions import *
from pyspark.sql import DataFrameStatFunctions, DataFrame
from pyspark.sql.types import *
from pyspark.ml import Pipeline
from pyspark.ml.feature import *
from pyspark.ml.classification import RandomForestClassifier, LogisticRegression
from pyspark.ml.regression import GeneralizedLinearRegression
from pyspark.ml.evaluation import MulticlassClassificationEvaluator
from pyspark.ml.tuning import CrossValidator, ParamGridBuilder
import re as re
import time

#arranque de Spark
conf = SparkConf()
conf.set("spark.driver.memory", "16g")
conf.set("spark.driver.cores", 4)
conf.set("spark.driver.memoryOverhead", 0.9)
conf.set("spark.executor.memory", "32g")
conf.set("spark.executor.cores", 12)
conf.set("spark.jars", "/home/jaa6766")
sc = SparkContext(master = "local[14]", sparkHome="/usr/local/spark/",
                  appName="tarea-mge-7", conf=conf)
spark = SQLContext(sc)
##############
#carga de datos
flights = spark.read.csv("hdfs://172.17.0.2:9000/data/flights/flights.csv",
                         inferSchema=True,
                         header=True)
flights = flights.cache()
flights.show(2)
flights.printSchema()
####################
#feature engineering
#columnas más relevantes
relevant_cols = ["YEAR", "MONTH", "DAY", "DAY_OF_WEEK", "AIRLINE",
                "FLIGHT_NUMBER", "ORIGIN_AIRPORT", "DESTINATION_AIRPORT",
                "SCHEDULED_DEPARTURE", "DEPARTURE_TIME", "SCHEDULED_TIME",
                "AIR_TIME", "DISTANCE", "SCHEDULED_ARRIVAL", "CANCELLED",
                "DEPARTURE_DELAY"]
relevant_cols

### selección de columnas
flights = flights.select(relevant_cols)
### descartar nulos
flights = flights.na.drop().cache()
### dividir en sets de entrenamiento y set de pruebas
flights.write.parquet("hdfs://172.17.0.2:9000/tmp/flights-tmp.parquet", mode="overwrite")
flights.show(2)

#por eficiencia he visto que es mejor guardarlo
flights = spark.read.parquet("hdfs://172.17.0.2:9000/tmp/flights-tmp.parquet")
flights = flights.cache()
#dividir el set
(train, test) = flights.randomSplit([0.7, 0.3], seed=175904)
train2 = train.sample(fraction=0.001, seed=175904)

#guardamos por eficiencia, nuevamente
train.write.parquet("hdfs://172.17.0.2:9000/tmp/train-tmp.parquet", mode="overwrite")
test.write.parquet("hdfs://172.17.0.2:9000/tmp/test-tmp.parquet", mode="overwrite")
train = spark.read.parquet("hdfs://172.17.0.2:9000/tmp/train-tmp.parquet")
test = spark.read.parquet("hdfs://172.17.0.2:9000/tmp/test-tmp.parquet")
train = train.cache()
test = test.cache()

#descartando categóricas y la que vamos a predecir: "DEPARTURE_DELAY"
features = [ x for x in train.schema.names if x not in
            [ "DEPARTURE_DELAY", "AIRLINE", "ORIGIN_AIRPORT", "DESTINATION_AIRPORT"] ]
#agregando las columnas que ya van a ser indizadas
features.append("airline_indexer")
features.append("origin_indexer")
features.append("destination_indexer")
print("Features que se tomarán para la predicción: ", features)

#indización de columnas: de categóricas a texto
li0 = StringIndexer(inputCol="AIRLINE",
                    outputCol="airline_indexer",
                    handleInvalid="skip") \
    .fit(flights)
li1 = StringIndexer(inputCol="ORIGIN_AIRPORT",
                    outputCol="origin_indexer",
                    handleInvalid="skip") \
    .fit(flights)
li2 = StringIndexer(inputCol="DESTINATION_AIRPORT",
                    outputCol="destination_indexer",
                    handleInvalid="skip") \
    .fit(flights)
li  = StringIndexer(inputCol="DEPARTURE_DELAY",
                    outputCol="delay_indexer",
                    handleInvalid="skip") \
    .fit(flights)
#este ensamble es requerido para usar los clasificadores y regresores de Spark ML
va0 = VectorAssembler() \
    .setInputCols(features) \
    .setOutputCol("features")
pca = PCA(k=10, inputCol="features", outputCol="features_pca")
lc = IndexToString(inputCol="prediction", outputCol="predictedLabel",
                   labels=li.labels)
######################
# modelos
# regresión logística
lr = LogisticRegression(featuresCol="features_pca",
                        labelCol="delay_indexer",
                        #maxIter=10, elasticNetParam=0.8,
                        regParam=0.3,
                        family="multinomial",
                        predictionCol="prediction")
#creación del pipeline, este no se va a utilizar en el Magic Loop
pipeline1 = Pipeline(stages=[li0, li1, li2, va0, pca, li, lr, lc])
# regresión lineal
glr = GeneralizedLinearRegression(featuresCol="features_pca",
                        labelCol="delay_indexer", family="Gaussian")
#creación del pipeline, este no se va a utilizar en el Magic Loop
pipeline2 = Pipeline(stages=[li0, li1, li2, va0, pca, li, glr, lc])

#####################
# Magic Loop
def magic_loop3(pipelines, grid, train, test, cvfolds=3):
    best_score = 0.0 #symbolic high value :-)
    best_grid = None #inicializar la variable
    #este loop inicia las pruebas secuenciales de los pipelines:
    #es relevante que no sólo soporta 2, sino se va en cada uno
    #de los que estén presentes en la lista
    for pipe in pipelines:
        try:
            #quiero que se desligue, para no modificar (al final, usa poco RAM)
            pipe = pipe.copy()
            #etapas del pipeline
            stages = pipe.getStages()
            #obtener el predictor (el motor ML)
            predictor = [stage for stage in stages
                if "pyspark.ml.classification" in str(type(stage)) or
                 "pyspark.ml.regression" in str(type(stage))][0]
            predictor_i = stages.index(predictor)
            stringer = [stage for stage in pipe.getStages()
                if "pyspark.ml.feature.StringIndexer" in str(type(stage))][0]
            if DEBUG: print("pipeline:\n%s\n\n"%stages)
            if DEBUG: print("predictor=%s (index %s, type %s), stringer=%s (%s)\n"%
                  (predictor, stages.index(predictor), type(predictor),
                  stringer, type(stringer)))
            #dado que no son predicciones susceptibles a cambios en el CV
            prepipe = Pipeline(stages=stages[0:(predictor_i)])
            if DEBUG: print("pre pipeline:\n%s\n\n"%prepipe.getStages())
            print("Starting fit on prepipe...")
            #modelo para el prepipeline
            prepipem = prepipe.fit(train)
            print("Starting transform on prepipe...")
            train2 = prepipem.transform(train)
            test2 = prepipem.transform(test)
            #el motor ML sí es susceptible CV
            postpipe = Pipeline(stages=stages[(predictor_i):len(stages)])
            if DEBUG: print("post pipeline:\n%s\n\n"%postpipe.getStages())
            #creación del Cross Validator
            gridcv = CrossValidator(estimator=postpipe,
                          estimatorParamMaps=grid,
                          evaluator=MulticlassClassificationEvaluator(labelCol="DEPARTURE_DELAY"),
                          numFolds=cvfolds)
            #extraemos el nombre y le quitamos la parte "fea" que devuelve type()
            predictr = [str(type(stage)) for stage in pipe.getStages()
            if "pyspark.ml.classification" in str(type(stage)) or
             "pyspark.ml.regression" in str(type(stage))][0]
            predictr = re.sub("^<class 'pyspark\.ml\.(classification|regression)\.([a-zA-Z0-9]+)'>",
                              "\\2",
                              predictr)
            #creación del modelo
            print("Starting fit on %s..."%predictr)
            gridcvm = gridcv.fit(train2)
            #aplicación del modelo
            print("Starting transform on %s..."%predictr)
            preds = gridcvm.transform(test2)
            #obtenemos el evaluador para el uso en la medición de errores
            ev = gridcvm.getEvaluator()
            #obtenemos la métrica del error
            metric = ev.getMetricName()
            print("Starting error calculation on %s..."%predictr)
            #obtenemos el valor del eror
            error  = ev.evaluate(preds)
            print("Error %s: %f"% (metric, error))
            #si es mejor que el modelo pasado, lo guardamos. El último
            #guardado será el que devuelva esta función
            if error > best_score:
                print("%s is the best model so far: %f (%s)"%(predictr, error, metric))
                best_grid = gridcvm
        #manejo de errores y horrores
        except Exception as e:
            print('Error during Magic Loop:', e)
        continue
    return best_grid
#parametros para el magic loop
paramGrid = ParamGridBuilder() \
                .addGrid(glr.family, ["Gaussian", "Poisson", "Tweedie"]) \
                .addGrid(glr.maxIter, [1, 2, 3]) \
                .addGrid(lr.maxIter, [1, 2, 3]) \
                .addGrid(lr.elasticNetParam, [0.1,0.2,0.3]) \
                .build()
magic = magic_loop3(pipelines, paramGrid, train, test, 3)
#obtengo el pipeline que devolvió el magic loop
best_model = magic.getEstimator()
#obtenemos el paso del clasificador
best_estimator = best_model.getStages()[0]
#guardo en una variable los parámetros más adecuados
best_estimator_params = best_estimator.extractParamMap()
#obtener el evaluador para medir errores
ev0 = magic.getEvaluator()
###################
#medición del error
#impresión de los parámetros y el mejor modelo
print("%s (best model) Parameters: maxIter=%d, elasticNetParam=%f"%
      (re.sub("^<class 'pyspark\.ml\.(classification|regression)\.([a-zA-Z0-9]+)'>",
                              "\\2", (str)((type(best_estimator)))),
       best_estimator_params[best_estimator.maxIter],
       best_estimator_params[best_estimator.elasticNetParam]))
#aquí probé como se veían los errores
preds = magic.transform(Pipeline(stages=pipeline1.getStages()[0:6]).fit(test).transform(test))
print("Error %s: %f"% (ev0.getMetricName(), ev0.evaluate(preds)))


from io import BytesIO
from time import time

import pybase16384 as pybs

pybs.is_64bits()
with open("input.pcm", "rb") as f:
    data = f.read()

st = time()
for i in range(10):
    pybs.encode_file(BytesIO(data), open("output2.pcm", "wb"), True, len(data) // 7)
print(f"耗时： {time() - st}")


"""
The :mod:`!xml_prm.parser` module contains the parsers that instatiate a :mod:`probrem` instance from the XML specifications. There is parser for

* Data Interface specification, :class:`DataInterfaceParser`
* PRM specifcation, :class:`PRMparser`
* Loading model parameters stored on disk, :class:`LocalDistributionParser`

"""

from prm.relationalschema import Entity, Relationship, UncertainRelationship
from prm.attribute import *
from prm.dependency import Dependency, UncertainDependency
from prm.localdistribution import *
from prm import prm

import data.datainterface as DI
from data.datainterface import datasetinterfaceFactory
from data.datainterface import DataSetInterface
import data.aggregation

import xml.parsers.expat

import logging

import numpy

'''
Elements defined for PRM specification
'''
prmEl = 'PRM'
prmEl_name = 'name'
prmEl_di = 'datainterface'
rsEl = 'RelationalSchema'
entsEl = 'Entities'
entiEl = 'Entity'
entiEl_name = 'name'
relsEl = 'Relationships'
relEl = 'Relationship'
relEl_name = 'name'
relEl_foreign = 'foreign'
relEl_k = 'k'
relEl_type = 'type'
relEl_description = 'description'
attrEl = 'Attribute'
attrEl_name = 'name'
attrEl_type = 'type'
attrEl_description = 'description'
attrEl_pk = 'pk'
attrEl_hidden = 'hidden'
depstEl = 'DependencyStructure'
depEl = 'Dependency'
depEl_name = 'name'
depEl_parent = 'parent'
depEl_child = 'child'
depEl_constraint = 'constraint'
depEl_agg = 'aggregator'
depEl_refun = 'refun'
locDistsEl = 'LocalDistributions'
locDistEl = 'LocalDistribution'
locDistEl_attr = 'attribute'
locDistEl_file = 'file'
locDistEl_parents = 'parents'

'''
Elements defined for a local distribution specification
'''
tabCPDEl = 'TabularCPD'
tabCPDEl_file = 'file'
treeCPDEl = 'TreeCPD'

'''
Elements defined for a data interface specification
'''
diEl = 'DataInterface'
diEl_name = 'name'
crossValEl = 'Crossvalidation'
dsEl = 'Dataset'
dsEl_type = 'type'
dsEl_path = 'path'

class PRMparser:
    """
    The PRM model is also specified in XML and saved in for example *./PRMexample.xml*::

        <?xml version="1.0" ?>
        <PRM name="PRMexample"  datainterface="./DIexample.xml" >
        	<RelationalSchema>
        		<Entities>
        			<Entity name="A">
        				<Attribute name="Aa" type="Binary"/>
        			</Entity>
        			<Entity name="B">
        				<Attribute name="Ba" type="Integer" description="1,20"/>
        			</Entity>
                    [......]
        		</Entities>
        		<Relationships>
        			<Relationship name="AB" foreign="A.pk,B.pk" type="1:n">
        				<Attribute name="ABa" type="Binary"/>
        			</Relationship>
        			[......]
        		</Relationships>
        	</RelationalSchema>
        	<DependencyStructure>
        		<Dependency name="Aa_Ba" child="A.Aa" parent="B.Ba" constraints="A.pk=B.pk"  aggregator='AVG'/>
        		[......]
        	</DependencyStructure>
        	<LocalDistributions>
        		<LocalDistribution attribute='A.Aa' file='./localdistributions/Da_Aa.xml'/>
        		<LocalDistribution attribute='B.Ba' file='./localdistributions/Ba_Aa.xml'/>
        		<LocalDistribution attribute='AB.ABa' file='./localdistributions/Ca_Aa.xml'/>
        	</LocalDistributions>
        </PRM>

    A list of all imporant xml tags and xml attributes. Note the somewhat confusing double use of the word attributes, on one hand xml attributes and on the other hand the probabilistic PRM :mod:`prm.attribute`.

    **PRM**

        * `name` : Freely chosen name for PRM model
        * `datainterface` : relative path to the data interface specification, optional

    **Entity**

        * `name` : The name has to correspond with the corresponding table name in the relational database

    **Relationship**

        * `name` : The name has to correspond with the corresponding table name in the relational database
        * `foreign` : A list, separated by a coma, of the foreign keys of the relationship. The foreign attributes can be probabilistic, but usually the primary key of an entity serves as foreign key. The primary key of an entity can be defined as probilistic attribute, but this entails that the domain is all the entries in one database table. This doesn't scale well at all, but sometimes this can be desired behavior. In case the foreign key is not probabilistic and thus most likely a primary key, the special keyword `pk` can be used to refer to a primary key of an entity. The `pk` keyword defaults to `entityname_id` as the name of the primary key. Thus if there is a table `Professor`, `Professor.pk' would refer `Professor.professor_id`.
        * `type` :  This is only used in the context of reference uncertainty. A relationship can be of type `k:n`,`n:k`.
        * `k` : A fixed parameter indicating an uncertain relationship (i.e. reference uncertainty) of type `n:k`.

    **Attribute**

        * `name` : The name has to be correspond to the data field in the relational database. Also, it has to be unique among the attributes of the entity/relationship it is part of.
        * `type` : The type must be either

            * `Binary`, instantiates an attribute of type :class:`~!.prm.attribute.BinaryAttribute`
            * `Integer`, instantiates an attribute of type :class:`~!.prm.attribute.IntegerAttribute`
            * `Enumerated`, instantiates an attribute of type :class:`~!.prm.attribute.EnumeratedAttribute`
            * `NotProbabilisticAttribute`, instantiates an attribute of type :class:`~!.prm.attribute.NotProbabilisticAttribute`

        * `description` : Specifies the domain of the attribute

            * For `Binary` not required
            * For `Integer`, e.g. "1,5" results in [1,2,3,4,5] (including 5)
            * For `Enumerated`, coma separated list of domain, e.g. "1,4,78" results in [1,4,78]
            * For `NotProbabilisticAttribute` not required

        * `pk` : Optional. If you choose to use the primary key, set pk="1". Note the remark in the `Relationship` tag for more information

    **Dependency**

        * `name` : Freely chosen name for probabilistic dependency
        * `parent` : Parent attribute, referenced by its full name. e.g. `Professor.fame`
        * `child` : Child attribute, referenced by its full name. e.g. `Student.success`
        * `constraint` : Optional. A coma separated list of constraints that can be applied to the data interface. Most commonly used are normal slotchains, e.g. "Professor.pk=Advisor.professor_id,Advisor.student_id=Student.pk". If no constraint is given, ProbReM will apply a depth-first-search to find a slot chain from child to parent using :meth:`.computeSlotChain`
        * `aggregator` : If the constraint on the dependency leads to one child attribute object having multiple parent attribute objects, the values of the parent attribute objects have to be aggregated as the CPD for the child allows only one value for that parent. The module :mod:`.aggregation` implements different such methods, e.g. `AVG`, `MAX`, `MIN`, `MODE`
        * `refun` : Indicator if the dependency has an uncertain relationship in the slotchain ('1','True' or 'T')

    **LocalDistribution**
        See :class:`LocalDistributionParser`
    """

    def __init__(self):
        '''
        See the documentation for the prm class
        '''
        self.entities = {}
        self.relationships = {}
        self.attributes_temp = {}
        self.dependencies = {}
        self.attributes = {}

        #temp variables
        self.currentER = None # we keep track of the currently handled entity/relationship because we attach the attributes (attributes_temp) at the end
        self.names = [] #we only allow unique names (makes things easier and is not unreasonable given that a person designs the prm specification for now)
        self.foreign = []

        ''' We need a different parser for parsing local distributions which have been specified '''
        self.localDistParser = LocalDistributionParser()

    def start_element(self,name, attrs):
        '''
        Encountered a starting tag of an element. Instances of entities, relationships, attributes
        and dependencies are created.
        '''

        ''' Avoid duplicate names for all elements  (Specifications should not become too big for now)'''
        if 'name' in attrs:
            if attrs['name'] in self.names:
                raise Exception('Error in PRM spec (duplicate name):%s'%attrs['name'])
            else:
                self.names.append(attrs['name'])

        ''' Set PRM name '''
        if name==prmEl:
            self.prmName = attrs[prmEl_name]
            self.di = None
            if prmEl_di in attrs:
                self.di = attrs[prmEl_di]

        ''' Create Entity class '''
        if name==entiEl:
            self.currentER = Entity(attrs[entiEl_name])
            self.entities[attrs[entiEl_name]] = self.currentER

        ''' Create Relationship class '''
        if name==relEl:
            # see comment in end_element handler for a relationship.
            self.foreign = attrs[relEl_foreign]

            # Reference Uncertainty introduces uncertain relationships
            if relEl_type in attrs:

                nTok = None
                reltype = attrs[relEl_type]
                if reltype == 'n:k':
                    nTok = True
                elif reltype == 'k:n':
                    nTok = False
                else:
                    raise Exception('Relationship type (`k:n` or `n:k`)')

                # k in `n:k`
                k = 1
                if relEl_k in attrs:
                    k = attrs[relEl_k]

                self.currentER = UncertainRelationship(name = attrs[relEl_name], nTok = nTok, k = k )
            else:
                self.currentER = Relationship( name = attrs[relEl_name] )

            self.relationships[attrs[relEl_name]] = self.currentER

        ''' Create Attribute class '''
        if name==attrEl:
            name = attrs[attrEl_name]
            er = self.currentER
            atype = attrs[attrEl_type]

            attrDef = None
            if attrEl_description in attrs:
                attrDef = attrs[attrEl_description]

            attribute = attributeFactory(name, er , atype, attrDef )

            # Adding the new attribute object to data structures
            self.attributes_temp[attribute.fullname] = attribute
            self.attributes[attribute.fullname] = attribute

            #primary key if defined in xml specification
            if attrEl_pk in attrs:
                pk = attrs[attrEl_pk]
                if pk=="1" or pk=="T" or pk=="True":
                    self.currentER.pk.append(attribute)

        ''' Create a dependency class '''
        if name==depEl:
            #parent must either be an entity or relationship attribute

            name = attrs[depEl_name]
            parent = self.attributes[attrs[depEl_parent]]
            child = self.attributes[attrs[depEl_child]]
            aggr = None
            if depEl_agg in attrs:
                aggr = data.aggregation.aggregators[attrs[depEl_agg]]
            constraint = None
            if depEl_constraint in attrs:
                constraint=attrs[depEl_constraint]

            refun = False
            if depEl_refun in attrs:
                refunS = attrs[depEl_refun]
                if refunS=="1" or refunS=="T" or refunS=="True":
                    refun = True

            if refun:
                self.dependencies[name] = UncertainDependency(name=name, parent=parent, child=child, constraint=constraint,aggregator=aggr,attributes=self.attributes)

                # The slotchain has been defined, we can now determine wheter there is reference uncertainty in
                # that dependency. If the dependency starts or ends with an uncertain relationship it is not
                # reference uncertainty

                # Find what direction the slotchain points
                childFirst = False
                if self.dependencies[name].child.erClass == self.dependencies[name].slotchain[0]:
                    childFirst = True

                # logging.debug(self.dependencies[name].slotchain)

                for er in self.dependencies[name].slotchain[1:-1]:
                    if er.isUncertainRelationship():
                        # extracting uncertain relationship
                        self.dependencies[name].uncertainRelationship = er

                # determining the direction of the uncertain relationship
                if self.dependencies[name].uncertainRelationship.nTok:
                    self.dependencies[name].nAttribute = self.dependencies[name].child
                    self.dependencies[name].kAttribute = self.dependencies[name].parent
                    self.dependencies[name].nIsParent = False

                else: # = k:n
                    self.dependencies[name].nAttribute = self.dependencies[name].parent
                    self.dependencies[name].kAttribute = self.dependencies[name].child
                    self.dependencies[name].nIsParent = True

            else:
                self.dependencies[name] = Dependency(name=name, parent=parent, child=child, constraint=constraint,aggregator=aggr, attributes=self.attributes)

            #we add information about the dependencies to the child/parent attributes
            child.dependenciesChild.append(self.dependencies[name])
            parent.dependenciesParent.append(self.dependencies[name])

            #we update the attribute.parents list of all of the child attribute for the given dependency
            child.parents.append(parent)

        ''' Loading the local distributions where they are available '''
        if name==locDistEl:
            attribute = self.attributes[attrs[locDistEl_attr]]
            distFile = attrs[locDistEl_file]
            #Parsing the local distribution for attribute from the file distFile
            self.localDistParser.parseLocalDistribution(attribute, distFile)

    def end_element(self,name):
        '''
        This method is a handler that is called if the parser encounters an tag that ends
        an element block.  When we encounter a entity or relationship block, we attach the
        attributes to the entity or relationship and reset attributes_temp for the next block
        '''

        if name==entiEl:
            '''
            The primary key of an entity or relationship class can be implicit. It's name is
            'entityname_id' and it is non-probabilistic. At this point, when
            all the attributes have been instantiated, we check whether the implicit integer primary
            key has been overwritten (for example to make a probabilistic id attribute). The attribute
            is overwritten by specifying an attribute with  pk='1' in the the xml representation.
            '''

            if self.currentER.pk == []:

                attr_name = self.currentER.name.lower()+'_id'
                attribute = attributeFactory( name=attr_name, er=self.currentER , type='NotProbabilistic', attrDef=None)
                self.attributes_temp[attribute.fullname] = attribute
                self.attributes[attribute.fullname] = attribute
                self.currentER.pk.append(attribute)

            self.currentER.pk_string = [self.currentER.pk[0].fullname]
            # assigning all attributes and resetting the temp dict for the attributes
            self.currentER.attributes = self.attributes_temp
            self.attributes_temp = {}

        if name==relEl:
            '''
            A relationship class has a primary key stored in self.pk that can consist of a set of foreign keys that are attributes of the connecting entities (usually their primary key attributes). The primary key can also be a simple attribute. It is very important that the primary key is unique as it will be used to identify individual nodes of the ground Bayesian network.
            The foreignkeys are defined in the xml-attribute 'foreign="E1.a1,E2.a2,.."' where Ex.ax can also be Ex.pk in which case the primary key of Ex is chosen. The fk set is are instances of Foreign Attribute stored
                -as a list of fa in self.pk
                -as a list of fa.fullname in self.pk_string
                -as a dict {key=Entity: foreignattribute}

            At this point all names must be unique, therefore the attribute name contains all information. We can only configure this after we instantiated all attributes (= when closing the relationship tag).
            '''

            # Is there an attribute defined to be the primary key we don't take the set of foreign keys as primary key
            pk_defined = False
            if len(self.currentER.pk) > 0:
                pk_defined = True

            # e.g. 'User.user_id:buyer_id,User.user_id:seller_id' where the first part identifies the attribute and the second is an optional alias name
            self.foreign = [fk.split(':') for fk in self.foreign.split(',')]

            for rpk in self.foreign:
                #add foreign key for the relationship primary key is always defined
                fk = rpk[0]
                #an alias is only defined if the relationship primary key name is different from the foreign key name
                # (entity,attribute) , e.g. User.item_id or User.pk
                (ent,a) = fk.split('.')
                #Entity instance
                entI = self.entities[ent]
                #Foreign Attribute to be instanciated
                fa = None
                #Target Attribute (=Foreign Key)
                ta = None
                if a == attrEl_pk: #use primary key of ent
                    ta = entI.pk[0]

                else:  #a is the name of the attribute in entity ent
                    ta = entI.attributes[fk]
                    #create Foreign Attribute

                #an alias is only defined if the relationship primary key name is different from the foreign key name
                alias = ta.name
                if len(rpk)==2:
                    alias = rpk[1]

                #create Foreign Attribute
                fa = ForeignAttribute(name=alias,target=ta,er=self.currentER)
                #adding fa to dict of attributes
                self.attributes[fa.fullname] = fa
                #adding fa to self.pk of relationship instance not already defined
                if not pk_defined:
                    self.currentER.pk.append(fa)

                #adding fa to foreign dict (entity:[foreignattribute])
                if entI not in self.currentER.foreign:
                    self.currentER.foreign[entI] = [fa]
                else:
                    self.currentER.foreign[entI].append(fa)

                #adding relationship reference to the dict of the connected entities
                entI.relationships[self.currentER.name] = self.currentER

            #list of the names of the foreign keys of the relationship
            self.currentER.pk_string = ['%s'%(pk_i.fullname) for pk_i in self.currentER.pk]
            #list of entities associated with the current relationship
            self.currentER.entities = self.currentER.foreign.keys()

            # Reference Uncertainty
            if self.currentER.isUncertainRelationship():
                logging.debug("TODO: UPDATE DOCUMENTATION FOR REFERENCE UNCERTAINTY")
                # Creating the exist attribute for the uncertain relationship
                ex = ExistAttribute(name='exist',er=self.currentER)
                # Adding ex to dict of attributes
                self.attributes[ex.fullname] = ex
                self.attributes_temp[ex.fullname] = ex
                self.currentER.existAttribute = ex

                # self.foreign is the list of foreign attributes has 2 entries in this case,
                # if the relationship is of type `n:k`, the first one refers to the
                # n attribute, the second to the k attribute

                foreignEntities = [None, None]
                for i,rpk in enumerate(self.foreign):
                    # rpk = [ pk_name , alias_name ]
                    fk = rpk[0]
                    # (entity,attribute) , e.g. User.item_id or User.pk
                    (ent,a) = fk.split('.')
                    entI = self.entities[ent]

                    foreignEntities[i] = entI

                if self.currentER.nTok:
                    self.currentER.nEnitity = foreignEntities[0]
                    self.currentER.kEnitity = foreignEntities[1]
                else: # = k:n
                    self.currentER.nEnitity = foreignEntities[1]
                    self.currentER.kEnitity = foreignEntities[0]

            # assigning all attributes and resetting the temp dict for the attributes
            self.currentER.attributes = self.attributes_temp
            self.attributes_temp = {}

    def parsePRM(self,xmlfile):
        '''
        Parse given PRM specification passed as 'xmlfile'. The parser will create all Entity,
        Relationship,Attribute and Dependency instances. Finally it will return a fully defined PRM
        instance
        '''

        '''
        The expat parser which will be used. We create a new expat parser for every file that we
        handle because ParseFile(f) can't be called more than once apparently
        '''
        prmParser = xml.parsers.expat.ParserCreate()
        prmParser.StartElementHandler = self.start_element
        prmParser.EndElementHandler = self.end_element

        ''' Parse given PRM specification '''
        prmParser.ParseFile(open(xmlfile))

        ''' We display a list of attributes that have no CPD after parsin. These distributions will have to be learned from data'''
        noCPD = [attr.fullname for attr in self.attributes.values() if attr.CPD is None and attr.probabilistic]
        if noCPD != []:
            logging.info('Local Distribution(s) missing for:'," ".join(noCPD))

        ''' Finally we instantiate the Probabilistic Relational Model and return it '''
        prm.name = self.prmName
        prm.entities = self.entities

        prm.relationships = self.relationships

        prm.attributes = self.attributes

        prm.dependencies = self.dependencies

        prm.datainterface = self.di

        prm.topoSortAttributes = topologicalSort([a for a in self.attributes.values() if a.probabilistic])

class LocalDistributionParser:
    '''
    The local distribution parser loads a the model parameters that have been saved to disk. Naturally this can only be done if the probabilistic structure and also the data itself have not changed. :meth:`learners.cpdlearners.CPDTabularLearner.learnCPDsFull` can be called with the `saveDistributions=True`. The required XML specification along with the .nlp (numpy.array format) is saved in `./localdistributions/xxx.xml` and the required XML will be printed to the standard output, e.g. ::

        <LocalDistribution attribute='A.Aa' file='./localdistributions/Da_Aa.xml'/>

    After adding that output to the `<LocalDistributions>` tag in the PRM specification, the next time the model is loaded - granted that the structure is the same - the local distribution will be loaded from disk.
    '''
    def __init__(self):

        ''' The attribute that we parse the distribution for '''
        self.attribute = None

        ''' Dictionary that maps the xml elements to the method that handles them'''
        self.elementsMappings = {locDistEl:self.localDist,
                                 tabCPDEl : self.tabCPD,
                                 treeCPDEl : self.treeCPD}

        '''
        Temporary storage for attributes that are passed with an element
        Note the possible confusion with the word attribute: The prm defines
        attributes (self.attribute) that we learn distribtions for, and the arguments
        passed with an xml starting tag are also called attributes (self.attrs)
        '''
        self.attrs = None

    def start_element(self,name, attrs):

        # store the attributes of the element so that all methods can access them
        self.attrs = attrs

        self.elementsMappings[name]()

    def localDist(self):
        ''' Create CPD for '''
        #raise exception if the name of attribute specified in the file doesn't match the one passed by the parser
        if self.attribute.fullname != self.attrs[locDistEl_attr]:
            raise Exception("Local Distribution attribute names don't match: %s!=%s"%(self.attribute.name,self.attrs[locDistEl_attr]))

    def tabCPD(self):

        cpdfile = self.attrs[tabCPDEl_file]

        self.attribute.CPD = CPDTabular(self.attribute)

        '''
        Note: the cpdMatrix is overwritten. The only thing that is
        checked is whether the dimensions of the matrices match.

        If the order of the parent assignments is different in the
        prm than in the file, the cpd will be false. Note that if
        you load a prm, the parent ordering in the attribute.parents
        list will always be the same as long as the xml specification
        of the prm is not changed.

        '''
        cpdM = numpy.load(cpdfile)

        if cpdM.shape == self.attribute.CPD.cpdMatrix.shape:
            self.attribute.CPD.cpdMatrix = cpdM
        else:
            logging.debug("WARNING: Matrix dimensions of loaded CPD  %s don't match (%s vs %s)"%(cpdfile,cpdM.shape,self.attribute.CPD.cpdMatrix.shape))
            raise Exception("Matrix dimensions of loaded CPD  %s don't match"%cpdfile)

        #calculating the cumulative distribution
        self.attribute.CPD.computeCumulativeDist()
        self.attribute.CPD.computeLogDists()

    def treeCPD(self):
        raise Exception('TreeCPDs are not implemented yet')

    def end_element(self,name):
        ''' Nothing needs to be done when we encounter an end tag of an element'''
        pass

    def parseLocalDistribution(self,attr,distFile):

        self.attribute = attr

        # we create a new expat parser for every file that we handle because ParseFile(f) can't be called more than once apparently
        localDistParser = xml.parsers.expat.ParserCreate()
        localDistParser.StartElementHandler = self.start_element
        localDistParser.EndElementHandler = self.end_element

        try:
            localDistParser.ParseFile(open(distFile))
        except:
            logging.debug('ERROR: Local distribution %s could not be loaded.'%distFile)
            #raise Exception('ERROR: Local distribution %s could not be loaded.'%distFile)

class DataInterfaceParser:
    '''
    The data interface is specified in XML and saved in for example *./DIexample.xml*::

        <?xml version="1.0" ?>
        <DataInterface name="DIexample">
            <Crossvalidation folds='1'>
                    <Dataset type='SQLite' path='./data/database.sqlite'/>
            </Crossvalidation>
        </DataInterface>

    A list of all imporant xml tags and xml attributes. Note the somewhat confusing double use of the word attributes, on one hand xml attributes and on the other hand the probabilistic PRM :mod:`prm.attribute`.

    **DataInterface**

        * `name` : Freely chosen name for PRM model

    **Crossvalidation**

        * `folds` : Number of folds used for cross validation. The data has to be split up on the database level, otherwise the different folds would have to be accessed by querying one database which decreases the performance. This feature has not been tested, if no cross validation is desired, just use `folds="1"`

    **Dataset**

        * `type` : Type of database the interface is connecting to. Currently only `SQLite` is supported, :class:`SQLiteDI`.
        * `path` : The path to the database file

    '''
    def __init__(self):

        ''' Dictionary that maps the xml elements to the method that handles them'''
        self.elementsMappings = {diEl:self.datainterface,
                                 crossValEl : self.crossvalidation,
                                 diEl+'_end': self.createDI,
                                 dsEl : self.dataset
                                 }

        ''' global variables
        diEl = 'DataInterface'
        diEl_name = 'name'
        crossValEl = 'Crossvalidation'
        dsEl = 'Dataset'
        dsEl_type = 'type'
        dsEl_path = 'path'
        '''

        # the instance of the datainterface
        self.diI = None
        self.name = 'Unamed_DI'
        self.ditype = 'Unamed_Type'
        # the datasets associated with the datainterface
        self.DSI = []

        '''
        Temporary storage for attributes that are passed with an element
        Note the possible confusion with the word attribute: The prm defines
        attributes that we learn distribtions for, and the arguments
        passed with an xml starting tag are also called attributes (self.attrs)
        '''
        self.attrs = None

    def start_element(self,name, attrs):
        '''Excecuted if the parser encounters an start tag for an element'''
        # store the attributes of the element so that all methods can access them
        self.attrs = attrs
        if name in self.elementsMappings:
            self.elementsMappings[name]()

    def datainterface(self):
        if diEl_name in self.attrs:
            self.name = self.attrs[diEl_name]

    def crossvalidation(self):
        self.ditype = 'CrossValidation'

    def dataset(self):
        path = self.attrs[dsEl_path]
        ditype = self.attrs[dsEl_type]
        dsi = datasetinterfaceFactory(path,ditype)

        self.DSI.append(dsi)

    def createDI(self):
        '''All information about the datasets has been collected and stored in self.DSI (datasetinterface),
        the module :mod:`.datainterface` can now be initializded'''

        DI.name = self.name
        DI.ditype = self.ditype
        DI.DSI = self.DSI
        DI.computeTrainingSets()

    def end_element(self,name):
        '''Excecuted if the parser encounters an end tag for an element'''
        com = name+'_end'
        if com in self.elementsMappings:
            self.elementsMappings[com]()

    def parseDataInterface(self,diFile):

        # we create a new expat parser for every file that we handle because ParseFile(f) can't be called more than once apparently
        diParser = xml.parsers.expat.ParserCreate()
        diParser.StartElementHandler = self.start_element
        diParser.EndElementHandler = self.end_element

        diParser.ParseFile(open(diFile))

        return self.diI


from setuptools import setup, find_packages
import re
import os
import sys
import subprocess

ROOT = os.path.dirname(__file__)

def get_long_description():
    with open(os.path.join(ROOT, 'README.md'), encoding='utf-8') as f:
        return f.read()

def get_version():
    VERSION_RE = re.compile(r'''VERSION\s+=\s+['"]([0-9.]+)['"]''')
    init = open(os.path.join(ROOT, 'bibsearch', 'bibsearch.py')).read()
    return VERSION_RE.search(init).group(1)

setup(
    name = 'bibsearch',

    # Versions should comply with PEP440.  For a discussion on single-sourcing
    # the version across setup.py and the project code, see
    # https://packaging.python.org/en/latest/single_source_version.html
    version = get_version(),

    description = 'Download, manage, and search a BibTeX database.',
    long_description = get_long_description(),

    # The project's main homepage.
    url = 'https://github.com/mjpost/bibsearch',

    author = 'David Vilar, Matt Post',
    author_email='david.vilar@gmail.com, post@cs.jhu.edu',
    maintainer_email='david.vilar@gmail.com, post@cs.jhu.edu',

    license = 'Apache License 2.0',

    python_requires = '>=3',

    # Alternatively, if you want to distribute just a my_module.py, uncomment this:
    # py_modules = ["bibsearch", "bibdb", "bibutils", "config"],
    packages = find_packages(),

    package_data = { 'bibsearch': ['manual.1'] },

    # List run-time dependencies here.  These will be installed by pip when
    # your project is installed. For an analysis of "install_requires" vs pip's
    # requirements files see:
    # https://packaging.python.org/en/latest/requirements.html
    install_requires = ['typing', 'tqdm', 'pyaml', 'stop-words', 'pybtex', 'feedparser'],

    # List additional groups of dependencies here (e.g. development
    # dependencies). You can install these using the following syntax,
    # for example:
    # $ pip install -e .[dev,test]
    extras_require = {},

    # To provide executable scripts, use entry points in preference to the
    # "scripts" keyword. Entry points provide cross-platform support and allow
    # pip to create the appropriate form of executable for the target platform.
    entry_points={
        'console_scripts': [
            'bibsearch = bibsearch.bibsearch:main',
        ],
    },

    # See https://pypi.python.org/pypi?%3Aaction=list_classifiers
    classifiers = [
        # How mature is this project? Common values are
        #   3 - Alpha
        #   4 - Beta
        #   5 - Production/Stable
        'Development Status :: 5 - Production/Stable',

        # Indicate who your project is intended for
        'Intended Audience :: Developers',
        'Intended Audience :: Science/Research',
        'Topic :: Scientific/Engineering',
        'Topic :: Text Processing',

        # Pick your license as you wish (should match "license" above)
        'License :: OSI Approved :: Apache Software License',

        # Specify the Python versions you support here. In particular, ensure
        # that you indicate whether you support Python 2, Python 3 or both.
        'Programming Language :: Python :: 3 :: Only',
    ],

    # What does your project relate to?
    keywords = ['computer science, LaTeX, BibTeX'],
)


import sqlite3
import logging
from variables import schema
from string import Template
from utilities import func_name
from os.path import isfile, isdir, join
from PIL import Image
from io import BytesIO

def connect_database(config, logger, **kwargs):
    logger = logging.getLogger(func_name())
    logger.debug(kwargs)

    database = config.database

    if not isfile(database):
        logger.warning("Use create database instead")
        return False
    connection = sqlite3.connect(database)
    sqlite_select_Query = "select sqlite_version();"
    cursor = connection.cursor()
    cursor.execute(sqlite_select_Query)
    record = cursor.fetchall()

    logger.debug("SQLite Database Version is: %s ", record)
    return connection

def create_table(logger, table_name, table_schema, connection, **kwargs):
    logger = logging.getLogger(func_name())
    logger.debug(table_name, table_schema, kwargs)

    cursor = connection.cursor()

    field_list = ""
    sql_template = Template("""CREATE TABLE  '${table_name}' (\n${field_list});""")
    for field, field_type in table_schema.items():
        field_list += "\t"
        field_list += " \t\t".join([field, field_type])
        field_list += ",\n"
    sql_create = sql_template.substitute(table_name=table_name, field_list=field_list[:-2])
    logger.debug(sql_create)
    cursor.execute(sql_create)

    cursor.close()

def create_database(config, logger, schema_list, **kwargs):
    """
    Create a database if none exists

    """
    database = config.database
    logger = logging.getLogger(func_name())
    logger.debug(database, schema_list, kwargs)

    connection = sqlite3.connect(database)
    sqlite_select_Query = "select sqlite_version();"
    cursor = connection.cursor()
    cursor.execute(sqlite_select_Query)
    record = cursor.fetchall()
    logger.info("Creating Database {}".format(database))
    logger.debug("SQLite Database Version is: ", record)

    # Put a list of tables from Globals to create schema
    cursor.execute("PRAGMA journal_mode=WAL;")
    logger.debug("Modifying database for WAL access")
    # Create all Schemas if new
    for schema in schema_list:
        table_name = list(schema.keys())[0]
        table_schema = schema[table_name]

        logger.debug('creating table: ', table_name)

        create_table(logger, table_name, table_schema, connection)

    cursor.close()
    connection.close()

def create_table(logger, table_name, table_schema, connection, **kwargs):
    logger = logging.getLogger(func_name())
    logger.debug(table_name, table_schema, kwargs)

    cursor = connection.cursor()

    field_list = ""
    sql_template = Template("""CREATE TABLE  '${table_name}' (\n${field_list});""")
    for field, field_type in table_schema.items():
        field_list += "\t"
        field_list += " \t\t".join([field, field_type])
        field_list += ",\n"
    sql_create = sql_template.substitute(table_name=table_name, field_list=field_list[:-2])
    logger.debug(sql_create)
    cursor.execute(sql_create)

    cursor.close()

def insert_into_db(logger, connection, table, record, **kwargs):
    """
    record is a dictionary
    """
    logger.debug('record filename {}'.format(record['filename_full']))
    logger.info('Writing record to table %s',table)
    logger = logging.getLogger(func_name())
    logger.debug(table, record, **kwargs)
    column_names = list()
    values = list()

    if not connection:
        logger.warning("Connection not valid")
        return False

    cursor = connection.cursor()
    sql_template = Template("""INSERT INTO ${table} (${columns}) VALUES (${values});""")
    column_names = ",".join(list(record.keys()))
    logger.debug(column_names)
    data_values = tuple(record.values())
    num_columns = len(list(record.keys()))
    values = ",".join([' ?'] * num_columns)
    sql_insert = sql_template.substitute(table=table, columns=column_names, values=values)
    logger.debug('columns_names %s', column_names)
    logger.debug("data values %s", data_values)
    logger.debug(sql_insert)
    logger.debug('columns_names',column_names)
    cursor.execute(sql_insert, data_values)
    connection.commit()

    return cursor.lastrowid

def insertImage(logger, connection, table_name, filename, name, image):
    print('inserting image into table {}'.format(table_name))
    try:
        sqlite_insert_blob_query = """ INSERT INTO {table_name}
                                  (filename, userid, headshot_image) VALUES (?, ?, ?)""".format(
            table_name=table_name)

        cursor = connection.cursor()
        # Convert data into tuple format
        data_tuple = (filename, name, image)
        cursor.execute(sqlite_insert_blob_query, data_tuple)
        connection.commit()
        print("Image and information {info} inserted successfully as a BLOB into a table {table_name}".format(info=name,
                                                                                table_name=table_name))

    except sqlite3.Error as error:
        print("Failed to insert blob data into sqlite table", error)

def get_named_faces(logger,connection,table_name):
    named_faces = dict()
    logger = logging.getLogger(func_name())
    logger.debug("get named faces from table %s", table_name)
    cursor = connection.cursor()
    sql_template = Template("""select * from  ${table} order by user_id;""")
    sql = sql_template.substitute(table_name=table_name)
    cursor.execute(sql)
    records = cursor.fetchall()
    for record in records:
        pass


import os
import random

import pygame

from data.base_modifier import BaseModifier
from data.colors import Colors

class Fortifier(BaseModifier):
    def __init__(self, game):
        self.name = "fortifier"
        self.game = game
        self.block_positions = self.get_block_positions()
        self.image = pygame.image.load(os.path.join("resources", "fortifier.png"))

    def set_icon_location(self):
        possible_locations = list(
            set(self.game.board_coordinates)
            - set(self.game.snake.body)
            - set(self.block_positions)
        )
        self.position = random.choice(possible_locations)
        x, y = self.position
        self.coordinates = (x * self.game.block_size, y * self.game.block_size)

    def get_block_positions(self):
        top_positions = [(i, 0) for i in range(20)]
        right_positions = [(19, i) for i in range(20)]
        bottom_positions = [(i, 19) for i in range(20)]
        left_positions = [(0, i) for i in range(20)]
        return [*top_positions, *right_positions, *bottom_positions, *left_positions]

    def check_block_hit(self):
        for block in self.block_positions:
            if self.game.snake.overlaps(block, head_only=True):
                self.game.end()

    def hide(self):
        self.position = []
        self.coordinates = []

    def draw(self):
        if self.coordinates:
            self.game.game_surface.blit(self.image, self.coordinates)

    def draw_blocks(self):
        for x, y in self.block_positions:
            rect = pygame.Rect(
                x * self.game.block_size,
                y * self.game.block_size,
                self.game.block_size,
                self.game.block_size,
            )
            pygame.draw.rect(self.game.game_surface, Colors.BLUE, rect, 0)


# -*- coding: utf-8 -*-
from lxml import etree

oaidcns = "{http://www.openarchives.org/OAI/2.0/oai_dc/}"
dcns = "{http://purl.org/dc/elements/1.1/}"

def getEmptyOaidcDict():
	oaidcdict = {
		"title": "",
		"description": "",
		"subject": [],
		"publisher": "",
		"format": "",
		"identifier": "",
		"language": "",
		"rights": "" }
	return oaidcdict

def makeOAIDC(oaidcdict):
	oaidc = etree.Element(oaidcns + 'dc', nsmap={"oai_dc": 'http://www.openarchives.org/OAI/2.0/oai_dc/', "dc": "http://purl.org/dc/elements/1.1/", "xsi": "http://www.w3.org/2001/XMLSchema-instance"})

	if oaidcdict["title"]:
		oaidc.append(makeElement("title",oaidcdict["title"]))

	if oaidcdict["description"]:
		oaidc.append(makeElement("description",oaidcdict["description"]))

	if oaidcdict["subject"]:
		for s in oaidcdict["subject"]:
			oaidc.append(makeElement("subject",s))

	if oaidcdict["publisher"]:
		oaidc.append(makeElement("publisher",oaidcdict["publisher"]))

	if oaidcdict["format"]:
		oaidc.append(makeElement("format",oaidcdict["format"]))

	if oaidcdict["identifier"]:
		oaidc.append(makeElement("identifier",oaidcdict["identifier"]))

	if oaidcdict["language"]:
		oaidc.append(makeElement("language",oaidcdict["language"]))

	if oaidcdict["rights"]:
		oaidc.append(makeElement("rights",oaidcdict["rights"]))

	s = etree.tostring(oaidc, pretty_print=True)
	return s

def makeElement(element,value):
	e = etree.Element(dcns + element)
	e.text = value
	return e


import pandas as pd
import requests

API_KEY = 'Bearer e3920e0f60c88136aa66dca3bbfbae7f-bb9c32f4d0ab0fc0106137b0467a4394'

HEADERS = {
    'Content-Type': 'application/json',
    'Authorization': API_KEY
}

# format: Currency1_Currency2
INSTRUMENT = 'EUR_USD'

# e.g. 'S5', 'M15', 'H1', 'D', 'W', 'M'
# https://developer.oanda.com/rest-live-v20/instrument-df/ for full options
GRANULARITY = 'M30'

# format: '2016-06-01T15:00:00.000000000Z'
# leave None if using COUNT
#FROM_DATE = '2020-01-01T15:00:00.000000000Z'

# format: '2016-10-17T15:00:00.000000000Z'
# leave None for present time
#TO_DATE = '2020-05-01T15:00:00.000000000Z'

# (1 - 5000) must be in quotes e.g. '100'
# leave None if using dates, else fill with number of past candlesticks you want up to present time
COUNT = None

# 'practice' or 'trade'
ACCOUNT_TYPE = 'practice'

def export_ba_data():
    try:
        r = requests.get("https://api-fx{}.oanda.com/v3/instruments/{}/candles?price=BA".format(
            ACCOUNT_TYPE, INSTRUMENT), headers=HEADERS, params=PARAMS).json()

        df = pd.DataFrame(([candle['time'], candle['ask']['o'], candle['ask']['h'], candle['ask']['l'], candle['ask']['c'], candle['volume']] for candle in r['candles']),
                          columns=['Time', 'Open', 'High', 'Low', 'Close', 'Volume'])
        print(FROM_DATE)
        print(TO_DATE)
        # df.to_csv('ask_data.csv', index=False)
        return df
    except:
        print('request failed, likely that candlestick count is out of range (max=50000)')

#####################################
from_year = 2020
from_month = 1
to_year = 2021
to_month = 1
#####################################
first_time = 1

for y in range(from_year, to_year+1):
    if ((y == to_year) and (y == from_year)):
        m_range_min = from_month
        m_range_max = to_month-1
    elif(y == from_year):
        m_range_min = from_month
        m_range_max = 12
    elif(y == to_year):
        m_range_min = 1
        m_range_max = to_month-1
    else:
        m_range_min = 1
        m_range_max = 12

    for m in range(m_range_min, m_range_max+1):
        m1 = m
        m2 = m+1
        if(m1 >= 10):
            FROM_DATE = str(y)+'-'+str(m1)+'-01T15:00:00.000000000Z'
        else:
            FROM_DATE = str(y)+'-0'+str(m1)+'-01T15:00:00.000000000Z'
        if(m2 >= 10):
            if(m2 == 13):
                TO_DATE = str(y+1)+'-0'+str(1)+'-01T15:00:00.000000000Z'
            else:
                TO_DATE = str(y)+'-'+str(m2)+'-01T15:00:00.000000000Z'
        else:
            TO_DATE = str(y)+'-0'+str(m2)+'-01T15:00:00.000000000Z'
            #TO_DATE = '2020-05-01T15:00:00.000000000Z'
        PARAMS = {
            'from': FROM_DATE,
            'to': TO_DATE,
            'count': COUNT,
            'granularity': GRANULARITY
        }
        # print(FROM_DATE)
        # print(TO_DATE)
        if(first_time):
            df = export_ba_data()
            first_time = 0
        else:
            df_aux = export_ba_data()
            df = df.append(df_aux)

df.to_csv('ask_data.csv', index=False)
print('bid/ask data exported to "ask_data.csv"')


from PyQt6.QtCore import QTimer

from UM.Application import Application
from UM.Logger import Logger
from UM.Scene.SceneNode import SceneNode
from UM.Scene.Iterator.BreadthFirstIterator import BreadthFirstIterator
from UM.Math.Vector import Vector
from UM.Scene.Selection import Selection
from UM.Scene.SceneNodeSettings import SceneNodeSettings

from cura.Scene.ConvexHullDecorator import ConvexHullDecorator

from cura.Operations import PlatformPhysicsOperation
from cura.Scene import ZOffsetDecorator

import random  # used for list shuffling

class PlatformPhysics:
    def __init__(self, controller, volume):
        super().__init__()
        self._controller = controller
        self._controller.getScene().sceneChanged.connect(self._onSceneChanged)
        self._controller.toolOperationStarted.connect(self._onToolOperationStarted)
        self._controller.toolOperationStopped.connect(self._onToolOperationStopped)
        self._build_volume = volume
        self._enabled = True

        self._change_timer = QTimer()
        self._change_timer.setInterval(100)
        self._change_timer.setSingleShot(True)
        self._change_timer.timeout.connect(self._onChangeTimerFinished)
        self._move_factor = 1.1  # By how much should we multiply overlap to calculate a new spot?
        self._max_overlap_checks = 10  # How many times should we try to find a new spot per tick?
        self._minimum_gap = 2  # It is a minimum distance (in mm) between two models, applicable for small models

        Application.getInstance().getPreferences().addPreference("physics/automatic_push_free", False)
        Application.getInstance().getPreferences().addPreference("physics/automatic_drop_down", True)

    def _onSceneChanged(self, source):
        if not source.callDecoration("isSliceable"):
            return

        self._change_timer.start()

    def _onChangeTimerFinished(self):
        if not self._enabled:
            return

        app_instance = Application.getInstance()
        app_preferences = app_instance.getPreferences()
        app_automatic_drop_down = app_preferences.getValue("physics/automatic_drop_down")
        app_automatic_push_free = app_preferences.getValue("physics/automatic_push_free")

        root = self._controller.getScene().getRoot()
        build_volume = app_instance.getBuildVolume()
        build_volume.updateNodeBoundaryCheck()

        # Keep a list of nodes that are moving. We use this so that we don't move two intersecting objects in the
        # same direction.
        transformed_nodes = []

        nodes = list(BreadthFirstIterator(root))

        # Only check nodes inside build area.
        nodes = [node for node in nodes if (hasattr(node, "_outside_buildarea") and not node._outside_buildarea)]

        # We try to shuffle all the nodes to prevent "locked" situations, where iteration B inverts iteration A.
        # By shuffling the order of the nodes, this might happen a few times, but at some point it will resolve.
        random.shuffle(nodes)
        for node in nodes:
            if node is root or not isinstance(node, SceneNode) or node.getBoundingBox() is None:
                continue

            bbox = node.getBoundingBox()

            # Move it downwards if bottom is above platform
            move_vector = Vector()

            if node.getSetting(SceneNodeSettings.AutoDropDown, app_automatic_drop_down) and not (node.getParent() and node.getParent().callDecoration("isGroup") or node.getParent() != root) and node.isEnabled(): #If an object is grouped, don't move it down
                z_offset = node.callDecoration("getZOffset") if node.getDecorator(ZOffsetDecorator.ZOffsetDecorator) else 0
                move_vector = move_vector.set(y = -bbox.bottom + z_offset)

            # If there is no convex hull for the node, start calculating it and continue.
            if not node.getDecorator(ConvexHullDecorator) and not node.callDecoration("isNonPrintingMesh") and node.callDecoration("getLayerData") is None:
                node.addDecorator(ConvexHullDecorator())

            # only push away objects if this node is a printing mesh
            if not node.callDecoration("isNonPrintingMesh") and app_automatic_push_free:
                # Do not move locked nodes
                if node.getSetting(SceneNodeSettings.LockPosition):
                    continue

                # Check for collisions between convex hulls
                for other_node in BreadthFirstIterator(root):
                    # Ignore root, ourselves and anything that is not a normal SceneNode.
                    if other_node is root or not issubclass(type(other_node), SceneNode) or other_node is node or other_node.callDecoration("getBuildPlateNumber") != node.callDecoration("getBuildPlateNumber"):
                        continue

                    # Ignore collisions of a group with it's own children
                    if other_node in node.getAllChildren() or node in other_node.getAllChildren():
                        continue

                    # Ignore collisions within a group
                    if other_node.getParent() and node.getParent() and (other_node.getParent().callDecoration("isGroup") is not None or node.getParent().callDecoration("isGroup") is not None):
                        continue

                    # Ignore nodes that do not have the right properties set.
                    if not other_node.callDecoration("getConvexHull") or not other_node.getBoundingBox():
                        continue

                    if other_node in transformed_nodes:
                        continue  # Other node is already moving, wait for next pass.

                    if other_node.callDecoration("isNonPrintingMesh"):
                        continue

                    overlap = (0, 0)  # Start loop with no overlap
                    current_overlap_checks = 0
                    # Continue to check the overlap until we no longer find one.
                    while overlap and current_overlap_checks < self._max_overlap_checks:
                        current_overlap_checks += 1
                        head_hull = node.callDecoration("getConvexHullHead")
                        if head_hull:  # One at a time intersection.
                            overlap = head_hull.translate(move_vector.x, move_vector.z).intersectsPolygon(other_node.callDecoration("getConvexHull"))
                            if not overlap:
                                other_head_hull = other_node.callDecoration("getConvexHullHead")
                                if other_head_hull:
                                    overlap = node.callDecoration("getConvexHull").translate(move_vector.x, move_vector.z).intersectsPolygon(other_head_hull)
                                    if overlap:
                                        # Moving ensured that overlap was still there. Try anew!
                                        move_vector = move_vector.set(x = move_vector.x + overlap[0] * self._move_factor,
                                                                      z = move_vector.z + overlap[1] * self._move_factor)
                            else:
                                # Moving ensured that overlap was still there. Try anew!
                                move_vector = move_vector.set(x = move_vector.x + overlap[0] * self._move_factor,
                                                              z = move_vector.z + overlap[1] * self._move_factor)
                        else:
                            own_convex_hull = node.callDecoration("getConvexHull")
                            other_convex_hull = other_node.callDecoration("getConvexHull")
                            if own_convex_hull and other_convex_hull:
                                overlap = own_convex_hull.translate(move_vector.x, move_vector.z).intersectsPolygon(other_convex_hull)
                                if overlap:  # Moving ensured that overlap was still there. Try anew!
                                    temp_move_vector = move_vector.set(x = move_vector.x + overlap[0] * self._move_factor,
                                                                       z = move_vector.z + overlap[1] * self._move_factor)

                                    # if the distance between two models less than 2mm then try to find a new factor
                                    if abs(temp_move_vector.x - overlap[0]) < self._minimum_gap and abs(temp_move_vector.y - overlap[1]) < self._minimum_gap:
                                        temp_x_factor = (abs(overlap[0]) + self._minimum_gap) / overlap[0] if overlap[0] != 0 else 0 # find x move_factor, like (3.4 + 2) / 3.4 = 1.58
                                        temp_y_factor = (abs(overlap[1]) + self._minimum_gap) / overlap[1] if overlap[1] != 0 else 0 # find y move_factor

                                        temp_scale_factor = temp_x_factor if abs(temp_x_factor) > abs(temp_y_factor) else temp_y_factor

                                        move_vector = move_vector.set(x = move_vector.x + overlap[0] * temp_scale_factor,
                                                                      z = move_vector.z + overlap[1] * temp_scale_factor)
                                    else:
                                        move_vector = temp_move_vector
                            else:
                                # This can happen in some cases if the object is not yet done with being loaded.
                                # Simply waiting for the next tick seems to resolve this correctly.
                                overlap = None

            if not Vector.Null.equals(move_vector, epsilon = 1e-5):
                transformed_nodes.append(node)
                op = PlatformPhysicsOperation.PlatformPhysicsOperation(node, move_vector)
                op.push()

        # After moving, we have to evaluate the boundary checks for nodes
        build_volume.updateNodeBoundaryCheck()

    def _onToolOperationStarted(self, tool):
        self._enabled = False

    def _onToolOperationStopped(self, tool):
        # Selection tool should not trigger an update.
        if tool.getPluginId() == "SelectionTool":
            return

        if tool.getPluginId() == "TranslateTool":
            for node in Selection.getAllSelectedObjects():
                if node.getBoundingBox() and node.getBoundingBox().bottom < 0:
                    if not node.getDecorator(ZOffsetDecorator.ZOffsetDecorator):
                        node.addDecorator(ZOffsetDecorator.ZOffsetDecorator())

                    node.callDecoration("setZOffset", node.getBoundingBox().bottom)
                else:
                    if node.getDecorator(ZOffsetDecorator.ZOffsetDecorator):
                        node.removeDecorator(ZOffsetDecorator.ZOffsetDecorator)

        self._enabled = True
        self._onChangeTimerFinished()


from collections import OrderedDict
from braindecode.datautil.signalproc import exponential_running_standardize
from braindecode.datautil.trial_segment import create_signal_target_from_raw_mne
import logging
import numpy as np
from braindecode.datasets.bbci import BBCIDataset
from braindecode.datautil.signalproc import highpass_cnt
from braindecode.mne_ext.signalproc import mne_apply, resample_cnt
import h5py
import os

log = logging.getLogger(__name__)
log.setLevel('INFO')
logging.basicConfig(level=logging.INFO)

def main():
    data_dir = '/home/no316758/data/high-gamma-dataset/data'
    data_dir = '/Users/sebas/code/_eeg_data/high-gamma-dataset/data'
    output_dir = os.path.abspath(
        os.path.join(os.path.dirname(__file__), '../..',
                     'data/hgd_processed_low_cut_4hz'))

    # Check ouput dir exists and possibly create it
    parent_output_dir = os.path.abspath(os.path.join(output_dir, os.pardir))
    assert os.path.exists(parent_output_dir), \
        "Parent directory of given output directory does not exist"
    if not os.path.exists(output_dir):
        os.makedirs(output_dir)

    # Get file paths:
    #   subject_ids: list of indices in range [1, ..., 14]
    subject_ids = [x for x in range(1, 15)]

    train_data_paths = [data_dir + f"/train/{subject_id}.mat" for subject_id in
                        subject_ids]
    test_data_paths = [data_dir + f"/test/{subject_id}.mat" for subject_id in
                       subject_ids]

    # Process and save data
    low_cut_hz = 4
    debug = False
    save_processed_high_gamma_datatset(train_data_paths, test_data_paths,
                                       output_dir, low_cut_hz, debug)

def save_processed_high_gamma_datatset(train_filenames, test_filenames,
                                       output_dir, low_cut_hz, debug=False):
    for train_filename, test_filename in zip(train_filenames, test_filenames):
        log.info("Processing data...")
        full_train_set = process_bbci_data(train_filename,
                                           low_cut_hz=low_cut_hz,
                                           debug=debug)
        test_set = process_bbci_data(test_filename, low_cut_hz=low_cut_hz,
                                     debug=debug)
        subject_id = train_filename.split('/')[-1].split('.')[0]
        log.info("Saving processed data...")
        with h5py.File(f'{output_dir}/{subject_id}_train.h5', 'w') as h5file:
            h5file.create_dataset(f'{subject_id}_train_X',
                                  data=list(full_train_set.X))
            h5file.create_dataset(f'{subject_id}_train_y',
                                  data=list(full_train_set.y))
        with h5py.File(f'{output_dir}/{subject_id}_test.h5', 'w') as h5file:
            h5file.create_dataset(f'{subject_id}_test_X', data=list(test_set.X))
            h5file.create_dataset(f'{subject_id}_test_y', data=list(test_set.y))
        log.info(f"Done processing data subject {subject_id}\n")

def process_bbci_data(filename, low_cut_hz, debug=False):
    """
    As taken from: https://github.com/robintibor/high-gamma-dataset
    :param filename: data from subject (train or test)
    :param low_cut_hz: 0 or 4
    :param debug:
    :return: SignalAndTarget
    """
    load_sensor_names = None
    if debug:
        load_sensor_names = ['C3', 'C4', 'C2']

    # We loaded all sensors to always get same cleaning
    #   results independent of sensor selection
    # There is an inbuilt heuristic that tries to use
    #   only EEG channels and that definitely
    #   works for datasets in our paper
    loader = BBCIDataset(filename, load_sensor_names=load_sensor_names)

    log.info("Loading raw data...")
    cnt = loader.load()

    # Cleaning: First find all trials that have absolute microvolt values
    # larger than +- 800 inside them and remember them for removal later
    log.info("Cutting trials...")

    marker_def = OrderedDict([('Right Hand', [1]), ('Left Hand', [2],),
                              ('Rest', [3]), ('Feet', [4])])
    clean_ival = [0, 4000]

    set_for_cleaning = create_signal_target_from_raw_mne(cnt, marker_def,
                                                         clean_ival)

    clean_trial_mask = np.max(np.abs(set_for_cleaning.X), axis=(1, 2)) < 800

    log.info("Clean trials: {:3d}  of {:3d} ({:5.1f}%)".format(
        np.sum(clean_trial_mask),
        len(set_for_cleaning.X),
        np.mean(clean_trial_mask) * 100))

    # now pick only sensors with C in their name
    # as they cover motor cortex
    C_sensors = ['FC5', 'FC1', 'FC2', 'FC6', 'C3', 'C4', 'CP5',
                 'CP1', 'CP2', 'CP6', 'FC3', 'FCz', 'FC4', 'C5', 'C1', 'C2',
                 'C6',
                 'CP3', 'CPz', 'CP4', 'FFC5h', 'FFC3h', 'FFC4h', 'FFC6h',
                 'FCC5h',
                 'FCC3h', 'FCC4h', 'FCC6h', 'CCP5h', 'CCP3h', 'CCP4h', 'CCP6h',
                 'CPP5h',
                 'CPP3h', 'CPP4h', 'CPP6h', 'FFC1h', 'FFC2h', 'FCC1h', 'FCC2h',
                 'CCP1h',
                 'CCP2h', 'CPP1h', 'CPP2h']
    if debug:
        C_sensors = load_sensor_names
    cnt = cnt.pick_channels(C_sensors)

    # Further preprocessings as descibed in paper
    log.info("Resampling...")
    cnt = resample_cnt(cnt, 250.0)
    log.info("Highpassing...")
    cnt = mne_apply(
        lambda a: highpass_cnt(
            a, low_cut_hz, cnt.info['sfreq'], filt_order=3, axis=1),
        cnt)
    log.info("Standardizing...")
    cnt = mne_apply(
        lambda a: exponential_running_standardize(a.T, factor_new=1e-3,
                                                  init_block_size=1000,
                                                  eps=1e-4).T,
        cnt)

    # Trial interval, start at -500 already,
    # since improved decoding for networks
    ival = [-500, 4000]

    dataset = create_signal_target_from_raw_mne(cnt, marker_def, ival)
    dataset.X = dataset.X[clean_trial_mask]
    dataset.y = dataset.y[clean_trial_mask]
    return dataset

if __name__ == '__main__':
    main()