import os.path
from pathlib import Path
from typing import List, Union
from urllib.parse import quote

from renku.core.metadata.immutable import Immutable
from renku.core.models.calamus import JsonLDSchema, Nested, fields, prov, renku
from renku.core.utils.git import get_object_hash

_entity_cache = {}

class Entity(Immutable):
    """Represent a file."""

    __slots__ = ("checksum", "path")

    def __init__(self, *, checksum: str, id: str = None, path: Union[Path, str], **kwargs):
        path = str(path)

        assert id is None or isinstance(id, str)
        assert not os.path.isabs(path), f"Entity is being created with absolute path: '{path}'"

        id = id or Entity.generate_id(checksum, path)
        super().__init__(checksum=checksum, id=id, path=path, **kwargs)

    def __eq__(self, other):
        if self is other:
            return True
        if not isinstance(other, Entity):
            return False
        return self.checksum == other.checksum and self.path == other.path

    def __hash__(self):
        return hash((self.checksum, self.path))

    @staticmethod
    def generate_id(checksum: str, path: Union[Path, str]) -> str:
        """Generate an Entity identifier."""
        quoted_path = quote(str(path).strip("/"))

        return f"/entities/{checksum}/{quoted_path}"

    @classmethod
    def from_revision(cls, client, path: Union[Path, str], revision: str = None) -> "Entity":
        """Return dependency from given path and revision."""
        revision = revision or "HEAD"
        assert isinstance(revision, str), f"Invalid revision: {revision}"

        # FIXME: Implement the cache properly: It's mainly used in migrations
        global _entity_cache
        key = (revision, str(path))
        cached_entry = _entity_cache.get(key)
        if cached_entry:
            return cached_entry

        # TODO: What checksum we get at "HEAD" if object is staged but not committed
        checksum = get_object_hash(repo=client.repo, revision=revision, path=path)
        # NOTE: If object was not found at a revision it's either removed or exists in a different revision; keep the
        # entity and use revision as checksum
        checksum = checksum or revision
        # TODO: What would be checksum for a directory if it's not committed yet.
        id = cls.generate_id(checksum=checksum, path=path)

        absolute_path = client.path / path
        if str(path) != "." and absolute_path.is_dir():
            members = cls.get_directory_members(client, revision, absolute_path)
            entity = Collection(id=id, checksum=checksum, path=path, members=members)
        else:
            entity = cls(id=id, checksum=checksum, path=path)

        _entity_cache[cached_entry] = entity

        return entity

    @classmethod
    def get_directory_members(cls, client, revision, absolute_path: Path) -> List["Entity"]:
        """Return first-level files/directories in a directory."""
        members: List[Entity] = []

        for member in absolute_path.iterdir():
            if member.name == ".gitkeep":
                continue

            member_path = str(member.relative_to(client.path))

            try:
                assert all(member_path != m.path for m in members)

                entity = cls.from_revision(client, member_path, revision)
                # NOTE: If a path is not found at a revision we assume that it didn't exist at that revision
                if entity:
                    members.append(entity)
            except KeyError:
                pass

        return members

class Collection(Entity):
    """Represent a directory with files."""

    __slots__ = ("members",)

    def __init__(self, *, checksum: str, id: str = None, path: Union[Path, str], members: List[Entity] = None):
        members = tuple(members) if members else ()
        super().__init__(checksum=checksum, id=id, path=path, members=members)

class EntitySchema(JsonLDSchema):
    """Entity Schema."""

    class Meta:
        """Meta class."""

        rdf_type = [prov.Entity]
        model = Entity

    checksum = fields.String(renku.checksum, missing=None)
    id = fields.Id()
    path = fields.String(prov.atLocation)

class CollectionSchema(EntitySchema):
    """Entity Schema."""

    class Meta:
        """Meta class."""

        rdf_type = prov.Collection
        model = Collection

    members = Nested(prov.hadMember, [EntitySchema, "CollectionSchema"], many=True)


from __future__ import print_function
from sklearn.utils import shuffle
from sklearn.model_selection import train_test_split
from sklearn import preprocessing

import numpy as np
import pandas as pd
import cv2
import matplotlib.pyplot as plt
from scipy.misc import *
from os import listdir
from os.path import isfile, join
import tensorflow as tf

from tensorflow import keras
from tensorflow.keras import layers
from tensorflow.keras.models import Sequential

# set display defaults
plt.rcParams['figure.figsize'] = (10, 10)
plt.rcParams['image.interpolation'] = 'nearest'
plt.rcParams['image.cmap'] = 'gray'
LANGUAGE = "Python"
def repeat(LANGUAGE,numE) :
    path = "B:\\data\\curated.tar\\curated\\"

    character_curated = []
    if LANGUAGE == "Python":
        character_curated = [ord(c) for c in '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabdefghnqrt():<=>[]+-*/'+ "'" ]
    elif LANGUAGE == "Java":
        character_curated = [ord(c) for c in '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabdefghnqrt();<=>[]+-*{}/!'+ "'" ]
    elif LANGUAGE == "Lisp":
        character_curated = [ord(c) for c in '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabdefghnqrt()<=>[]+-*/!'  + "'"]

    #character_curated = [ord(c) for c in '!"#$%&' + "'" + '()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[]^_' + "'" +'abcdefghijklmnopqrstuvwxyz{|}~' ]
    def img_label_load(data_path, num_classes=None):
        data = pd.read_csv(data_path, header=None)
        data_rows = len(data)
        if not num_classes:
            num_classes = len(data[0].unique())
        img_size = int(np.sqrt(len(data.iloc[0][1:])))
        imgs = np.transpose(data.values[:, 1:].reshape(data_rows, img_size, img_size, 1),
                            axes=[0, 2, 1, 3])
        labels = keras.utils.to_categorical(data.values[:, 0], len(character_curated))  # one-hot encoding vectors
        return imgs / 255., labels
    eX, ey = img_label_load('B:\data\csv\emnist-balanced-train.csv')
    etestX, etestY = img_label_load('B:\data\csv\emnist-balanced-test.csv')

    print([chr(i) for i in character_curated])
    img_size = 28
    X = []
    y = []
    for i in character_curated:
        path_img = path + str(i) + '/'
        for file_name in [f for f in listdir(path_img) if isfile(join(path_img, f))]:
            img = cv2.imread(path_img + file_name, 0)
            img = cv2.resize(img,(img_size, img_size), interpolation = cv2.INTER_AREA)
            X += [img/255.]
            t = []
            for j in character_curated:
                if j == i:
                    t.append(1)
                else:
                    t.append(0)
            y += [t]

    X = np.array(X, dtype=np.float64)
    y = np.array(y, dtype=np.float32)
    np.append(X,eX)
    np.append(y,ey)

    def prepare_data_char_subset(X, y):

        X_select = X
        y_select = y
        #y_select = np.reshape(y_select[0],len(character_curated))
        # Shuffle
        X_select, y_select = shuffle(X_select, y_select, random_state=1)

        # Separate train test
        X_train, X_test, y_train, y_test = train_test_split(X_select, y_select, test_size=0.20, stratify=y)
        #print(X_train.shape)
        X_train = np.reshape(X_train, (X_train.shape[0],  X_train.shape[1], X_train.shape[2],1))
        X_test = np.reshape(X_test, (X_test.shape[0],  X_test.shape[1], X_test.shape[2],1))

        #print('Train shape: ', X_train.shape, y_train.shape)
        #print('Test shape: ', X_test.shape, y_test.shape)
        #print('Num classes: ', len(set(y_train)))
        #print('Classes: ', set(y_train))

        return X_train, y_train, X_test, y_test

    X, y, X_test, y_test = prepare_data_char_subset(X, y)
    f = plt.figure()
    a = f.add_subplot(1,4,2)
    max = 0
    #print(X.shape)
    #print(y.shape)
    #print("afterprepared")
    model = keras.models.Sequential()
    model.add(keras.layers.Conv2D(filters=12, kernel_size=(5,5), strides=2, activation='relu', input_shape=(img_size,img_size,1)))
    model.add(keras.layers.Dropout(.5))
    model.add(keras.layers.Conv2D(filters=18, kernel_size=(3,3) , strides=2, activation='relu'))
    model.add(keras.layers.Dropout(.5))
    model.add(keras.layers.Conv2D(filters=24, kernel_size=(2,2), activation='relu'))
    model.add(keras.layers.Flatten())
    model.add(keras.layers.Dense(units=150, activation='relu'))
    model.add(keras.layers.Dense(units=len(character_curated), activation='softmax')) #this is probably current error units =

    model.compile(loss='categorical_crossentropy', optimizer='Adam', metrics=['accuracy'])
    #model.summary()
    #for layer in model.layers:
    #    print(layer.get_output_at(0).get_shape().as_list())

    dataset = tf.data.Dataset.from_tensor_slices(X)
    data_generator = keras.preprocessing.image.ImageDataGenerator(validation_split=.2)
    test_data_generator = data_generator.flow(X, y)
    training_data_generator = data_generator.flow(X, y, subset='training')
    validation_data_generator = data_generator.flow(X_test, y_test, subset='validation')
    history = model.fit(training_data_generator, epochs=numE, verbose=2, steps_per_epoch=500)
    model.evaluate(test_data_generator)
    print("Finished" + LANGUAGE)
    model.save(LANGUAGE + str(numE)  + 'model.h5')
num = 10
while(num < 1000000):
    repeat("Python",num)
    repeat("Java",num)
    repeat("Lisp",num)
    num *= 10

#def run_prediction(idx):
#    result = np.argmax(model.predict(X_test[idx:idx+1]))
 #   print('Prediction: ', result ,' Char: ', chr(character_curated[result]))

#for i in range(100):
#    run_prediction(i)
#    plt.imshow(X_test[i].reshape([28, 28]), cmap='Greys_r')
 #   plt.show()


# -*- coding: utf-8 -*-
from flask import Flask, render_template, redirect, url_for
from flask.ext.login import LoginManager, login_required
from flask.ext import login
from flask.ext.sqlalchemy import SQLAlchemy
from flask.ext.babel import Babel, lazy_gettext, gettext, refresh, ngettext
#import app.modules.usuario.models
#import app.modules.usuario.controller

app = Flask(__name__)
app.config.from_object('config')
app.config['BABEL_DEFAULT_LOCALE'] = 'pt_BR'
""" Translate """
babel = Babel(app)
db = SQLAlchemy()
db.init_app(app)

@app.route('/')
@app.route('/index.html')
@login_required
def inicio():
  return render_template('index.html', user=login.current_user)

@app.route('/teamunifacs')
def team_unifacs_view():
	membros = [
		('eduardo.jpg', u'Eduardo Júnior', u'Analista e Desenvolvedor', u'Irecê - BA', u'ej@eduardojunior.com',
			'(74) 9932-6161'),
		('robert-300.jpg', u'Eduardo Júnior', u'Analista e Desenvolvedor', u'Irecê - BA', u'ej@eduardojunior.com',
			'(74) 9932-6161'),
		('robert-300.jpg', u'Eduardo Júnior', u'Analista e Desenvolvedor', u'Irecê - BA', u'ej@eduardojunior.com',
			'(74) 9932-6161'),
		('robert-300.jpg', u'Eduardo Júnior', u'Analista e Desenvolvedor', u'Irecê - BA', u'ej@eduardojunior.com',
			'(74) 9932-6161'),
		('robert-300.jpg', u'Eduardo Júnior', u'Analista e Desenvolvedor', u'Irecê - BA', u'ej@eduardojunior.com',
			'(74) 9932-6161'),
		('robert-300.jpg', u'Eduardo Júnior', u'Analista e Desenvolvedor', u'Irecê - BA', u'ej@eduardojunior.com',
			'(74) 9932-6161'),
	]
	return render_template('teamunifacs.html', membros=membros, user=login.current_user)

@app.route('/install')
def install():
	db.drop_all()
	db.create_all()
	from app.modules.adm.models.usuario import Usuario, Modulo
	usuario = Usuario()
	usuario.nome = 'admin'
	usuario.login = 'admin'
	usuario.set_senha('admin')
	db.session.add(usuario)

	modulos = [
		'adm',
	]
	for m in modulos:
		mod = Modulo()
		mod.nome = m
		db.session.add(mod)

	db.session.commit()

	return "install..."

@app.before_first_request
def initialize_database():
	db.create_all()

@app.errorhandler(404)
def not_found(error):
    return render_template('404.html'), 404

from app.modules.modelmodule.views import mod as modelmoduleModule
app.register_blueprint(modelmoduleModule)

from app.modules.adm.controllers import mod as modulo_adm
app.register_blueprint(modulo_adm)

from app.modules.auth.controllers import mod as modulo_auth
app.register_blueprint(modulo_auth)

def init_login():
	from app.modules.adm.models.usuario import Usuario
	login_manager = login.LoginManager()
	login_manager.setup_app(app)
	# Create user loader function
	@login_manager.user_loader
	def load_user(user_id):
		return db.session.query(Usuario).get(user_id)

	@login_manager.unauthorized_handler
	def unauthorized():
		''' redirecionar para tela de login '''
		return redirect(url_for('auth.login_view'))

""" INICIANDO LOGIN """
init_login()

# Later on you'll import the other blueprints the same way:
#from app.comments.views import mod as commentsModule
#from app.posts.views import mod as postsModule
#app.register_blueprint(commentsModule)
#app.register_blueprint(postsModule)


import os
import time
import random
import codecs
import numpy as np
import tensorflow as tf
from tensorflow.keras.utils import to_categorical
import keras
from keras import backend as K
from keras.layers import Input
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import confusion_matrix
import matplotlib.pyplot as plt
from matplotlib.font_manager import FontProperties

from create_data import get_ns

def set_seed(seed):
    os.environ['PYTHONHASHSEED'] = str(seed)
    os.environ['TF_DETERMINISTIC_OPS'] = '1'
    random.seed(seed)
    np.random.seed(seed)
    tf.random.set_seed(seed)

def get_folds_data(num_people, feat_time, kfold_num, img_rows, img_cols):
    folds_data = {}
    folds_data_person = {}
    for nb in range(num_people):
        one_falx_1 = falx[nb * 3:nb * 3 + 3]
        one_falx_1 = one_falx_1.reshape((-1, feat_time, img_rows, img_cols, 5))

        # ###============= random select ============####
        # permutation = np.random.permutation(one_y_1.shape[0])
        # one_falx_2 = one_falx_1[permutation, :]
        # one_falx = one_falx_2[0:3400]
        # one_y_2 = one_y_1[permutation, :]
        # one_y = one_y_2[0:3400]
        # ###============= random select ============####

        one_y = one_y_1
        one_falx = one_falx_1[:, :, :, :, 1:5]

        # print(one_y.shape)
        # print(one_falx.shape)
        # x_train, x_test, y_train, y_test = train_test_split(one_falx, one_y, test_size=0.25)
        kfold = StratifiedKFold(n_splits=kfold_num, shuffle=True, random_state=seed)
        for fi, (train, test) in enumerate(kfold.split(one_falx, one_y.argmax(1))):
            # x_train = one_falx[train]
            # y_train = one_y[train]
            x_test = one_falx[test]
            y_test = one_y[test]

            if fi in folds_data:
                # folds_data[fi]['x_train'] = np.concatenate([folds_data[fi]['x_train'], x_train], axis=0)
                # folds_data[fi]['y_train'] = np.concatenate([folds_data[fi]['y_train'], y_train], axis=0)
                folds_data[fi]['x_test'] = np.concatenate([folds_data[fi]['x_test'], x_test], axis=0)
                folds_data[fi]['y_test'] = np.concatenate([folds_data[fi]['y_test'], y_test], axis=0)
            else:
                folds_data[fi] = {}
                # folds_data[fi]['x_train'] = x_train
                # folds_data[fi]['y_train'] = y_train
                folds_data[fi]['x_test'] = x_test
                folds_data[fi]['y_test'] = y_test

            # folds_data_person[fi][nb]['x_train'] = x_train
            # folds_data_person[fi][nb]['y_train'] = y_train
            folds_data_person[fi] = {nb: {}}
            folds_data_person[fi][nb]['x_test'] = x_test
            folds_data_person[fi][nb]['y_test'] = y_test

    return folds_data, folds_data_person

def plot_confusion_matrix(output_path, y_true, y_pred, labels=["0", "1", "-1"]):
    cm = confusion_matrix(y_true, y_pred)
    cm_norm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
    cm_norm = cm_norm * 100
    cm_norm = np.around(cm_norm, decimals=2)  # 保留两位小数

    plt.matshow(cm_norm, cmap=plt.cm.Reds)  # 根据最下面的图按自己需求更改颜色
    # plt.colorbar()

    for i in range(len(cm_norm)):
        for j in range(len(cm_norm)):
            plt.annotate('%s' % cm_norm[j, i], xy=(i, j), horizontalalignment='center', verticalalignment='center')

    # plt.tick_params(labelsize=15) # 设置左边和上面的label类别如0,1,2,3,4的字体大小。

    font_label = FontProperties(fname='./data/SimHei.ttf', size=13)
    plt.ylabel('实际类别', fontproperties=font_label)
    plt.xlabel('预测类别', fontproperties=font_label)

    font_ticks = FontProperties(fname='./data/SimHei.ttf', size=11)
    xlocations = np.array(range(len(labels)))
    plt.xticks(xlocations, labels, fontproperties=font_ticks)
    plt.yticks(xlocations, labels, fontproperties=font_ticks)

    # plt.show()
    plt.savefig('%s/confusion_matrix.png' % output_path)
    plt.close('all')
    print('=================================================================')
    print('=================================================================')
    print('=================================================================')
    print('=================================================================')

if __name__ == '__main__':
    # os.environ['CUDA_VISIBLE_DEVICES'] = '1'

    data_path = './data/model/'
    output_path = './runtime_all/'
    num_people = 1
    num_classes = 3
    kfold_num = 5
    img_rows, img_cols, num_chan = 8, 9, 4
    model_type = 'cnn'  # cnn, densenet, resnet
    feat_time = 1
    se_head = False
    se_tail = False
    seed = 1
    labels_name = ['中性', '积极', '消极']  # 0, 1, -1

    set_seed(seed)

    runtime_path = '%s/t%s/%s' % (output_path, feat_time, model_type)
    if se_head:
        runtime_path += '_se_head'
    if se_tail:
        runtime_path += '_se_tail'

    img_size = (img_rows, img_cols, num_chan)
    falx = np.load('%s/X_89_t%s.npy' % (data_path, feat_time))
    y = np.load('%s/y_89_t%s.npy' % (data_path, feat_time))

    one_y_1 = np.array([y[:get_ns(feat_time)]] * 3).reshape((-1,))
    one_y_1 = to_categorical(one_y_1, num_classes)

    people_acc_list = {}
    label_gold = []
    label_pred = []
    folds_data, folds_data_person = get_folds_data(num_people, feat_time, kfold_num, img_rows, img_cols)
    with codecs.open('%s/test.log' % runtime_path, 'w', 'utf-8') as fout:
        for fi in folds_data:
            start = time.time()
            K.clear_session()

            x_test = folds_data[fi]['x_test']
            y_test = folds_data[fi]['y_test']

            fold_path = '%s/f%s' % (runtime_path, fi)

            input_list = [Input(shape=img_size) for _ in range(feat_time)]
            if os.path.exists('%s/best.h5' % fold_path):
                model = keras.models.load_model('%s/best.h5' % fold_path)
            else:
                model = keras.models.load_model('%s/last.h5' % fold_path)

            # evaluate the model
            scores = model.evaluate(
                [x_test[:, nf] for nf in range(feat_time)],
                y_test, verbose=1
            )

            accuracy = (scores[1] * 100)
            print("%.2f%%" % accuracy)
            fout.write('fold acc： %s\n' % accuracy)

            # 画图
            y_pred = model([x_test[:, nf] for nf in range(feat_time)])
            y_pred = K.eval(tf.argmax(y_pred, axis=1, output_type=tf.int32)).tolist()
            y_true = np.argmax(y_test, axis=1).tolist()
            plot_confusion_matrix(fold_path, y_true, y_pred, labels=labels_name)
            label_gold.extend(y_true)
            label_pred.extend(y_pred)

            people_acc = {}
            for nb in folds_data_person[fi]:
                person_x = folds_data_person[fi][nb]['x_test']
                person_y = folds_data_person[fi][nb]['y_test']
                person_y_pred = model([person_x[:, nf] for nf in range(feat_time)])
                person_y_pred = K.eval(tf.argmax(person_y_pred, axis=1, output_type=tf.int32)).tolist()
                person_y_true = np.argmax(person_y, axis=1).tolist()
                person_acc = np.mean(np.equal(np.array(person_y_pred), np.array(person_y_true)))
                people_acc[nb] = person_acc

                if nb in people_acc_list:
                    people_acc_list[nb].append(person_acc)
                else:
                    people_acc_list[nb] = [person_acc]

            with codecs.open('%s/people_accuracy.log' % fold_path, 'w', 'utf-8') as fout_per:
                for nb in people_acc:
                    fout_per.write('%s: %s\n' % (nb, people_acc[nb]))

            # print("all acc: {}".format(all_acc))
            # print('mean acc: %s, std: %s' % (np.mean(all_acc), np.std(all_acc)))
            # fout.write('mean acc: %s, std: %s\n' % (np.mean(all_acc), np.std(all_acc)))
            # acc_list.append(np.mean(all_acc))
            # std_list.append(np.std(all_acc))
            # print("进度： {}".format(nb))
            # all_acc = []
            end = time.time()
            print("%.2f" % (end - start))  # run time

        plot_confusion_matrix(runtime_path, label_gold, label_pred, labels=labels_name)
        with codecs.open('%s/people_accuracy.log' % runtime_path, 'w', 'utf-8') as fout_peo:
            for nb in people_acc_list:
                fout_peo.write('%s: %s\n' % (nb, people_acc_list[nb]))


from datetime import date
from helpers.db import *
from helpers.classes import *

def make_list_of_clubs(query_result):
    clubs = []
    for row in query_result:
        clubs.append(Club(row))
    return clubs

def make_list_of_events(query_result):
    events = []
    for row in query_result:
        events.append(Event(row))
    return events

def make_list_of_events_for_overview(query_result):
	events = []
	for row in query_result:
		events.append(Event_for_overview(row))
	return events

def user_has_administrative_rights(user_id, club_id):
	if user_id:
		conn = get_db_connection()
		cur = conn.cursor()
		cur.execute(
			"""
			SELECT EXISTS(SELECT 1 FROM member_rights WHERE (user_id = {user_id} AND club_id = {club_id} AND right_id = 2 and is_active = true) OR (user_id = {user_id} AND right_id = 3) and is_active = true) AS user_is_admin
			""".format(user_id=user_id, club_id=club_id)
		)
		user_is_admin = cur.fetchone()
		return user_is_admin[0]
	return False

def user_is_member(user_id, club_id):
	if user_id:
		conn = get_db_connection()
		cur = conn.cursor()
		cur.execute(
			"""
			SELECT EXISTS(SELECT 1 FROM member_rights WHERE user_id = {user_id} AND club_id = {club_id} AND right_id IN (1,2) AND is_active = true) AS user_is_member
			""".format(user_id=user_id, club_id=club_id)
		)
		user_is_member = cur.fetchone()
		return user_is_member[0]
	return False

def string_append(source, input):
	("Source: %d", source)
	if bool(source):
		return (source + ", " + input)
	else:
		return input

def signup_for_event_is_open(event):
	return event.signup_opens <= date.today() <= event.signup_closes

def determine_signup_status_for_event(event):
	open = False
	# If there is a limit for number of participants
	if event.max_spots is not None and not event.is_cancelled:
		# Check that date is within period, and that there are remaining spots
		if signup_for_event_is_open(event) and event.max_spots > 0:
			open = True
	else:
		# Check that date is within period
		if signup_for_event_is_open(event) and not event.is_cancelled:
			open = True
	return open


# pip3 install pulsectl
import pyttsx3
import pulsectl
import random

# Call this to get a 'say' function which reuses underlying engine instance
def get_sayer():
	engine = pyttsx3.init()
	engine.setProperty('volume', 0.2)  # Stop it blasting us when we have paused a movie
	pulse = pulsectl.Pulse()

	def say(msg):
		print(msg)
		if not is_running(pulse):
			choose_voice(engine)
			engine.say(msg)
			engine.runAndWait()

	return say

def is_running(pulse):
	sinks = pulse.sink_list()
	return any(sink.state == 'running' for sink in sinks)

def choose_voice(engine):
	voices = engine.getProperty('voices')
	voice_ids = []
	for voice in voices:
		id = voice.id
		if id.startswith('english'):
			# English:
			voice_ids.append(id)
			# Chicks - only works on Linux:
			voice_ids.append(f"{id}+f1")
			voice_ids.append(f"{id}+f2")
			voice_ids.append(f"{id}+f3")
			voice_ids.append(f"{id}+f4")
	engine.setProperty('voice', random.choice(voice_ids))


#!/usr/bin/env python3

import itertools
import json

import click
import cv2

def read_result_file(file_path):
    video_frame_table = {}
    with open(file_path, "r") as file:
        video_meta = json.loads(file.readline())
        for line in file:
            if line is None:
                break
            record = json.loads(line)
            frame_index = record["frame_index"]
            video_frame_table[frame_index] = record
    return (video_meta, video_frame_table)

@click.command()
@click.argument("result_file_path", type=click.Path(exists=True))
@click.argument("output_file_path", type=click.Path(exists=False))
def main(result_file_path, output_file_path):
    video_meta, video_frame_table = read_result_file(result_file_path)

    result = [
        [
            frame_index,
            [
                [
                    face["score"],
                    [
                        face["boundingBox"]["x1"],
                        face["boundingBox"]["y1"],
                        face["boundingBox"]["x2"],
                        face["boundingBox"]["y2"],
                    ],
                    [
                        [
                            point["x"],
                            point["y"],
                        ]
                        for point in face["keyPoints"]
                    ],
                    [
                        face["attributes"]["sex"],
                        face["attributes"]["age"],
                    ],
                ]
                for face in video_frame_table[frame_index]["result"]["response"][
                    "faces"
                ]
            ],
        ]
        for frame_index in range(len(video_frame_table))
    ]

    with open(output_file_path, "w") as file:
        json.dump(
            result, file, sort_keys=True, separators=(",", ":"), ensure_ascii=False
        )

if __name__ == "__main__":
    main()


from datetime import datetime
import config
from db import sql_connect

def update_user_info(user_id, tg_id=None, notification_time=None, min_days_num=None, max_days_num=None,
                     time_to_gift_flg=None):
    if user_id is None:
        config.logger.error("Can not add a user info without user_id")
        return

    if tg_id is None and notification_time is None and min_days_num is None and max_days_num is None \
            and time_to_gift_flg is None:
        config.logger.error("Can't update user_id: " + str(user_id) + " info. No arguments passed.")
        return

    conn = sql_connect.create_connection()
    cursor = conn.cursor()

    cursor.execute("""
        SELECT count(*)
        FROM user
        WHERE user_id = ? and deleted_flg = '0'
        """, (user_id,))

    if cursor.fetchone()[0] == 0:
        config.logger.error("user_id " + str(user_id) + " does not exist")
        cursor.close()
        conn.close()
        return

    if tg_id is not None:
        cursor.execute("""
            UPDATE user
            SET tg_id = ?
            WHERE user_id = ? and deleted_flg = '0'
            """, (tg_id, user_id))

    if notification_time is not None:
        cursor.execute("""
            UPDATE user
            SET notification_time = ?
            WHERE user_id = ? and deleted_flg = '0'
            """, (notification_time, user_id))

    if min_days_num is not None:
        cursor.execute("""
            UPDATE user
            SET min_days_num = ?
            WHERE user_id = ? and deleted_flg = '0'
            """, (min_days_num, user_id))

    if max_days_num is not None:
        cursor.execute("""
            UPDATE user
            SET max_days_num = ?
            WHERE user_id = ? and deleted_flg = '0'
            """, (max_days_num, user_id))

    if time_to_gift_flg is not None:
        cursor.execute("""
            UPDATE user
            SET time_to_gift_flg = ?
            WHERE user_id = ? and deleted_flg = '0'
            """, (time_to_gift_flg, user_id))

    config.logger.info("user_id: " + str(user_id) + " tg_id: " + str(tg_id) + " info updated")

    conn.commit()
    cursor.close()
    conn.close()

def disable_user(user_id):
    if user_id is None:
        config.logger.error("Can not disable user without user_id")
        return

    conn = sql_connect.create_connection()
    cursor = conn.cursor()

    cursor.execute("""
        UPDATE user
        SET deleted_flg = '1'
        WHERE user_id = ?
        """, (user_id, ))

    config.logger.info("user_id: " + str(user_id) + " disabled")

    conn.commit()
    cursor.close()
    conn.close()

def add_user(tg_id=None):
    if tg_id is None:
        config.logger.error("Can not add a user without telegram id")
        return

    conn = sql_connect.create_connection()
    cursor = conn.cursor()

    cursor.execute("""
        SELECT user_id, deleted_flg
        FROM user
        WHERE tg_id = ?
        """, (tg_id,))

    row = cursor.fetchone()

    if row is None:
        cursor.execute("""
            SELECT coalesce(max(user_id), 0) + 1
            FROM user
            """)
        user_id = cursor.fetchone()[0]

        cursor.execute("""
            INSERT INTO user (user_id, tg_id, time_to_gift_flg, deleted_flg, processed_dttm)
            VALUES (?, ?, ?, ?, ?)
            """, (user_id, tg_id, '0', '0', datetime.now()))

        config.logger.info("user_id: " + str(user_id) + " tg_id: " + str(tg_id) + " added")
    elif row['deleted_flg'] == '0':
        config.logger.error("tg_id: " + str(tg_id) + " already exists")
        cursor.close()
        conn.close()
        return None
    elif row['deleted_flg'] == '1':
        cursor.execute("""
            UPDATE user
            SET deleted_flg = '0'
            WHERE tg_id = ?
            """, (tg_id,))

        user_id = row['user_id']
        config.logger.info("tg_id: " + str(tg_id) + " enabled")

    conn.commit()
    cursor.close()
    conn.close()

    return user_id

def add_gift(user_id, gift_dt):
    if user_id is None:
        config.logger.error("Can not add a gift info without user_id")
        return

    conn = sql_connect.create_connection()
    cursor = conn.cursor()

    cursor.execute("""
        UPDATE gift
        SET is_active_flg = '0'
        WHERE user_id = ?
        """, (user_id,))

    cursor.execute("""
    INSERT INTO gift (user_id, gift_dt, is_active_flg, processed_dttm)
    VALUES (?, ?, ?, ?)
    """, (user_id, gift_dt, '1', datetime.now()))

    config.logger.info("user_id: " + str(user_id) + " gift_dt: " + str(gift_dt) + " added")

    conn.commit()
    cursor.close()
    conn.close()

def add_notification_log(user_id, notification_time, sent_flg):
    if user_id is None:
        config.logger.error("Can not log info without user_id")
        return

    conn = sql_connect.create_connection()
    cursor = conn.cursor()

    cursor.execute("""
        UPDATE notification_log
        SET is_active_flg = '0'
        WHERE user_id = ?
        """, (user_id,))

    cursor.execute("""
    INSERT INTO notification_log (user_id, notification_time, sent_flg, is_active_flg, processed_dttm)
    VALUES (?, ?, ?, ?, ?)
    """, (user_id, notification_time, sent_flg, '1', datetime.now()))

    config.logger.info("user_id: " + str(user_id) + " notification log added")

    conn.commit()
    cursor.close()
    conn.close()

def add_random_log(user_id, since_latest_gift_days_cnt, random_days_cnt):
    if user_id is None:
        config.logger.error("Can not log info without user_id")
        return

    conn = sql_connect.create_connection()
    cursor = conn.cursor()

    cursor.execute("""
        UPDATE random_log
        SET is_active_flg = '0'
        WHERE user_id = ?
        """, (user_id,))

    cursor.execute("""
    INSERT INTO random_log (user_id, since_latest_gift_days_cnt, random_days_cnt, is_active_flg, processed_dttm)
    VALUES (?, ?, ?, ?, ?)
    """, (user_id, since_latest_gift_days_cnt, random_days_cnt, '1', datetime.now()))

    config.logger.info("user_id: " + str(user_id) + " random log added")

    conn.commit()
    cursor.close()
    conn.close()


__author__ = 'kristukat'

import numpy as np
import wx
from wx.lib import newevent
from wx.lib.floatcanvas import FloatCanvas
from functools import reduce

import logging
logger = logging.getLogger('pom')

SelectEvent, EVT_RECT_SELECT = newevent.NewEvent()
ReorderEvent, EVT_RECT_REORDER = newevent.NewEvent()

class Map(list):
    def __getitem__(self, item):
        if type(item) == tuple:
            for k in self:
                if k.gridpos == item:
                    return k
            raise IndexError(item)
        else:
            return super(Map, self).__getitem__(item)

class PlotOrderPanel(wx.Panel):
    def __init__(self, parent, **kwargs):
        wx.Panel.__init__(self, parent, **kwargs)

        self.Canvas = FloatCanvas.FloatCanvas(self, BackgroundColor="LIGHT BLUE")
        self.Canvas.InitAll()

        MainSizer = wx.BoxSizer(wx.VERTICAL)
        MainSizer.Add(self.Canvas, 4, wx.EXPAND)
        self.SetSizer(MainSizer)

        self.Bind(wx.EVT_SIZE, self.OnSize)

    def OnSize(self, evt):
        evt.Skip()
        self.Canvas.ZoomToBB()

    def Log(self, text):
        print(text)

    def OnLeftUp(self, event):
        if self.moving is not None:
            frm = self.moving
            self.moving.PutInBackground()
            self.moving.Text.PutInBackground()
            self.moving.SetFillColor('White')
            self.moving = None
            if self.target is not None:
                to = self.target
                self.target.SetFillColor('White')
                self.Canvas.Draw(Force=True)
                self.target = None
                self.reorder(frm, to)
            self.redraw_rects()

    def add_rect(self, row, col):
        xy = np.asarray([col,row])*[1,-1]
        rect = self.Canvas.AddRectangle(xy, (0.8,0.8), LineWidth = 1, FillColor = 'White')
        rect.Name = ''
        rect.gridpos = (row,col)
        self.__rects.append(rect)

        t = self.Canvas.AddText(rect.Name, xy+[0.3,0.4], Size = 9, Position = 'cc')
        rect.Text = t
        rect.Bind(FloatCanvas.EVT_FC_LEFT_DOWN, self.RectGotHitLeft)
        rect.Bind(FloatCanvas.EVT_FC_ENTER_OBJECT, self.RectMouseOver)
        rect.Bind(FloatCanvas.EVT_FC_LEAVE_OBJECT, self.RectMouseLeave)

    def reorder(self, frm, to):
        to.gridpos, frm.gridpos = frm.gridpos, to.gridpos
        wx.PostEvent(self, ReorderEvent(swap=(frm.gridpos, to.gridpos)))

    def redraw_rects(self, removed=[]):
        for r in removed:
            self.Canvas.RemoveObject(r.Text)
            self.Canvas.RemoveObject(r)

        for r in self.__rects:
            xy = np.asarray(r.gridpos[::-1])*[1,-1]
            r.SetPoint(xy)
            r.Text.SetText(r.Name)
            r.Text.SetPoint(xy+[0.3,0.4])
        self.Canvas.Draw(Force=True)

    @property
    def shape(self):
        return self.__shape
    @shape.setter
    def shape(self, shape):
        if np.asarray(shape).sum() < 2:
            shape = (1,1)
        remove = []
        for r in range(self.__shape[0],shape[0]): # only if new shape has more rows than current
            for c in range(self.__shape[1]):
                self.add_rect(r,c)
            self.__shape = shape[0],self.__shape[1]
        for c in range(self.__shape[1], shape[1]):
            for r in range(self.__shape[0]):
                self.add_rect(r,c)
            self.__shape = self.__shape[0],shape[1]
        for n in range(len(self.__rects)-1,-1,-1):
            if self.__rects[n].gridpos[0] >= shape[0]:
                remove.append(self.__rects.pop(n))
                self.__shape = shape[0], self.__shape[1]
        for n in range(len(self.__rects)-1,-1,-1):
            if self.__rects[n].gridpos[1] >= shape[1]:
                remove.append(self.__rects.pop(n))
                self.__shape = self.__shape[0], shape[1]
        if self.selection in remove:
            self.selection = self.__rects[0]
            self.Focus(self.__rects[0])
            wx.PostEvent(self, SelectEvent(pos=self.selection.gridpos, name=self.selection.Name))
        self.redraw_rects(remove)
        self.Canvas.ZoomToBB()

    @property
    def order(self):
        return [(q.gridpos, q.Name) for q in self.__rects if q.Name != '']

    def set_name(self, name, idx=None):
        if idx is None:
            self.selection.Name = name
            self.selection.Text.SetText(name)
        else:
            self.__rects[idx].Name = name
            self.__rects[idx].Text.SetText(name)
        self.Canvas.Draw(Force=True)

    def Init(self, shape=(1,1)):
        self.moving = None
        self.selection = None
        self.target = None

        self.__shape = shape
        self.__rects = Map()

        Canvas = self.Canvas

        for row,col in np.mgrid[0:shape[1],0:shape[0]].reshape((2,reduce(np.multiply, shape))).T:
            xy = np.asarray([col,row])*[1,-1]
            self.add_rect(row, col)

        self.Canvas.Bind(FloatCanvas.EVT_MOTION, self.OnDrag)
        self.Canvas.Bind(FloatCanvas.EVT_LEFT_UP, self.OnLeftUp)

        Canvas.ZoomToBB()
        self.redraw_rects()

        self.selection = self.__rects[0]

    def update_from_model(self, mpmodel):
        self.shape = mpmodel.shape
        for k,v in mpmodel.items():
            #print(k,v.name)
            self.__rects[k].Name = v.name

        self.redraw_rects()

        if self.selection is not None:
            self.Focus(self.__rects[0])
            wx.PostEvent(self, SelectEvent(pos=self.selection.gridpos, name=self.selection.Name))

    def OnDrag(self, evt):
        self.curr_coords = evt.Coords
        if self.moving is not None:
            x, y = evt.Coords
            ox, oy = self.moving.XY+self.offset
            self.moving.SetFillColor('Lime Green')
            self.moving.Move((x-ox,y-oy))
            self.moving.Text.Move((x-ox,y-oy))
            self.Canvas.Draw(Force=True)
            wx.GetApp().Yield(True)

    def RectGotHitLeft(self, Object):
        if self.selection == Object:
            return
        self.offset = self.curr_coords-Object.XY
        self.Focus(Object)
        self.selection = Object
        wx.PostEvent(self, SelectEvent(pos=Object.gridpos, name=Object.Name))
        self.moving = Object
        Object.PutInForeground()
        Object.Text.PutInForeground()

    def Focus(self, Object=None):
        if self.selection is not None:
            self.selection.SetLineColor('Black')
            self.selection.SetLineWidth(1)
        if Object is not None:
            Object.SetLineColor('Light Grey')
            Object.SetLineWidth(4)
        self.Canvas.Draw(Force=True)

    def RectMouseOver(self, Object):
        if Object == self.moving:
            return
        if self.moving is not None:
            Object.SetFillColor('Red')
            self.Canvas.Draw(Force=True)
            self.target = Object

    def RectMouseLeave(self, Object):
        if Object == self.moving:
            return
        if self.moving is not None:
            Object.SetFillColor('White')
            self.Canvas.Draw(Force=True)
            self.target = None

class PlotLayout(wx.Panel):
    def __init__(self, parent, **kwargs):
        super(PlotLayout, self).__init__(parent, **kwargs)

        self.setup_controls()
        self.layout()

        #self.pop.Bind(EVT_RECT_SELECT, self.OnSelect)
        self.ch_plot_pri.Bind(wx.EVT_CHOICE, self.OnChoice)
        self.ch_rows.Bind(wx.EVT_CHOICE, self.OnShape)
        self.ch_cols.Bind(wx.EVT_CHOICE, self.OnShape)

        wx.CallAfter(self.pop.Canvas.ZoomToBB)

    def setup_controls(self):
        self.pop = PlotOrderPanel(self, size=(200,150))
        self.pop.Init((1,1))

        self.ch_rows = wx.Choice(self, choices=[str(q) for q in range(1,6)])
        self.ch_rows.SetSelection(0)
        self.ch_cols = wx.Choice(self, choices=[str(q) for q in range(1,6)])
        self.ch_cols.SetSelection(0)

        self.ch_plot_pri = wx.Choice(self, choices=[''], name='pri')
        self.ch_plot_sec = wx.Choice(self, choices=[''], name='sec')

    def layout(self):
        b = wx.BoxSizer(wx.VERTICAL)
        fg = wx.FlexGridSizer(2, 2, 0, 0)
        fg.Add(wx.StaticText(self, label='Rows'), 0, wx.ALL|wx.ALIGN_CENTRE_VERTICAL, 5)
        fg.Add(self.ch_rows, 0, wx.EXPAND|wx.ALIGN_CENTRE_VERTICAL|wx.ALL, 5)
        fg.Add(wx.StaticText(self, label='Columns'), 0, wx.ALL|wx.ALIGN_CENTRE_VERTICAL, 5)
        fg.Add(self.ch_cols, 0, wx.EXPAND|wx.ALIGN_CENTRE_VERTICAL|wx.ALL, 5)
        hb = wx.BoxSizer(wx.HORIZONTAL)
        hb.Add(fg, 0, wx.EXPAND)
        hb.Add(self.pop, 1, wx.EXPAND)
        b.Add(hb, 1, wx.EXPAND)

        hb = wx.BoxSizer(wx.HORIZONTAL)
        hb.Add(wx.StaticText(self, label='Primary axis'), 0, wx.RIGHT|wx.ALIGN_CENTER_VERTICAL, 5)
        hb.Add(self.ch_plot_pri, 1, wx.EXPAND)
        hb.Add(wx.StaticText(self, label='Secondary axis'), 0, wx.LEFT|wx.ALIGN_CENTER_VERTICAL, 10)
        hb.Add(self.ch_plot_sec, 1, wx.LEFT|wx.EXPAND, 5)
        b.Add(hb, 0, wx.EXPAND|wx.ALL, 5)
        self.SetSizer(b)

    def update_from_model(self, mpmodel):
        print('plotlayout: update from model, mpmodel.shape',mpmodel.shape)
        rows,cols = mpmodel.shape

        self.ch_rows.SetSelection(max(0,rows-1))
        self.ch_cols.SetSelection(max(0,cols-1))

        def select(ctrl, uuid):
            if uuid is None:
                ctrl.Selection = 0
            else:
                for n in range(ctrl.Count):
                    if ctrl.GetClientData(n) == uuid:
                        ctrl.Selection = n

        if mpmodel.selected is not None:
            pd = mpmodel.selected

            select(self.ch_plot_pri, pd.plot_ref)
            select(self.ch_plot_sec, pd.plot_ref_secondary)
            self.ch_plot_sec.Enable(pd.plot_ref is not None)
        else:
            select(self.ch_plot_pri, None)
            select(self.ch_plot_sec, None)
            self.ch_plot_sec.Enable(False)

    def set_plot_choices(self, choices, clientdata):
        for ch in self.ch_plot_pri, self.ch_plot_sec:
            ch.Clear()
            for c,d in zip(choices,clientdata):
                ch.Append(c, d)

    @property
    def selection(self):
        sel = self.pop.selection
        return sel.gridpos if sel is not None else None

    def OnShape(self, evt):
        evt.Skip()
        self.pop.shape = self.ch_rows.GetSelection()+1,self.ch_cols.GetSelection()+1

    def OnSelect(self, evt):
        #TODO: remove
        logger.debug('plotlayout on select should not be called')
        evt.Skip()
        if evt.name != '':
            self.ch_plot_pri.SetSelection(self.ch_plot_pri.FindString(evt.name))
            self.ch_plot_sec.SetSelection(0)
        else:
            self.ch_plot_pri.SetSelection(0)
            self.ch_plot_sec.SetSelection(0)

    def OnChoice(self, evt):
        evt.Skip()
        sel = evt.GetEventObject().GetSelection()
        name = evt.GetEventObject().GetString(sel)
        self.pop.set_name(name if sel != 0 else '')

if __name__ == "__main__":
    class DemoFrame(wx.Frame):
        def __init__(self):
            super(DemoFrame, self).__init__(None)
            self.panel = PlotLayout(self)

    class DemoApp(wx.App):
        def __init__(self, *args, **kwargs):
            wx.App.__init__(self, *args, **kwargs)

        def OnInit(self):
            frame = DemoFrame()
            frame.Show()
            return True

    app = DemoApp(False)
    app.MainLoop()


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('twentyFour', '0025_auto_20141206_0526'),
    ]

    operations = [
        migrations.AddField(
            model_name='player',
            name='total_points',
            field=models.IntegerField(default=0),
            preserve_default=True,
        ),
        migrations.AlterField(
            model_name='game',
            name='card_sets',
            field=models.ManyToManyField(to=b'twentyFour.Cardset'),
        ),
    ]


import matplotlib.pyplot as plt
from sklearn.ensemble import RandomForestClassifier
import pandas as pd
import numpy as np
from sklearn.metrics import auc, precision_recall_curve, average_precision_score
from sklearn.metrics import roc_curve, f1_score, classification_report
from sklearn.metrics import precision_score, recall_score
from collections import defaultdict
import seaborn as sns
import csv, os
import pickle
from sklearn import cross_validation

BINARY = True

def plotCurves(fpr, tpr, cl, fn=None):
    if len(fpr) != 0:
        roc_auc = auc(fpr, tpr)
        plt.plot(fpr, tpr, 'b', label='AUC = %0.2f' % roc_auc)
    if fn == None:
        fn = 'all features'
    plt.title('ROC: Predicting {0} boundaries using {1}'.format(cl, fn))
    plt.legend(loc='lower right')
    plt.plot([0, 1], [0, 1], 'r--')
    ax = plt.gca()
    ax.grid(True)
    plt.ylabel('True Positive Rate')
    plt.xlabel('False Positive Rate')
    # plt.savefig('/Users/rohanpaul/Desktop/figs/pairwise_binary/{}_{}_cos.png'.format(cl, fn), bbox_inches='tight')
    plt.show()
    plt.close()

def OneVsAll():
    w = defaultdict(list)
    cells = ['HeLa', 'HMEC', 'HUVEC', 'NHEK', 'K562']
    for cell in cells:
        toconcat = []

        ocl = [k for k in cells if k != cell]
        for t in ocl:
            path = 'PairwiseFeatures(global)/{}_testData'.format(t)
            df = pd.read_csv(path, header=0, index_col=0, sep='\t')
            toconcat.append(df)
        train = pd.concat(toconcat)
        test = pd.read_csv('PairwiseFeatures(global)/{}_testData'.format(cell), header=0,
                           index_col=0, sep='\t')

        # parameters = {'n_estimators': [10, 100, 1000],
        #               'criterion': ['gini', 'entropy']}
        # rf = RandomForestClassifier(n_jobs=-1)
        # clf = GridSearchCV(rf, parameters)

        clf = RandomForestClassifier(n_estimators=200, criterion='gini',
                                     n_jobs=-1)

        Xtrain = train[train.columns.values[:-1]]
        ytrain = train['Class']
        Xtest = test[test.columns.values[:-1]]
        ytest = test['Class']

        clf.fit(Xtrain, ytrain)
        # print(clf.best_params_)
        scores = clf.predict_proba(Xtest)
        posScores = scores[:, 1].reshape(scores.shape[0])
        fpr, tpr, _ = roc_curve(ytest, posScores)

        ##############PLOT################

        roc_auc = round(auc(fpr, tpr), 3)
        print('AUROC for {}: {}'.format(cell, roc_auc))
        plt.plot(fpr, tpr, label='AUC for {} = {}'.format(roc_auc, cell))
    plt.title('One vs. All'.format(cell))
    plt.legend(loc='lower right')
    plt.plot([0, 1], [0, 1], 'r--')
    plt.ylabel('True Positive Rate')
    plt.xlabel('False Positive Rate')
    # plt.savefig('/Users/rohanpaul/Desktop/OnevAll.png'.format(cell), bbox_inches='tight')
    plt.show()
    plt.close()

def readData(cell, binary=True):
    """
    :param cell: The cell line for which train and test data should be fetched
    :param binary: Whether to use binary or continuous features
    replace with style:[binary | continuous | average | aggregate]
    :return: returns the training and testing data as pandas dataframes
    """

    # path = '../data/main/unexpandedFeatures/train/{}'.format(cell)
    path = '../data/main/mainFeatures/{}'.format(cell)
    extension = '.bin.txt'
    train, test = [], []
    if not binary:
        extension = '.cont.txt'

    #training data
    for file in [k for k in os.listdir(path) if k.endswith(extension)]:
        fname = os.path.join(path, file)
        train.append(pd.read_csv(fname, index_col=0, header=0, sep='\t'))

    # Using bag of boundaries method for testing data
    # i.e. using same tads for training and testing
    # path = '../data/main/unexpandedFeatures/test/'
    path = '../data/main/BOBTestFeatures'
    for file in [k for k in os.listdir(path) if k.endswith(extension) and cell in k]:
        fname = os.path.join(path, file)
        test.append(pd.read_csv(fname, index_col=0, header=0, sep='\t'))

    train = pd.concat(train)
    test = pd.concat(test)

    ####
    # Add number of other cell lines each boundary is present in feature
    train_tads = train.index
    test_tads = test.index

    with open('../data/pickle/pairwise_presence.pickle', 'rb') as f:
        df = pickle.load(f)
        df = df.drop(cell, axis=1)

    train_counts = []
    for t in train_tads:
        numTrue = len([k for k in df.loc[t] if k == True])
        if numTrue == 0:
            exit(1)
        else:
            train_counts.append(numTrue)

    test_counts = []
    for t in test_tads:
        numTrue = len([k for k in df.loc[t] if k == True])
        if numTrue == 0:
            exit(1)
        else:
            test_counts.append(numTrue)

    train_counts = np.array(train_counts)
    test_counts = np.array(test_counts)
    train['counts'] = (train_counts - min(train_counts))/(max(train_counts) - min(train_counts))
    test['counts'] = (test_counts - min(test_counts))/(max(test_counts) - min(test_counts))

    #testing data
    # path = '../data/main/mainFeatures/testData/'
    # for file in [k for k in os.listdir(path) if k.endswith(extension) and cell in k]:
    #     fname = os.path.join(path, file)
    #     test.append(pd.read_csv(fname, index_col=0, header=0, sep='\t'))
    #
    # pospath = '../data/main/mainFeatures/testData/'
    # for file in [k for k in os.listdir(pospath) if k.endswith(extension) and cell in k and 'pos' in k]:
    #     fname = os.path.join(pospath, file)
    #     test.append(pd.read_csv(fname, index_col=0, header=0, sep='\t'))
    #
    # #One end in all tads is fixed with a real boundary
    # negpath = '../data/main/fixedNegativeFeatures'
    # for file in [k for k in os.listdir(negpath) if k.endswith(extension) and cell in k]:
    #     fname = os.path.join(negpath, file)
    #     test.append(pd.read_csv(fname, index_col=0, header=0, sep='\t'))

    return train, test

def singleCellLine():
    w = defaultdict(list)
    for cl in ['HeLa', 'HMEC', 'HUVEC', 'NHEK', 'K562']:
        master = []
        with open('../PairwiseFeatures(global)/{}_testData'.format(cl), 'r') as f:
            reader = csv.reader(f, delimiter='\t')
            for line in reader:
                    master.append(list(line))
        header = master.pop(0)
        # master = master[1:]
        master = pd.DataFrame(master, columns=header)
        # master.set_index('featureID')
        y = master['Class']
        X = master.drop('Class', axis=1)
        cv = cross_validation.StratifiedKFold(y, n_folds=5, shuffle=True)
        y = y.values; X = X.drop('featureID', axis=1).values
        auroc = 0
        # print(X[:10]); exit()
        plt.plot([0, 1], [0, 1], 'r--')
        plt.xlabel('False Positive Rate')
        plt.title('{} Only ROC Curve'.format(cl))
        plt.ylabel('True Positive Rate')

        for fold, (train_index, test_index) in enumerate(cv):
            y_train, y_test = y[train_index], y[test_index]
            x_train, x_test = X[train_index], X[test_index]

            clf = RandomForestClassifier(n_estimators=200, criterion='gini',
                                         n_jobs=-1, max_depth=5)

            clf.fit(x_train, y_train)
            scores = clf.predict_proba(x_test)
            posScores = scores[:, 1].reshape(-1, 1).tolist()
            # for i, name in enumerate(master.columns.values[1:-1]):
            #     if name != 'Correlation':
            #         w[name[:-2]].append(clf.coef_[0][i])
            #     else:
            #         w[name].append(clf.coef_[0][i])
            fpr, tpr, _ = roc_curve(y_test, posScores, pos_label='1')
            auc_roc = round(auc(fpr, tpr), 3)
            plt.plot(fpr, tpr, label='ROC on fold {}: {}'.format(fold, auc_roc))
            print('AUC on fold {} for {}:'.format(fold, cl), auc_roc)
            auroc += auc(fpr, tpr)
        # plotCurves([], [], cl)
        plt.legend(loc='lower right')

        plt.show()
        print('Average AUC for {} = {}'.format(cl, auroc/5))

    # avw = []
    # for key, value in w.items():
    #     avw.append(sum(value)/len(value))
    # names = list(w.keys())
    # ind = np.arange(len(avw))
    # width = 0.35
    # p1 = plt.bar(ind, avw, width, color='0.5')
    # plt.ylabel('Feature Weights')
    # plt.title('Feature Importance')
    # plt.xticks(ind, names, rotation='vertical')
    # plt.show()
    # plt.close()

def classify(train, test, cl, binary):

    Xtrain = train[[k for k in train.columns.values if k != 'Class']]
    ytrain = train['Class']
    Xtest = test[[k for k in test.columns.values if k != 'Class']]
    ytest = test['Class']

    if not binary:  #normalize data
        Xtrain = (Xtrain - Xtrain.mean())/(Xtrain.max() - Xtrain.min())
        Xtest = (Xtest - Xtest.mean())/(Xtest.max() - Xtest.min())

    clf = RandomForestClassifier(n_estimators=200, criterion='gini',
                                 n_jobs=-1, class_weight='balanced',
                                 max_depth=6)
    clf.fit(Xtrain, ytrain)
    scores = clf.predict_proba(Xtest)
    prediction = clf.predict(Xtest)
    precision = precision_score(ytest, prediction)
    recall = recall_score(ytest, prediction)
    f1 = f1_score(ytest, prediction)

    print('{}\t{}\t{}\t{}'.format(cl, precision, recall, f1))

    posScores = scores[:, 1].reshape(-1, 1).tolist()
    weights = {}
    # for i, feature in enumerate(Xtrain.columns.values):
    #     if feature == 'Correlation':
    #         weights[feature] = clf.coef_[0][i]
    #     else:
    #         weights[feature[:-2]] = clf.coef_[0][i]
    fpr, tpr, _ = roc_curve(ytest, posScores)
    precision, recall, _ = precision_recall_curve(ytest, posScores)
    # print(cl)
    print('AUROC:', auc(fpr, tpr))
    aupr = average_precision_score(ytest, posScores)
    # print(classification_report(ytest, prediction))
    # print()
    # print('Feature weights:  ')
    # for f, w in weights.items():
    #     print(f, w)
    # plotCurves(fpr, tpr, cl)
    return fpr, tpr, precision, recall, aupr

def read21Data(cell):
    # path = '../data/main/unexpandedFeatures/train/{}'.format(cell)
    path = '../data/21/features/train/{}'.format(cell)
    train, test = [], []

    for subdir in os.listdir(path):
        for feature_file in os.listdir(os.path.join(path, subdir)):
            if feature_file.endswith('.features'):
                fname = os.path.join(path, subdir, feature_file)
                train.append(pd.read_csv(fname, index_col=0, header=0, sep='\t'))

    path = '../data/21/features/test/'
    for feature_file in os.listdir(path):
        if cell in feature_file and feature_file.endswith('.features'):
            fname = os.path.join(path, feature_file)
            test.append(pd.read_csv(fname, index_col=0, header=0, sep='\t'))

    train = pd.concat(train)
    test = pd.concat(test)

    ####
    # Add number of other cell lines each boundary is present in feature
    train_tads = train.index
    test_tads = test.index

    with open('../data/21/pickle/pairwise_presence.unexpandedcont.pickle', 'rb') as f:
        df = pickle.load(f)
        df = df.drop(cell, axis=1)

    train_counts = []
    for t in train_tads:
        numTrue = len([k for k in df.loc[t] if k == True])
        if numTrue == 0:
            print('here1')
            exit(1)
        else:
            train_counts.append(numTrue)

    test_counts = []
    for t in test_tads:
        numTrue = len([k for k in df.loc[t] if k == True])
        if numTrue == 0:
            print('here2')
            exit(1)
        else:
            test_counts.append(numTrue)

    train_counts = np.array(train_counts)
    test_counts = np.array(test_counts)
    train['counts'] = (train_counts - min(train_counts)) / (max(train_counts) - min(train_counts))
    test['counts'] = (test_counts - min(test_counts)) / (max(test_counts) - min(test_counts))

    # testing data
    # path = '../data/main/mainFeatures/testData/'
    # for file in [k for k in os.listdir(path) if k.endswith(extension) and cell in k]:
    #     fname = os.path.join(path, file)
    #     test.append(pd.read_csv(fname, index_col=0, header=0, sep='\t'))
    #
    # pospath = '../data/main/mainFeatures/testData/'
    # for file in [k for k in os.listdir(pospath) if k.endswith(extension) and cell in k and 'pos' in k]:
    #     fname = os.path.join(pospath, file)
    #     test.append(pd.read_csv(fname, index_col=0, header=0, sep='\t'))
    #
    # #One end in all tads is fixed with a real boundary
    # negpath = '../data/main/fixedNegativeFeatures'
    # for file in [k for k in os.listdir(negpath) if k.endswith(extension) and cell in k]:
    #     fname = os.path.join(negpath, file)
    #     test.append(pd.read_csv(fname, index_col=0, header=0, sep='\t'))

    return train, test

def main():
    weights = defaultdict(list)
    cells = 'AD AO CO GM12878 H1 HC IMR90 LG LI LV MES MSC NPC OV PA PO RV SB SX TRO'.split()
    for cell in cells:
        train, test = read21Data(cell)
        fpr, tpr, precision, recall, aupr = classify(train, test, cell, binary=BINARY)
        plt.subplot(1, 2, 1)
        plt.plot(fpr, tpr, label='ROC for {}: {}'.format(cell, round(auc(fpr, tpr), 2)))
        plt.subplot(1, 2, 2)
        plt.plot(precision, recall, label='AUPR for {}: {}'.format(cell, round(aupr, 3)))
        # plot ROC curves
    plt.subplot(1, 2, 1)
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
    plt.legend(loc='lower right')
    plt.title('Bag of Boundaries ROCs')
    plt.plot([0, 1], [0, 1], 'k--')
    # plot PR curves
    plt.subplot(1, 2, 2)
    plt.xlabel('Recall')
    plt.ylabel('Precision')
    plt.legend(loc='upper left')
    plt.title('Bag of Boundaries AUPRs')
    plt.show()

main()
# singleCellLine()
# OneVsAll()


# -*- coding: utf-8 -*-
#
#  http://www.privacyidea.org
#  2015-09-28 Initial writeup.
#             Cornelius Kölbel <cornelius@privacyidea.org>
#
# This code is free software; you can redistribute it and/or
# modify it under the terms of the GNU AFFERO GENERAL PUBLIC LICENSE
# License as published by the Free Software Foundation; either
# version 3 of the License, or any later version.
#
# This code is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU AFFERO GENERAL PUBLIC LICENSE for more details.
#
# You should have received a copy of the GNU Affero General Public
# License along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

import re
from OpenSSL import crypto
import binascii
from hashlib import sha256
import base64
import logging
import time
import struct
from cryptography.hazmat.primitives.asymmetric import ec
from cryptography.hazmat.primitives import hashes
from cryptography.exceptions import InvalidSignature

from privacyidea.lib.utils import (to_bytes, to_unicode, hexlify_and_unicode,
                                   urlsafe_b64encode_and_unicode)

__doc__ = """Helper functions for U2F protocol according to
https://fidoalliance.org/specs/fido-u2f-v1.0-nfc-bt-amendment-20150514/fido-u2f-raw-message-formats.html

This file is tested in tests/test_lib_tokens_utf.py
"""

log = logging.getLogger(__name__)

def url_decode(url):
    """
    Decodes a base64 encoded, not padded string as used in FIDO U2F

    :param url: base64 urlsafe encoded string
    :type url: basestring or bytes
    :return: the decoded string
    :rtype: bytes
    """
    # remove all non base64 characters (newline, CR) from the string before
    # calculating the padding length
    pad_len = -len(re.sub('[^A-Za-z0-9-_+/]+', '', to_unicode(url))) % 4

    padding = pad_len * "="
    res = base64.urlsafe_b64decode(to_bytes(url) + to_bytes(padding))
    return res

def url_encode(data):
    """
    Encodes a string base64 websafe and omits trailing padding "=".
    :param data: Some string
    :return: websafe b64 encoded string
    """
    url = urlsafe_b64encode_and_unicode(data)
    return url.strip("=")

def parse_response_data(resp_data):
    """
    According to https://fidoalliance.org/specs/fido-u2f-v1.0-nfc-bt-amendment-20150514/fido-u2f-raw-message-formats.html#authentication-response-message-success
    the response is made up of
    0:      user presence byte
    1-4:    counter
    5-:     signature

    :param resp_data: response data from the FIDO U2F client
    :type resp_data: hex string
    :return: tuple of user_presence_byte(byte), counter(int),
        signature(bytearray)
    """
    resp_data_bin = binascii.unhexlify(resp_data)
    user_presence = bytes((resp_data_bin[0], ))
    signature = resp_data_bin[5:]
    counter = struct.unpack(">L", resp_data_bin[1:5])[0]
    return user_presence, counter, signature

def parse_registration_data(reg_data, verify_cert=True):
    """
    returns the parsed registration data in a tuple
    attestation_cert, user_pub_key, key_handle, signature, description

     * attestation_cert is a x509 object
     * user_pub_key is a hex string
     * key_handle is a hex string
     * signature is a hex string
     * description is a basestring

    see
    https://fidoalliance.org/specs/fido-u2f-v1.0-nfc-bt-amendment
    -20150514/fido-u2f-raw-message-formats.html#registration-messages

    :param reg_data: base64 encoded registration data
    :param verify_cert: whether the attestation certificate should be verified
    :return: tuple
    """
    reg_data_bin = url_decode(reg_data)
    reserved_byte_value = reg_data_bin[0]  # must be 5
    if reserved_byte_value != 5:
        raise Exception("The registration data is in a wrong format. It must "
                        "start with 0x05")
    user_pub_key = reg_data_bin[1:66]
    key_handle_len = reg_data_bin[66]
    # We need to save the key handle
    key_handle = reg_data_bin[67:67+key_handle_len]

    certificate = reg_data_bin[67+key_handle_len:]
    attestation_cert = crypto.load_certificate(crypto.FILETYPE_ASN1,
                                               certificate)
    cert_len = len(crypto.dump_certificate(crypto.FILETYPE_ASN1,
                                           attestation_cert))
    # TODO: Check the issuer of the certificate
    issuer = attestation_cert.get_issuer()
    log.debug("The attestation certificate is signed by {0!r}".format(issuer))
    not_after = to_unicode(attestation_cert.get_notAfter())
    not_before = to_unicode(attestation_cert.get_notBefore())
    log.debug("The attestation certificate "
              "is valid from %s to %s" % (not_before, not_after))
    start_time = time.strptime(not_before, "%Y%m%d%H%M%SZ")
    end_time = time.strptime(not_after, "%Y%m%d%H%M%SZ")
    # check the validity period of the certificate
    if verify_cert:
        if start_time > time.localtime() or \
                        end_time < time.localtime():  #pragma no cover
            log.error("The certificate is not valid. {0!s} -> {1!s}".format(not_before,
                                                                  not_after))
            raise Exception("The time of the attestation certificate is not "
                            "valid.")

    # Get the subject as description
    subj_x509name = attestation_cert.get_subject()
    subj_list = subj_x509name.get_components()
    description = ""
    cdump = to_unicode(crypto.dump_certificate(crypto.FILETYPE_PEM, attestation_cert))
    log.debug("This attestation certificate registered: {0!s}".format(cdump))

    for component in subj_list:
        # each component is a tuple. We are looking for CN
        if component[0].upper() == b"CN":
            description = to_unicode(component[1])
            break

    signature = reg_data_bin[67+key_handle_len+cert_len:]
    return (attestation_cert, hexlify_and_unicode(user_pub_key),
            hexlify_and_unicode(key_handle), hexlify_and_unicode(signature),
            description)

def check_registration_data(attestation_cert, app_id,
                            client_data, user_pub_key,
                            key_handle, signature):
    """
    See example in fido spec
    https://fidoalliance.org/specs/fido-u2f-v1.0-nfc-bt-amendment-20150514/fido-u2f-raw-message-formats.html#registration-example

    In case of signature error an exception is raised

    :param attestation_cert: The Attestation cert of the FIDO device
    :type attestation_cert: x509 Object
    :param app_id: The appId
    :type app_id: str
    :param client_data: The ClientData
    :type client_data: str
    :param user_pub_key: The public key for this AppID
    :type user_pub_key: hex string
    :param key_handle: The keyHandle on the FIDO device
    :type key_handle: hex string
    :param signature: The signature of the registration request
    :type signature: hex string
    :return: Bool
    """
    app_id_hash = sha256(to_bytes(app_id)).digest()
    client_data_hash = sha256(to_bytes(client_data)).digest()
    reg_data = b'\x00' + app_id_hash + client_data_hash \
               + binascii.unhexlify(key_handle) + binascii.unhexlify(user_pub_key)
    try:
        crypto.verify(attestation_cert,
                      binascii.unhexlify(signature),
                      reg_data,
                      "sha256")
    except Exception as exx:
        raise Exception("Error checking the signature of the registration "
                        "data. %s" % exx)
    return True

def check_response(user_pub_key, app_id, client_data, signature,
                   counter, user_presence_byte=b'\x01'):
    """
    Check the ECDSA Signature with the given pubkey.
    The signed data is constructed from
     * app_id
     * user_presence_byte
     * counter and
     * client_data

    :param user_pub_key: The Application specific public key
    :type user_pub_key: hex string
    :param app_id: The AppID for this challenge response
    :type app_id: str
    :param client_data: The ClientData
    :type client_data: str
    :param counter: A counter
    :type counter: int
    :param user_presence_byte: User presence byte
    :type user_presence_byte: bytes
    :param signature: The signature of the authentication request
    :type signature: hex string
    :return:
    """
    res = True
    app_id_hash = sha256(to_bytes(app_id)).digest()
    client_data_hash = sha256(to_bytes(client_data)).digest()
    user_pub_key_bin = binascii.unhexlify(user_pub_key)
    counter_bin = struct.pack(">L", counter)
    signature_bin = binascii.unhexlify(signature)

    input_data = app_id_hash + user_presence_byte + counter_bin + client_data_hash

    try:
        vkey = ec.EllipticCurvePublicKey.from_encoded_point(ec.SECP256R1(),
                                                            user_pub_key_bin)
        vkey.verify(signature_bin, input_data, ec.ECDSA(hashes.SHA256()))
    except (ValueError, TypeError) as e:
        log.error("Could not load application specific public key!")
        log.debug('{0!s}'.format(e))
        res = False
    except InvalidSignature:
        log.error("Bad signature for app_id {0!s}".format(app_id))
        res = False
    return res

def x509name_to_string(x509name):
    """
    converts a X509Name to a string as in a DN

    :param x509name: The X509Name object
    :return:
    """
    components = x509name.get_components()
    return ",".join(["{0}={1}".format(to_unicode(c[0]), to_unicode(c[1])) for c in components])


from django import forms
from django.forms import ModelForm

from countries.models import City, Country

class ChainedCountryForm(ModelForm):
    """
    Ricorda di importare lo script 'js/chained-country.js' in Media
    quando erediti la classe
    """

    class Meta:
        fields = "__all__"

    def __init__(self, *args, **kwargs):
        # print("countries.forms.ChainedCountryForm")
        prefix = kwargs.get('prefix')
        data = kwargs.get('data')
        country_id = None
        # city_id = None
        if prefix and data:
            country_id = data.get(prefix+'-country')
            # city_id = data.get(prefix+'-city')
        super(ChainedCountryForm, self).__init__(*args, **kwargs)

        # se i campi non sono modificabili
        if not self.fields.get("country") and not self.fields.get("city"):
            return

        try:
            self.initial['country'] = kwargs['instance'].country.id
        except:
            # non esiste ancora istanza dell'oggetto
            pass
        country_list = [('', '---------')] + [(i.id, i.name) for i in Country.objects.all()]

        try:
            self.initial['city'] = kwargs['instance'].city.id
            city_init_form = [(i.id, i.name) for i in City.objects.filter(
                country=kwargs['instance'].country
            )]
        except:
            city_init_form = [('', '---------')]
            pass

        # Override the form, add onchange attribute to call the ajax function
        country_attrs = self.fields['city'].widget.attrs
        country_attrs.update(id="id_country")
        country_attrs.update(onchange="getCities(this)")
        self.fields['country'].widget = forms.Select(
            attrs=country_attrs,
            choices=country_list,
        )

        if country_id:
            # print("istanza non esistente")
            city_init_form = [('', '---------')]
            city_init_form_append = [
                (i.id, "%s (%s)" % (i.name, i.sigla_prov)) for i in City.objects.filter(
                    country=Country.objects.get(pk=country_id)).order_by('name')
            ]
            city_init_form = city_init_form + city_init_form_append
        elif self.initial.get('city'): #
            # print("istanza esistente")
            self.initial['city'] = kwargs['instance'].city.id
            city_init_form = [(i.id, "%s (%s)" % (i.name, i.sigla_prov)) for i in City.objects.filter(
                country=kwargs['instance'].country
            )]

        else:
            city_init_form = [('', '---------')]
            city_init_form_append = []
            if self.initial.get('country'):
                city_init_form_append = [
                        (i.id, "%s (%s)"%(i.name,i.sigla_prov)) for i in City.objects.filter(
                        country=kwargs['instance'].country).order_by('name')
                        ]
            city_init_form = city_init_form + city_init_form_append

        city_attrs = self.fields['city'].widget.attrs
        city_attrs.update(id="id_city")
        self.fields['city'].widget = forms.Select(
            attrs=city_attrs,
            choices=city_init_form
        )


# Load and test embedding model
# model_name = 'word2vec-kowiki.model'	# Word2Vec model
# C> wv_test.py 'word2vec-kowiki.model'
import pandas as pd
from pykrx import stock
from gensim.models import Word2Vec
import sys
from datetime import datetime

def search_relation_stock(model,company_name):
    today = datetime.today()
    tickers = stock.get_market_ticker_list(today, market="KOSPI") + stock.get_market_ticker_list(
        today, market="KOSDAQ")
    company_list = []

    for ticker in tickers:
        name = stock.get_market_ticker_name(ticker)
        company_list.append(name)
    stock_df = pd.DataFrame(columns=["company_name","similarity"])
    stock_df["company_name"] = company_list

    for i in stock_df.index:
        try:
            stock_df.iloc[i,1] = model.wv.similarity(u'{0}'.format(company_name), u'{0}'.format(stock_df.iloc[i,0]))
        except:
            stock_df.iloc[i,1] = -99

    print(stock_df.sort_values("similarity",ascending=False).head(5))
if __name__ == "__main__":
    # if len(sys.argv) < 2:
    #     print("C> wv_test.py word2vec.model")
    #     exit()
    # print("\nLoading Korean word embedding vectors for 'KMA tokenized text file'.\n")
    # model_name = sys.argv[1]  # Word2Vec model -- 'word2vec-kowiki.model'
    model_name = "dataFile/word2vec-KCC_news_mecab.model"

    model = Word2Vec.load(model_name)
    search_relation_stock(model,"삼성전자")
    search_relation_stock(model,"카카오")

    # print(model.wv.get_vector(u'배우'))
    # print(model.wv.get_vector(u'여배우'))
    #
    # print(model.wv.similarity(u'LG', u'SK하이닉스'))
    # print(model.wv.similarity(u'카카오', u'카카오뱅크'))
    # print(model.wv.similarity(u'카카오페이', u'카카오뱅크'))
    # print(model.wv.similarity(u'카카오', u'삼성전자'))
    # print(model.wv.similarity(u'현대차', u'기아'))
    # print("한미약품 관련 종목")
    # print(model.wv.most_similar('카카오'))
    # search_relation_stock(model,"반도체")

    # print(model.wv.similarity(u'배우', u'남자'))
    # print(model.wv.similarity(u'남자', u'여배우'))

    # print(model.wv.most_similar(positive=[u'카카오'], topn=3))
    # print(model.wv.most_similar(positive=[u'현대차', u'기아'],  topn=3))
    # print(model.wv.most_similar(positive=[u'현대차', u'기아'], negative=[u'카카오'], topn=3))


"""Module for testing the Categories."""

import os
import unittest

import pytest

from app import create_app, db
from tests.helpers import get_json, user1, admin, cat1, cat2

class CategoryTestCase(unittest.TestCase):
    """Category tests."""

    def setUp(self):
        self.route = '/categories'
        os.environ['ADMIN_EMAIL'] = admin['email']
        os.environ['ADMIN_PASSWORD'] = admin['password']
        self.app = create_app("testing")
        self.client = self.app.test_client()
        with self.app.app_context():
            db.session.close()
            db.drop_all()
            db.create_all()

    def login(self):
        """Login a user for other tests."""
        login_res = self.client.post('/users/login', data=admin)
        self.assertEqual(login_res.status_code, 200)

    def test_create_category(self):
        """Test for category creation."""
        self.login()
        res = self.client.post(self.route, data=cat1)
        result = get_json(res)
        self.assertEqual(res.status_code, 201)
        self.assertEqual(result['name'], 'codility')

    def create_category(self):
        """Create a category for other tests."""
        self.login()
        res = self.client.post(self.route, data=cat1)
        assert res.status_code == 201

    def test_invalid_create_category(self):
        """Check invalid user inputs."""
        self.create_category()
        res = self.client.post(self.route, data=cat1)
        assert res.status_code == 409
        assert get_json(res)['message'] == 'Category already exists.'

    def test_get_categories(self):
        """Test that all categories can be retrieved."""
        self.login()
        add_cat_res = self.client.post(self.route, data=cat2)
        self.assertEqual(add_cat_res.status_code, 201)

        res = self.client.get(self.route)
        result = get_json(res)
        self.assertEqual(res.status_code, 200)
        self.assertEqual(result[0]['name'], 'hackerrank')

    def test_get_category(self):
        """Test that a category can be retrieved."""
        self.create_category()
        res = self.client.get(self.route + '/1')
        assert res.status_code == 200
        assert get_json(res)['name'] == 'codility'

    def test_update_category(self):
        """Test updating a category."""
        self.create_category()
        res = self.client.put(self.route + '/1', data=cat2)
        assert res.status_code == 200
        assert get_json(res)['name'] == 'hackerrank'

    def test_delete_category(self):
        """Test deleting a category."""
        self.create_category()
        res = self.client.delete(self.route + '/1')
        assert res.status_code == 200
        assert get_json(res)['message'] == "Category successfully deleted."

    def test_only_admin_can_create_update_or_delete_category(self):
        """Test that only the Admin can create, update or delete category."""
        res = self.client.post('/users/register', data=user1)
        self.assertEqual(res.status_code, 201)
        login_res = self.client.post('/users/login', data=user1)
        self.assertEqual(login_res.status_code, 200)

        res = self.client.post(self.route, data=cat1)
        assert res.status_code == 403
        assert get_json(res)['message'] == 'Unauthorized.'

        self.create_category()
        self.client.post('/users/logot')
        self.client.post('/users/login', data=user1)

        res = self.client.put(self.route + '/1', data=cat2)
        assert res.status_code == 403

        res = self.client.delete(self.route + '/1')
        assert res.status_code == 403

    def test_invalid_id_for_get_update_and_delete_of_category(self):
        """Return 404 if item does not exist."""
        self.login()
        res = self.client.get(self.route + '/22')
        assert res.status_code == 404
        assert get_json(res)['message'] == 'Category does not exist.'

        res = self.client.put(self.route + '/22', data=cat2)
        assert res.status_code == 404

        res = self.client.delete(self.route + '/22')
        assert res.status_code == 404

    def tearDown(self):
        with self.app.app_context():
            db.session.remove()
            db.drop_all()

if __name__ == "__main__":
    pytest.main()


import numpy as np
import glob, os, argparse
import scipy.misc as smc
from tqdm import tqdm
import matplotlib.pyplot as plt

import config_res as config

total = config.net_params['total_frames']
total_train = config.net_params['total_frames_train']
total_validation = config.net_params['total_frames_validation']
partition_limit = config.net_params['partition_limit']

IMG_HT = config.depth_img_params['IMG_HT']
IMG_WDT = config.depth_img_params['IMG_WDT']
batch_size = config.net_params['batch_size']

dataset = np.loadtxt(config.paths['dataset_path_full'], dtype = str)

dataset_train = dataset[:total_train]
dataset_validation = dataset[total_train:total]

def shuffle():
    np.random.shuffle(dataset_train)
    np.random.shuffle(dataset_validation)

def load(p_no, mode):

    if(mode == "train"):
        dataset_part = dataset_train[p_no*partition_limit:(p_no + 1)*partition_limit]
    elif(mode == "validation"):
        dataset_part = dataset_validation[p_no*partition_limit:(p_no + 1)*partition_limit]
    source_file_names = dataset_part[:,0]
    target_file_names = dataset_part[:,1]
    source_image_names = dataset_part[:,2]
    target_image_names = dataset_part[:,3]
    transforms = np.float32(dataset_part[:,4:])

    target_container = np.zeros((partition_limit, IMG_HT, IMG_WDT, 1), dtype = np.float32)
    source_container = np.zeros((partition_limit, IMG_HT, IMG_WDT, 1), dtype = np.float32)
    source_img_container = np.zeros((partition_limit, IMG_HT, IMG_WDT, 3), dtype = np.float32)
    target_img_container = np.zeros((partition_limit, IMG_HT, IMG_WDT, 3), dtype = np.float32)
    transforms_container = np.zeros((partition_limit, 4, 4), dtype = np.float32)

    c_idx = 0
    for s_name, t_name, img_source_name, img_target_name, transform in tqdm(zip(source_file_names, target_file_names, source_image_names, target_image_names, transforms)):

        warped_ip = np.float32(smc.imread(s_name, True))
        warped_ip[0:5,:] = 0.0 ; warped_ip[:,0:5] = 0.0 ; warped_ip[IMG_HT - 5:,:] = 0.0 ; warped_ip[:,IMG_WDT-5:] = 0.0 ;
        warped_ip = (warped_ip - 40.0)/40.0
        source_container[c_idx, :, :, 0] = warped_ip

        target_ip = np.float32(smc.imread(t_name, True))
        target_ip[0:5,:] = 0.0 ; target_ip[:,0:5] = 0.0 ; target_ip[IMG_HT - 5:,:] = 0.0 ; target_ip[:,IMG_WDT-5:] = 0.0 ;
        target_ip = (target_ip - 40.0)/40.0
        target_container[c_idx, :, :, 0] = target_ip

        source_img = np.float32(smc.imread(img_source_name))
        source_img[0:5,:,:] = 0.0 ; source_img[:,0:5,:] = 0.0 ; source_img[IMG_HT - 5:,:,:] = 0.0 ; source_img[:,IMG_WDT-5:,:] = 0.0 ;
        source_img = (source_img - 127.5)/127.5
        source_img_container[c_idx, :, :, :] = source_img

        target_img = np.float32(smc.imread(img_target_name))
        target_img[0:5,:,:] = 0.0 ; target_img[:,0:5,:] = 0.0 ; target_img[IMG_HT - 5:,:,:] = 0.0 ; target_img[:,IMG_WDT-5:,:] = 0.0 ;
        target_img = (target_img - 127.5)/127.5
        target_img_container[c_idx, :, :, :] = target_img

        transforms_container[c_idx, :, :] = np.linalg.inv(transform.reshape(4,4))
        c_idx+=1

    source_container = source_container.reshape(partition_limit/batch_size, batch_size, IMG_HT, IMG_WDT , 1)
    target_container = target_container.reshape(partition_limit/batch_size, batch_size, IMG_HT, IMG_WDT , 1)
    source_img_container = source_img_container.reshape(partition_limit/batch_size, batch_size, IMG_HT, IMG_WDT, 3)
    target_img_container = target_img_container.reshape(partition_limit/batch_size, batch_size, IMG_HT, IMG_WDT, 3)
    transforms_container = transforms_container.reshape(partition_limit/batch_size, batch_size, 4, 4)

    return source_container, target_container, source_img_container, target_img_container, transforms_container


# Assume you have access to a function toss_biased() which returns 0 or 1 with
# a probability that's not 50-50 (but also not 0-100 or 100-0). You do not know
# the bias of the coin.

# Write a function to simulate an unbiased coin toss.

from random import random

BIAS = 0.66

def toss_biased():
    return random() > BIAS

# Say, the probability of getting heads, H, is P. Thus we can deduce that tails, T,
# would be equal to 1 - P. We can compute two consectuive coin flips as follows:

# HH: P * P
# HT: P * (1 - P)
# TH: (1 - P) * P
# TT: (1 - P) * (1 - P)

# Since multiplication is commutative, we find that flipping heads and then tails
# has the same probability of flipping tails, then heads

def toss_fair():
    t1, t2 = toss_biased(), toss_biased()

    if t1 and not t2:
        return True
    elif not t1 and t2:
        return False
    else:
        return toss_fair()

# Test to prove our probabilites match close to 50/50
from collections import defaultdict

c = defaultdict(int)
for i in range(10000):
    c[toss_fair()] += 1

print(c)


"""Frequently used data.

Data:
    driver_settings: settings of selenium chromedriver.

    prefs: preferences to set accepting languages, disable images and
    javascript for faster loading.

    urls: links to pages.

    xpaths: groupped xpaths to allow scrapping elements from the webpages.

    Typical usage examples:
    >>> self.driver.get(urls['login'])
"""
from typing import Any, Dict, Tuple

from selenium.webdriver.common.keys import Keys

driver_settings: Tuple[str, ...] = (
    '--profile-directory=Default',
    '--disable-plugins-discovery',
    '--start-maximized',
    '--log-level=3',
    'user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) \
    AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.137 Safari/537h.36',
)

prefs: Dict[str, Any] = {
    'profile.default_content_setting_values': {'images': 2},
    'intl.accept_languages': 'pl-PL',
    'profile.managed_default_content_settings.javascript': 2,
}

urls: Dict[str, str] = {
    'login': 'https://example.com/pages/enter.php',
    'ptc': 'https://example.com/pages/ptcontest.php?startpos=',
    'ptc_paid': 'https://example.com/pages/ptc.php?startpos=',
    'inbox': 'https://example.com/pages/inbox.php',
    'starting_page': 'https://example.com/pages/startowa.php',
    'ublock_settings': 'chrome-extension://cjpalhdlnbpafiamejdnhcphjbkeiagm/1p-filters.html',
    'earnings': 'https://example.com/pages/earnings.php',
}

xpaths: Dict[str, Dict[str, str]] = {
    'login': {
        'username_input': 'html/body/div[@class="container"]/div[@class="content"]\
            /form[@class="form-login"]/table/tbody/tr/td/input[@name="username"]',
        'password_input': 'html/body/div[@class="container"]/div[@class="content"]\
            /form[@class="form-login"]/table/tbody/tr/td/input[@name="password"]',
        'login_button': 'html/body/div[@class="container"]/div[@class="content"]\
            /form[@class="form-login"]/table/tbody/tr/td/input[@name="submit"]',
    },
    'enter': {
        "username": 'html/body/div[@class="green-box"]/div[@class="container-top"]\
            /div[@class="user"]/a[@href="https://example.com/pages/profil.php"]/span'
    },
    'ptc': {
        'anchor': 'html/body/div[@class="container"]/div[@class="content"]/div[@class="box"]\
            /div[@class="box-content-row"]/div[@class="title" and not(center)]\
                /a[@target="_ptc" and not(img)]',
        'anchor_alt': 'html/body/div[@class="container"]/div[@class="content"]/div[@class="box"]\
            /div[@class="box-content-row"]/div[@class="title"]/a[@target="_ptc" and img]',
    },
    'inbox': {
        'anchor': 'html/body/div[@class="container"]/div/div[@class="content-width referers"]\
            /div[@class="inbox_table"]/form/table/tbody/tr/td[2]/a',
    },
    'starting_page': {
        'user_starting_page': 'html/body/div[@class="container"]/div[@class="content"]/p/b',
    },
    'ublock_settings': {
        'save_button': '//*[@id="userFiltersApply"]',
    },
    'earnings': {
        'money': 'html/body/div[@class="container"]/div[@class="content"]/div[@class="inbox_table"]/table/tbody/tr/td/b',
        'points': 'html/body/div[@class="container"]/div[@class="content"]/div[@class="inbox_table"]/table/tbody/tr[4]/td[@class="wages_point"]',
    },
}

ANTICHEAT_WORDS = (
    'sprawdza',
    'uwaga',
    'weryfik',
    'zatwier',
    'potwier',
)

PARTIAL_INBOX_LINK = 'http://example.com/scripts/runner.php?EA'
UBLOCK_SEND_KEYS = '##iframe{0}*.css{0}*.ico'.format(Keys.ENTER)


from ....common.game_objects.tokens.token import Token
from src.client.utils.image_transform import crop_to
import cv2
import os
import numpy
from src.client.utils.config import config
from src.client.utils.debug import debugger

CASCADE_PATH = os.path.join('src', 'client', 'utils', 'detection', 'model', 'token_cascade.xml')

def centre_coordinate(anchor, size):
    (x, y) = anchor
    (w, h) = size
    return x + (w / 2), y + (h / 2)

def detect_color(piece_image):
    if piece_image is not None:
        # Filter background
        threshold = config.get_property(['client', 'color', 'token_background_threshold'])
        black_pixel = numpy.array([0, 0, 0])
        threshold_filter = numpy.array([threshold, threshold, threshold])
        mask = cv2.inRange(piece_image, black_pixel, threshold_filter)
        threshold_image = cv2.bitwise_and(piece_image, piece_image, mask=mask)

        # Calculate average color and filter out black pixels
        r, g, b, count = 0, 0, 0, 0
        for row in threshold_image:
            for pixel in row:
                if not numpy.array_equal(pixel, black_pixel):
                    b += pixel[0]
                    g += pixel[1]
                    r += pixel[2]
                    count += 1
        if count > 0:
            return int(b / count), int(g / count), int(r / count)  # bgr color
        else:
            return None
    return None

def classify_color(bgr_color):
    colors = config.get_property(['client', 'color', 'classes'])
    proximity = dict()
    for color in colors:
        color_class_bgr = colors.get(color)
        color_class_vector = numpy.array([color_class_bgr.get('b'), color_class_bgr.get('g'), color_class_bgr.get('r')])
        distance = numpy.linalg.norm(numpy.array(bgr_color) - color_class_vector)
        proximity[distance] = color
    nearst_color = proximity.get(sorted(proximity.keys())[0])
    return nearst_color

class TokenDetection:
    def __init__(self, board_dimensions):
        self.board_dimensions = board_dimensions
        self.detected_pieces = []
        self.cascade = cv2.CascadeClassifier(CASCADE_PATH)

    def find_pieces(self, board_image, board_anchor):
        [debugger.update_variables(f'detected_{color}', '') for color in config.get_property(['client', 'color', 'classes'])]
        if board_image is not None:
            identified_pieces_rects = self.cascade.detectMultiScale(
                board_image
            )
            offset_x, offset_y = board_anchor
            for rectangle in identified_pieces_rects:
                (x, y) = (rectangle[0], rectangle[1])
                (dx, dy) = (rectangle[2], rectangle[3])
                piece_image = crop_to(board_image, (x, y), (x + dx, y + dy))
                color_bgr = detect_color(piece_image)

                if color_bgr:
                    # Known color class for a token
                    color_class = classify_color(color_bgr)
                    classified_token = False

                    token_anchor_coordinate = (x + offset_x, y + offset_y)
                    token_coordinate = centre_coordinate(token_anchor_coordinate, (dx, dy))

                    for token in self.detected_pieces:
                        if token.get_color() == color_class:
                            token.update_coordinate(token_coordinate)
                            debugger.update_annotation(color_class, token_anchor_coordinate, (dx, dy))
                            classified_token = True
                            # TODO work to include if it is nearby, instead of just the color
                    if not classified_token:
                        # new Token
                        token = Token(color_class, token_coordinate)
                        debugger.update_annotation(color_class, token_anchor_coordinate, (dx, dy))
                        self.detected_pieces.append(token)
                else:
                    # No color detected, or image all black
                    pass

            # Handle missing tokens (one no longer on the board)
            for t in self.detected_pieces:
                t.update_counter()
                if t.get_last_update() > config.get_property(['client', 'token_update_timeout']):
                    # token gone missing
                    debugger.remove_annotation(t.get_color())
                    debugger.remove_annotation(f'{t.get_color()}_coord')
                    self.detected_pieces.remove(t)

            return self.detected_pieces
        return None