'''
script for dealing with word embeddings
'''

import torch
from torch import nn
import numpy as np
from tqdm import tqdm

'''
initialize embedding tensor with values from the uniform distribution

input param:
    input_embedding: embedding tensor
'''
def init_embedding(input_embedding):
    bias = np.sqrt(3.0 / input_embedding.size(1))
    nn.init.uniform_(input_embedding, -bias, bias)

'''
load pre-trained word embeddings (Glove) for words in the word map

input params:
    emb_file: path to the file containing embeddings (stored in Glove format)
    word_map: word map

return:
    embeddings in the same order as the words in the word map, dimension of embeddings
'''
def load_embeddings(emb_file, word_map):

    # find embedding dimension
    with open(emb_file, 'r') as f:
        emb_size = len(f.readline().split(' ')) - 1
        num_lines = len(f.readlines())

    vocab = set(word_map.keys())

    # create tensor to hold embeddings, initialize
    embeddings = torch.FloatTensor(len(vocab), emb_size)
    init_embedding(embeddings)

    # read embedding file
    for line in tqdm(open(emb_file, 'r'), total = num_lines, desc = 'Loading embeddings'):
        line = line.split(' ')

        emb_word = line[0]
        embedding = list(map(lambda t: float(t), filter(lambda n: n and not n.isspace(), line[1:])))

        # ignore word if not in train_vocab
        if emb_word not in vocab:
            continue

        embeddings[word_map[emb_word]] = torch.FloatTensor(embedding)

    return embeddings, emb_size


"""
FA 16: Lab 14 OOP (Barista lab)
Main program
    - creates a Barista
    - Barista takes orders
    - when the barista is done taking orders, make the drinks
    - Continue taking orders until user chooses to exit
"""
from Barista import Barista

def main():
    name = input("Please enter your name:\n> ")
    barista = Barista(name)
    print("\n~Welcome to the Coffee Shop!~\n")

    cont = True
    while cont:
        # Take 5 orders
        while barista.isAcceptingOrders():
            barista.takeOrder()
        # Make drinks
        barista.makeDrinks()

        # Ask to continue
        response = input("Would you like to continue? (y/n)\n> ")
        while response.lower() != "y" and response.lower() != "n":
            print("*Invalid input, please try again!")
            response = input("Would you like to continue? (y/n)\n> ")
        cont = (response.lower() == "y")

    print("Goodbye, please come again!")

main()


# -*- coding: utf-8 -*-
from __future__ import unicode_literals
import json
import os
import sys
from django.http import HttpResponse
from django.shortcuts import render
import base64
from urllib import parse
# Create your views here.
from django.shortcuts import render, redirect
#from models import InterationRe,IndirectRe,Node,UserInfo

sys.path.append(os.path.dirname(os.path.abspath(__file__)))

print("********************", os.getcwd())
from agraph.server import agraph, genRelations,uri2str,uri2name
from datamap.tree import *
from datamap.entry import Execute
from pythonfun import fundef,Cls,ClsNameSpace,ModuleLoad
default_scope_func = {'fundef':fundef,'Cls':Cls, 'ClsNameSpace':ClsNameSpace,'ModuleLoad':ModuleLoad}

import logging
import logging.config
import time

log_filename = "logging.log"
logging.basicConfig(level=logging.DEBUG,
    format='[%(asctime)s] %(levelname)s [%(funcName)s: %(filename)s, %(lineno)d] %(message)s',
    datefmt='%Y-%m-%d %H:%M:%S',
    filemode='a',
	filename = log_filename)

def auth(func):
	def inner(request):
		if request.user.is_authenticated:
			userinfo = {
				'username':request.user.username
			}
			return func(request, userinfo)
		else:
			return  render(request, 'registration/login.html', context={'next': request.path})
	return inner
@auth
def index(request, userinfo):
	return render(request, 'jsmind/index.html', context = {'userinfo':userinfo})

str2cmd = lambda strs:'for expr in [%s]:exec(expr)'%strs

	#return {param['name']:param['root'],param['children']:[roots.append(parent) or insertChild(parent)  for parent in parents if not parent in childrens and not parent in roots]}
	#insertChild = lambda s: not s in parents and  {param['name']:s} or {param['name']:s, param['children']:[pairs.add(parents[i], childrens[i]) or insertChild(childrens[i])  for i in range(0,len(parents)) if parents[i] == s and not (parents[i], childrens[i]) in pairs]}
	#return {param['name']:param['root'],param['children']:[roots.append(parent) or insertChild(parent)  for parent in parents if not parent in childrens and not parent in roots]}

@auth
def repoClass(request, userinfo):
	catalog = request.GET.get('catalog', "")
	repo = request.GET.get('repo',"owl")
	lang = request.GET.get('lang', "zh")
	relation = request.GET.get('rel', "rdfs:subClassOf")
	cmdlist = ['query = """SELECT ?s ?o ?sName ?oName WHERE { \
				?s  rdfs:subClassOf  ?o; \
					rdfs:label ?sName FILTER (lang(?sName) = "zh"). \
					?o rdfs:label ?oName FILTER (lang(?oName) = "zh")"""' , 'agraph.repoOpen(catalog, repo)',
					'elementmap = lambda elem : (uri2str(elem, "sName"), uri2str(elem, "oName"),uri2name(elem,"s"), uri2name(elem,"o"))',
					'elementgen = lambda elem : elem[2][0] and elem[3][0] and parents.append(elem[3][0] + \':"%s"\'%elem[1]) or childrens.append(elem[2][0] + \':"%s"\'%elem[0])',
					'parents ,childrens,roots = ([],[],[])',
					'genRelations(elementmap, elementgen, query_lang)',
					'len(parents) or genRelations(elementmap, elementgen, query)',
					'data = pairs2Tree(parents, childrens)',
					'viewstyle = "height:100%"']
	#print [catalog, repo, lang, relation]
	#agraph.repoOpen(catalog, repo)

	#query_lang = """
	#	SELECT ?s ?o ?sName ?oName WHERE {
	#		?s  %s  ?o;
	#			rdfs:label ?sName FILTER (lang(?sName) = "%s").
	#		?o rdfs:label ?oName FILTER (lang(?oName) = "%s")
	#	}"""%(relation, lang, lang)
	#
	#query ="""
	#	SELECT ?s ?o ?sName ?oName WHERE {
	#		?s  %s  ?o;
	#			rdfs:label ?sName.
	#		?o rdfs:label ?oName
	#	}"""%relation

	elementmap = lambda elem : (uri2str(elem, "sName"), uri2str(elem, "oName"),uri2name(elem,"s"), uri2name(elem,"o"))
	elementgen = lambda elem : elem[2][0] and elem[3][0] and parents.append(elem[3][0] + ':"%s"'%elem[1]) or childrens.append(elem[2][0] + ':"%s"'%elem[0])
	parents ,childrens,roots = ([],[],[])
	genRelations(elementmap, elementgen, query_lang)
	len(parents) or genRelations(elementmap, elementgen, query)

	data = pairs2Tree(parents, childrens)
	logging.info(data)
	viewstyle = "height:100%"

	return

@auth
def repoConn(request, userinfo):

	isjson = request.GET.get('isjson', False)
	catalog = request.GET.get('catalog', "")
	repo = request.GET.get('repo','sys')
	query = request.GET.get('query',"""
							SELECT ?s ?p ?o {
								?s  ?p  ?o;}
							""")
	print [catalog, repo, query]
	agraph.repoOpen(catalog, repo)

	parents ,childrens= ([],[])
	elementmap = lambda elem : (uri2name(elem,"s"), uri2name(elem,"o"))
	elementgen = lambda elem : elem[0][1] and elem[1][1] and (parents.append(elem[0]) or childrens.append(elem[1]))
	genRelations(elementmap, elementgen, query)
	logging.info(parents)
	logging.info(childrens)
	data = pairs2TreeWithCatalog(parents, childrens)
	if isjson:
		return HttpResponse(json.dumps(data), content_type="application/json")
	else:
		viewstyle = "height:100%"
		return render(request, 'jsmind/grapdep.html', context = {'viewstyle':viewstyle, 'data':json.dumps(data), 'userinfo':userinfo})

@auth
def viz(request, userinfo):
	viewstyle = "height:100%"
	isjson = request.GET.get('isjson', False)
	catalog = request.GET.get('catalog', "")
	repo = request.GET.get('repo','sys')
	query = request.GET.get('query',"""
							SELECT ?s ?p ?o {
								?s  ?p  ?o;}
							""")
	print [catalog, repo, query]
	agraph.repoOpen(catalog, repo)

	parents ,childrens= ([],[])
	elementmap = lambda elem : (uri2name(elem,"s"), uri2name(elem,"o"))
	elementgen = lambda elem : elem[0][1] and elem[1][1] and (parents.append(elem[0]) or childrens.append(elem[1]))
	genRelations(elementmap, elementgen, query)
	logging.info(parents)
	logging.info(childrens)
	data = treeWithCatalog2dot(parents, childrens)

	if isjson:
		return HttpResponse(data, content_type="application/json")
	else:
		viewstyle = "height:100%"
		return render(request, 'jsmind/viz.html', context = {'viewstyle':viewstyle, 'data':data, 'userinfo':userinfo})
@auth
def showclass(request, userinfo):
	return render(request, 'jsmind/showclass/showobj.html')

@auth
def cal(request, userinfo):
	exe = Execute([('Public', 'D:/pythonLib/public.owl'),('Module','D:/pythonLib/module.owl')])
	cal = request.POST.getlist('cals[]',[])
	scope = {'request':request, 'userinfo':userinfo, 'user':request.user}
	scope.update(default_scope_func)

	if not len(cal):
		cal_bs64 = request.POST.get('cals',request.GET.get('cals',""))
		cal_str = str(base64.b64decode(cal_bs64),encoding = "utf-8")
		print(cal_str)
		cal = json.loads(cal_str)
	print('................',cal,'................')
	exe.execute("\n".join(cal), scope)
	print(scope['data'])
	if 'data_type' in scope:
		print(scope['data_type'])
		return HttpResponse(scope['data'], content_type=scope['data_type'])#"application/json"
	else:
		return scope['data']

@auth
def call(request, userinfo):
	scope = {'request':request, 'userinfo':userinfo, 'user':request.user}
	scope.update(default_scope_func)

	path = parse.unquote(request.POST.get('path',request.GET.get('path',"")))
	args_base64 = parse.unquote(request.POST.get('args',request.GET.get('args',"")))
	exe = Execute([('Public', 'D:/pythonLib/public.owl'),('Module','D:/pythonLib/module.owl'),('callmodule', path)])
	args = str(base64.b64decode(args_base64),encoding = "utf-8")
	print('---------------', path,args,'---------------')
	exe.execute('self.ontos["callmodule"].'+args, scope)
	print(scope['data'])
	if 'data_type' in scope:
		print(scope['data_type'])
		return HttpResponse(scope['data'], content_type=scope['data_type'])#"application/json"
	else:
		return scope['data']

@auth
def edit(request, userinfo):

	catalog = request.GET.get('catalog', "")
	repo = request.GET.get('repo',"owl")
	lang = request.GET.get('lang', "zh")
	relation = request.GET.get('rel', "?p")

	agraph.repoOpen(catalog, repo)
	query_lang = """
	SELECT ?s ?o ?sName ?oName WHERE {
		?s  %s  ?o;
			rdfs:label ?sName FILTER (lang(?sName) = "%s").
		?o rdfs:label ?oName FILTER (lang(?oName) = "%s")
	}"""%(relation, lang, lang)

	#query ="""
	#	SELECT ?s ?o ?sName ?oName WHERE {
	#		?s  %s  ?o;
	#			rdfs:label ?sName.
	#		?o rdfs:label ?oName
	#	}"""%relation

	query ="""
		SELECT ?s ?o  WHERE {
			?s  %s  ?o;
		}"""%relation

	elementmap = lambda elem : (uri2name(elem,"s"), uri2name(elem,"o"))
	elementgen = lambda elem : elem[0][1] and elem[1][1] and parents.append(elem[0][0] + ':"%s"'%elem[0][1]) or childrens.append(elem[1][0] + ':"%s"'%elem[1][1])

	parents ,childrens,roots = ([],[],[])
	#genRelations(elementmap, elementgen, query_lang)
	len(parents) or genRelations(elementmap, elementgen, query)

	logging.info(parents)
	logging.info(childrens)
	print("get data",parents)

	data = pairs2Tree(parents, childrens, name="topic")

	temp = {}
	#extendChild = lambda data, i :  temp.update(data['children'][i]) or data['children'].__setitem__(i, {}) or data['children'][i].update(temp)
	data = json.loads(json.dumps(data)) #to split the same node to more.

	updateid = lambda data,id : data.update({'id':"%s%s"%(data['topic'],id)}) or (data.has_key('children') and [updateid(data['children'][i], data['id']) for i in xrange(len(data['children']))])
	updateid(data,'0')

	#logging.info(data)
	return render(request, 'jsmind/jsmind.html', context = {'userinfo':userinfo,'data':json.dumps(data)})

@auth
def getcenter(request, auth):
	if request.is_ajax():
		rsp = {'errorcode': 100, 'detail': 'Get success'}
		id = request.POST.get('type_id')
		return HttpResponse(json.dumps(rsp), content_type="application/json")
	else:
		return HttpResponse("error")

#create a relationship
@auth
def rel_create(request, userinfo):
	if request.is_ajax():
		rsp = {'errorcode': 100, 'detail': 'Get success'}
		request.POST.get('start')
		request.POST.get('end')

		return HttpResponse(json.dumps(rsp), content_type="application/json")
	else:
		return HttpResponse("error")

#delete a relationship  should build a forget mechanism don't delte the data immedaitally
@auth
def rel_del(request, userinfo):
	if request.is_ajax():
		rsp = {'errorcode': 100, 'detail': 'Get success'}
		id = request.POST.get('type_id')
		return HttpResponse(json.dumps(rsp), content_type="application/json")
	else:
		return HttpResponse("error")

@auth
def node_create(request, userinfo):
	if request.is_ajax():
		rsp = {'errorcode': 100, 'detail': 'Get success'}
		id = request.POST.get('type_id')
		return HttpResponse(json.dumps(rsp), content_type="application/json")
	else:
		return HttpResponse("error")

@auth
def node_delete(request, userinfo):
	if request.is_ajax():
		rsp = {'errorcode': 100, 'detail': 'Get success'}
		id = request.POST.get('type_id')
		return HttpResponse(json.dumps(rsp), content_type="application/json")
	else:
		return HttpResponse("error")


#!/usr/bin/env python
# vim: ai ts=4 sts=4 et sw=4

##
##
## @author UWANTWALI ZIGAMA Didier
## d.zigama@pivotaccess.com/zigdidier@gmail.com
##

__author__="Zigama Didier"
__date__ ="$Nov 22, 2017 1:29:30 PM$"

from controller.main import RSMSRWController
from model.privilege import Privilege
from model.role import Role
from util.mch_util import average, makedict, makecol

class PrivilegeController(RSMSRWController):

    def __init__(self, navb):
        self.navb = navb

    def get_total(self):
        cnds    = self.navb.conditions()
        exts = {}
        cols = ['COUNT(*) AS total']
        total = Privilege.get_privileges_summary(cnds, cols)[0]
        self.navb.kw = {}
        return total

    def get_stats(self):
        cnds    = self.navb.conditions()
        cols    = ['COUNT(*) AS total']
        exts    = {}
        attrs   = []
        nat     = Privilege.get_privileges_summary(cnds, cols)
        roles   = Role.get_roles()
        privileges = Privilege.get_privileges()
        self.navb.kw = {}
        return [roles, privileges, nat, attrs]

    def get_tables(self):
        cnds    = self.navb.conditions()#;print cnds
        if self.navb.kw.get("search") and self.navb.kw.get("identity"):
            mkw     = self.navb.kw.get('identity').strip()
            if self.navb.kw.get("search") == 'code': cnds.update({"code = %s": mkw })
        exts = {}
        attrs = []
        cnds, markup, cols = self.navb.neater_tables(cnds = cnds, extras = [
                                              ('indexcol', 'ID'),
                                              ('code',              'Code'),
                                              ('name', 'Name'),

                                            ])

        markup.update({'indexcol': lambda x, _, __: 'View' % (x), })

        DESCRI = []
        INDICS = []
        group = "Privilege"
        title = "Privilege List"

        dcols = [x[0] for x in cols]
        nat = Privilege.fetch_privileges_table(cnds, dcols)
        desc  = 'Privileges%s' % (' (%s)' % (self.navb.find_descr(DESCRI + INDICS ,
                                                                        self.navb.kw.get('privilege')
                                                                    )
					                            )
                                    )
        #print INDICS, title, group, attrs, "NAT: ", nat[0].__dict__
        self.navb.kw = {}
        return (title, desc, group, attrs, markup, cols, nat)

    def register_privileges(self):
        cnds    = self.navb.conditions()
        message = ''
        privilege = None
        privileges = self.navb.kw.get("privileges")
        if type(privileges) in [str, unicode]: privileges = [privileges]
        assigned_privileges = []
        for p in privileges:
            if self.navb.kw.get("role") or self.navb.kw.get('pk'):
                formdata = {
                                "privilege_pk": int(p),
                                "user_pk": int(self.navb.kw.get('pk')) if self.navb.kw.get('pk') else None,
                                "role_pk": int(self.navb.kw.get('role')) if self.navb.kw.get('role') else None,
                            }

                #print "\nFORM: ", formdata, "\n"
                message, privilege = Privilege.get_or_assign(formdata)
                assigned_privileges.append(privilege)

        return [message, assigned_privileges]


"""

s = UserSession(ub)
s.bind_to_env(env)

u = s.user()
s.setuser(u)

"""
from werkzeug.utils import dump_cookie, parse_cookie
import random
import string
import datetime

class UserSessionMiddleware(object):
    def __init__(self, app, userbase,
                 cookie_name='user_session',
                 cookie_age=None,
                 cookie_expires=None,
                 cookie_path='/',
                 cookie_domain=None,
                 cookie_secure=None,
                 cookie_httponly=False,
                 environ_key='userbase'):

        self.app = app
        self.userbase = userbase
        self.cookie_name = cookie_name
        self.cookie_age = cookie_age
        self.cookie_expires = cookie_expires
        self.cookie_path = cookie_path
        self.cookie_domain = cookie_domain
        self.cookie_secure = cookie_secure
        self.cookie_httponly = cookie_httponly
        self.environ_key = environ_key

    def __call__(self, environ, start_response):

        cookie = parse_cookie(environ)
        user_cookie = cookie.get(self.cookie_name, None)

        usersession = UserSession(self.userbase)
        usersession.set_cookie_value(user_cookie)

        environ[self.environ_key + ".session"] = usersession
        environ[self.environ_key + ".userbase"] = self.userbase

        def injecting_start_response(status, headers, exc_info=None):

            cookie_value = usersession.get_cookie_value()

            cookie_str = dump_cookie(
                self.cookie_name,
                cookie_value if cookie_value is not None else "",
                self.cookie_age,
                None if cookie_value is not None else datetime.datetime.now() - datetime.timedelta(days=1),
                self.cookie_path,
                self.cookie_domain,
                self.cookie_secure,
                self.cookie_httponly)

            headers.append(('Set-Cookie', cookie_str))

            self.userbase.store_changes()

            return start_response(status, headers, exc_info)

        return self.app(environ, injecting_start_response)

class UserSession(object):

    def __init__(self, userbase):
        self.userbase = userbase
        self._user = None
        self._cookie = None

    def set_cookie_value(self, cookie):
        self._cookie = cookie

    def get_cookie_value(self):
        return self._cookie

    def user(self):

        if self._user is None:
            self.__find_user()

        return self._user

    def setuser(self, user):
        self._user = user
        self.update_user_key()

    def clear(self):
        self._cookie = None
        self._user = None

    def update_user_key(self):
        key = self.__generate_key()
        self._user.setfield('sessionkey', key)
        self.__format_cookie(self._user.uid(), key)

    def __find_user(self):
        uid, key = self.__parse_cookie()

        if uid is None:
            self._user = None
            return

        user = self.userbase.getuser(uid)

        if user is None:
            return

        user_key = user.field('sessionkey')

        if user_key == key:
            self._user = user

            #print "Changing user key"
            #self.update_user_key()

        else:
            self._user = None

    def __parse_cookie(self):
        if self._cookie is None:
            return (None, None)

        parts = self._cookie.rsplit(":",1)

        if len(parts) == 2:
            uid = parts[0]
            key = parts[1]
            return uid, key

        else:
            return (None, None)

    def __format_cookie(self, uid, key):
        self._cookie = "%s:%s" % (uid, key)

    def __generate_key(self):
        pool = string.letters + string.digits
        key = ''.join(random.choice(pool) for i in xrange(40))
        return key


#!/usr/bin/env python

import calnet.fit_calnet_ori_with_nonlinear_bidi_optogenetics as fcowo
import sys
import pyute as ut

calnet_data_fold = '/Users/dan/Documents/notebooks/mossing-PC/shared_data/calnet_data/'

#ca_data_file = calnet_data_fold+'rs_vm_denoise_200605.npy'
ca_data_file = calnet_data_fold+'rs_200828.npy'
opto_silencing_data_file = calnet_data_fold+'vip_halo_data_for_sim.npy'
opto_activation_data_file = calnet_data_fold+'vip_chrimson_data_for_sim.npy'

weights_base = '/Users/dan/Documents/notebooks/mossing-PC/shared_data/calnet_data/weights/weights_200721e/'
good_fits = ['006','010','012','014','018','019','020','021','023','025','030',\
             '035','042','045','049','058','060','064','066','076','078','083']
weights_files = [weights_base+fit+'.npy' for fit in good_fits]

weights_base = '/Users/dan/Documents/notebooks/mossing-PC/shared_data/calnet_data/weights/weights_200722a/'
good_fits = ['003','012','014','025','028','029','031','048','059','060',\
             '066','072','079','081','088','089','090','094']
weights_files = weights_files + [weights_base+fit+'.npy' for fit in good_fits]

weights_base = '/Users/dan/Documents/notebooks/mossing-PC/shared_data/calnet_data/weights/weights_200723a/'
good_fits = ['000','001','002','004','011','025','026','027','029','032',\
             '043','045','064','065','069','070','072','076','080','086',\
            '087','089','093','096']
weights_files = weights_files + [weights_base+fit+'.npy' for fit in good_fits]

weights_base = '/Users/dan/Documents/notebooks/mossing-PC/shared_data/calnet_data/weights/weights_200724a/'
good_fits = ['000','003','004','017','018','028','029','034','037','044',\
             '046','050','054','058','060','062','066','069','072','079',\
            '086','087','091','092','097']
weights_files = weights_files + [weights_base+fit+'.npy' for fit in good_fits]

weights_base = '/Users/dan/Documents/notebooks/mossing-PC/shared_data/calnet_data/weights/weights_200725a/'
good_fits = ['002','003','005','010','015','034','042','070','072','073',\
             '074','076','079','080','085','087','089','092','094','102',\
            '103','105','118','120','123','132','134','137','138','141',\
            '143','145','147','161','164','166','171','174','180','182',\
            '185','192','193','199']
weights_files = weights_files + [weights_base+fit+'.npy' for fit in good_fits]

weights_files_base = '/Users/dan/Documents/notebooks/mossing-PC/shared_data/calnet_data/weights/'
weights_files = [weights_files_base+'weights_200901a/%03d.npy'%ifile for ifile in range(67)]

init_files = weights_files
ntries = len(init_files)

if __name__=="__main__":
    weight_base = sys.argv[1]

    ut.mkdir(calnet_data_fold+'weights/'+weight_base)

    weights_files = [calnet_data_fold+'weights/'+weight_base+'/%03d.npy'%itry for itry in range(ntries)]

    for itry in range(ntries):
        fcowo.fit_weights_and_save(weights_files[itry],ca_data_file=ca_data_file,opto_silencing_data_file=opto_silencing_data_file,opto_activation_data_file=opto_activation_data_file,allow_var=False,fit_s02=True,constrain_isn=True,l2_penalty=0.1,init_noise=0.1,init_W_from_lsq=True,scale_init_by=1,init_W_from_file=True,init_file=init_files[itry],correct_Eta=True)


from base_form import BaseForm
from widgets import TextField
from cairo import ImageSurface, Context, FontOptions, FORMAT_ARGB32
import math
import datetime
from io import BytesIO

class Page5_1(BaseForm):
    ACTION = "page5_1"
    VIEW = "page5_1.tpl"

    def create_widgets(self):
        panels_data = []
        for row in self.db.select("select ID, NAME, HEIGHT from web_stat_panels order by ID"):
            panels_data += [self._create_panel(str(row[0]), str(row[1], "utf-8"), str(row[2]))]

        tf = TextField("STATISTIC_PANELS", "".join(panels_data))
        self.add_widget(tf)

        tf = TextField("START_TIME", str(round(datetime.datetime.now().timestamp())))
        self.add_widget(tf)

        """
        try:
            self.copy_var_changes_to_mem()
        except Exception as e:
            print(e)
        """

    def _create_panel(self, key, name, height):
        lb1, lb2, lb3, lb4 = ["-- нет --"] * 4
        for row in self.db.select("select v.ID, CONCAT(v.COMM, ' [', v.NAME, ']'), p.SERIES_1, p.SERIES_2, p.SERIES_3, p.SERIES_4 "
                                  "  from web_stat_panels p, core_variables v "
                                  " where (p.SERIES_1 = v.ID "
                                  "     or p.SERIES_2 = v.ID "
                                  "     or p.SERIES_3 = v.ID "
                                  "     or p.SERIES_4 = v.ID)"
                                  "    and p.ID = %s" % (key)):
            sers = row[2:6]
            lab = str(row[1], "utf-8")
            if row[0] == sers[0]:
                lb1 = lab
            elif row[0] == sers[1]:
                lb2 = lab
            elif row[0] == sers[2]:
                lb3 = lab
            elif row[0] == sers[3]:
                lb4 = lab

        f = open("views/stat_panel.tpl", "r")
        tmpl = f.read()
        f.close()
        tmpl = tmpl.replace("@ID@", key)
        tmpl = tmpl.replace("@LABEL@", name)
        tmpl = tmpl.replace("@HEIGHT@", height)
        tmpl = tmpl.replace("@LABEL_1@", lb1)
        tmpl = tmpl.replace("@LABEL_2@", lb2)
        tmpl = tmpl.replace("@LABEL_3@", lb3)
        tmpl = tmpl.replace("@LABEL_4@", lb4)
        return tmpl

    def query(self, query_type):
        if query_type == "append_panel":
            self.db.IUD("insert into web_stat_panels (NAME) values ('%s')" % ("Новая панель"))
            self.db.commit()
            return self._create_panel(str(self.db.lastID()), "Новая панель", "200")
        elif query_type == "del_panel":
            self.db.IUD("delete from web_stat_panels where ID = %s" % (self.param("key")))
            self.db.commit()
            return "OK"
        elif query_type == "panel_img":
            self.content_type = "image/png"
            return self._paint_panel(self.param('key'), self.param('width'), self.param('height'), self.param('panel_h'))
        elif query_type == "panel_cursor":
            sql = ("select * from (select c.VALUE, %s NUM "
                                  "  from core_variable_changes c, web_stat_panels p "
                                  " where p.ID = %s "
                                  "   and p.SERIES_%s = c.VARIABLE_ID "
                                  "   and c.CHANGE_DATE < FROM_UNIXTIME(%s) "
                                  "order by c.CHANGE_DATE desc limit 1) a")
            s = []
            for i in range(1, 5):
                s += [sql % (i, self.param('key'), i, self.param('pos')), " union "]

            s = "".join(s[:-1])

            res = [",", ",", ",", ","]
            for rec in self.db.select("select b.* from (%s) b order by b.NUM " % (s)):
                res[rec[1] - 1] = "%s," % (round(rec[0], 2))

            return "".join(res)

    def _get_one_val(self, series, min_max_vals, min_x, max_x):
        def min_x_val(key):
            for v in min_max_vals:
                if v[0] == key:
                    #print(v[1])
                    return v[1]
            return min_x

        def max_x_val(key):
            for v in min_max_vals:
                if v[0] == key:
                    #print(v[2])
                    return v[2]
            return max_x

        res = [[], [], [], []]
        sql = []
        for i in range(4):
            if series[i] > 0:
                sql_prev = ("select a.* from ("
                            "select UNIX_TIMESTAMP(CHANGE_DATE) D, VALUE, VARIABLE_ID, ID, 1 "
                            "  from core_variable_changes "
                            " where VARIABLE_ID = %s "
                            "   and CHANGE_DATE < FROM_UNIXTIME(%s) "
                            "order by CHANGE_DATE desc limit 1) a" % (series[i], min_x_val(series[i])))

                sql_next = ("select a.* from ("
                            "select UNIX_TIMESTAMP(CHANGE_DATE) D, VALUE, VARIABLE_ID, ID, 2 "
                            "  from core_variable_changes "
                            " where VARIABLE_ID = %s "
                            "   and CHANGE_DATE > FROM_UNIXTIME(%s) "
                            "order by CHANGE_DATE limit 1) a" % (series[i], max_x_val(series[i])))
            if len(sql) > 0:
                sql += [" union "]

            try:
                sql += ["%s union %s" % (sql_prev, sql_next)]
            except:
                pass

        for row in self.db.select("".join(sql)):
            try:
                i = series.index(row[2])
                res[i] += [row]
            except Exception as e:
                #print("{}".format(e.args))
                pass
        return res
    """
    def _get_one_val(self, series, min_x, max_x):
        res = [[], [], [], []]
        sql = []
        for i in range(4):
            if series[i] > 0:
                sql_prev = ("select a.* from ("
                            "select UNIX_TIMESTAMP(CHANGE_DATE) D, VALUE, VARIABLE_ID, ID, 1 "
                            "  from core_variable_changes "
                            " where VARIABLE_ID = %s "
                            "   and CHANGE_DATE < FROM_UNIXTIME(%s) "
                            "order by CHANGE_DATE desc limit 1) a" % (series[i], min_x))

                sql_next = ("select a.* from ("
                            "select UNIX_TIMESTAMP(CHANGE_DATE) D, VALUE, VARIABLE_ID, ID, 2 "
                            "  from core_variable_changes "
                            " where VARIABLE_ID = %s "
                            "   and CHANGE_DATE > FROM_UNIXTIME(%s) "
                            "order by CHANGE_DATE limit 1) a" % (series[i], max_x))
            if len(sql) > 0:
                sql += [" union "]

            try:
                sql += ["%s union %s" % (sql_prev, sql_next)]
            except:
                pass

        for row in self.db.select("".join(sql)):
            try:
                i = series.index(row[2])
                res[i] += [row]
            except:
                pass
        return res
    """

    def _paint_panel(self, key, width, height, panel_h):
        self.db.IUD("update web_stat_panels set HEIGHT = %s where ID = %s" % (panel_h, key))
        self.db.commit()

        color_x_line = (0.7, 0.7, 0.7)
        color_x_line_2 = (0.9, 0.9, 0.9)
        color_y_line = (0.9, 0.9, 0.9)
        color_y_line_date = (0.7, 0.7, 0.7)
        color_border = (0.5, 0.5, 0.5)

        left = 40
        right = 10;
        bottom = 15

        var_ids = "0";
        series = [0, 0, 0, 0]
        typ = 0
        for row in self.db.select("select SERIES_1, SERIES_2, SERIES_3, SERIES_4, TYP "
                                  "  from web_stat_panels "
                                  " where ID = " + str(key)):
            series = row
            var_ids = "%s, %s, %s, %s" % row[0:4]
            typ = row[4]

        width, height = int(width), int(height)
        if width <= 0:
            width = 300
        if height <= 0:
            height = 150

        interval = self.param('range')

        delta_x = 1
        if interval == "-6 hour":
            delta_x = 6 * 3600
        elif interval == "-12 hour":
            delta_x = 12 * 3600
        elif interval == "-1 day":
            delta_x = 24 * 3600
        elif interval == "-3 day":
            delta_x = 3 * 24 * 3600
        elif interval == "-7 day":
            delta_x = 7 * 24 * 3600
        elif interval == "-14 day":
            delta_x = 14 * 24 * 3600
        elif interval == "-30 day":
            delta_x = 30 * 24 * 3600
        elif interval == "-90 day":
            delta_x = 3 * 30 * 24 * 3600
        elif interval == "-180 day":
            delta_x = 6 * 30 * 24 * 3600
        elif interval == "-360 day":
            delta_x = 12 * 30 * 24 * 3600

        min_x = int(self.param('start')) - delta_x // 2
        max_x = min_x + delta_x

        min_x_q = min_x - delta_x# // 100
        max_x_q = max_x + delta_x# // 100

        max_y = -9999
        min_y = 9999

        # Делаем полную выборку данных. Выкидываем подозрительные точки и
        # собираем статистику.
        prev_vals = [-9999] * 4
        chart_data = [[], [], [], []]

        zoom_step = delta_x / (width * 5)
        if zoom_step < 1 or typ != 0:
            zoom_step = 1

        x_min_max_values = []
        mi_x = max_x_q
        ma_x = min_x_q
        tt = -100

        for row in self.db.select("select UNIX_TIMESTAMP(CHANGE_DATE) D, MIN(VALUE) + (MAX(VALUE) - MIN(VALUE)) VALUE, VARIABLE_ID "
                                  "  from core_variable_changes "
                                  " where VARIABLE_ID in (%s) "
                                  "   and CHANGE_DATE >= FROM_UNIXTIME(%s) "
                                  "   and CHANGE_DATE <= FROM_UNIXTIME(%s) "
                                  " group by 3, ROUND(UNIX_TIMESTAMP(CHANGE_DATE) / %s)"
                                  " order by 3, 1 " % (var_ids, min_x_q, max_x_q, zoom_step)):
            ind = series.index(row[2])
            chart_data[ind] += [row]
            if row[0] > min_x and row[0] < max_x:
                max_y = max(max_y, row[1])
                min_y = min(min_y, row[1])

            if tt == -100:
                tt = row[2]

            if tt != row[2]:
                v = [tt, mi_x, ma_x]
                x_min_max_values += [v]
                mi_x = max_x_q
                ma_x = min_x_q
                tt = row[2]

            if row[0] < mi_x:
                mi_x = row[0]
            if row[0] > ma_x:
                ma_x = row[0]

        if tt != -1:
            v = [tt, mi_x, ma_x]
            x_min_max_values += [v]

        #print(x_min_max_values)
        #print(series)

        """
        for row in self.db.select("select UNIX_TIMESTAMP(CHANGE_DATE) D, VALUE, VARIABLE_ID, ID "
                                  "  from core_variable_changes "
                                  " where VARIABLE_ID in (" + var_ids + ") "
                                  "   and CHANGE_DATE >= FROM_UNIXTIME(%s) "
                                  "   and CHANGE_DATE <= FROM_UNIXTIME(%s) "
                                  "order by CHANGE_DATE " % (min_x_q, max_x_q)):
        """
        """
        try:
            self.db.IUD("set @rn := 0")

            sql = ("select UNIX_TIMESTAMP(CHANGE_DATE) D, VALUE, VARIABLE_ID, ID "
                   "  from core_variable_changes "
                   " where VARIABLE_ID in (" + var_ids + ") "
                   "   and CHANGE_DATE >= FROM_UNIXTIME(%s) "
                   "   and CHANGE_DATE <= FROM_UNIXTIME(%s) "
                   "order by CHANGE_DATE " % (min_x_q, max_x_q))

            for c in self.db.select("select count(*) c "
                                    "  from core_variable_changes "
                                    " where VARIABLE_ID in (" + var_ids + ") "
                                    "   and CHANGE_DATE >= FROM_UNIXTIME(%s) "
                                    "   and CHANGE_DATE <= FROM_UNIXTIME(%s) " % (min_x_q, max_x_q)):
                cou = c[0]
                ccc = 1000 * 4
                if cou > ccc:
                    sql = ("select UNIX_TIMESTAMP(CHANGE_DATE) D, VALUE, VARIABLE_ID, ID, @rn := @rn + 1 rownum "
                           "  from core_variable_changes "
                           " where VARIABLE_ID in (" + var_ids + ") "
                           "   and CHANGE_DATE >= FROM_UNIXTIME(%s) "
                           "   and CHANGE_DATE <= FROM_UNIXTIME(%s) "
                           "having mod(rownum, %s) = 0 "
                           "order by VARIABLE_ID, CHANGE_DATE " % (min_x_q, max_x_q, math.ceil(cou / ccc)))

            for row in self.db.select(sql):
                ind = series.index(row[2])
                prev_vals[ind] = row[1]

                if abs(prev_vals[ind] - row[1]) < 10:
                    chart_data[ind] += [row]
                    if row[0] > min_x and row[0] < max_x:
                        max_y = max(max_y, row[1])
                        min_y = min(min_y, row[1])
                prev_vals[ind] = row[1]
        except:
            pass
        """

        if min_y is None or max_y is None or min_y == 9999 or max_y == -9999 or min_y == max_y:
            max_y = 1
            min_y = 0

        min_y = math.floor(min_y)
        max_y = math.ceil(max_y)

        if typ == 2:
            if min_y < 0 and max_y < 0:
                max_y = 0
            elif min_y > 0 and max_y > 0:
                min_y = 0

        # Определяем цвета
        colors = [[1, 0, 0], [0, 0.65, 0.31], [0, 0, 1], [1, 0, 1]]

        off_y = (max_y - min_y) / 10
        min_y -= off_y
        max_y += off_y

        try:
            kx = ((max_x - min_x) / (width - left - right))
            ky = ((max_y - min_y) / (height - bottom))
            if ky == 0:
                ky = 1
        except:
            kx, ky = 1, 1

        img = ImageSurface(FORMAT_ARGB32, width, height)
        ctx = Context(img)

        width -= right
        ctx.set_line_width(1)

        # Рисуем сетку

        ctx.set_font_size(12)
        try:
            b_w, b_h = ctx.text_extents("00-00-0000")[2:4]

            # Метки на оси Y
            count = math.ceil(max_y) - math.ceil(min_y)
            space_count = math.ceil(count / ((height - bottom) / (b_h * 1.5)))
            sc = 0
            for i in range(math.ceil(min_y), math.ceil(max_y)):
                if sc == 0:
                    y = height - bottom + (min_y - i) / ky
                    ctx.set_source_rgb(*(color_x_line))
                    ctx.move_to(left, y)
                    ctx.line_to(width, y)
                    ctx.stroke()
                    ctx.set_source_rgb(0, 0, 0)
                    num = str(i)
                    tw, th = ctx.text_extents(num)[2:4]
                    ctx.move_to(left - 5 - tw, y + th // 2)
                    ctx.show_text(num)
                    sc = space_count
                sc -= 1

            # Метки на оси Х

            x_step = 3600
            if (interval == "-6 hour" or
                interval == "-12 hour" or
                interval == "-1 day"):
                # Дополнительно метки часов
                x_step = 3600
                for i in range(math.ceil(min_x / x_step), math.ceil(max_x / x_step)):
                    x = (i * x_step - min_x) / kx + left
                    ctx.set_source_rgb(*(color_x_line_2))
                    ctx.move_to(x, 0)
                    ctx.line_to(x, height - bottom)
                    ctx.stroke()
                    num = datetime.datetime.fromtimestamp(i * x_step).strftime('%H')
                    tw, th = ctx.text_extents(num)[2:4]
                    ctx.move_to(x + 2, height - bottom - 3)
                    ctx.set_source_rgb(*(color_x_line))
                    ctx.show_text(num)

            x_step = 3600 * 24

            space_count = 1
            count = math.ceil(max_x / x_step) - math.ceil(min_x / x_step)
            try:
                if (width / count) < b_w:
                    space_count = 2
            except:
                pass

            sc = 0
            tz = 3600 * 2
            tx_prev = -100
            for i in range(math.ceil(min_x / x_step), math.ceil(max_x / x_step) + 1):
                if sc == 0:
                    d_i = datetime.datetime.fromtimestamp(i * x_step)
                    x = (i * x_step - min_x - tz) / kx + left
                    ctx.set_source_rgb(0, 0, 0)
                    num = d_i.strftime('%d-%m-%Y %H')

                    x -= (int(d_i.strftime('%H')) * 3600 - tz) / kx

                    tw, th = ctx.text_extents(num)[2:4]
                    tx = x - tw // 2
                    ctx.move_to(tx, height - bottom + th + 5)
                    if tx - tx_prev > tw:
                        ctx.show_text(num)
                        tx_prev = tx
                        ctx.set_source_rgb(*(color_y_line_date))
                    else:
                        ctx.set_source_rgb(*(color_y_line))
                    if x >= left and x < width:
                        ctx.move_to(x, 0)
                        ctx.line_to(x, height - bottom)
                        ctx.stroke()
                    sc = space_count
                sc -= 1
        except Exception as e:
            pass

        # Рисуем верхний и правый бордер

        ctx.set_source_rgb(*color_border)
        ctx.move_to(left, 0)
        ctx.line_to(width, 0)
        ctx.line_to(width, height - bottom)
        ctx.stroke()

        #Рисуем сами графики

        ctx.rectangle(left, 0, width - left, height)
        ctx.clip()

        is_first = True
        currVarID = -1
        prevX = -1;

        if typ == 0: # Линейная
            for ind in range(4):
                """
                if len(chart_data[ind]) > 0:
                    for i in range(len(chart_data[ind]) - 1):
                        chart_data[ind][i] = list(chart_data[ind][i])
                        r1 = chart_data[ind][i]
                        r2 = chart_data[ind][i + 1]
                        chart_data[ind][i][0] += (r2[0] - r1[0]) / 2
                        chart_data[ind][i][1] += (r2[1] - r1[1]) / 2
                """
                ctx.set_source_rgb(*colors[ind])
                is_first = True
                for row in chart_data[ind]:
                    x = (row[0] - min_x) / kx + left
                    y = height - bottom - (row[1] - min_y) / ky

                    if is_first:
                        ctx.move_to(x, y)
                    else:
                        if row[0] - prevX > 100000:
                            ctx.move_to(x, y)
                        else:
                            ctx.line_to(x, y)

                    prevX = row[0]
                    is_first = False
                ctx.stroke()

        elif typ == 1: # Точечная
            for ind in range(4):
                if chart_data[ind]:
                    ctx.set_source_rgb(*colors[ind])
                    for row in chart_data[ind]:
                        x = (row[0] - min_x) / kx + left
                        y = height - bottom - (row[1] - min_y) / ky
                        ctx.rectangle(x - 3, y - 3, 6, 6)
                    ctx.fill()
        elif typ == 2: # Столбчатая
            cy = height - bottom - (-min_y) / ky
            for ind in range(4):
                for row in chart_data[ind]:
                    if currVarID != row[2]:
                        ctx.fill()
                        for i in range(4):
                            if series[i] == row[2]:
                                ctx.set_source_rgb(*colors[i])
                    x = (row[0] - min_x) / kx + left
                    y = height - bottom - (row[1] - min_y) / ky
                    ctx.rectangle(x - 5, y, 10, cy - y)

                    currVarID = row[2]
                ctx.fill()
        else: # Линейчастая
            #one_vals = self._get_one_val(series, min_x_q, max_x_q)
            one_vals = self._get_one_val(series, x_min_max_values, min_x_q, max_x_q)
            for ind in range(4):
                if series[ind]:
                    is_now = True
                    for r in one_vals[ind]:
                        if r[4] == 1:
                            chart_data[ind].insert(0, r)
                        else:
                            chart_data[ind] += [r]
                            is_now = False

                    if is_now:
                        if len(chart_data[ind]) > 0:
                            r = list(chart_data[ind][len(chart_data[ind]) - 1])
                            r[0] = datetime.datetime.now().timestamp()
                            chart_data[ind] += [r]

                color = colors[ind]
                color_fill = color.copy()
                color_fill += [0.3]
                is_first = True
                y0 = height - bottom + min_y / ky
                for row in chart_data[ind]:
                    x = (row[0] - min_x) / kx + left
                    y = height - bottom - (row[1] - min_y) / ky

                    if is_first:
                        is_first = False
                    else:
                        ctx.set_source_rgb(*color)
                        ctx.move_to(prevX, prevY)
                        ctx.line_to(x, prevY)
                        ctx.line_to(x, y)
                        ctx.stroke()
                        ctx.set_source_rgba(*color_fill)
                        rx, ry, rw, rh = prevX, y0, x - prevX, prevY - y0
                        ctx.rectangle(rx, ry, rw, rh)
                        ctx.fill()

                    prevX, prevY = x, y

        # Рисуем оси

        ctx.set_source_rgb(0, 0, 0)
        ctx.move_to(left, 0)
        ctx.line_to(left, height - bottom)
        ctx.line_to(width, height - bottom)
        ctx.stroke()

        # ---------------------------

        del ctx

        byt = BytesIO()
        img.write_to_png(byt)
        byt.seek(0)
        return byt.read()

    def copy_var_changes_to_mem(self):
        min_mem = None
        for rec in self.db.select("select MIN(ID) from core_variable_changes_mem"):
            min_mem = rec[0]

        if min_mem != None:
            min_mem_sql = " where ID < %s " % (min_mem)
        else:
            min_mem_sql = ""

        i = 0
        for rec in self.db.select("select ID, VARIABLE_ID, UNIX_TIMESTAMP(CHANGE_DATE), VALUE "
                                  "  from core_variable_changes "
                                  " %s" % (min_mem_sql)):
            try:
                if rec[3] == None:
                    rec[3] = "null"
                self.db.IUD("insert into core_variable_changes_mem "
                            "   (ID, VARIABLE_ID, CHANGE_DATE, VALUE) "
                            "values "
                            "   (%s, %s, FROM_UNIXTIME(%s), %s)" % rec)
                if i > 100:
                    self.db.commit()
                    i = 0
            except Exception as e:
                print(e)
                print(rec)
            i += 1
        self.db.commit()


import datetime
from functools import reduce
from itertools import groupby
from operator import itemgetter

import pandas as pd
from IPython.display import display_html, display_svg

def matrix_svg(value_counts, ranges=(3, 5, 10), display=True):

    MONTHS = ('Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep',
              'Oct', 'Nov', 'Dec')

    def get_color(count):
        if count == 0:
            return '#eee'
        elif count < ranges[0]:
            return '#c6e48b'
        elif count < ranges[1]:
            return '#7bc96f'
        elif count < ranges[2]:
            return '#239a3b'
        else:
            return '#196127'

    def build_svg_day(weekday, date, month, count, x):
        fmt = ('<rect class="day" width="10" height="10" x="{x}" y="{y}" ' +
               'fill="{color}" data-count="{count}" data-date="{date}"></rect>')
        y = weekday * 12
        color = get_color(count)
        return fmt.format(x=x, y=y, color=color, count=count, date=date)

    def build_svg_week(week, days):
        svg = '<g transform="translate({x}, 0)">'.format(x=week * 13)
        x = 13 - week
        for day in days:

            svg += build_svg_day(*day, x)
        return svg + '</g>'

    def build_svg_month(x, month):
        return '<text x="{x}" y="{y}" class="month">{month}</text>'.format(
            x=x, y=-10, month=month)

    def build_svg(dates, width=674, height=104):

        def list_date_tuples(start, end, counter):
            today = datetime.date.today()

            def _init_date_tuple(date):
                return (date.weekday(), date, MONTHS[date.month - 1],
                        counter(date))

            return [_init_date_tuple(d)
                    for d in (today - datetime.timedelta(days=start + x)
                    for x in reversed(range(end)))]

        def build_counter(xs, ys):
            def _inner(x):
                t = pd.Timestamp(x)
                return ys[xs.index(t)] if t in xs else 0
            return _inner

        # build the counter from `dates` -- this allows us to count how many
        # tickets are linked to a given `date`
        counter = build_counter(*list(zip(*dates)))

        # build a weeekday, date, month tuple list
        days = list_date_tuples(0, 365, counter)

        # ensure we start with a 1st day of the week -- a.k.a. Monday
        days = list_date_tuples(365, days[0][0], counter) + days

        def split(acc, curr):
            xs, i = acc
            if curr[0] == 0:
                return xs + [[curr]], i + 1
            xs[i] = xs[i] + [curr]
            return xs, i

        weeks = list(reduce(split, days, ([[]], 0))[0])

        def reducer(y, x):
            if len(y) > 0:
                prev = y[-1]
                if prev[1] == x[1]:
                    return y + [(prev[0] + 1, x[1])]
            return y + [x]

        months = [max(x[1]) for x in groupby(reduce(reducer, [(1, x[2])
                  for x in days if x[0] == 0], []), itemgetter(1))]

        svg = ('<svg width="{width}" height="{height}" ' +
               'class="js-calendar-graph-svg">').format(width=width,
                                                        height=height)
        svg += ''''''
        svg += '<g transform="translate(16, 20)">'
        for i, week in enumerate(weeks):
            svg += build_svg_week(i, week)
        x = 12 + (months[0][0] * 12 if months[0][0] < 3 else 0)
        for i, month in enumerate(months):
            n, name = month
            if n > 2:
                svg += build_svg_month(x, name)
                x += n * 12

        svg += '''
        <text text-anchor="start" class="wday" dx="-14" dy="8"
            style="display: none;">Mon</text>
        <text text-anchor="start" class="wday" dx="-14" dy="20">Tue</text>
        <text text-anchor="start" class="wday" dx="-14" dy="32"
            style="display: none;">Wed</text>
        <text text-anchor="start" class="wday" dx="-14" dy="44">Thu</text>
        <text text-anchor="start" class="wday" dx="-14" dy="57"
            style="display: none;">Fri</text>
        <text text-anchor="start" class="wday" dx="-14" dy="69">Sat</text>
        <text text-anchor="start" class="wday" dx="-14" dy="81"
            style="display: none;">Sun</text>'''
        svg += '</g>'
        svg += '</svg>'
        return svg

    result = build_svg(value_counts.items())

    return display_svg(result, raw=True) if display else result

def matrix_chart(value_counts, title, ranges=(3, 5, 10)):

    style = '''
    '''
    html = '''

{svg}

{title}

                Less

                More

'''
    return display_html(
        style + html.format(
            title=title, svg=matrix_svg(value_counts, ranges), display=False),
        raw=True)


class Student(object):

    def __init__(self, first_name, last_name):
        self.first_name = first_name
        self.last_name = last_name

scott = Student('Scott',  'Robinson')
print(scott.last_name)

class Teacher(object):

    @classmethod
    def from_string(cls, name_str):
        first_name, last_name = map(str, name_str.split(' '))
        cls.first_name = first_name
        cls.last_name = last_name
        cls.len_name = cls.len_name(name_str)
        return cls()

    @staticmethod
    def len_name(name_str):
        return len(name_str)

dave = Teacher.from_string('Dave Chapelle')
print(dave.last_name, dave.len_name)


import pandas as pd
import numpy as np
from fxdayu_data.data.decorators import value_wrapper
import itertools

def match(iterable, *iters):
    for ex in map(itertools.product, iterable, *iters):
        for e in ex:
            yield e

def multi_from_frame(frame):
    names = []
    items = []
    for name, item in frame.iteritems():
        names.append(name)
        items.append(item.values)

    return pd.MultiIndex.from_arrays(items, names=names)

multi_match = value_wrapper(list, pd.MultiIndex.from_tuples)(match)

def frame_2_multi_series(frame, names=None):
    if isinstance(frame, pd.DataFrame):
        value = np.concatenate(frame.values)
        index = pd.MultiIndex.from_product((frame.index, frame.columns), names=names)
        return pd.Series(value, index)

    else:
        raise TypeError('type of frame should be pandas.DataFrame')

def frames_2_multi_frame(names=None, **frames):
    name, frame = frames.popitem()
    index = pd.MultiIndex((frame.index, frame.columns), names=names)
    dct = {name: np.concatenate(frame.values)}
    shape = frame.shape
    for name_, frame_ in frames.items():
        if frame_.shape == shape:
            dct[name_] = np.concatenate(frame_.values)
        else:
            raise ValueError(
                'shape not match: shape of %s is %s but shape of %s is %s' % (name_, frame_.shape, name, shape)
            )

    return pd.DataFrame(dct, index)

def tsf_2_multi_frame(time_frame, columns=None, names=None):
    first = []
    second = []
    values = []
    for time, frame in time_frame:
        if isinstance(frame, pd.DataFrame):
            first.append([time])
            second.append(frame.index)
            values.append(frame.values)
        else:
            raise TypeError('type of frame in time_frame should be pandas.DataFrame')

    index = multi_match(first, second)
    if names:
        index = index.set_names(names)
    frame = np.concatenate(values)
    return pd.DataFrame(frame, index, columns)

def make_multi_frame(data, **kwargs):
    return pd.DataFrame(
        {name: np.concatenate(func(data).values) for name, func in kwargs.items()},
        pd.MultiIndex.from_product([data.index, data.columns])
    )

def panel_2_multi_frame(panel, names=None):
    if isinstance(panel, pd.Panel):
        return pd.DataFrame(
            {minor: np.concatenate(panel.minor_xs(minor).values) for minor in panel.minor_axis},
            pd.MultiIndex.from_product((panel.major_axis, panel.items), names=names)
        )
    else:
        raise (TypeError("Type of panel should be pandas.Panel"))

def roll_panel(panel, window=1, axis=1):
    if isinstance(panel, pd.Panel):
        slicer = [slice(None), slice(None), slice(None)]
        index = getattr(panel, ['items', 'major_axis', 'minor_axis'][axis])
        for i in range(window, len(index)):
            slicer[axis] = slice(i-window, i)
            yield index[i-1], panel.iloc[tuple(slicer)]
    else:
        raise (TypeError("Type of panel should be pandas.Panel"))


import copy

class SudokuPuzzle():
    #constructor
    def __init__(self):
        #create a 2d array filled with 0 that will represent the game board
        self.coordinateValue = [[0 for i in range(9)] for j in range(9)]
        #this will hold the possbile domains for every single tile
        self.coordinateDomains = {}
        self.input_file = ""
        self.output_file = ""
    #create the puzzle and give it the correct input and output files
    def create_puzzle(self, inputObj, outputObj):
        input_lines = inputObj.readlines()
        #split every single line and add the respective number as a string into the 2d array
        for i in range(9):
            temp_num_list = input_lines[i].strip().split()
            self.coordinateValue[i] = temp_num_list
            for j in range(9):
                self.coordinateValue[i][j] = int(self.coordinateValue[i][j])
        #for every single tile on the board, add the respective possible domains into coordinateDomains
        for i in range(9):
            for j in range(9):
                #if the tile already is filled, it has no possible domains
                if(self.coordinateValue[i][j] != 0):
                    self.coordinateDomains[str(i) + str(j)]  = []\
                #else if it is an empty tile, it can be filled with any number from 1-9
                #   (we will remove the possible domains in a later step)
                else:
                    self.coordinateDomains[str(i) + str(j)] = [1,2,3,4,5,6,7,8,9]
        self.input_file = inputObj
        self.output_file = outputObj

    #add the solution into the output text and print out the solution
    def print_solution(self):
        for row in self.coordinateValue:
            line = " ".join(str(coord) for coord in row)
            print(line)
            self.output_file.write(line + "\n")
        print("\n")

    #check if this is in the overlapping regions
    def checkInOverlappingBlock(self,x,y):
            return ((0<x<4 or 4<x<8) and (0<y<4 or 4<y<8))

    #forward checking algorithm
    def forwardCheck(self, x, y, coord_value):

        #remove domains for row
        for i in range(9):
            temp_arr = self.coordinateDomains[str(x) + str(i)]
            if(int(coord_value) in temp_arr):
                temp_arr.remove(int(coord_value))
            if(len(temp_arr) == 0 and self.coordinateValue[x][i] == 0):
                return False
            if(self.coordinateValue[x][i] == self.coordinateValue[x][y] and i != y):
                return False
            self.coordinateDomains[str(x) + str(i)] = temp_arr

        #remove domains for column
        for i in range(9):
            temp_arr = self.coordinateDomains[str(i) + str(y)]
            if(int(coord_value) in temp_arr):
                temp_arr.remove(coord_value)
            if(len(temp_arr) == 0 and self.coordinateValue[i][y] == 0):
                return False
            if(self.coordinateValue[i][y] == self.coordinateValue[x][y] and i != x):
                return False
            self.coordinateDomains[str(i) + str(y)] = temp_arr

        #remove domains for non-overlapping block
        startx = x - (x % 3)
        endx = startx + 3
        starty = y - (y % 3)
        endy = starty + 3

        for i in range(startx, endx, 1):
            for j in range(starty, endy, 1):
                temp_arr = self.coordinateDomains[str(i) + str(j)]
                if(int(coord_value) in temp_arr):
                    temp_arr.remove(coord_value)
                if(len(temp_arr) == 0 and self.coordinateValue[i][j] == 0):
                    return False
                if(self.coordinateValue[i][j] == self.coordinateValue[x][y] and i != x):
                   return False
                self.coordinateDomains[str(i) + str(j)] = temp_arr

        #remove domain in overlapping block
        if (self.checkInOverlappingBlock(x,y)):
            if x <= 3:
                startx = 1
                endx = 4
            else:
                startx = 5
                endx = 8
            if y <= 3:
                starty = 1
                endy = 4
            else:
                starty = 5
                endy = 8

            for i in range(startx, endx, 1):
                for j in range(starty, endy, 1):
                    temp_arr = self.coordinateDomains[str(i) + str(j)]
                    if(int(coord_value) in temp_arr):
                        temp_arr.remove(coord_value)
                    if(len(temp_arr) == 0 and self.coordinateValue[i][j] == 0):
                        return False
                    if(self.coordinateValue[i][j] == self.coordinateValue[x][y] and i != x):
                        return False
                    self.coordinateDomains[str(i) + str(j)] = temp_arr
        return True

    def backwardCheck(self):
        #if there are no empty spaces, then you are done
        if self.noEmptySpaces():
            return True
        #select the next tile to fill
        coord = self.chooseNextCoordinate()
        #_____________________________________
        for i in range(1,10):
            #_____________________________________
            if i in self.coordinateDomains[coord[0]+coord[1]]:
                temp = copy.deepcopy(self.coordinateDomains)
                self.coordinateValue[int(coord[0])][int(coord[1])] = i
                self.coordinateDomains[coord[0]+coord[1]] = []
                #_____________________________________
                for row in self.coordinateValue:
                    line = " ".join(str(coord) for coord in row)
                #_____________________________________
                if not self.forwardCheck(int(coord[0]), int(coord[1]), i):
                    self.coordinateValue[int(coord[0])][int(coord[1])] = 0
                    self.coordinateDomains = temp
                    continue
                #_____________________________________
                if self.backwardCheck():
                    return True
                #_____________________________________
                self.coordinateValue[int(coord[0])][int(coord[1])] = 0
                self.coordinateDomains = temp
        return False

    #exit if there are no empty spaces to fill
    def noEmptySpaces(self):
        for i in range(9):
            for j in range(9):
                if(int(self.coordinateValue[i][j]) == 0):
                    return False
        return True

    #choose the next tile/coordinate to fill
    def chooseNextCoordinate(self):
        #MRV heuristic

        #__________________________
        emptyCoordinateDomains = {key:domain for key, domain in self.coordinateDomains.items() if len(domain) > 0}
        #sort the dictionary by the length of the domain list
        sort_coordinate_domains = sorted(emptyCoordinateDomains.items(), key=lambda x: len(x[1]))
        #find the tiles with the smallest number of possible domains left
        tempLength = len(sort_coordinate_domains[0][1])
        tempArray = []
        for elem in sort_coordinate_domains:
            if(len(elem[1]) == tempLength):
                tempArray.append(elem[0])
            else:
                break

        #Degree heuristic part
        #_______________________________
        tempArrayUnassignedNeighborNum = []
        for coord in tempArray:
            coord_x = int(coord) % 10
            coord_y = int(coord) // 10
            counter = 0

            #____________________________
            #columns
            checked_coord_ls = []
            for i in range(9):
                if coord_x != i and self.coordinateValue[coord_x][i] == 0:
                    checked_coord_ls.append((coord_x, i))
                    counter+=1
            #rows
            for i in range(9):
                if coord_y != i and self.coordinateValue[i][coord_y] == 0:
                    checked_coord_ls.append((i, coord_y))
                    counter+=1
            #non-overlapping block
            startx = coord_x  - (coord_x  % 3)
            endx = startx + 3
            starty = coord_y  - (coord_y % 3)
            endy = starty + 3
            for i in range(startx, endx, 1):
                for j in range(starty, endy, 1):
                    if (i,j) not in checked_coord_ls and self.coordinateValue[i][j] == 0:
                            counter+=1
            #overlapping block
            if (self.checkInOverlappingBlock(coord_x, coord_y)):
                if coord_x <= 3:
                    startx = 1
                    endx = 4
                else:
                    startx = 5
                    endx = 8
                if coord_y <= 3:
                    starty = 1
                    endy = 4
                else:
                    starty = 5
                    endy = 8

                for i in range(startx, endx, 1):
                    for j in range(starty, endy, 1):
                        if (i,j) not in checked_coord_ls and self.coordinateValue[i][j] == 0:
                                counter+=1
            #________________________________________
            tempArrayUnassignedNeighborNum.append((coord,counter))
        #___________________________________________________
        sort_UNN = sorted(tempArrayUnassignedNeighborNum, key=lambda x: x[1], reverse=True)
        return (sort_UNN[0][0][0], sort_UNN[0][0][1])

    #solve the puzzle
    def solve_puzzle(self):
        for i in range(9):
            for j in range(9):
                if self.coordinateValue[i][j] != 0:
                    if not (self.forwardCheck(i,j, self.coordinateValue[i][j])):
                        self.output_file.write("The puzzle is unsolvable!" + "\n")
                        quit()
        #___________________________
        self.backwardCheck()
        self.print_solution()

i1= open("Input1.txt", "r")
o1 = open("Output1.txt", "w+")
i2= open("Input2.txt", "r")
o2 = open("Output2.txt", "w+")
i3= open("Input3.txt", "r")
o3 = open("Output3.txt", "w+")

#this bottom 2 was just because we had additional input files to check if
#   the program early exists if no solution was found
'''
i4= open("Input4.txt", "r")
o4 = open("Output4.txt", "w+")

i5= open("Input5.txt", "r")
o5 = open("Output5.txt", "w+")
'''

solver = SudokuPuzzle()
solver.create_puzzle(i1, o1)
solver.solve_puzzle()
solver.create_puzzle(i2, o2)
solver.solve_puzzle()
solver.create_puzzle(i3, o3)
solver.solve_puzzle()
'''
solver.create_puzzle(i4, o4)
solver.solve_puzzle()
solver.create_puzzle(i5, o5)
solver.solve_puzzle()
'''
i1.close()
o1.close()
i2.close()
o2.close()
i3.close()
o3.close()

'''
i4.close()
o4.close()
i5.close()
o5.close()
'''


from datetime import datetime, timedelta
from collections import defaultdict
from pprint import pprint

AVERAGE_WINDOW = 7
NUM_SAMPLES = 7

def increasing(samples):
    last = 0
    for i, d in sorted(samples.items(), reverse=True):
        if d < (last * 1.1):
            return False
        last = d
    return True

def make_deltas(filename):
    fh = open(filename, 'r')
    ref_data = defaultdict(lambda: defaultdict(int))
    for line in fh:
        lemma, sample, hits = line.split('\t')
        ref_data[lemma][int(sample)] = int(hits)
    fh.close()

    for lemma, samples in ref_data.items():
        diffs = []
        for sample in range(1, NUM_SAMPLES):
            try:
                diffs.append(samples[sample-1] - samples[sample])
            except:
                pass
        if filter(lambda d: d<100, diffs):
            continue

        if not increasing(samples):
            continue
        print [lemma, diffs, samples]
        #print [lemma, diffs]

    #pprint(ref_data)

if __name__ == '__main__':
    make_deltas('samples/windows.tsv')


from contextlib import contextmanager
import os
import pickle
import shutil
from unittest.mock import MagicMock
import pytest

from tts_wrapper import (
    AwsCredentials, GoogleTTS, MicrosoftTTS, PollyTTS, WatsonTTS)

SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))

TMP_DIR = '/tmp/tts-wrapper'
TMP_SPEECH = os.path.join(TMP_DIR, 'speech.wav')
TEST_DATA_DIR = os.path.join(SCRIPT_DIR, 'data')

def load_pickle(path):
    with open(path, 'rb') as f:
        return pickle.load(f)

def load_resp_wav():
    with open(os.path.join(TEST_DATA_DIR, 'test.wav'), 'rb') as f:
        return f.read()

def check_audio_file(path):
    assert os.path.exists(path), f'{path} does not exists'
    assert os.path.getsize(path) > 0

@contextmanager
def managed_tts(cls, *args, _filename=TMP_SPEECH, **kwargs):
    tts = cls(*args, **kwargs)
    assert not os.path.exists(_filename)
    try:
        yield tts
    finally:
        check_audio_file(_filename)
        if os.path.exists(_filename):
            os.remove(_filename)

def setup_module():
    if os.path.exists(TMP_DIR):
        shutil.rmtree(TMP_DIR)
    os.makedirs(TMP_DIR)

def test_polly():
    resp = load_pickle(os.path.join(TEST_DATA_DIR, 'polly.pickle'))
    creds = AwsCredentials('AWS_ID', 'AWS_KEY', region='us-east-1')

    with managed_tts(PollyTTS, creds=creds) as tts:
        tts.polly_client = MagicMock()
        synth_resp = tts.polly_client.synthesize_speech.return_value
        synth_resp['AudioStream'] = MagicMock()
        synth_resp['AudioStream'].read.return_value = resp

        tts.synth('hello world', TMP_SPEECH)

def patch_microsoft_tts(mocker, tts):
    resp = load_resp_wav()
    mock_post = mocker.patch('requests.post')
    mocked_resp = mock_post.return_value
    mocked_resp.status_code = 200
    mocked_resp.content = resp

    tts._fetch_access_token = MagicMock()
    tts._fetch_access_token.return_value = 'mocked-access-token'

def test_microsoft(mocker):
    with managed_tts(MicrosoftTTS, creds='fakecreds') as tts:
        patch_microsoft_tts(mocker, tts)
        tts.synth('hello world', TMP_SPEECH)

def test_microsoft_repeated_synth(mocker):
    with managed_tts(MicrosoftTTS, creds='fakecreds') as tts:
        patch_microsoft_tts(mocker, tts)
        tts.synth('hello world', TMP_SPEECH)
        check_audio_file(TMP_SPEECH)
        os.remove(TMP_SPEECH)
        tts.synth('bye world', TMP_SPEECH)

def test_google(mocker):
    resp = load_resp_wav()
    mocked_client = mocker.patch(
        'google.cloud.texttospeech.TextToSpeechClient')
    mocked_client.return_value.synthesize_speech.return_value.audio_content = resp

    with managed_tts(GoogleTTS) as tts:
        tts.synth('hello world', TMP_SPEECH)

def test_watson(mocker):
    resp = load_resp_wav()
    with managed_tts(WatsonTTS, api_key='api_key', api_url='api_url') as tts:
        tts.client = MagicMock()
        tts.client.synthesize.return_value.get_result.return_value.content = resp
        tts.synth('hello world', TMP_SPEECH)


import cv2
import numpy as np
from sklearn import mixture
import sys
from time import time
start = time()

# Enter key parameters for the preprocessing
global height, width
dimen_limit = 200
img = cv2.imread(sys.argv[1])
height, width, _ = img.shape
large = max(height,width)

for i in range(1,100):
    if large/i < dimen_limit:
        break

# img = cv2.equalizeHist(img)
# img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY).reshape(height,width,1)
# img = np.concatenate((img,img,img),axis=2)
# img = cv2.cvtColor(img, cv2.COLOR_BGR2YUV)
# img = cv2.medianBlur(img,5)

def DS(img): # Downsampling

    img_small = cv2.resize(img,None,fx=1/i, fy=1/i, interpolation = cv2.INTER_CUBIC)
    dimen_1, dimen_2, _ = img_small.shape
#     img_flat = cv2.Canny(img_small.copy(),10,10).flatten().reshape(-1,1) # Edge filter
    img_flat = np.concatenate((img_small[:,:,0].flatten().reshape(-1,1),
                               img_small[:,:,1].flatten().reshape(-1,1),
                               img_small[:,:,2].flatten().reshape(-1,1)),axis=1)

    return img_flat, dimen_1, dimen_2

def GMM(img_flat, dimen_1, dimen_2): # GMM clustering

    gmm_tmp = mixture.GaussianMixture(n_components=2, covariance_type='full', max_iter=500)
    gmm_tmp.fit(img_flat)
    tmp = np.argmax(gmm_tmp.predict_proba(img_flat),axis=1).reshape(-1,1)
    tmp0 = tmp.reshape(dimen_1, dimen_2).astype('int16')

    return tmp0

img_flat, dimen_1, dimen_2 = DS(img)
tmp0 = GMM(img_flat, dimen_1, dimen_2)

# from matplotlib import pyplot as plt
# plt.imshow(cv2.resize(np.ceil(img_gmm*255), (width,height)), cmap="hot")
# plt.show()

np.savetxt("0.csv", tmp0, delimiter=",",fmt='%1d')

print('Time used',time()-start)


import argparse
import logging

from fontTools import merge, subset
from fontTools import configLogger
from fontTools.ttLib import TTFont

def main():
    parser = argparse.ArgumentParser(description="Merge Aref Ruqaa fonts.")
    parser.add_argument("file1", metavar="FILE", help="input font to process")
    parser.add_argument("file2", metavar="FILE", help="input font to process")
    parser.add_argument(
        "--out-file", metavar="FILE", help="output font to write", required=True
    )

    args = parser.parse_args()

    configLogger(level=logging.ERROR)

    merger = merge.Merger()
    font = merger.merge([args.file1, args.file2])

    # Drop incomplete Greek support.
    unicodes = set(font.getBestCmap().keys()) - set(range(0x0370, 0x03FF))

    options = subset.Options()
    options.set(
        layout_features="*",
        layout_scripts=["arab", "latn", "DFLT"],
        name_IDs="*",
        name_languages="*",
        notdef_outline=True,
        glyph_names=False,
        recalc_average_width=True,
    )
    subsetter = subset.Subsetter(options=options)
    subsetter.populate(unicodes=unicodes)
    subsetter.subset(font)

    font.save(args.out_file)

if __name__ == "__main__":
    main()


#!/usr/bin/env python3
import json
import pickle
import pathlib

import numpy as np
import pandas as pd
from sklearn import preprocessing
from sklearn.model_selection import train_test_split, RandomizedSearchCV
from sklearn.metrics import auc, accuracy_score, confusion_matrix, mean_squared_error
import xgboost as xgb
import argparse
import logging

logging.basicConfig(format='%(asctime)s.%(msecs)03d %(levelname)s {%(module)s} [%(funcName)s : %(lineno)d] %(message)s', datefmt='%Y-%m-%d,%H:%M:%S', level=logging.INFO)
logger = logging.getLogger(__name__)

def _check_path(path):
    p = pathlib.Path(path)
    if not p.exists():
        logger.info(f"Path {path} does not exist. Creating...")
        p.parent.absolute().mkdir(parents=True, exist_ok=True)
        return False
    else:
        logger.info(f"Path {path} already exists.")
        return True

def load_data(features_file_path, labels_file_path):
    X = np.loadtxt(features_file_path, delimiter=',')
    y = pd.read_csv(labels_file_path)

    assert X.shape[0] == y['MTurk_label'].shape[0]

    return X, [int(d) for d  in y['MTurk_label'].to_list()]

def train(X, y,
    cv_results_file_path,
    model_save_path,
    mlpipeline_metrics_path,
    mlpipeline_ui_metadata_path,
    test_score_path,
    test_size=0.3):

    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=test_size, stratify=y)

    estimator = xgb.XGBClassifier(
        objective="binary:logistic",
        eval_metric="logloss",
        use_label_encoder=False,
        random_state=42)

    parameters={
        'max_depth': range(2, 5, 1),
        'n_estimators': range(10, 100, 10),
        'learning_rate': [0.1, 0.05, 0.01]
    }

    random_search = RandomizedSearchCV(
        estimator=estimator,
        param_distributions=parameters,
        scoring='accuracy',
        random_state=22,
        return_train_score=True
        )

    search_result = random_search.fit(X_train, y_train)

    cv_results = search_result.cv_results_
    best_params = search_result.best_params_
    best_score = search_result.best_score_
    best_model = search_result.best_estimator_

    cv_results = pd.DataFrame(cv_results)

    _check_path(cv_results_file_path)
    cv_results.to_csv(cv_results_file_path)

    logger.info(f"Model: {estimator}")
    logger.info(f"Best CV score is {best_score}")
    logger.info(f"Best parameters are {best_params}")

     # Viz of cv results
    _check_path(mlpipeline_ui_metadata_path)
    metadata = {
        'outputs' : [{
            'type': 'table',
            'storage': 'inline',
            'format': 'csv',
            'header': ['idx'] + list(cv_results.columns.values),
            'source': cv_results.to_csv(header=False)
        }]
    }
    with open(mlpipeline_ui_metadata_path, 'w') as f:
        json.dump(metadata, f)

    # Save model
    _check_path(model_save_path)
    pickle.dump(best_model, open(model_save_path, 'wb'))

    # Save metrics
    _check_path(mlpipeline_metrics_path)
    y_pred = best_model.predict(X_test)
    p = pathlib.Path(mlpipeline_metrics_path)
    metrics = {
        'metrics': [
            {
            'name': 'accuracy_score',
            'numberValue': accuracy_score(y_test, y_pred),
            'format': "PERCENTAGE"
            }
        ]
    }
    with open(mlpipeline_metrics_path, 'w') as f:
        json.dump(metrics, f)

    logger.info(f"Test score is {accuracy_score(y_test, y_pred)}")
    test_score = {'score': accuracy_score(y_test, y_pred)}
    _check_path(test_score_path)
    with open(test_score_path, 'w') as f:
        json.dump(test_score, f)

if __name__ == "__main__":
    try:
        # The component must be stateless
        # All inputs are not hard coded but passed in as params
        parser = argparse.ArgumentParser()
        parser.add_argument('--features_file_path', type=str, action='store')
        parser.add_argument('--labels_file_path', type=str, action='store')
        parser.add_argument('--cv_results_file_path', type=str, action='store')
        parser.add_argument('--model_file_path', type=str, action='store')
        parser.add_argument('--kfp_metrics_path', type=str, action='store')
        parser.add_argument('--mlpipeline_ui_metadata_path', type=str, action='store')
        parser.add_argument('--test_score_path', type=str, action='store')

        FLAGS = parser.parse_args()

        X, y = load_data(FLAGS.features_file_path, FLAGS.labels_file_path)
        train(X, y,
            FLAGS.cv_results_file_path,
            FLAGS.model_file_path,
            FLAGS.kfp_metrics_path,
            FLAGS.mlpipeline_ui_metadata_path,
            FLAGS.test_score_path
            )

    except Exception as e:
        logger.exception(e)


import boto3, json

def lambda_handler(event, context):

    client = boto3.client('codebuild')

    response = client.start_build(
        projectName="paulhastings-staging",
        artifactsOverride={
            'type': 'CODEPIPELINE'
        }
    )
    return "Paul Hastings AWS Staging build triggered"

    print(response)


#!/usr/bin/env python
"""
####################################################################################
    # -*- coding: utf-8 -*-
    # Author  : Thomas Neuer (tneuer)
    # Creation Date : 2019-11-26 23:06:11
    # Description :
####################################################################################
"""
import os
import re
import sys
sys.path.insert(1, '../Preprocessing')
import json
import pickle

import numpy as np
import pandas as pd
import tensorflow as tf
import initialization as init
import matplotlib.pyplot as plt

from functionsOnImages import padding_zeros, clip_outer, get_layout, build_image, build_images, savefigs, separate_images
from functionsOnImages import build_histogram, get_energies, get_max_energy, get_number_of_activated_cells
from functionsOnImages import get_center_of_mass_x, get_center_of_mass_y, get_std_energy, get_energy_resolution
from functionsOnImages import get_center_of_mass_r
from functionsOnImages import build_histogram_HTOS, crop_images

from TrainedCycleGenerator import TrainedCycleGenerator
from TrainedIm2Im import TrainedIm2Im

def atoi(text):
    return int(text) if text.isdigit() else text

def natural_keys(text):
    return [ atoi(c) for c in re.split(r'(\d+)', text) ]

if __name__ == "__main__":

    nr_test_hist = 2000
    batch_size = 100
    result_path = "../../Results/"
    include_all = ["ServerTemp/B2DmunuTracker/1Good"]
    save_path = "../../Results/ServerTemp/B2DmunuTracker/1Good/Summary.pdf"
    models = [
            "{}/{}".format(folder, name) for folder in include_all for name in os.listdir(result_path+"/"+folder)
                    if os.path.isdir("{}/{}".format(result_path+"/"+folder, name))
    ]
    models.sort(key=natural_keys)

    # colnames = [r"$\sum_{i} E_{Ti}$", "MaxEnergy [GeV]", "MaxEnergy", "StdEnergy [GeV]", "X CoM", "Y CoM", "Tracker-GAN"]
    colnames = [
        r"$\sum_{i} E_{Ti}$ [GeV]", r"$\max (E_{Ti})$ [GeV]", r"$\sum_{i} [E_{Ti} > 6MeV]$",
        r"std($E_{Ti}$) [GeV]", r"$E_{T,Tracker} - \sum_{i} E_{Ti}$ [MeV]"
    ]
    fig, axes = plt.subplots(nrows=len(models), ncols=len(colnames)+4, figsize=(12*len(models), 6*len(models)), facecolor='w', edgecolor='k')
    fig.subplots_adjust(wspace=0.2, hspace=0.3)
    figs = [fig]

    with open("../../Data/B2Dmunu/TestingPurpose/calo_images.pickle", "rb") as f:
        mc_data = pickle.load(f)
    with open("../../Data/B2Dmunu/TestingPurpose/tracker_images.pickle", "rb") as f:
        tracker_images = pickle.load(f)
    with open("../../Data/B2Dmunu/TestingPurpose/tracker_events.pickle", "rb") as f:
        tracker_events = pickle.load(f)
        tracker_real_ET = tracker_events["real_ET"].apply(sum).to_numpy()[-nr_test_hist:]
    with open("../../Data/PiplusLowerP/LargeSample/ProcessedScaler.pickle", "rb") as f:
        scaler = pickle.load(f)
        calo_scaler = scaler["Calo"]
    print("Test data loaded.")

    for model_idx, model in enumerate(models):
        print(""""
              #######################################################################
              ######## Evaluating Model {} / {}:  {}
              #######################################################################
              """.format(model_idx+1, len(models), model))

        image_shape = [64, 64, 1]
        mc_data_images_m = padding_zeros(mc_data[-nr_test_hist:], top=6, bottom=6).reshape(-1, 64, 64) / calo_scaler
        tracker_images_m = padding_zeros(tracker_images[-nr_test_hist:], top=6, bottom=6).reshape([-1, *image_shape]) / calo_scaler

        #####################################################################################################
        # Model loading
        #####################################################################################################
        model_path = "{}/{}/".format(result_path, model)
        meta_path = model_path + "TFGraphs/"
        config_path = model_path + "config.json"
        if "Keras" in model:
            print(model_path)
            generator_file = [f for f in os.listdir(model_path+"/ModelSave") if f.startswith("Generator")]
            assert len(generator_file) == 1, "Ambiguous generator file."
            with open(model_path+"/ModelSave/"+generator_file[0], "rb") as f:
                Generator = pickle.load(f)

            nr_batches = int(nr_test_hist/batch_size)
            generated_images = []
            for i in range(nr_batches):
                print("Generate", i, "/", nr_batches)
                start = i*batch_size
                end = (i+1)*batch_size
                batch_generated_images = Generator.predict(tracker_images_m[start:end]).reshape([-1, 64, 64])
                generated_images.extend(batch_generated_images)

        else:
            Generator = TrainedIm2Im(path_to_meta=meta_path, path_to_config=config_path)
            nr_batches = int(nr_test_hist/batch_size)
            generated_images = Generator.generate_batches(inputs=tracker_images_m, batch_size=batch_size)

        generated_images = np.array(generated_images)
        generated_images = clip_outer(images=generated_images, clipval=1/4)

        #####################################################################################################
        # Build figures
        #####################################################################################################
        # use_functions = {get_energies: {"energy_scaler": calo_scaler/1000}, get_max_energy: {"energy_scaler": calo_scaler/1000, "maxclip": 6.12},
        #                 get_number_of_activated_cells: {"threshold": 5/calo_scaler},
        #                 get_std_energy: {"energy_scaler": calo_scaler/1000, "threshold": 0.005/calo_scaler},
        #                 get_center_of_mass_x: {"image_shape": image_shape}, get_center_of_mass_y: {"image_shape": image_shape},
        #                 get_energy_resolution: {"real_ET": tracker_real_ET, "energy_scaler": calo_scaler}}
        # colnames = ["Enery [GeV]", "MaxEnergy [GeV]", "Cells", "StdEnergy [GeV]", "X CoM", "Y CoM", "Tracker-GAN"]
        use_functions = {get_energies: {"energy_scaler": calo_scaler/1000}, get_max_energy: {"energy_scaler": calo_scaler/1000, "maxclip": 6.12},
                        get_number_of_activated_cells: {"threshold": 5/calo_scaler},
                        get_std_energy: {"energy_scaler": calo_scaler/1000, "threshold": 5/calo_scaler},
                        get_energy_resolution: {"real_ET": tracker_real_ET, "energy_scaler": calo_scaler}}

        assert mc_data_images.shape == generated_images.shape == tracker_images_m.shape, "Shape mismatch."

        axes[model_idx, -1].scatter(tracker_real_ET / 1000, get_energies(mc_data_images, energy_scaler=calo_scaler/1000),
                                    label="Geant4", alpha=0.05)
        axes[model_idx, -1].scatter(tracker_real_ET / 1000, get_energies(generated_images, energy_scaler=calo_scaler/1000),
                                    label="Im2Im", alpha=0.05)
        axes[model_idx, -1].set_xlabel("Tracker [GeV]")
        axes[model_idx, -1].set_ylabel("Reconstructed [GeV]")
        axes[model_idx, -1].legend()

        build_histogram_HTOS(true=mc_data_images, fake=generated_images,
                             energy_scaler=calo_scaler, threshold=3600, real_ET=tracker_real_ET,
                            labels=["Geant4", "Im2Im", "CGAN"], ax1=axes[model_idx, -3], ax2=axes[model_idx, -2])

        # Resolution Geant vs Generated
        resolution_geant_nn = (get_energies(mc_data_images) - get_energies(generated_images))
        is_small_discrepancy = np.abs(resolution_geant_nn)<5
        resolution_geant_nn_moderate = resolution_geant_nn[is_small_discrepancy]
        axes[model_idx, -4].hist(resolution_geant_nn_moderate, bins=40, histtype="step")
        axes[model_idx, -4].set_title(r"$\sum_{i} E_{Ti, Geant4} - \sum_{i} E_{Ti, Im2Im}$ [MeV]")
        resolution_mean = np.mean(resolution_geant_nn_moderate)
        resolution_std = np.std(resolution_geant_nn_moderate)
        textstr = '\n'.join((
                r'$\mu=%.2f$' % (resolution_mean, ),
                r'$\sigma=%.2f$' % (resolution_std, )
        ))
        props = dict(boxstyle='round', facecolor='wheat', alpha=0.5)
        axes[model_idx, -4].text(0.05, 0.95, textstr, transform=axes[model_idx, -4].transAxes, fontsize=14,
                                 verticalalignment='top', bbox=props)

        for func_idx, (func, params) in enumerate(use_functions.items()):
            build_histogram(true=mc_data_images, fake=generated_images,
                            function=func, name=colnames[func_idx], epoch="", folder=None, ax=axes[model_idx, func_idx],
                            labels=["Geant4", "Im2Im", "CGAN"], **params)
            if func_idx == 0:
                fs = {"2": 12, "3": 15, "7": 10, "8": 10, "9": 12, "11": 13, "12": 12, "13": 12, "14": 13, "17": 13, "18": 13}
                axes[model_idx, func_idx].set_ylabel(model, fontsize=fs[str(len(models))])
            if model_idx != 0:
                axes[model_idx, func_idx].set_title("")
        idx = 9
        use_functions[get_center_of_mass_r] = {"image_shape": image_shape}
        use_functions[get_energy_resolution] = {"real_ET": tracker_real_ET[idx], "energy_scaler": calo_scaler}
        figs.append(Generator.build_simulated_events(condition=tracker_images_m[idx],
                                 tracker_image=tracker_images_m[idx].reshape([image_shape[0], image_shape[1]]),
                                 calo_image=mc_data_images[idx],
                                 cgan_image=None,
                                 n=500,
                                 eval_functions=use_functions,
                                 title=model
        )[0])

        tf.reset_default_graph()

    savefigs(figures=figs, save_path=save_path)


# -*- coding: utf-8 -*-
# $Author   : LiuShiWen
# $Date     : 2021/7/2 2:59 下午

from configparser import ConfigParser
import os
import sys

project_path = str(os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)))
if sys.platform.startswith("win32"):
    conf_path = os.path.join(project_path, 'Config/config.ini').replace('/', '\\')
    log_path = os.path.join(project_path, 'OutPuts/Logs').replace('/', '\\')
    report_path = os.path.join(project_path, 'OutPuts/Reports').replace('/', '\\')
    img_path = os.path.join(project_path, 'OutPuts/Images').replace('/', '\\')
    UITestData_path = os.path.join(project_path, 'Data/UITestData').replace('/', '\\')
    APITestData_path = os.path.join(project_path, 'Data/APITestData').replace('/', '\\')
else:
    conf_path = os.path.join(project_path, 'Config/.config.ini')
    log_path = os.path.join(project_path, 'OutPuts/Logs')
    report_path = os.path.join(project_path, 'OutPuts/Reports')
    img_path = os.path.join(project_path, 'OutPuts/Images')
    UITestData_path = os.path.join(project_path, 'Data/UITestData')
    APITestData_path = os.path.join(project_path, 'Data/APITestData')
if not os.path.exists(conf_path):
    raise FileNotFoundError("请检查配置文件路径是否存在")
if not os.path.exists(log_path):
    raise FileNotFoundError("请检查日志文件路径是否存在")
if not os.path.exists(report_path):
    raise FileNotFoundError("请检查报告路径是否存在")
if not os.path.exists(img_path):
    raise FileNotFoundError("请检查照片路径是否存在")
if not  os.path.exists(UITestData_path):
    raise FileNotFoundError("请检查UI测试数据路径是否存在")
if not  os.path.exists(APITestData_path):
    raise FileNotFoundError("请检查API测试数据路径是否存在")

class Config(object):
    def __init__(self, conf_path=conf_path, encode="utf-8"):
        if os.path.exists(conf_path):
            self.__cfg_file = conf_path
        else:
            # 此处做其他异常处理或创建配置文件操作
            raise Exception
        self.__config = ConfigParser()
        self.__config.read(self.__cfg_file, encoding=encode)

    # 获取配置文件的所有section
    def get_sections(self):
        return self.__config.sections()

    # 获取指定section的所有option
    def get_options(self, section_name):
        if self.__config.has_section(section_name):
            return self.__config.options(section_name)
        else:
            raise Exception

    # 获取指定section下option的value值
    def get_option_value(self, section_name, option_name):
        if self.__config.has_option(section_name, option_name):
            return self.__config.get(section_name, option_name)

    # 获取指定section下的option的键值对
    def get_all_items(self, section):
        if self.__config.has_section(section):
            return self.__config.items(section)

    # 打印配置文件所有的值
    def print_all_items(self):
        for section in self.get_sections():
            print("[" + section + "]")
            for K, V in self.__config.items(section):
                print(K + "=" + V)

    # 增加section
    def add_new_section(self, new_section):
        if not self.__config.has_section(new_section):
            self.__config.add_section(new_section)
            self.__update_cfg_file()

    # 增加指定section下option
    def add_option(self, section_name, option_key, option_value):
        if self.__config.has_section(section_name):
            self.__config.set(section_name, option_key, option_value)
            self.__update_cfg_file()

    # 删除指定section
    def del_section(self, section_name):
        if self.__config.has_section(section_name):
            self.__config.remove_section(section_name)
            self.__update_cfg_file()

    # 删除指定section下的option
    def del_option(self, section_name, option_name):
        if self.__config.has_option(section_name, option_name):
            self.__config.remove_option(section_name, option_name)
            self.__update_cfg_file()

    # 更新指定section下的option的值
    def update_option_value(self, section_name, option_key, option_value):
        if self.__config.has_option(section_name, option_key):
            self.add_option(section_name, option_key, option_value)

    # 私有方法:操作配置文件的增删改时，更新配置文件的数据
    def __update_cfg_file(self):
        with open(self.__cfg_file, "w") as f:
            self.__config.write(f)

def getEmailOptionValues():
    config = Config()
    email_data = []
    email_username = config.get_option_value('email','username')
    email_authorization_code = config.get_option_value('email','authorization_code')
    email_data.append(email_username)
    email_data.append(email_authorization_code)
    return email_data

if __name__ == '__main__':
    print(getEmailOptionValues())


"""Task01 - HWResult"""
import datetime
from collections import defaultdict

class DeadlineError(Exception):
    """New type of Exception"""

class Person:
    """Initializes name and lastname of Person"""

    def __init__(self, last_name, first_name):
        self.last_name = last_name
        self.first_name = first_name

class Homework:
    """Describe text of hw and deadline"""

    def __init__(self, text, deadline):
        self.text = text
        self.deadline = datetime.timedelta(days=deadline)
        self.created = datetime.datetime.now()

    def is_active(self):
        return datetime.datetime.now() - self.created < self.deadline

class HomeworkResult:
    """Returns result of hw with info: who does hw, which hw has done, and what a solution of hw"""

    def __init__(self, student, homework, solution):
        self.author = student
        if not isinstance(homework, Homework):
            raise TypeError("You gave a not Homework object")
        self.homework = homework
        self.solution = solution
        self.created = datetime.datetime.now()

class Student(Person):
    """Does any homework and returns result of hw"""

    def do_homework(self, homework, solution):
        if homework.is_active():
            return HomeworkResult(self, homework, solution)
        raise DeadlineError("You are late")

class Teacher(Person):
    """Gives any homework, checks it, saves it in homework_done or delete it from homework_done"""

    homework_done = defaultdict(set)

    @classmethod
    def create_homework(cls, text, days):
        return Homework(text, days)

    @classmethod
    def check_homework(cls, homework_result):
        if len(homework_result.solution) > 5:
            cls.homework_done[homework_result.homework].add(homework_result)
            return True
        return False

    @classmethod
    def reset_results(cls, homework=None):
        if not isinstance(homework, Homework):
            cls.homework_done.clear()
        else:
            del cls.homework_done[homework]

if __name__ == "__main__":
    opp_teacher = Teacher("Shadrin", "Daniil")
    advanced_python_teacher = Teacher("Smetanin", "Aleksandr")

    lazy_student = Student("Petrov", "Roman")
    good_student = Student("Sokolov", "Lev")

    oop_hw = opp_teacher.create_homework("Learn OOP", 1)
    docs_hw = opp_teacher.create_homework("Read docs", 5)

    result_1 = good_student.do_homework(oop_hw, "I have done this hw")
    result_2 = good_student.do_homework(docs_hw, "I have done this hw too")
    result_3 = lazy_student.do_homework(docs_hw, "done")
    try:
        result_4 = HomeworkResult(good_student, "fff", "Solution")
    except Exception:
        print("There was an exception here")
    opp_teacher.check_homework(result_1)
    temp_1 = opp_teacher.homework_done

    advanced_python_teacher.check_homework(result_1)
    temp_2 = Teacher.homework_done
    assert temp_1 == temp_2

    print(opp_teacher.check_homework(result_2))
    print(opp_teacher.check_homework(result_3))

    print(Teacher.homework_done[oop_hw])
    Teacher.reset_results()