# -*- coding: UTF-8 -*-

import re, json, hashlib, datetime

import lara.nlp

# Intents Class
class Intents:

	# STATIC REGULAR EXPRESSIONS
	prefixes			= r'(?:(?i)'+('|'.join(["abba","alá","át","be","bele","benn","el","ellen","elő","fel","föl","hátra","hozzá","ide","ki","körül","le","meg","mellé","neki","oda","össze","rá","szét","túl","utána","vissza"]))+')?'
	typo_prefixes		= r'(?:(?i)'+('|'.join(["aba","ala","at","be","bele","ben","el","elen","elo","fel","fol","hatra","hoza","ide","ki","korul","le","meg","mele","neki","oda","osze","ra","szet","tul","utana","visza"]))+')?'
	pattern_noun		= r'(?i)a?i?n?(?:[aáeéioóöőuúü]?[djknmrst])?(?:[abjhkntv]?[aáeéioóöőuúü]?[lgkntz]?)?(?:[ae][kt])?'
	typo_pattern_noun	= r'(?i)a?i?n?(?:[aeiou]?[djknmrst])?(?:[abjhkntv]?[aeiou]?[lgkntz]?)?(?:[ae][kt])?'
	pattern_adj			= r'(?i)(?:[aeoóöő]?s)?(?:[aáeéoó]?b{0,2})(?:[ae]?[nk])?(?:(?:[aáeéioóöőuúü]?[dklmnt])?(?:[aáeéioóöőuúü]?[klnt]?)?)'
	typo_pattern_adj	= r'(?i)(?:[aeo]?s)?(?:[aeo]?b?)(?:[ae]?[nk])?(?:(?:[aeiou]?[dklmnt])?(?:[aeiou]?[klnt]?)?)'
	pattern_verb		= r'(?i)(?:h[ae][st])?(?:[eaá]?s{0,2}d?|[aáeéo]tt)?(?:(?:[jntv]|[eo]?g[ae]t+)?(?:[aeioöuü]n?[dklmt]|n[aáeéi]k?|sz|[aái])?(?:t[aáeéou][dkmt]?(?:ok)?)?)?(?:(?:t[ae]t)?(?:h[ae]t(?:[jnt]?[aáeéou](?:[dkm]|t[eéo]k)?)?t*)|[aáeé]?z?ni)?'
	typo_pattern_verb	= r'(?i)(?:h[ae][st])?(?:[eaá]?s?d?|[aeo]t)?(?:(?:[jntv]|[eo]?g[ae]t)?(?:[aeiou]n?[dklmt]|n[aei]k?|sz|[ai])?(?:t[aeou][dkmt]?(?:ok)?)?)?(?:(?:t[ae]t)?(?:h[ae]t(?:[jnt]?[aeou](?:[dkm]|t[eo]k)?)?t?)|[ae]?z?ni)?'

	##### CONSTRUCTOR #####
	def __init__(self, new_intents={}, is_raw=False):
		self.intents	= {}
		if not isinstance(new_intents, str) and not isinstance(new_intents, dict) and self.__class__.__name__ == new_intents.__class__.__name__:
			raise ValueError('Unsupported value for Intents.')
		if new_intents:
			if is_raw:
				self.raw(new_intents)
			else:
				self.add(new_intents)

	##### DATA MODEL #####
	def __repr__(self):
		return "<Lara Intents Parser instance at {0}>".format(hex(id(self)))

	def __str__(self):
		return json.dumps(self.intents)

	def __len__(self):
		return len(self.intents.keys())

	def __eq__(self, other):
		if self.__class__.__name__ == other.__class__.__name__:
			return (self.intents==other.intents)
		elif isinstance(other, bool):
			return (len(self.intents)!=0)==other
		return False

	def __ne__(self, other):
		return not self.__eq__(other)

	def __add__(self, other):
		if other:
			tmp = Intents(self.intents)
			if self.__class__.__name__ == other.__class__.__name__:
				tmp.add(other.intents)
			elif isinstance(other,dict):
				tmp.add(other)
			return tmp
		return self

	##### CLASS FUNCTIONS #####

	# Add dict of intents
	def add(self, new_intents={}):
		for key, value in new_intents.items():
			if key not in self.intents:
				self.intents[key]	= []
			for item in new_intents[key]:
				if item not in self.intents[key]:
					item	= self._generate(item)
					if item not in self.intents[key]:
						self.intents[key].append(item)

	# Add (previously cached) raw intents without further optimization
	def raw(self, new_intents):
		if new_intents:
			if isinstance(new_intents, str):
				new_intents	= json.loads(new_intents)
			elif new_intents.__class__.__name__=='Intents':
				new_intents	= json.loads(str(new_intents))
			if isinstance(new_intents,dict):
				self.intents	= new_intents.copy()
			else:
				raise ValueError('Unsupported value: %s' % (new_intents))

	# Add default values and fill in optional parameters for a single intent
	def _generate(self, item):
		if 'stem' not in item:
			raise KeyError('Intent declaration missing compulsory "stem" key.')
		if not item['stem'] or not isinstance(item['stem'], str):
			raise ValueError('Invalid value for "stem".')

		if 'wordclass' not in item:
			item['wordclass']		= 'special'
		elif item['wordclass'] not in ('special','noun','verb','adjective','regex','emoji'):
			if item['wordclass']=='ADJ':
				item['wordclass']	= 'adjective'
			elif isinstance(item['wordclass'],str) and item['wordclass'].lower() in ('noun','verb'):
				item['wordclass']	= item['wordclass'].lower()
			else:
				item['wordclass']	= 'special'
		if 'typo_stem' not in item:
			if item['wordclass'] in ('regex','emoji'):
				item['typo_stem']	= item['stem']
			else:
				item['typo_stem']	= lara.nlp.trim(lara.nlp.strip_accents(lara.nlp.remove_double_letters(item['stem'])))

		if 'prefix' not in item:
			if item['wordclass']	== 'verb':
				item['prefix']		= r'(?i)'+Intents.prefixes
				item['typo_prefix']	= r'(?i)'+Intents.typo_prefixes
			elif item['wordclass']	== 'adjective':
				item['prefix']		= r'(?i)(?:leg(?:esleg)?)?'
				item['typo_prefix']	= r'(?i)(?:leg(?:esleg)?)?'
			else:
				item['prefix']		= r''
				item['typo_prefix']	= r''
		elif not item['prefix']:
			item['prefix']		= r''
			item['typo_prefix']	= r''
		else:
			if 'typo_prefix' not in item:
				if isinstance(item['prefix'],list):
					item['typo_prefix']	= r'(?:'+lara.nlp.strip_accents('|'.join(item['prefix']))+')?'
				else:
					item['typo_prefix']	= r''+lara.nlp.strip_accents(item['prefix'])
			else:
				if isinstance(item['typo_prefix'],list):
					item['typo_prefix']	=  [re.escape(prefix) for prefix in item['typo_prefix']]
					item['typo_prefix']	= r'(?:'+('|'.join(item['typo_prefix']))+')?' #prefix?
				else:
					item['typo_prefix']	= r''+(item['typo_prefix'])
			if isinstance(item['prefix'],list):
				item['prefix']		= r'(?:'+('|'.join(item['prefix']))+')?'
			else:
				item['prefix']		= r''+(item['prefix'])

		if 'affix' not in item or not item['affix']:
			item['affix']		= r''
			item['typo_affix']	= r''
		else:
			if 'typo_affix' not in item:
				if isinstance(item['affix'],list):
					item['typo_affix']	= r'(?:'+lara.nlp.strip_accents('|'.join(item['affix']))+')?'
				else:
					item['typo_affix']	= r''+lara.nlp.strip_accents(item['affix'])
			else:
				if isinstance(item['typo_affix'],list):
					item['typo_affix']	=  [re.escape(affix) for affix in item['typo_affix']]
					item['typo_affix']	= r'(?:'+('|'.join(item['typo_affix']))+')?'
				else:
					item['typo_affix']	= r''+(item['typo_affix'])
			if isinstance(item['affix'],list):
				item['affix']		= r'(?:'+('|'.join([affix+'{1,2}' for affix in item['affix']]))+')?'
			else:
				item['affix']		= r''+(item['affix'])

		if 'match_stem' not in item:
			item['match_stem']	= True
		if 'ignorecase' not in item:
			item['ignorecase']	= True
		if 'boundary' not in item:
			if item['wordclass']=='emoji':
				item['boundary']	= False
			else:
				item['boundary']	= True

		if 'with' in item:
			if 'score' not in item:
				item['score']		= 0
			new_items	= []
			for sub_item in item['with']:
				sub_item	= self._generate(sub_item)
				if sub_item not in new_items:
					new_items.append(sub_item)
			item['with']	= new_items[:]
		else:
			item['with']	= []
			if 'score' not in item:
				item['score']		= 1
		if 'without' in item:
			new_items	= []
			for sub_item in item['without']:
				sub_item	= self._generate(sub_item)
				if sub_item not in new_items:
					new_items.append(sub_item)
			item['without']	= new_items[:]
		else:
			item['without']	= []

		if 'typo_score' not in item:
			item['typo_score']= item['score']

		if item['score']<0:
			raise ValueError('Value of "score" can not be less than 0')
		if item['typo_score']<0:
			raise ValueError('Value of "typo_score" can not be less than 0')

		# cache pattern
		if item['wordclass'] in ('regex','emoji'):
			item['pattern']			= r''+item['stem']+item['affix']
			item['typo_pattern']	= r''+item['typo_stem']+item['typo_affix']
		else:

			item['pattern']			= r'(?:'+re.escape(item['stem'])+r'{1,2}'+item['affix']+r')'
			scramble				= item['typo_stem']
			if len(scramble)>3:
				typo		= [scramble[1:-1]]
				for i in range(len(scramble)-3):
					typo.append(re.escape(scramble[1:i+1]+scramble[i+2]+scramble[i+1]+scramble[i+3:-1]))
				is_consonant= lara.nlp.is_consonant(scramble[-1])
				scramble	= re.escape(scramble[0])+'(?:'+('|'.join(typo))+')(?:'+re.escape(scramble[-1])
				scramble	= '[\s\-]?'.join(scramble.split('\ '))
				if item['wordclass'] == 'adjective' and is_consonant:
					scramble	+= '|[bB])'
				else:
					scramble	+=')'
			else:
				scramble				= re.escape(scramble)
				scramble	= '[\s\-]?'.join(scramble.split('\ '))
			item['typo_pattern']	= r'(?:'+scramble+item['typo_affix']+')'
			if not item['ignorecase']:
				item['pattern']			= r'(?s)'+item['pattern']
				item['typo_pattern']	= r'(?s)'+item['typo_pattern']

			if item['wordclass'] == 'noun':
				item['pattern']			+= Intents.pattern_noun
				item['typo_pattern']	+= Intents.typo_pattern_noun
			elif item['wordclass'] == 'adjective':
				item['pattern']			+= Intents.pattern_adj
				item['typo_pattern']	+= Intents.typo_pattern_adj
			elif item['wordclass'] == 'verb':
				item['pattern']			+= Intents.pattern_verb
				item['typo_pattern']	+= Intents.typo_pattern_verb

		item['pattern']			= item['prefix']+item['pattern']
		item['typo_pattern']	= item['typo_prefix']+item['typo_pattern']

		return item

	# Get all matches from text
	def match(self, text="", zeros=False):
		if text:
			score		= self._get_score(text)
			final_score	= {}
			for key, value in score.items():
				if value or zeros:
					final_score[key]	= value
			return final_score
		else:
			return {}

	# Get set of matches from text
	def match_set(self, text=""):
		if text:
			score		= self._get_score(text,False)
			final_score	= {}
			for key, value in score.items():
				if value:
					final_score[key]	= value
			return set(list(final_score.keys()))
		else:
			return set()

	# Remove matches from text
	def clean(self, text=""):
		if text:
			return self._get_clean_text(text)
		else:
			return ""

	# Returns text without the inflected forms of matched intents
	def _get_clean_text(self, text):
		text		= lara.nlp.trim(text)
		typo_text	= lara.nlp.strip_accents(lara.nlp.remove_double_letters(text))
		fix_text	= text
		if text:
			for key, value in self.intents.items():
				ignore	= False
				allow	= -1
				for item in self.intents[key]:
					if 'without' in item and len(item['without']):
						for without in item['without']:
							if 'stem' in without and self._match_pattern(text,without)[0]:
								ignore	= True
							elif 'typo_stem' in without and self._match_pattern(typo_text,without,True)[0]:
								ignore	= True
					if 'with' in item and len(item['with']):
						if allow == -1:
							allow	= 0
						for with_ in item['with']:
							if 'stem' in with_ and self._match_pattern(text,with_)[0]:
								allow	= 1
							elif 'typo_stem' in with_ and self._match_pattern(typo_text,with_,True)[0]:
								allow	= 1
				if not ignore and allow in (-1,1):
					for item in self.intents[key]:
						if 'stem' in item:
							fix_text		= self._match_pattern(fix_text,item,False,True)
						if 'typo_stem' in item:
							fix_text		= self._match_pattern(fix_text,item,True,True)
		return fix_text

	# Get score for intents in text
	def _get_score(self, text, greedy=True):
		text		= lara.nlp.trim(text)
		score		= {}
		if text:
			typo_text	= lara.nlp.strip_accents(lara.nlp.remove_double_letters(text))
			for key, value in self.intents.items():
				for item in self.intents[key]:
					found	= False

					result		= self._match_pattern(text,item)
					found		= found or result[0]
					if key not in score:
						score[key]	= 0
					score[key]	+= result[1]
					result		= self._match_pattern(typo_text,item,True)
					found		= found or result[0]
					if key not in score:
						score[key]	= 0
					score[key]	+= result[1]

					if found and item['with']:
						for sub_item in item['with']:
							if key not in score:
								score[key]	= 0
							found	= self._match_pattern(text,sub_item)
							if found[0]:
								score[key]	+=found[1]
							found	= self._match_pattern(typo_text,sub_item,True)
							if found[0]:
								score[key]	+=found[1]

					if found and item['without']:
						if key in score and score[key]:
							for sub_item in item['without']:
								if self._match_pattern(text,sub_item)[0]:
									score[key]	= 0
								elif self._match_pattern(typo_text,sub_item,True)[0]:
									score[key]	= 0

					if not greedy and key in score and score[key] > 0:
						break
		return score

	# Find an intent in text
	def _match_pattern(self, text, item, is_clean=False, delete=False):
		if text:
			if is_clean:
				select		= 'typo_'
			else:
				select		= ''
			if item['boundary']:
				boundary	= r'\b'
			else:
				boundary	= r''

			if item['ignorecase']:
				matches	= _re.findall(boundary+r'('+item[select+'pattern']+r')'+boundary,re.IGNORECASE,text)
			else:
				matches	= _re.findall(boundary+r'('+item[select+'pattern']+r')'+boundary,None,text)

			if matches:
				if delete:
					tmp	= text
					for match in matches:
						if not isinstance(match,str):
							match	= match[0]
						if item['match_stem'] or (item['ignorecase'] and match.lower() != item[select+'stem'].lower()) or (match.lower() != item[select+'stem']):
							tmp	= _re.sub(boundary+r'('+re.escape(match)+r')'+boundary,re.IGNORECASE,'',tmp)
					return tmp
				else:
					if not item['match_stem']:
						if item['ignorecase']:
							stem_matches	= _re.findall(boundary+r'('+re.escape(item[select+'stem'])+r')'+boundary,re.IGNORECASE,text)
						else:
							stem_matches	= _re.findall(boundary+r'('+re.escape(item[select+'stem'])+r')'+boundary,None,text)
						if stem_matches:
							if len(matches) <= len(stem_matches):
								return (False, 0)
							return (True,(len(matches)-len(stem_matches))*item[select+'score'])
					return (True,len(matches)*item[select+'score'])
		if delete:
			return text
		return (False,0)

	# Returns dictionary with N best matching intents with the highest value
	def match_best(self, text, n=1):
		if text:
			score	= self.match(text)
			if score:
				best_candidates	= sorted(score, key=score.get, reverse=True)
				best_candidates	= best_candidates[:(min(len(best_candidates),n))]
				return {item:score[item] for item in best_candidates}
		return {}

# Extract Class
class Extract:

	##### CONSTRUCTOR #####
	def __init__(self, text=''):
		if not isinstance(text, str):
			raise ValueError('Constructor only accepts strings.')
		elif text:
			self.text	= text
			self.ntext	= self._convert_numbers(self.text)
			self._text_	= ' '+self.text+' '		# some complex regular expressions were easier to write for padded text
			self._ntext_= ' '+self.ntext+' '	# some complex regular expressions were easier to write for padded text

	##### DATA MODEL #####
	def __repr__(self):
		return "<Lara Extract Parser instance at {0}>".format(hex(id(self)))

	def __str__(self):
		return self.text

	def __len__(self):
		return len(self.text)

	def __eq__(self, other):
		if self.__class__.__name__ == other.__class__.__name__:
			return (self.text==other.text)
		elif isinstance(other, bool):
			return (len(self.text)!=0)==other
		elif isinstance(other, str):
			return self.text==other
		return False

	def __ne__(self, other):
		return not self.__eq__(other)

	def __add__(self, other):
		if other:
			if self.__class__.__name__ == other.__class__.__name__:
				self.text	+= other.text
			elif isinstance(other,str):
				self.text	+= other
			self._text_	= ' '+self.text+' '
			self.ntext	= self._convert_numbers(self.text)
			self._ntext_= ' '+self.ntext+' '
			return self
		return self

	##### CLASS FUNCTIONS #####

	# extract list #hashtags from text
	def hashtags(self,normalize=True):
		if self.text:
			matches	= _re.findall(r'#([\w\d]+(?:[\w\d_\-\']+[\w\d]+)+)\b', None, self.text)
			if normalize:
				return ['#{0}'.format(hashtag.lower()) for hashtag in matches]
			else:
				return ['#{0}'.format(hashtag) for hashtag in matches]
		return []

	# extract list of @hashtags from text
	def mentions(self):
		if self.text:
			return _re.findall(r'(?<![\w\d\_])(\@[\w\d_]+(?:[\w\d_\-\'\.]+[\w\d_]+)+)\b', None, self._text_)
		return []

	# extract list of http://urls/ from text
	def urls(self):
		if self.text:
			return _re.findall(r'\b((?:https?\:[\/\\]{2}(?:w{3}\.)?|(?:w{3}\.))(?:[\w\d_\-]+\.\w{2,})(?:[\/\\](?:[\w\d\-_]+[\/\\]?)*)?(?:\?[^\s]*)?(?:\#[^\s]+)?)', re.IGNORECASE, self.text)
		return []

	# extract list of smileys :) from text
	def smileys(self):
		if self.text:
			return _re.findall(r'(?:[\:\;\=]\-*[DdXxCc\|\[\]\(\)3]+[89]*)|(?:[\(\)D\[\]\|]+\-*[\:\;\=])', None, self.text)
		return []

	# extract digits with n places
	def digits(self,n=0,normalize=True,convert=True):
		results	= []
		if self.text:
			matches	= _re.findall(r'((?:\d[\-\.\,\s]?)+)', re.IGNORECASE, self.ntext if convert else self.text)
			for item in matches:
				original= item
				item	= lara.nlp.trim(''.join(e for e in item if e.isdigit()))
				if n<=0 or len(item)==n:
					if normalize:
						results.append(item)
					else:
						results.append(original.strip())
		return results

	# extract (decimal) numbers
	def numbers(self,decimals=True,convert=True):
		if self.text:
			if decimals:
				matches	= _re.findall(r'((?:-\s?)?(?:\d\s?)+(?:[\.\,]\d+[^\.\,])?)', re.IGNORECASE, self._ntext_ if convert else self._text_)
				return [float(''.join(number.replace(',','.').split())) for number in matches]
			else:
				matches	= _re.findall(r'(?<![\.\,\d])(\-?(?:\d\s?)+(?![\.\,]\d))\D', re.IGNORECASE, self._ntext_ if convert else self._text_)
				return [int(''.join(number.split())) for number in matches]
		return []

	# extract percentages
	def percentages(self,normalize=True):
		if self.text:
			if normalize:
				matches	= _re.findall(r'((?:\d+(?:[\,\.]\d+)?|[\,\.]\d+))\s?(?:\%|sz[aá]zal[eé]k)', re.IGNORECASE, self.text)
				results	= []
				for item in matches:
					item = item.replace(',','.')
					if item.startswith('.'):
						item='0'+item
					if '.' in item:
						places=len(item.split('.')[1])+2
					else:
						places=2
					item = str("{:."+str(places)+"f}").format(float(item)/100.00)
					results.append(float(item))
				return results
			else:
				return _re.findall(r'((?:\d+(?:[\,\.]\d+)?|[\,\.]\d+)\s?(?:\%|sz[aá]zal[eé]k))', re.IGNORECASE, self.text)
		return []

	# extract phone numbers
	def phone_numbers(self,normalize=True,convert=True):
		results	= []
		if self.text:
			matches	= _re.findall(r'((?:\(?(?:\+36|0036|06)[\s\-\\\/]?)?\(?\d{1,2}\)?[\s\-\\\/]?\d(?:\d[\s\-\\\/]?){5}\d)', re.IGNORECASE, self.ntext if convert else self.text)
			if not normalize:
				return matches
			for item in matches:
				item	= lara.nlp.trim(''.join(e for e in item if e.isdigit()))
				if item.startswith('36') or item.startswith('06'):
					item	= item[2:]
				elif item.startswith('0036'):
					item	= item[4:]
				if len(item)==8:
					item	= item[0]+' '+item[1:]
				else:
					item	= item[0:2]+' '+item[2:]
				results.append('+36 '+item)
		return results

	# extract list of common Hungarian date formats from text without further processing them
	def dates(self,normalize=True,convert=True):
		results	= []
		if self.text:
			now = datetime.datetime.now()
			matches	= _re.findall(r'((\d{2})?(\d{2}([\\\/\.\-]\s?|\s))([eé]v\s?)?(\d{1,2}([\\\/\.\-]\s?|\s)(h[oó](nap)?\s?)?)?(\d{1,2}))\W*([aáeéio][ikn]|nap)?\b', re.IGNORECASE, self.text)
			for item in matches:
				match	= re.sub('([eé]v|h[oó]|nap)', '', item[0])
				parts	= list(filter(None,re.split(r'\W', match+'-')))
				if len(parts)==3:
					if int(parts[1])<=12:
						if normalize:
							if len(parts[0])==4:
								results.append(parts[0]+'-'+parts[1].zfill(2)+'-'+parts[2].zfill(2))
							else:
								results.append('20'+parts[0]+'-'+parts[1].zfill(2)+'-'+parts[2].zfill(2))
						else:
							results.append(item[0])
				elif len(parts)==2:
					if normalize:
						if len(parts[0])==4:
							results.append(parts[0]+'-'+parts[1].zfill(2)+'-??')
						elif int(parts[0])>12:
							results.append('20'+parts[0]+'-'+parts[1].zfill(2)+'-??')
						else:
							results.append(str(now.year)+'-'+parts[0].zfill(2)+'-'+parts[1].zfill(2))
					else:
						results.append(item[0])
			matches	= _re.findall(r'((\d{2}(\d{2})?)?\W{1,2}(jan|feb|m[aá]r|[aá]pr|m[aá]j|j[uú][nl]|aug|sz?ep|okt|nov|dec|[ivx]+\W{0,2})\w{0,10}(\W{1,2}h[aoó][nv]?\w{0,7})?(\W{1,2}\d{1,2})?)\b', re.IGNORECASE, self.ntext if convert else self.text)
			for item in matches:
				match	= item[0].lower()
				year	= ''
				day		= ''
				switch	= False
				for char in match:
					if switch:
						if char.isdigit():
							day		+=char
					else:
						if char.isdigit():
							year	+=char
						else:
							switch	= True
				if not year and not day:
					continue
				if not year:
					year	= str(now.year)
				elif len(year)==2:
					year	= '20'+year
				if not day:
					day		= '??'
				elif len(day)==1:
					day		= '0'+day
				month	= ''
				if 'jan' in match:
					month	= '01'
				elif 'feb' in match:
					month	= '02'
				elif 'mar' in match or 'már' in match:
					month	= '03'
				elif 'apr' in match or 'ápr' in match:
					month	= '04'
				elif 'maj' in match or 'máj' in match:
					month	= '05'
				elif 'jun' in match or 'jún' in match:
					month	= '06'
				elif 'jul' in match or 'júl' in match:
					month	= '07'
				elif 'aug' in match:
					month	= '08'
				elif 'sep' in match or 'szep' in match:
					month	= '09'
				elif 'okt' in match:
					month	= '10'
				elif 'nov' in match:
					month	= '11'
				elif 'dec' in match:
					month	= '12'
				else:
					roman	= ''
					for char in match:
						if char in ('i','v','x'):
							roman	+= char
						elif char.isalpha():
							roman	= ''
							break
					if not roman:
						continue
					if 'v' in roman:
						if roman.startswith('v'):
							month	= str(4+len(roman)).zfill(2)
						else:
							month	= str(6-len(roman)).zfill(2)
					elif 'x' in roman:
						if roman.startswith('x'):
							month	= str(9+len(roman)).zfill(2)
						else:
							month	= str(11-len(roman)).zfill(2)
					else:
						month	= str(len(roman)).zfill(2)
				if month and month!='00':
					if normalize:
						results.append(year+'-'+month+'-'+day)
					else:
						results.append(item[0])
		return results

	# extract times like 12:00 or délután 4
	def times(self,normalize=True,convert=True,current=-1):
		if self.text:
			matches	= _re.findall(r'((?:ma\s?|holnap(?:\s?ut[aá]n)?\s?|tegnap(?:\s?el[oöő]t+)?\s?)?(?:reggel\s?|hajnal(?:i|ban)?\s?|d[eé]lel[oöő]t+\s?|d\.?e\.?\s?|d[eé]lut[aá]n\s?|d\.?u\.?\s?|este\s?|[eé]j+el\s?)?\,?\s?(?:[12345]?\d\s?perc+el\s)?(?:(?:h[aá]rom)?negyed\s?|f[eé]l\s?)?[012]?\d\s?(?:\:\s?|\-?kor\s?|[oó]r[aá]\w{0,3}\s?)?(?:el[oöő]t+\s?|ut[aá]n\s?)?(?:[0123456]?\d[\-\s]?(?:kor|perc\w{0,3})?)?\,?\s?(?:ma\s?|holnap(?:\s?ut[aá]n)?\s?|tegnap(?:\s?el[oöő]t+)?\s?)?(?:reggel\s?|hajnal(?:i|ban)?\s?|d[eé]lel[oöő]t+\s?|d\.?e\.?\s?|d[eé]lut[aá]n\s?|d\.?u\.?\s?|este\s?|[eé]j+el\s?)?)', re.IGNORECASE, self._ntext_ if convert else self._text_)
			if normalize:
				results	= []
				for item in matches:
					if len(item.strip())>2:
						item	= ' '+item+' '
						hour	= "00"
						minute	= "00"
						pm		= False
						zero	= False
						elott	= False
						hour_matches 	= _re.findall(r'\D([012]?\d(?!\d))\D*?(?!perc)(?:\:|\-?kor|[oó]r[aá])?', re.IGNORECASE, item)
						minute_matches 	= _re.findall(r'(?!negyed|f[eé]l)\D([0123456]?\d(?!\d))\D*?(?![oó]ra)(?:\-?kor|perc)?', re.IGNORECASE, item)
						quarter_matches	= _re.findall(r'((?:h[aá]rom)?negyed|f[eé]l)', re.IGNORECASE, item)
						am_matches		= _re.findall(r'(reggel|hajnal|d[eé]lel[oöő]t|d\.?e\.?)', re.IGNORECASE, item)
						pm_matches		= _re.findall(r'(d[eé]lut[aá]n|d\.?u\.?|este|[eé]j+el)', re.IGNORECASE, item)
						if len(hour_matches) in (1,2):
							if len(hour_matches)==1:
								if len(minute_matches)==1:
									hour	= (hour_matches[0])
									minute	= "00"
								elif len(minute_matches)==2:
									if (hour_matches[0])==(minute_matches[0]):
										hour	= (hour_matches[0])
										minute	= (minute_matches[1])
									else:
										hour	= (hour_matches[0])
										minute	= (minute_matches[0])
							else:
								if len(minute_matches) == 2:
									if (hour_matches[0])==(minute_matches[1]):
										hour	= (hour_matches[0])
										minute	= (minute_matches[0])
									else:
										hour	= (hour_matches[0])
										minute	= (minute_matches[1])
								elif len(minute_matches) == 1:
									if (hour_matches[0])==(minute_matches[0]):
										hour	= (hour_matches[1])
										minute	= "00"
									else:
										hour	= (hour_matches[0])
										minute	= (minute_matches[0])
								else:
									hour	= (hour_matches[0])
									if len(hour_matches) == 2:
										minute	= (hour_matches[1])
							if hour[0]=='0':
								zero	= True
							hour	= int(hour)
							minute	= int(minute)
							if hour>24 and minute<24:
								minute, hour	= hour, minute
							if minute>60:
								minute	= 0
							if _re.findall(r'(el[oöő]t+)', re.IGNORECASE, item):
								if minute:
									if not _re.findall(r'(el[oöő]t+.+?perc)', re.IGNORECASE, item):
										hour, minute	= minute, hour
									elott	= True
									hour	-= 1
									minute	= 60-minute
							if _re.findall(r'(perccel.+?ut[aá]n+)', re.IGNORECASE, item):
								hour, minute	= minute, hour
								hour	= hour
							if quarter_matches:
								if quarter_matches[0] in ('fel','fél'):
									if not elott:
										hour	-= 1
									minute	+= 30
								elif quarter_matches[0] in ('haromnegyed','háromnegyed'):
									if not elott:
										hour	-= 1
									minute	+= 45
								elif quarter_matches[0] in ('negyed'):
									if not elott:
										hour	-= 1
									minute	+= 15
							if not zero:
								if pm_matches:
									pm	= True
								elif not am_matches:
									if current:
										if current>=0:
											now	= current
										else:
											now	= datetime.datetime.now().hour
										if 'holnap' in item and hour<9:
											pm = True
										elif hour<12 and now>hour:
											pm = True
								if pm and hour<=12:
									hour	+= 12
							hour	%= 24
							minute	%= 60

							results.append(str(hour).zfill(2)+':'+str(minute).zfill(2))
				return results
			else:
				return [item.strip() for item in matches if len(item.strip())>2]
		return []

	# extract list of time durations TODO: 3 óra 2 perc -> [3h+2s] instead of [3h,2s]
	def durations(self,normalize=True,convert=True):
		if self.text:
			matches	= _re.findall(r'\b((?:(?:(?:\d\s?)+(?:[\.\,]\d+)?\s(?:(?:[eé]s\s)?(?:f[eé]l|(?:h[aá]rom)?negyed)\s)?(?:(?:(?:t[ií]zed|sz[aá]zad|ezred)?m[aá]sod)?perc\w{0,3}|[oó]r[aá]\w{0,3}|nap\w{0,3}|7|h[eé]t\w{0,3}|h[oó]nap\w{0,3}|[eé]v\w{0,3})(?:\s(?:m[uú]lva|r[aá]|(?:ez)?el[oöő]t+|el[oöő]b+|k[eé]s[oö]b+|bel[uü]l|h[aá]tr(?:a|[eé]bb)|vissza|el[oöő]re))?)(?:\W*(?:[eé]s|meg)\W*)?)+)', re.IGNORECASE, self.ntext if convert else self.text)
			if normalize:
				results	= []
				now = datetime.datetime.now()
				for item in matches:
					sub_matches	= _re.findall(r'\b((?:(?:\d\s?)+(?:[\.\,]\d+)?\s(?:(?:[eé]s\s)?(?:f[eé]l|(?:h[aá]rom)?negyed)\s)?(?:(?:(?:t[ií]zed|sz[aá]zad|ezred)?m[aá]sod)?perc\w{0,3}|[oó]r[aá]\w{0,3}|nap\w{0,3}|7|h[eé]t\w{0,3}|h[oó]nap\w{0,3}|[eé]v\w{0,3})(?:\s(?:m[uú]lva|r[aá]|(?:ez)?el[oöő]t+|el[oöő]b+|k[eé]s[oö]b+|bel[uü]l|h[aá]tr(?:a|[eé]bb)|vissza|el[oöő]re))?))', re.IGNORECASE, item)
					val			= 0
					for sub_item in sub_matches:
						match	= sub_item.lower().replace(',','.')
						sval	= ''
						for char in match:
							if char.isdigit() or char=='.':
								sval+=char
							else:
								break
						sval	= float(sval)
						mpx		= 1
						if 'tized' in match or 'tízed' in match:
							mpx		= 0.1
						elif 'szazad' in match or 'század' in match:
							mpx		= 0.01
						elif 'ezred' in match:
							mpx		= 0.001
						elif 'masod' in match or 'másod' in match:
							mpx		= 1
						elif 'perc' in match:
							mpx		= 60
						elif 'or' in match or 'ór' in match:
							mpx		= 3600
						elif 'ho' in match or 'hó' in match:
							if now.month in (1,3,5,7,8,10,12):
								mpx		= 86400 * 31
							elif now.month==2:
								if now.year%400==0 or now.year%100==0 or now.year%4==0:
									mpx		= 86400 * 29
								else:
									mpx		= 86400 * 28
							else:
								mpx		= 86400 * 30
						elif 'nap' in match:
							mpx		= 86400
						elif 'het' in match or 'hét' in match or '7' in match:
							mpx		= 604800
						elif 'ev' in match or 'év' in match:
							if now.year%400==0 or now.year%100==0 or now.year%4==0:
								mpx		= 86400 * 366
							else:
								mpx		= 86400 * 365
						sval	*= mpx
						if 'fel' in match or 'fél' in match:
							sval		+= mpx*.5
						elif 'negyed' in match:
							if 'háromnegyed' in match or 'haromnegyed' in match:
								sval		+= mpx*.75
							else:
								sval		+= mpx*.25
						val	+= sval
					if _re.findall(r'(el[oöő]t+|el[oöő]b+|h[aá]tr(?:a|[eé]bb)|vissza)', re.IGNORECASE, self.text):
						val		*= -1
					results.append(val)
				return results
			else:
				return matches
		return []

	# extract list of common currencies from text (including $ € £ ￥ and forints)
	def currencies(self,normalize=True,convert=True):
		if self.text:
			matches	= _re.findall(r'((?:(?:\$|€|£|￥)\s?(?:\d(?:[\s\.,]\d)?)+)|(?:(?:\d(?:[\s\.,]\d)?)+\s?(?:\.\-|\$|€|£|￥|huf\b|ft\b|forint\w{0,3}|[jy]en\w{0,3}|font\w{0,3}|doll[aá]r\w{0,3}|eur[oó]?\w{0,3}|gbp\b|usd\b|jpy\b))(?:\,?\s(?:[eé]s\s)?\d+\s?(?:cent|fill[eé]r)\w{0,3})?)', re.IGNORECASE, self.ntext if convert else self.text)
			if normalize:
				results	= []
				for item in matches:
					if 'cent' in item or 'fill' in item:
						switch	= False
						amount	= ""
						cent	= ""
						for char in item:
							if char != ' ':
								if not switch and not char.isdigit():
									switch	= True
								elif not switch and char.isdigit():
									amount	+= char
								elif switch and char.isdigit():
									cent	+= char
						if not amount:
							amount	= "0"
						if not cent:
							cent	= "0"
					else:
						amount	= item
						cent	= "0"
					amount	= ''.join([e for e in amount.replace(',','.') if e.isdigit() or e=='.'])
					if amount[-1]=='.':
						amount	= amount[:-1]
					if '.' in amount:
						amount	= float(amount)
					else:
						amount	= float(amount)+float("0."+cent)
					currency= 'HUF'
					item	= item.lower()
					if '$' in item or 'doll' in item or 'usd' in item:
						currency= 'USD'
					elif '€' in item or 'eur' in item:
						currency= 'EUR'
					elif '£' in item or 'font' in item or 'gbp' in item:
						currency= 'GBP'
					elif '￥' in item or 'yen' in item or 'jen' in item or 'jpy' in item:
						currency= 'JPY'
					results.append(str(amount)+' '+currency)
				if not results:
					return [str(item)+' HUF' for item in self.numbers()]
				return results
			else:
				if not matches:
					return [str(item) for item in self.numbers()]
				return matches
		return []

	# extract commands and arguments from text: "/help lara" will return ('help',['lara'])
	def commands(self):
		if self.text and self.text[0] == '/':
			commands				= (lara.nlp.trim(str(self.text[1:]))).split(" ")
			if len(commands)>1:
				return (commands[0],commands[1:])
			else:
				return (commands[0],[])
		return ('',[])

	# extract list of emojis from text via https://gist.github.com/naotokui/
	def emojis(self):
		if self.text:
			return _re.findall("["
					   u"\U0001F600-\U0001F64F"  # emoticons
					   u"\U0001F300-\U0001F5FF"  # symbols & pictographs
					   u"\U0001F680-\U0001F6FF"  # transport & map symbols
					   u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
					   "]", re.UNICODE, self.text)
		return []

	# extract e-mail addresses
	def emails(self):
		if self.text:
			return _re.findall(r'\b([\w\d\-\_\.]+\@[\w\d\-\_\.]+\.\w{2,4}(?:\.\w{2,4})?)\b', re.IGNORECASE, self.text)
		return []

	# returns the approx. number of words in text
	def number_of_words(self):
		if self.text:
			return len(self.tokens())
		return 0

	# tokenizes text
	def tokenizer(self):
		if self.text:
			return lara.nlp.tokenizer(self.text)
		return []

	# True if word has a digit in it
	def hasDigits(self):
		return any(char.isdigit() for char in self.text)

	# Converts text representation of numbers to digits
	def _convert_numbers(self,text):
		if text:
			#fix		= _re.sub(r'(?<=\d)\s+(?=\d)',re.IGNORECASE,'',text.lower())
			matches	= _re.findall(r'((?:m[ií]n[uú]sz\s?|negat[ií]v\s?)?(?:(?:(?:(?:(?:t[ií]z|h[uú]sz|harminc)(?:[eo]n)?)?(?:nulla|egy|els[eoöő]|k[eé]t+[oöő]?|m[aá]sod(?:ik)?|h[aá]rom|harmadik|n[eé]gy|[oö]t|hat|h[eé]t|nyolc|kilenc)(?:v[ae]n)?)(?:milli[aá]rd|milli[oó]|ezer|sz[aá]z)?\W*)|(?:milli[aá]rd|milli[oó]|ezer|sz[aá]z|t[ií]z|h[uú]sz|harminc|nulla|z[eé]r[oó])\W*)+(?:[aeoö]dik)?(?:j?[aáeéi]+n?)?)\b', re.IGNORECASE, text.lower())
			results	= {}
			for match in matches:
				value	= 0
				minusc	= _re.findall(r'(m[ií]n[uú]sz\s?|negat[ií]v\s?)', re.IGNORECASE, match)
				if minusc:
					minus	= -1
					minusm	= minusc[0]
				else:
					minus	= 1
					minusm	= ''
				parts	= _re.findall(r'((?:(?:(?:(?:t[ií]z|h[uú]sz|harminc)(?:[eo]n)?)?(?:nulla|egy|els[eoöő]|k[eé]t+[oöő]?|m[aá]sod(?:ik)?|h[aá]rom|harmadik|n[eé]gy|[oö]t|hat|h[eé]t|nyolc|kilenc)(?:v[ae]n)?)(?:milli[aá]rd|milli[oó]|ezer|sz[aá]z)?|(?:milli[aá]rd|milli[oó]|ezer|sz[aá]z|t[ií]z|h[uú]sz|harminc|nulla|z[eé]r[oó]))\W*)', re.IGNORECASE, match)
				values	= []
				for part in parts:
					val			= 0
					scale		= 1
					if 'nulla' in part:
						val	= 0
					elif _re.findall(r'milli[aá]rd', re.IGNORECASE, part):
						scale	= 1000000000
					elif _re.findall(r'milli[oó]', re.IGNORECASE, part):
						scale	= 1000000
					elif _re.findall(r'ezer', re.IGNORECASE, part):
						if _re.findall(r'sz[aá]z', re.IGNORECASE, part):
							scale	= 100000
						elif _re.findall(r'v[ae]n', re.IGNORECASE, part):
							scale	= 10000
						else:
							scale	= 1000
					elif _re.findall(r'sz[aá]z', re.IGNORECASE, part):
						scale	= 100
					if _re.findall(r'v[ae]n', re.IGNORECASE, part):
						elem		= re.split('v[ae]n',part)
						if len(elem)==2:
							val		= self._convert_numbers_helper(elem[1],0)*10
							val		+= self._convert_numbers_helper(elem[0],0)
						else:
							val		= self._convert_numbers_helper(elem[0],0)
						if 'ezer' in part:
							scale	= max(scale,10)
						else:
							scale	= max(scale*10,10)
					else:
						if _re.findall(r't[ií]z', re.IGNORECASE, part):
							val		= 10
							scale	= max(scale,1)
						elif _re.findall(r'h[uú]sz', re.IGNORECASE, part):
							val		= 20
							scale	= max(scale,1)
						elif _re.findall(r'harminc', re.IGNORECASE, part):
							val		= 30
							scale	= max(scale,1)
						if 'nulla' not in part:
							if not _re.findall(r't[ií]z|h[uú]sz|harminc|v[ae]n', re.IGNORECASE, part):
								val		+= 	self._convert_numbers_helper(part,1)
							else:
								val		+= 	self._convert_numbers_helper(part,0)
					values.append([part,val*scale])
				number	= 0
				maxn	= 0
				name	= ''
				for n in reversed(values):
					old	 = name
					name = n[0] + name
					if n[1]<maxn:
						if len(str(n[1]))==3:
							if len(str(maxn))==4:
								n[1] 	*= int(pow(10,len(str(maxn))-1))
							else:
								n[1] 	*= int(pow(10,len(str(maxn))-2))
						else:
							n[1] 	*= int(pow(10,len(str(maxn))-len(str(n[1]))+1))
					elif len(str(n[1]))==len(str(maxn)):
						number	*=minus
						name	= old
						if self._convert_numbers_name(name):
							results[self._convert_numbers_name(minusm+name)]=number
						name	= n[0]
						number	= 0
					if not _re.findall(r'\b(milli[oó]|ezer)\b', re.IGNORECASE, n[0]):
						number	+= n[1]
					maxn	= pow(10,len(str(n[1]))-1)
				number	*=minus
				if self._convert_numbers_name(name):
					results[self._convert_numbers_name(minusm+name)]=number

			swap 	= sorted(results.items(), key=lambda x: x[1], reverse=True)
			for item in swap:
				text		= _re.sub(r'\b('+re.escape(item[0])+r')(?:[aeoö]dik?)?(?:j?[aáeéi]+n?)?\b', re.IGNORECASE, str(item[1]), text)
			return text
		return ''

	def _convert_numbers_name(self,name):
		newname=''
		for char in name:
			if char.isalnum() or char in (' ','.',',','-'):
				newname	+= char
		if newname and not newname[-1].isalnum():
			newname	= newname[:-1]
		return newname.strip()

	def _convert_numbers_helper(self,match,default):
		if _re.findall(r'(k[eé]t+[oöő]?|m[aá]sod(ik)?)', re.IGNORECASE, match):
			return 2
		elif _re.findall(r'(harmadik|h[aá]rom)', re.IGNORECASE, match):
			return 3
		elif _re.findall(r'n[eé]gy', re.IGNORECASE, match):
			return 4
		elif 'egy' in match or 'els' in match:
			return 1
		elif _re.findall(r'[oö]t', re.IGNORECASE, match):
			return 5
		elif _re.findall(r'hat', re.IGNORECASE, match):
			return 6
		elif _re.findall(r'h[eé]t', re.IGNORECASE, match):
			return 7
		elif _re.findall(r'nyolc', re.IGNORECASE, match):
			return 8
		elif _re.findall(r'kilenc', re.IGNORECASE, match):
			return 9
		return default

# Wrapper Class for Regular Expression Caching
class _re:

	compile_cache		= {}
	findall_cache		= {}

	# cache re.compile() outputs
	def compile(e_hash,expression,flags):
		flags_str	= str(flags)
		if flags_str not in _re.compile_cache:
			_re.compile_cache[flags_str]	= {}
		if e_hash not in _re.compile_cache[flags_str]:
			if flags:
				cache									= re.compile(r''+expression,flags)
				_re.compile_cache[flags_str][e_hash]	= cache
			else:
				cache									= re.compile(r''+expression)
				_re.compile_cache[flags_str][e_hash]	= cache

	# cache re.compile().findall() outputs
	def findall(expression,flags,text):
		e_hash			= hashlib.sha1(str(expression).encode("utf-8")).hexdigest()
		_re.compile(e_hash,expression,flags)
		t_hash			= hashlib.sha1(str(text).encode("utf-8")).hexdigest()
		flags_str		= str(flags)
		if flags_str not in _re.findall_cache:
			_re.findall_cache[flags_str]	= {}
		if t_hash not in _re.findall_cache[flags_str]:
			_re.findall_cache[flags_str][t_hash]	= {}
		if e_hash not in _re.findall_cache[flags_str][t_hash]:
			_re.findall_cache[flags_str][t_hash][e_hash]	= _re.compile_cache[flags_str][e_hash].findall(text)
		return _re.findall_cache[flags_str][t_hash][e_hash]

	# user cached re.compile() for re.sub()
	def sub(expression,flags,repl,string):
		e_hash			= hashlib.sha1(str(expression).encode("utf-8")).hexdigest()
		_re.compile(e_hash,expression,flags)
		flags_str		= str(flags)
		return _re.compile_cache[flags_str][e_hash].sub(repl,string)

	# clean cache
	def flush(self):
		_re.compile_cache	= {}
		_re.findall_cache	= {}


# Databricks notebook source
createTable =  """
  CREATE TABLE IF NOT EXISTS edw_lnd.inv_balance
  (
    HeaderFlag String,
    InterfaceActionFlag String,
    StoreCode String,
    SKUNbr String,
    Quantity Double,
    EDW_ACTV_FLG String,
    EDW_CRT_TS Timestamp,
    EDW_UPD_TS Timestamp,
    EDW_HASH_CHK String
  )
  USING DELTA
  LOCATION '{0}'
"""

# COMMAND ----------

dbutils.notebook.run("/Users/svceimdbrx@columbia.com/edw_admin/streaming/simplestreaming", 0,
                     {
                       "inputPath": "/mnt/entadls/native/internal/inventory/balance/landing/",
                       "schemaPath": "/mnt/entadls/native/internal/inventory/balance/landingschema/jsonschema.py",
                       "checkPointPath": "/mnt/entadls/native/internal/inventory/balance/checkpoint/_checkpoint",
                       "outputPath": "/mnt/entadls/native/internal/inventory/balance/delta/",
                       "maxFilesPerTrigger": "5",
                       "createSchema": """CREATE SCHEMA IF NOT EXISTS edw_lnd""",
                       "tableCreate": createTable
                     }
                    )

# COMMAND ----------

filepath="dbfs:/mnt/entadls/curated/internal/product/itochu/"
filedir="/dbfs/mnt/entadls/curated/internal/product/itochu/"
blob_file="/dbfs/mnt/eimastrg/bods/inv_bal_itochu.txt"

# COMMAND ----------

df=spark.sql(""" select
KUNNR,
SKU,
CREATE_DATE,
CREATE_TIME,
QUANTITY from (
SELECT
'RCN951612' as KUNNR,
SKUNbr as SKU,
date_format(current_timestamp,'yyyyMMdd') as CREATE_DATE,
date_format(current_timestamp,'HHmmss') as CREATE_TIME,
Quantity as QUANTITY,
row_number() over (partition by SKUNbr order by EDW_UPD_TS desc) as Rank
FROM edw_lnd.inv_balance
WHERE EDW_UPD_TS = (SELECT MAX(EDW_UPD_TS) FROM edw_lnd.inv_balance)
) where Rank=1
""")

# COMMAND ----------

#Generate file
import glob
import os
import time
import shutil
from datetime import datetime
import subprocess

filename=filedir+"inv_bal_itochu.txt"

outputCount = "{0} Records to be written to file".format(df.count())

print(outputCount)

df.coalesce(1).write.format('com.databricks.spark.csv').mode("overwrite").options(header="true",sep="|").option("treatEmptyValuesAsNulls", "false").option("quote", "\u0000").save(filepath+"inv_bal_itochu.tmp")

FileList=glob.glob(filedir+"inv_bal_itochu.tmp/*.csv")

print("file:"+FileList[0])

shutil.copyfile(FileList[0],filename)

shutil.rmtree(filedir+"inv_bal_itochu.tmp")

if os.stat(filename).st_size == 0:
  file = open(filename,'w')
  file.write("KUNNR|SKU|CREATE_DATE|CREATE_TIME|QUANTITY\n")
  file.close()

# COMMAND ----------

#Copy file to Blob Storage
shutil.copy(filename,blob_file)

# COMMAND ----------

# Create Archive Directory if not exists

if not os.path.exists(filedir+"Archive"):
    os.makedirs(filedir+"Archive")

# COMMAND ----------

#Archive Files and rename file
timestamp = datetime.now().strftime('%Y%m%d%H%M%S')
shutil.copy(filename,filedir+"Archive/inv_bal_itochu_"+timestamp+".txt")

# COMMAND ----------
# Delete Files from Archive Folder older than 90 days
now = time.time()
files = [os.path.join(filedir+"Archive/", filename) for filename in os.listdir(filedir+"Archive/")]
for filename in files:
    if (now - os.stat(filename).st_mtime) > 7776000:
        command = "rm {0}".format(filename)
        subprocess.call(command, shell=True)


import numpy as np
from yahoo_finance import Share
import copy
import transaction
import datetime
from dateutil import parser
import pprint

class Portfolio:
    """
    t = type Transaction

    Create new portfolio given old portfolio and a Transaction object. When
    last_portfolio is None, it means that our old portfolio was empty. Also,
    keep an instance variable start_date with the date of the transaction.
    """

    def __init__(self, t=None, last_portfolio=None, init_cash=None):
        if t is not None:
            # dictionary self.holdings: map from symbol (string) to qty (# shares)
            if last_portfolio is not None:
                self.holdings = copy.deepcopy(last_portfolio.holdings)
                if t.direction is not None:
                    qty = t.direction * t.quantity
                    if t.symbol in self.holdings:
                        self.holdings[t.symbol] += qty
                        if self.holdings[t.symbol] == 0:
                            del self.holdings[t.symbol]
                    else:
                        self.holdings[t.symbol] = qty
                        print("Shorting stock: {}".format(t.symbol) if t.direction == -1 else "Setting stock: {}".format(t.symbol))
                else:
                    self.holdings['cash'] += t.amt
            else: #this loop will never be activated with our current code
                print(t.direction, t.quantity, t.date)
                self.holdings = {}
                qty = t.direction * t.quantity
                self.holdings[t.symbol] = qty
                self.holdings['cash'] = init_cash
            if t.direction:
                comm = 0
                if t.comm:
                    comm = t.comm
                self.holdings['cash'] -= (qty * t.price + comm)
            self.start_date = t.date
        else:
            self.holdings = {}
            start = open('2015_start.csv', 'r')
            start_lines = start.readlines()
            for starts in start_lines:
                info = starts.split(',') #change to ints here
                if info[0] == 'cash':
                    self.holdings['cash'] = float(info[1])
                else:
                    self.holdings[info[0]] = int(info[1])
            # set the start date manually here
            self.start_date = parser.parse('12/31/2014')
            print(self.holdings)

    # storage for prices from yahoo finance
    # Prices is a dictionary of dictionaries; each inner dictionary consists of (date, value) pairs for a ticker
    prices = {}

    """
    Given date, calculate the value of the portfolio. Verifies that the
    date is after the start_date.
    """

    def calculateValue(self, date):
        value = 0.
        date = date.strftime('%Y-%m-%d')
        for sym,qty in self.holdings.items():
            if sym == 'cash':
                value += qty
            else:
                if sym not in Portfolio.prices:
                    stock = Share(sym)
                    # get prices until today
                    string_today = datetime.date.today().strftime('%Y-%m-%d')
                    # print(string_today, date, sym)
                    price_list = stock.get_historical(date, string_today)
                    # pprint.PrettyPrinter(depth = 6).pprint(price_list)
                    Portfolio.prices[sym] = {item['Date']: float(item['Close']) for item in price_list}

                # find price for the date of interest
                if date in Portfolio.prices[sym]:
                    close_price = Portfolio.prices[sym][date]
                    value += close_price * qty
                else:
                    # I still don't know why this would ever happen. What's going on?
                    print(date, sym)
                    return None
        return value


##############################################################################
#
#    Copyright (C) 2017 - OutTech (<http://www.outtech.com.br>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from odoo import fields, models, _, api
from calendar import monthrange
from datetime import datetime
import base64
import pytz

class ConfirpReport(models.Model):
    _name = 'confirp.report'

    def _get_company(self):
        company_id = self.env.user.company_id
        return company_id

    company_id = fields.Many2one('res.company', 'Company', required=True, default=_get_company, readonly=True)
    journal_id = fields.Many2one('account.journal', 'Journal', required=True)
    period_id = fields.Date(string='Date Start', required=True)
    period_end_id = fields.Date(string='Date End', required=True)

    def cofirp_report(self):
        dict_data = {'header': {}, 'detail': {}, 'footer': {}}
        confirp_file = []
        debit = ''
        credit = ''

        move_date = datetime.strptime(self.period_id, "%Y-%m-%d")
        start_date = datetime.strptime('%s-%s-%s' % (move_date.year, move_date.month, '01'), '%Y-%m-%d')
        month_range = monthrange(move_date.year, move_date.month)
        final_date = datetime.strptime('%s-%s-%s' % (move_date.year, move_date.month, month_range[1]), '%Y-%m-%d')

        account_move_ids = self.env['account.move'].search([('company_id','=',self.env.user.company_id.id),('date','>=',start_date),('date','<=',final_date),('journal_id','=',self.journal_id.id)])
        for account_move in account_move_ids:
            company_name = self.env.user.company_id.name[0:4]
            account_date = datetime.strptime(account_move.date, "%Y-%m-%d")
            account_day = account_date.day
            account_month = account_date.month
            account_year = account_date.year
            journal = account_move.journal_id

            if journal.type == 'bank':
                if journal.bank_id:
                    bank_code = journal.bank_id.bic
            else:
                bank_code = ''

            # move_ids = self.env['account.move.line'].search([('move_id','=',account_move.id)])
            for move in account_move.line_ids:
                if move.credit:
                    credit = move.credit
                if move.debit:
                    debit = move.debit

            report_line = {account_move.name: {
                'CodEmpresa': company_name,
                'DiaLancamento': account_day,
                'MesLancamento': account_month,
                'AnoLancamento': account_year,
                'NumNotaFiscal': '',
                'ContaDebito': 54321,
                'EntradaSaida': 'E',
                'Banco': bank_code,
                'CentroCusto': '',
                'Valor': debit,
                'Historico': account_move.ref,
                'PadraoSistema': '',
                'ContaCredito': 12345,
            }}
            dict_data['detail'].update(report_line)

        confirp_report_txt = self.env['file.export.layout'].create_file('RelatorioConfirp', dict_data)

        obj_ir_attachment = self.env['ir.attachment']
        w_timezone = self.env['res.users'].browse(self._uid).tz
        now = datetime.now(pytz.timezone(w_timezone))

        src_ir_attachment = obj_ir_attachment.search([('res_model', '=', 'confirp.report'), ('res_id', '=', self.id)])

        filename = 'CONFIRP_%04d%02d%02d_%02d%02d%02d.txt' % (now.year,
                                                          now.month,
                                                          now.day,
                                                          now.hour,
                                                          now.minute,
                                                          now.second)

        if src_ir_attachment:
            attach_val = {
                'name': filename,
                'datas_fname': filename,
                'datas': base64.b64encode(confirp_report_txt.encode('ascii', 'replace'))
            }
            obj_ir_attachment.write(src_ir_attachment, attach_val)
        else:
            attach_vals = {'name': filename,
                           'datas_fname': filename,
                           'datas': base64.b64encode(confirp_report_txt.encode('ascii', 'replace')),
                           'file_type': format,
                           'res_model': 'confirp.report',
                           'res_id': self._uid
                           }

            obj_ir_attachment.create(attach_vals)

        confirp_report_download = self.env['confirp.report.download'].create({
            'data': base64.b64encode(confirp_report_txt.encode('ascii', 'replace')), 'filename': filename})

        return {
            'type': 'ir.actions.act_window',
            'res_model': 'confirp.report.download',
            'view_mode': 'form',
            'view_type': 'form',
            'res_id': confirp_report_download.id,
            'views': [(False, 'form')],
            'target': 'new'
        }

class ConfirpReportDownload(models.Model):
    _name = 'confirp.report.download'

    filename = fields.Char('Filename', invisible=True, filename="filename")
    data = fields.Binary('File', readonly=True)


# -*- coding: UTF-8 -*-
# **********************************************************************************#
#     File:
#   Author: Myron
# **********************************************************************************#
import operator
import numpy as np
from copy import copy
from .. base import *
from ... import logger
from ... core.clock import clock
from ... core.schema import SchemaType
from ... core.enum import SecuritiesType
from ... trade.order import PMSOrder, OrderState, OrderStateMessage
from ... trade.cost import Commission
from ... trade.position import PMSPosition
from ... trade.trade import PMSTrade
from ... utils.dict_utils import DefaultDict, CompositeDict
from ... utils.linked_list_utils import LinkedList, Node
from ... utils.error_utils import Errors
from ... utils.decorator_utils import mutex_lock, scramble_redis_lock
from ... data.database_api import load_adjust_close_price, get_current_st_securities
from ... data.redis_base import redis_queue, RedisCollection

class SecurityBroker(object):

    pool = DefaultDict(LinkedList)
    commission = Commission()
    ST = None
    ST_key = None
    pre_close_price = None
    pre_close_price_key = None
    nodes_positions = DefaultDict()

    def __new__(cls, *args, **kwargs):
        """
        Single instance security broker
        """
        if not hasattr(cls, '_instance'):
            cls._instance = super(SecurityBroker, cls).__new__(cls)
        return cls._instance

    @classmethod
    def get_st_securities(cls):
        """
        Get st securities
        """
        if cls.ST is None or cls.ST_key != clock.current_date:
            cls.ST = get_current_st_securities()
            cls.ST_key = clock.current_date
        return cls.ST

    def get_pre_close_price(self):
        """
        Get pre_close price
        """
        if self.pre_close_price is None or self.pre_close_price_key != clock.current_date:
            self.pre_close_price = load_adjust_close_price(clock.previous_trading_date)
            self.pre_close_price_key = clock.current_date
        return self.pre_close_price

    @scramble_redis_lock()
    def prepare(self):
        """
        load active orders into order pool
        """
        order_today = query_by_securities_('redis', SchemaType.order, clock.current_date,
                                           securities_type=SecuritiesType.SECURITY)
        # active orders group by symbol, submit_time as key
        symbol_orders = CompositeDict()
        for order_schema in order_today.itervalues():
            for order in order_schema.orders.itervalues():
                if not order.order_time:
                    continue
                if order.state \
                        in OrderState.ACTIVE \
                        and order.order_time.split(' ')[0] == clock.current_date.strftime('%Y-%m-%d'):
                    symbol_orders[order.symbol].update({order: order.order_time})
        for orders in symbol_orders.itervalues():
            # order by order time
            order_list = sorted(orders.items(), key=operator.itemgetter(1))
            self.accept_orders([e[0] for e in order_list])

    @mutex_lock
    def accept_orders(self, orders):
        """
        Accept orders
        """
        if isinstance(orders, PMSOrder):
            orders = [orders]
        for order in orders:
            current_order_node = Node(order)
            self.pool[order.symbol].append(current_order_node)

    @mutex_lock
    def transact_minute(self, bar_data):
        """
        Transact for orders by minute bar database
        """
        linked_list = self.pool[bar_data.security_id]
        if not linked_list.get_length():
            return

        # Query chosen positions from redis
        symbol_portfolios = linked_list.recursive(formula=(lambda x, y: x | y),
                                                  formatter=(lambda x: {x.obj.portfolio_id}))
        self.nodes_positions = query_from_('redis', SchemaType.position, portfolio_id=symbol_portfolios)

        # Transact
        volume_ceiling = {e: bar_data.total_volume*100 for e in symbol_portfolios}
        feedback = linked_list.traversal(func=_transact_security_bar, bar_data=bar_data, volume_ceiling=volume_ceiling,
                                         pre_close_price=self.get_pre_close_price(),
                                         nodes_positions=self.nodes_positions)
        changed_orders = map(lambda x: x[0], feedback)
        original_nodes = map(lambda x: x[1], feedback)

        # Update changed orders to redis
        update_('redis', SchemaType.order, changed_orders)

        # del zero amount position
        for position_schema in self.nodes_positions.itervalues():
            if position_schema.positions.get(bar_data.security_id) == 0:
                del position_schema.positions[bar_data.security_id]

        # Dump changed positions to redis
        dump_to_('redis', SchemaType.position, self.nodes_positions)
        self.nodes_positions.clear()

        # Remove inactive orders from linked list
        active_index = [index for index, order in enumerate(changed_orders)
                        if order.state in OrderState.ACTIVE]
        active_orders = list(np.array(changed_orders)[active_index])
        active_nodes = list(np.array(original_nodes)[active_index])
        for node in set(original_nodes) - set(active_nodes):
            linked_list.delete(node)
        linked_list.traversal(func=_synchronize_node, orders=active_orders)

    @classmethod
    def calculate_stock_commission(cls, price, direction):
        """
        Calculate stock commission.

        Args:
            price(float): price
            direction(int): direction
        """
        return cls.commission.calculate_stock_commission(price, direction)

    def post_trading_day(self):
        """
        Post trading days
        """
        self.clear()

    def clear(self):
        """
        Clear info
        """
        self.pool.clear()
        self.nodes_positions.clear()

    @staticmethod
    def playback_daily(order_schema, position_schema, daily_data):
        """
        Transact daily database

        Args:
            order_schema(obj): order schema
            position_schema(obj): position schema
            daily_data(dict): daily database

        Returns:
            (obj, obj): order, position
        """
        return _playback_security_daily(order_schema, position_schema, daily_data)

def _transact_security_bar(node, bar_data, volume_ceiling, pre_close_price, nodes_positions):
    """
    Transact using bar database

    Args:
        node(node): node
        bar_data(database): bar database
    """
    order = copy(node.obj)
    portfolio = nodes_positions.get(order.portfolio_id)
    if portfolio is None:
        _change_order_state(order, OrderState.ERROR, OrderStateMessage.INVALID_PORTFOLIO)
        msg = order.__repr__()
        logger.error('[SECURITY] [TRANSACTION FAILED]'+msg)
        return order, node
    if order.state == OrderState.CANCEL_SUBMITTED:
        _change_order_state(order, OrderState.CANCELED, OrderStateMessage.CANCELED)
        return order, node
    if order.order_amount == 0:
        _change_order_state(order, OrderState.FILLED, OrderStateMessage.FILLED)
        return order, node

    act_price = bar_data.open_price
    pre_close_price = pre_close_price.get(order.symbol) or act_price
    high_price = bar_data.high_price
    low_price = bar_data.low_price
    volume = volume_ceiling.get(order.portfolio_id, 0)
    if volume == 0:
        _change_order_state(order, OrderState.OPEN, OrderStateMessage.NO_AMOUNT)
        return order, node
    limit_pct = 0.0483 if order.symbol in SecurityBroker.get_st_securities() else 0.0993
    if order.direction == 1 and portfolio.cash < act_price * order.order_amount:
        _change_order_state(order, OrderState.REJECTED, OrderStateMessage.NO_ENOUGH_CASH)
        return order, node

    # order.order_type is fixed equal to 'market' currently
    if order.order_type != 'market':
        raise Errors.INVALID_ORDER_TYPE
    commission = order.direction * SecurityBroker.calculate_stock_commission(act_price, order.direction)
    cost_price = act_price + commission
    if order.direction > 0:
        if high_price == low_price and high_price / pre_close_price > 1 + limit_pct:
            _change_order_state(order, target_message=OrderStateMessage.UP_LIMIT)
            return order, node   # 全分钟涨停

        available = \
            min(order.open_amount, volume, portfolio.cash / cost_price) * 1. if cost_price > 0 else 0
        available = int(available) // 100 * 100
    else:
        if order.symbol not in portfolio.positions:
            _change_order_state(order, target_message=OrderStateMessage.SELLOUT)
            return order, node

        if high_price == low_price and high_price / pre_close_price < 1 - limit_pct:
            _change_order_state(order, target_message=OrderStateMessage.DOWN_LIMIT)
            return order, node   # 全分钟跌停

        symbol_available_amount = portfolio.positions[order.symbol].available_amount
        available = min(order.open_amount, volume, symbol_available_amount)
        available = int(available)
        symbol_available_amount -= available

    if available == 0:
        _change_order_state(order, OrderState.OPEN, OrderStateMessage.SELLOUT)
        return order, node

    order._filled_time = ' '.join([portfolio.date, bar_data.bar_minute])
    order._transact_price = (act_price * available + order.transact_price * order.filled_amount) / \
                            (order.filled_amount + available)
    order._filled_amount = order.filled_amount + available
    order._commission = order.commission + abs(available * commission)
    order._turnover_value = order.turnover_value + available * act_price
    pms_trade = PMSTrade(order.order_id, order.symbol, order.direction, order.offset_flag, available,
                         act_price, order.filled_time, commission, None, order.portfolio_id)
    redis_queue.put([pms_trade.to_dict()], key=RedisCollection.trade)
    portfolio.cash -= order.direction * cost_price * available
    if order.filled_amount == order.order_amount:
        _change_order_state(order, OrderState.FILLED, OrderStateMessage.FILLED)
    elif order.filled_amount == 0:
        _change_order_state(order, OrderState.OPEN)
    else:
        _change_order_state(order, OrderState.PARTIAL_FILLED)

    volume_after = max((volume - available), 0)
    volume_ceiling.update({order.portfolio_id: volume_after})
    position = portfolio.positions.get(order.symbol)
    pre_amount = 0 if position is None else position.amount
    if pre_amount:
        position.amount += order.direction * available
        if order.direction == -1:
            position.total_cost = position.total_cost * position.amount / pre_amount if position.amount != 0 else 0
            position.available_amount -= available
        else:
            position.total_cost = position.total_cost + order.direction * available * cost_price
        position.cost = position.total_cost / position.amount if position.amount else 0
    else:
        if position is None:
            position = PMSPosition(order.symbol, 0, 0, 0.0)
            portfolio.positions[order.symbol] = position
        position.amount = order.direction * available
        position.total_cost = order.turnover_value + order.commission
        position.cost = position.total_cost / position.amount
    position.value = act_price * position.amount
    msg = order.__repr__()
    logger.debug('[SECURITY] [TRANSACTION]'+msg)
    return order, node

def _playback_security_daily(order_schema, position_schema, daily_data):
    """
    Transact securities by daily database

    Args:
        order_schema(obj): order schema
        position_schema(obj): position schema
        daily_data(dict): daily database

    Returns:
        (obj, obj): order, position
    """
    # todo. transact order_schema based on position_schema
    playback_order_schema, playback_position_schema = order_schema, position_schema
    return playback_order_schema, playback_position_schema

def _synchronize_node(node, orders):
    """
    Synchronize orders to linked list

    Args:
        node(node): node
        orders(list): orders
    """
    node.obj = orders.pop(0)

def _change_order_state(order, target_state=None, target_message=None):
    """
    更新当前order状态，暂不处理非active的Order

    Args:
        order (PMSOrder): 需要更新的order
        target_state (str): 目标状态
        target_message (str): 目标状态提示

    """
    if order is None:
        return
    if not isinstance(order, PMSOrder):
        raise Errors.INVALID_ORDER_OBJECT
    if target_state is not None:
        order._state = target_state
    if target_message is not None:
        order._state_message = target_message


def magfit(F,M):

    import numpy as np
    from scipy.optimize import curve_fit
    import matplotlib.pyplot as plt

    def func(x, a, b):
      return a*M + b

    y = func(M,2,4)

    popt, pcov = curve_fit(func,M,F)

    if(0):
        plt.figure()
        plt.plot(M,F,'bx',label='Data')
        plt.plot(M, func(M, *popt), 'k-',lw=1.5, label="fit")
        plt.title('Magnification Calibration')
        plt.xlabel('Magnification')
        plt.ylabel('$\Delta F$')
        plt.legend()
        plt.show()

    return popt[0],popt[1]


__author__ = 'similarface'
import sys, os, multiprocessing
from socket_stream_redirect import *
# def server1():
#     mypid = os.getpid()
#     conn = initListenerSocket()
#     file = conn.makefile('r')
#     for i in range(3):
#         data = file.readline().rstrip()
#         print('server %s got [%s]' % (mypid, data))

def server1():
    mypid=os.getpid()
    conn=initListenerSocket()
    file=conn.makefile('r')
    for i in range(3):
        data=file.readline().rstrip()
        print('Server %s got [%s] '%(mypid,data))

def client1():
    mypid=os.getpid()
    redirectOut()
    for i in range(3):
        print('client %s: %s' % (mypid, i))
        sys.stdout.flush()

if __name__ == '__main__':
    server=eval('server'+sys.argv[1])

    client=eval('client'+sys.argv[1])

    multiprocessing.Process(target=server).start()

    client()
    #import time; time.sleep(5)


# Bing Image Search API v7 - Image Scraper Application
# Adapted from Adrian Rosebrock's Bing image scraper
# Source: https://www.pyimagesearch.com/2018/04/09/how-to-quickly-build-a-deep-learning-image-dataset/

# Python libraries
import os
import logging
import argparse
import time
# 3rd Party Libraries
import cv2
import numpy as np
import requests
# User libraries
from keys.azure import subscription_key

logger = logging.getLogger()
logger.setLevel(logging.INFO)

ap = argparse.ArgumentParser()
ap.add_argument("-q", "--query", required=True,
    help="search query to search Bing Image API for")
ap.add_argument("-o", "--output", required=False,
    help="path to output directory of images")
ap.add_argument("-m", "--max", required=False,
    help="maximum number of search results")
ap.add_argument("-g", "--group", required=False,
    help="group size to process query")
ap.add_argument("-t", "--timeout", required=False,
    help="timeout in seconds to wait for an image to download")
ap.add_argument("--width", required=False,
    help="minimum required width")
ap.add_argument("--height", required=False,
    help="minimum required height")
ap.add_argument("-y", "--yolo", required=True,
    help="base path to YOLO directory")
ap.add_argument("--confidence", type=float, default=0.5,
    help="minimum probability to filter weak detections")
ap.add_argument("--threshold", type=float, default=0.3,
    help="threshold when applyong non-maxima suppression")
args = vars(ap.parse_args())

"""
# Load the image classifier
"""
# load the COCO class labels our YOLO model was trained on
labelsPath = os.path.sep.join([args["yolo"], "coco.names"])
LABELS = open(labelsPath).read().strip().split("\n")

# initialize a list of colors to represent each possible class label
np.random.seed(int(time.time()))
COLORS = np.random.randint(0, 255, size=(len(LABELS), 3),
                           dtype="uint8")

# derive the paths to the YOLO weights and model configuration
weightsPath = os.path.sep.join([args["yolo"], "yolov3.weights"])
configPath = os.path.sep.join([args["yolo"], "yolov3.cfg"])

# load our YOLO object detector trained on COCO dataset (80 classes)
print("[INFO] loading YOLO from disk...")
net = cv2.dnn.readNetFromDarknet(configPath, weightsPath)

# determine only the *output* layer names that we need from YOLO
ln = net.getLayerNames()
ln = [ln[i[0] - 1] for i in net.getUnconnectedOutLayers()]

"""
# Azure parameters
"""
# NOTE: you need a subscription key from Azure, place it in keys/azure.py with the identifier subscription_key
# Source: https://portal.azure.com
assert subscription_key
search_url = "https://api.cognitive.microsoft.com/bing/v7.0/images/search"

# Global Variables
if args["max"]:
    MAX_RESULTS = int(args["max"])
else:
    MAX_RESULTS = 250
if args["group"]:
    GROUP_SIZE = int(args["group"])
else:
    GROUP_SIZE = 50
if args["timeout"]:
    TIMEOUT = int(args["timeout"])
else:
    TIMEOUT = 30
if args["width"]:
    MIN_WIDTH = int(args["width"])
else:
    MIN_WIDTH = 0
if args["height"]:
    MIN_HEIGHT = int(args["height"])
else:
    MIN_HEIGHT = 0
if args["output"]:
    OUTPUT_DIR = args["output"]
else:
    OUTPUT_DIR = os.path.join('output', args["query"].replace(' ', '_').replace('\"', '').replace(':', '-').replace('.', 'p'))[:64]

"""
# Possible exceptions
"""
exception_set = [
    IOError, FileNotFoundError,
    requests.RequestException, requests.HTTPError,
    requests.ConnectionError, requests.Timeout,
    requests.exceptions.SSLError, requests.exceptions.ConnectTimeout
]
"""
# Possible image extensions
"""
extensions = [
    'jpg', 'jpeg', 'jpe', 'png', 'bmp',
    'pbm', 'pgm', 'ppm', 'sr', 'ras',
    'jp2', 'tiff', 'tif'
]
"""
# Create output directory
"""
# Source: https://stackoverflow.com/questions/273192/how-can-i-safely-create-a-nested-directory-in-python
if not os.path.exists(OUTPUT_DIR):
    os.makedirs(OUTPUT_DIR)

"""
# Search query
"""
term = args["query"]
headers = {"Ocp-Apim-Subscription-Key" : subscription_key}
params = {"q": args["query"], "imageType": "photo"}
logging.info("searching Bing API for '{}'".format(term))

try:
    response = requests.get(search_url, headers=headers, params=params)
    response.raise_for_status()
    search_results = response.json()

    estNumResults = min(search_results["totalEstimatedMatches"], MAX_RESULTS)
    logging.info("{} total results for '{}'".format(estNumResults, term))

    # Download images
    total = 0
    # Loop over total results in GROUP_SIZE batches
    for offset in range(0, estNumResults, GROUP_SIZE):
        logging.info("making request for group {}-{} of {}...".format(
            offset,
            offset+GROUP_SIZE,
            estNumResults
        ))
        params["offset"] = offset
        search = requests.get(search_url, headers=headers, params=params)
        search.raise_for_status()
        results = search.json()
        logging.info("saving images for group {}-{} of {}...".format(
            offset,
            offset+GROUP_SIZE,
            estNumResults
        ))
        # Loop over group batch
        for v in results["value"]:
            try:
                logging.info("fetching: {}".format(v["contentUrl"]))
                try:
                    r = requests.get(v["contentUrl"], timeout=TIMEOUT)
                except requests.ReadTimeout:
                    continue
                ext = v["contentUrl"][v["contentUrl"].rfind("."):]
                ext = ext.lower()

                if v["width"] < MIN_WIDTH or v["height"] < MIN_HEIGHT:
                    logging.warning("{}x{} is smaller than {}x{}, skipping: {}".format(
                        v["width"], v["height"],
                        MIN_WIDTH, MIN_HEIGHT,
                        v["contentUrl"])
                    )
                    continue
                # Check for valid extensions
                elif ext[1:] not in extensions:

                    ext = ext[:4]
                    if ext[1:] not in extensions:
                        # default to .jpg
                        ext = '.jpg'
                        logging.warning("invalid extension, assigning extension as jpg: {}".format(v["contentUrl"]))
                    else:
                        logging.warning("invalid extension, truncating extension: {}".format(v["contentUrl"]))
                # path in format 'outputPath/number.ext'
                p = os.path.sep.join(
                    [OUTPUT_DIR,
                     "{}{}".format(str(total).zfill(8), ext)]
                )
                # Write to disk
                f = open(p, "wb")
                f.write(r.content)
                f.close()
            except Exception as e:
                if type(e) in exception_set:
                    logging.warning("encountered exception: {}, skipping: {}".format(e, v["contentUrl"]))
                    continue
                else:
                    raise e
            """
            # verify it's a good image
            """
            image = cv2.imread(p)
            if image is None:
                logging.warning("deleting corrupted image: {}".format(p))
                os.remove(p)
                continue
            (H, W) = image.shape[:2]
            # construct a blob from the input frame and then perform a forward
            # pass of the YOLO object detector, giving us our bounding boxes
            # and associated probabilities
            blob = cv2.dnn.blobFromImage(
                image,
                1 / 255.0,  # sigma
                (416, 416),  # Resize to X by Y
                swapRB=True,  # BGR to RGB
                crop=False  # cropping
            )
            net.setInput(blob)
            layerOutputs = net.forward(ln)

            # initialize our lists of detected bounding boxes, confidences,
            # and class IDs, respectively
            boxes = []
            confidences = []
            classIDs = []

            # loop over each of the layer outputs
            for output in layerOutputs:
                # loop over each of the detections
                for detection in output:
                    # extract the class ID and confidence (i.e., probability)
                    # of the current object detection
                    scores = detection[5:]
                    classID = np.argmax(scores)
                    confidence = scores[classID]

                    # filter out weak predictions by ensuring the detected
                    # probability is greater than the minimum probability
                    if confidence > args["confidence"]:
                        # scale the bounding box coordinates back relative to
                        # the size of the image, keeping in mind that YOLO
                        # actually returns the center (x, y)-coordinates of
                        # the bounding box followed by the boxes' width and
                        # height
                        box = detection[0:4] * np.array([W, H, W, H])
                        (centerX, centerY, width, height) = box.astype("int")

                        # use the center (x, y)-coordinates to derive the top
                        # and and left corner of the bounding box
                        x = int(centerX - (width / 2))
                        y = int(centerY - (height / 2))

                        # update our list of bounding box coordinates,
                        # confidences, and class IDs
                        boxes.append([x, y, int(width), int(height)])
                        confidences.append(float(confidence))
                        classIDs.append(classID)

            # apply non-maxima suppression to suppress weak, overlapping
            # bounding boxes
            idxs = cv2.dnn.NMSBoxes(boxes, confidences, args["confidence"],
                                    args["threshold"])

            # Test to see if at least one person was detected
            if not any([LABELS[c] == 'person' for c in classIDs]):
                logging.warning("deleting image because no people found: {}".format(p))
                os.remove(p)
                continue

            # Found a good image, go to the next
            total += 1
except requests.HTTPError as e:
    # Sometimes you'll get this error if you entered the wrong subscription key
    # Source: https://blogs.msdn.microsoft.com/kwill/2017/05/17/http-401-access-denied-when-calling-azure-cognitive-services-apis/
    print(e)


import pytest
from selenium import webdriver
from selenium.webdriver.chrome.options import Options

def pytest_addoption(parser):
    parser.addoption('--language', action='store', default='ru', help='Chose language')

@pytest.fixture(scope="function")
def browser(request):

	# считываем язык и браузер (если есть)
	choosen_language = request.config.getoption("language")
    # инициализируем браузер chrome с нужными опциями
	opts = Options()
	opts.add_experimental_option('prefs', {'intl.accept_languages': choosen_language})
	print('\nstart Chrome browser...')
	browser = webdriver.Chrome(options=opts)
	yield browser
	# закрытие браузера после работы
	browser.quit()

unit3. Тестовые фреймворки/6. PyTest — параметризация, конфигурирование, плагины/Рецензии/2/Language_store-master/conftest.py


import numpy as np
import time
import matplotlib.pyplot as plt

def naive_calc(p,k):
    ws = np.array([],dtype=complex)
    for i in range(len(p)):
        w_i = np.exp(-2*np.pi*(i*1j)/k)
        ws = np.append(ws,w_i)
    ps = np.array([],dtype=complex)
    for i in range(len(p)):
        sumofp = complex(0.)
        for l in range(len(p)):
            w_i = ws[i] ** l
            sumofp = sumofp + w_i * p[l]
        ps = np.append(ps,sumofp)
    return ps

def fft_calc(p,ws):
    if (len(p) == 1):
        ps = np.array([complex(p[0] * ws[0])],dtype=complex)
    else :
        ps = np.empty(len(ws),dtype=complex)
        half_h = int(len(ws)/2)
        q = p[::2]
        s = p[1::2]
        qs = fft_calc(q,ws[::2])
        ss = fft_calc(s,ws[::2])
        for i in range(half_h):
            ps[i] = complex(qs[i] + ws[i] * ss[i])
        for i in range(half_h):
            ps[half_h+i] = complex(qs[i] - ws[i] * ss[i])
    return ps

def fft(p,k):
    ws = np.array([],dtype=complex)
    for i in range(len(p)):
        w_i = np.exp(-2*np.pi*(i*1j)/k)
        ws = np.append(ws,w_i)
    return fft_calc(p,ws)

def create_array(k):
    array = np.zeros(k,dtype=complex)
    for i in range(k):
        a = np.random.rand()
        b = np.random.rand()
        array[i] = a + b * 1j
    return array

if __name__ == "__main__" :
    fft_time = []
    naive_time = []
    for i in range(15):
        print("K = ", i)
        K = 2 ** i
        a = create_array(K)
        start = time.time()
        p1 = fft(a,K)
        elapsed_time = time.time() - start
        fft_time.append(elapsed_time)
        print ("fft_time:{0}".format(elapsed_time) + "[sec]")
        start = time.time()
        p2 = naive_calc(a,K)
        elapsed_time = time.time() - start
        naive_time.append(elapsed_time)
        print ("naive_time:{0}".format(elapsed_time) + "[sec]")
        print ("diff is ", np.linalg.norm(p1 - p2, ord = 2))
        print ("")

    plt.figure()
    plt.figure(figsize=(8,8))
    plt.plot(np.log2(fft_time),label="fft")
    plt.plot(np.log2(naive_time),label="naive")
    plt.axes().set_aspect('equal', 'datalim')
    plt.legend()
    plt.savefig("compare_time.png")


#! /usr/bin/env python

"""Multilayer Perceptron for drug response problem"""

from __future__ import division, print_function

import argparse
import csv
import logging
import sys

import numpy as np
import pandas as pd

from itertools import tee, islice

from keras import backend as K
from keras import metrics
from keras.models import Sequential
from keras.layers import Dense, Dropout, LocallyConnected1D, Convolution1D, MaxPooling1D, Flatten
from keras.callbacks import Callback, ModelCheckpoint, ProgbarLogger

from sklearn.preprocessing import Imputer
from sklearn.preprocessing import StandardScaler, MinMaxScaler, MaxAbsScaler

# For non-interactive plotting
import matplotlib as mpl
mpl.use('Agg')
import matplotlib.pyplot as plt

import p1b3

# Model and Training parameters

# Seed for random generation
SEED = 2016
# Size of batch for training
BATCH_SIZE = 100
# Number of training epochs
NB_EPOCH = 20
# Number of data generator workers
NB_WORKER = 1

# Percentage of dropout used in training
DROP = 0.1
# Activation function (options: 'relu', 'tanh', 'sigmoid', 'hard_sigmoid', 'linear')
ACTIVATION = 'relu'
LOSS = 'mse'
OPTIMIZER = 'sgd'
# OPTIMIZER = 'adam'

# Type of feature scaling (options: 'maxabs': to [-1,1]
#                                   'minmax': to [0,1]
#                                   None    : standard normalization
SCALING = 'std'
# Features to (randomly) sample from cell lines or drug descriptors
FEATURE_SUBSAMPLE = 500#0
# FEATURE_SUBSAMPLE = 0

# Number of units in fully connected (dense) layers
D1 = 1000
D2 = 500
D3 = 100
D4 = 50
DENSE_LAYERS = [D1, D2, D3, D4]

# Number of units per locally connected layer
C1 = 10, 10, 5       # nb_filter, filter_length, stride
C2 = 0, 0, 0         # disabled layer
# CONVOLUTION_LAYERS = list(C1 + C2)
CONVOLUTION_LAYERS = [0, 0, 0]
POOL = 10

MIN_LOGCONC = -5.
MAX_LOGCONC = -4.

CATEGORY_CUTOFFS = [0.]

np.set_printoptions(threshold=np.nan)
np.random.seed(SEED)

def get_parser():
    parser = argparse.ArgumentParser(prog='p1b3_baseline',
                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument("-v", "--verbose", action="store_true",
                        help="increase output verbosity")
    parser.add_argument("-a", "--activation", action="store",
                        default=ACTIVATION,
                        help="keras activation function to use in inner layers: relu, tanh, sigmoid...")
    parser.add_argument("-b", "--batch_size", action="store",
                        default=BATCH_SIZE, type=int,
                        help="batch size")
    parser.add_argument("-c", "--convolution", action="store", nargs='+', type=int,
                        default=CONVOLUTION_LAYERS,
                        help="integer array describing convolution layers: conv1_nb_filter, conv1_filter_len, conv1_stride, conv2_nb_filter, conv2_filter_len, conv2_stride ...")
    parser.add_argument("-d", "--dense", action="store", nargs='+', type=int,
                        default=DENSE_LAYERS,
                        help="number of units in fully connected layers in an integer array")
    parser.add_argument("-e", "--epochs", action="store",
                        default=NB_EPOCH, type=int,
                        help="number of training epochs")
    parser.add_argument("-l", "--locally_connected", action="store_true",
                        default=False,
                        help="use locally connected layers instead of convolution layers")
    parser.add_argument("-o", "--optimizer", action="store",
                        default=OPTIMIZER,
                        help="keras optimizer to use: sgd, rmsprop, ...")
    parser.add_argument("--drop", action="store",
                        default=DROP, type=float,
                        help="ratio of dropout used in fully connected layers")
    parser.add_argument("--loss", action="store",
                        default=LOSS,
                        help="keras loss function to use: mse, ...")
    parser.add_argument("--pool", action="store",
                        default=POOL, type=int,
                        help="pooling layer length")
    parser.add_argument("--scaling", action="store",
                        default=SCALING,
                        help="type of feature scaling; 'minabs': to [-1,1]; 'minmax': to [0,1], 'std': standard unit normalization; None: no normalization")
    parser.add_argument("--drug_features", action="store",
                        default="descriptors",
                        help="use dragon7 descriptors, latent representations from Aspuru-Guzik's SMILES autoencoder, or both, or random features; 'descriptors','latent', 'both', 'noise'")
    parser.add_argument("--feature_subsample", action="store",
                        default=FEATURE_SUBSAMPLE, type=int,
                        help="number of features to randomly sample from each category (cellline expression, drug descriptors, etc), 0 means using all features")
    parser.add_argument("--min_logconc", action="store",
                        default=MIN_LOGCONC, type=float,
                        help="min log concentration of dose response data to use: -3.0 to -7.0")
    parser.add_argument("--max_logconc", action="store",
                        default=MAX_LOGCONC, type=float,
                        help="max log concentration of dose response data to use: -3.0 to -7.0")
    parser.add_argument("--subsample", action="store",
                        default='naive_balancing',
                        help="dose response subsample strategy; None or 'naive_balancing'")
    parser.add_argument("--category_cutoffs", action="store", nargs='+', type=float,
                        default=CATEGORY_CUTOFFS,
                        help="list of growth cutoffs (between -1 and +1) seperating non-response and response categories")
    parser.add_argument("--train_samples", action="store",
                        default=0, type=int,
                        help="overrides the number of training samples if set to nonzero")
    parser.add_argument("--val_samples", action="store",
                        default=0, type=int,
                        help="overrides the number of validation samples if set to nonzero")
    parser.add_argument("--save", action="store",
                        default='save',
                        help="prefix of output files")
    parser.add_argument("--scramble", action="store_true",
                        help="randomly shuffle dose response data")
    parser.add_argument("--workers", action="store",
                        default=NB_WORKER, type=int,
                        help="number of data generator workers")

    return parser

def extension_from_parameters(args):
    """Construct string for saving model with annotation of parameters"""
    ext = ''
    ext += '.A={}'.format(args.activation)
    ext += '.B={}'.format(args.batch_size)
    ext += '.D={}'.format(args.drop)
    ext += '.E={}'.format(args.epochs)
    if args.feature_subsample:
        ext += '.F={}'.format(args.feature_subsample)
    if args.convolution:
        name = 'LC' if args.locally_connected else 'C'
        layer_list = list(range(0, len(args.convolution), 3))
        for l, i in enumerate(layer_list):
            nb_filter = args.convolution[i]
            filter_len = args.convolution[i+1]
            stride = args.convolution[i+2]
            if nb_filter <= 0 or filter_len <= 0 or stride <= 0:
                break
            ext += '.{}{}={},{},{}'.format(name, l+1, nb_filter, filter_len, stride)
        if args.pool and layer_list[0] and layer_list[1]:
            ext += '.P={}'.format(args.pool)
    for i, n in enumerate(args.dense):
        if n:
            ext += '.D{}={}'.format(i+1, n)
    ext += '.S={}'.format(args.scaling)

    return ext

def evaluate_keras_metric(y_true, y_pred, metric):
    objective_function = metrics.get(metric)
    objective = objective_function(y_true, y_pred)
    return K.eval(objective)

def evaluate_model(model, generator, samples, metric, category_cutoffs=[0.]):
    y_true, y_pred = None, None
    count = 0
    while count < samples:
        x_batch, y_batch = next(generator)
        y_batch_pred = model.predict_on_batch(x_batch)
        y_batch_pred = y_batch_pred.ravel()
        y_true = np.concatenate((y_true, y_batch)) if y_true is not None else y_batch
        y_pred = np.concatenate((y_pred, y_batch_pred)) if y_pred is not None else y_batch_pred
        count += len(y_batch)

    loss = evaluate_keras_metric(y_true.astype(np.float32), y_pred.astype(np.float32), metric)

    y_true_class = np.digitize(y_true, category_cutoffs)
    y_pred_class = np.digitize(y_pred, category_cutoffs)

    # theano does not like integer input
    acc = evaluate_keras_metric(y_true_class.astype(np.float32), y_pred_class.astype(np.float32), 'binary_accuracy')  # works for multiclass labels as well

    return loss, acc, y_true, y_pred, y_true_class, y_pred_class

def plot_error(y_true, y_pred, batch, file_ext, file_pre='save', subsample=1000):
    if batch % 10:
        return

    total = len(y_true)
    if subsample and subsample < total:
        usecols = np.random.choice(total, size=subsample, replace=False)
        y_true = y_true[usecols]
        y_pred = y_pred[usecols]

    y_true = y_true * 100
    y_pred = y_pred * 100
    diffs = y_pred - y_true

    bins = np.linspace(-200, 200, 100)
    if batch == 0:
        y_shuf = np.random.permutation(y_true)
        plt.hist(y_shuf - y_true, bins, alpha=0.5, label='Random')

    #plt.hist(diffs, bins, alpha=0.35-batch/100., label='Epoch {}'.format(batch+1))
    plt.hist(diffs, bins, alpha=0.3, label='Epoch {}'.format(batch+1))
    plt.title("Histogram of errors in percentage growth")
    plt.legend(loc='upper right')
    plt.savefig(file_pre+'.histogram'+file_ext+'.b'+str(batch)+'.png')
    plt.close()

    # Plot measured vs. predicted values
    fig, ax = plt.subplots()
    plt.grid('on')
    ax.scatter(y_true, y_pred, color='red', s=10)
    ax.plot([y_true.min(), y_true.max()],
            [y_true.min(), y_true.max()], 'k--', lw=4)
    ax.set_xlabel('Measured')
    ax.set_ylabel('Predicted')
    plt.savefig(file_pre+'.diff'+file_ext+'.b'+str(batch)+'.png')
    plt.close()

class MyLossHistory(Callback):
    def __init__(self, progbar, val_gen, test_gen, val_samples, test_samples, metric, category_cutoffs=[0.], ext='', pre='save'):
        super(MyLossHistory, self).__init__()
        self.progbar = progbar
        self.val_gen = val_gen
        self.test_gen = test_gen
        self.val_samples = val_samples
        self.test_samples = test_samples
        self.metric = metric
        self.category_cutoffs = category_cutoffs
        self.pre = pre
        self.ext = ext

    def on_train_begin(self, logs={}):
        self.best_val_loss = np.Inf
        self.best_val_acc = -np.Inf
        self.best_model = None

    def on_epoch_end(self, batch, logs={}):
        if float(logs.get('val_loss', 0)) < self.best_val_loss:
            self.best_model = self.model
            val_loss, val_acc, y_true, y_pred, y_true_class, y_pred_class = evaluate_model(self.best_model, self.val_gen, self.val_samples, self.metric, self.category_cutoffs)
            test_loss, test_acc, _, _, _, _ = evaluate_model(self.best_model, self.test_gen, self.test_samples, self.metric, self.category_cutoffs)
            self.progbar.append_extra_log_values([('val_acc', val_acc), ('test_loss', test_loss), ('test_acc', test_acc)])
            plot_error(y_true, y_pred, batch, self.ext, self.pre)
        self.best_val_loss = min(float(logs.get('val_loss', 0)), self.best_val_loss)
        self.best_val_acc = max(float(logs.get('val_acc', 0)), self.best_val_acc)

class MyProgbarLogger(ProgbarLogger):
    def on_train_begin(self, logs=None):
        super(MyProgbarLogger, self).on_train_begin(logs)
        self.verbose = 1
        self.extra_log_values = []

    def append_extra_log_values(self, tuples):
        for k, v in tuples:
            self.extra_log_values.append((k, v))

    def on_epoch_end(self, epoch, logs=None):
        logs = logs or {}
        for k in self.params['metrics']:
            if k in logs:
                self.log_values.append((k, logs[k]))
        for k, v in self.extra_log_values:
            self.log_values.append((k, v))
        if self.verbose:
            self.progbar.update(self.seen, self.log_values, force=True)

def main():
    parser = get_parser()
    args = parser.parse_args()
    print('Args:', args)

    loggingLevel = logging.DEBUG if args.verbose else logging.INFO
    logging.basicConfig(level=loggingLevel, format='')

    ext = extension_from_parameters(args)

    loader = p1b3.DataLoader(feature_subsample=args.feature_subsample,
                             scaling=args.scaling,
                             drug_features=args.drug_features,
                             scramble=args.scramble,
                             min_logconc=args.min_logconc,
                             max_logconc=args.max_logconc,
                             subsample=args.subsample,
                             category_cutoffs=args.category_cutoffs)

    gen_shape = None
    out_dim = 1

    model = Sequential()
    if args.convolution and args.convolution[0]:
        gen_shape = 'add_1d'
        layer_list = list(range(0, len(args.convolution), 3))
        for l, i in enumerate(layer_list):
            nb_filter = args.convolution[i]
            filter_len = args.convolution[i+1]
            stride = args.convolution[i+2]
            if nb_filter <= 0 or filter_len <= 0 or stride <= 0:
                break
            if args.locally_connected:
                model.add(LocallyConnected1D(nb_filter, filter_len, subsample_length=stride, input_shape=(loader.input_dim, 1), activation=args.activation))
            else:
                model.add(Convolution1D(nb_filter, filter_len, subsample_length=stride, input_shape=(loader.input_dim, 1), activation=args.activation))
            if args.pool:
                model.add(MaxPooling1D(pool_length=args.pool))
        model.add(Flatten())

    for layer in args.dense:
        if layer:
            model.add(Dense(layer, input_dim=loader.input_dim, activation=args.activation))
            if args.drop:
                model.add(Dropout(args.drop))
    model.add(Dense(out_dim))

    model.summary()
    model.compile(loss=args.loss, optimizer=args.optimizer)

    train_gen = p1b3.DataGenerator(loader, batch_size=args.batch_size, shape=gen_shape).flow()
    val_gen = p1b3.DataGenerator(loader, partition='val', batch_size=args.batch_size, shape=gen_shape).flow()
    val_gen2 = p1b3.DataGenerator(loader, partition='val', batch_size=args.batch_size, shape=gen_shape).flow()
    test_gen = p1b3.DataGenerator(loader, partition='test', batch_size=args.batch_size, shape=gen_shape).flow()

    train_samples = int(loader.n_train/args.batch_size) * args.batch_size
    val_samples = int(loader.n_val/args.batch_size) * args.batch_size
    test_samples = int(loader.n_test/args.batch_size) * args.batch_size

    train_samples = args.train_samples if args.train_samples else train_samples
    val_samples = args.val_samples if args.val_samples else val_samples

    checkpointer = ModelCheckpoint(filepath=args.save+'.model'+ext+'.h5', save_best_only=True)
    progbar = MyProgbarLogger()
    history = MyLossHistory(progbar=progbar, val_gen=val_gen2, test_gen=test_gen,
                            val_samples=val_samples, test_samples=test_samples,
                            metric=args.loss, category_cutoffs=args.category_cutoffs,
                            ext=ext, pre=args.save)

    model.fit_generator(train_gen, train_samples,
                        nb_epoch=args.epochs,
                        validation_data=val_gen,
                        nb_val_samples=val_samples,
                        verbose=0,
                        callbacks=[checkpointer, history, progbar],
                        pickle_safe=True,
                        nb_worker=args.workers)

if __name__ == '__main__':
    main()
    try:
        K.clear_session()
    except AttributeError:      # theano does not have this function
        pass