# ------------------------------------------------------------------------------
# Entities: classes for movement, etc
#
# All positions are in pixels
# All speeds are in pixels per second
# All accelerations (if I end up adding any) are in pixels per second per second
# ------------------------------------------------------------------------------

import events
import math
import misc
from misc import rectLeft, rectTop, rectRight, rectBottom
import pyglet
from pyglet.window import key
import random
import stats

COS_45_DEG = 0.7071
DEATH_DURATION_SEC = 1
DAMAGE_ANIM_DURATION_SEC = 1
LOOT_DURATION_SEC = 2
STUCK_DIST_SQ_LIMIT = 1
SCREEN_MINIMUM_Y = 50		# Text goes below this
ALERT_MAX_RANGE = 120

g_window = None

# ------------------------------------------------------------------------------
class Entity(object):
	_entities = []
	_new_entities = []

	# ------------------------------------------------------------------------------
	def __init__(self, img_fname, x=0, y=0, *args):
		self.name = 'unknown'
		self.x = float(x)		# left x
		self.y = float(y)		# *bottom* y
		self.vx = 0
		self.vy = 0
		self.rotation = 0
		self.alive = True
		self.dying = False
		self.die_off_screen = False
		self.force_on_screen = False
		self.collide_circle_duration = 0
		self.collide_rect_duration = 0
		self.stuck_duration = 0
		self.chat_col = (255, 255, 255, 255)

		self.image = misc.getImage(img_fname)
		self.width = self.image.width
		self.height = self.image.height
		self.half_width = self.width / 2
		self.half_height = self.height / 2
		self.sprite_left = 0
		self.sprite_right = self.width
		self.sprite_top = 0
		self.sprite_bottom = self.height
		self.updateRadius()
		self.sprite = pyglet.sprite.Sprite(self.image, 0, 0)
		self.updatePos()

		Entity._new_entities.append(self)

	# ------------------------------------------------------------------------------
	def updateRadius(self):
		relevant_width = self.sprite_right - self.sprite_left
		relevant_height = self.sprite_bottom - self.sprite_top
		self.radius = (relevant_width + relevant_height) / 4	# /2 for diameter to radius; /2 for average

	# ------------------------------------------------------------------------------
	def updatePos(self):
		misc.setSpriteRenderPos(self.sprite, self.x, self.y, self.rotation)
		self.updateBoundaries()

	# ------------------------------------------------------------------------------
	def updateBoundaries(self):
		self.circle = (self.x, self.y, self.radius)
		render_left = self.x - self.half_width
		render_top = self.y - self.half_height
		self.rect = (render_left + self.sprite_left,  render_top + self.sprite_top,
					 render_left + self.sprite_right, render_top + self.sprite_bottom)

	# ------------------------------------------------------------------------------
	left =   lambda self: rectLeft(self.rect)
	top =    lambda self: rectTop(self.rect)
	right =  lambda self: rectRight(self.rect)
	bottom = lambda self: rectBottom(self.rect)

	# ------------------------------------------------------------------------------
	def setSpriteRotationFromDir(self, dir_x, dir_y):
		rotation = misc.getRotationFromDir(dir_x, dir_y)
		if rotation is None:
			return

		self.rotation = rotation
		self.updatePos()

	# ------------------------------------------------------------------------------
	def circlesCollide(self, entity):
		return misc.circlesOverlap(self.circle, entity.circle)

	# ------------------------------------------------------------------------------
	def rectsCollide(self, entity):
		return misc.rectsOverlap(self.rect, entity.rect)

	# ------------------------------------------------------------------------------
	def startDying(self):
		self.dying = True
		self.death_timer = DEATH_DURATION_SEC
		self.vx = 0
		self.vy = 0

	# ------------------------------------------------------------------------------
	def limitRight(self, right_limit):
		overlap = self.right() - right_limit
		if overlap > 0:
			self.x -= overlap
			self.updateBoundaries()

	# ------------------------------------------------------------------------------
	def limitLeft(self, left_limit):
		overlap = left_limit - self.left()
		if overlap > 0:
			self.x += overlap
			self.updateBoundaries()

	# ------------------------------------------------------------------------------
	def limitBottom(self, bottom_limit):
		overlap = self.bottom() - bottom_limit
		if overlap > 0:
			self.y -= overlap
			self.updateBoundaries()

	# ------------------------------------------------------------------------------
	def limitTop(self, top_limit):
		overlap = top_limit - self.top()
		if overlap > 0:
			self.y += overlap
			self.updateBoundaries()

	# ------------------------------------------------------------------------------
	def update(self, dt):
		self.prev_x = self.x
		self.prev_y = self.y
		self.prev_circle = self.circle
		self.prev_rect = self.rect

		self.x += self.vx * dt
		self.y += self.vy * dt
		self.updateBoundaries()

		if self.dying:
			self.death_timer -= dt
			opacity = 255 * self.death_timer / DEATH_DURATION_SEC
			self.sprite.opacity = opacity
			if hasattr(self, 'vision_sprite'):
				base_col = self.vision_sprite.base_col
				self.vision_sprite.color = tuple([int(float(opacity) * col / 255) for col in base_col])
			if self.death_timer <= 0:
				self.alive = False

		else:
			# Kill some things when they're off-screen
			if self.die_off_screen:
				if self.x >= g_window.width or self.x < 0 or self.y >= g_window.height or self.y < SCREEN_MINIMUM_Y:
					self.alive = False

			# Lock some things on screen
			if self.force_on_screen:
				self.limitRight(g_window.width)
				self.limitLeft(0)
				self.limitBottom(g_window.height)
				self.limitTop(SCREEN_MINIMUM_Y)

			if not self.dying:
				self.checkCollisions(dt)

		self.updatePos()

	# ------------------------------------------------------------------------------
	def draw(self):
		self.sprite.draw()

	# ------------------------------------------------------------------------------
	def checkCollisions(self, dt):
		# Check for collisions if moving
		stuck = False
		if self.vx != 0 or self.vy != 0:
			for entity in Entity._entities:
				if entity is self:
					continue
				if isinstance(self, LivingEntity) and isinstance(entity, LivingEntity):
					if self.circlesCollide(entity):
						#print self.name, 'collided with', entity.name
						# It's much easier to just stop the moving entity than limiting it
						self.x = self.prev_x
						self.y = self.prev_y
						self.collide_circle_duration += dt
					else:
						self.collide_circle_duration = 0
				else:
					if self.rectsCollide(entity):
						#print '%s (%.1f, %.1f, %.1f, %.1f) overlaps with %s (%.1f, %.1f, %.1f, %.1f)' % \
						#		(self.name, self.rect[0], self.rect[1], self.rect[2], self.rect[3],
						#		 entity.name, entity.rect[0], entity.rect[1], entity.rect[2], entity.rect[3])
						if self.vx > 0 and rectRight(self.prev_rect) <= entity.left():
							self.limitRight(entity.left())
						elif self.vx < 0 and rectLeft(self.prev_rect) >= entity.right():
							self.limitLeft(entity.right())
						if self.vy > 0 and rectBottom(self.prev_rect) <= entity.top():
							self.limitBottom(entity.top())
						elif self.vy < 0 and rectTop(self.prev_rect) >= entity.bottom():
							self.limitTop(entity.bottom())

						self.collide_rect_duration += dt
					else:
						self.collide_rect_duration = 0

			if misc.distSqBetween(self.x, self.y, self.prev_x, self.prev_y) < STUCK_DIST_SQ_LIMIT:
				stuck = True

		if stuck:
			self.stuck_duration += dt
		else:
			self.stuck_duration = 0

	# ------------------------------------------------------------------------------
	def alertGuardsInRange(self):
		# Work out which guards are within a short distance of this entity
		for entity in Entity._entities:
			if entity is self or not isinstance(entity, GuardEntity):
				continue
			dist = misc.distBetween(self.x, self.y, entity.x, entity.y)
			if dist < ALERT_MAX_RANGE:
				# Alert the guard and make them come to this location
				entity.alertTo(self.x, self.y)

	# ------------------------------------------------------------------------------
	@classmethod
	def updateAll(cls, dt):
		[entity.update(dt) for entity in cls._entities]
		cls._entities = [entity for entity in cls._entities if entity.alive]
		cls._entities.extend(cls._new_entities)
		cls._new_entities = []

	# ------------------------------------------------------------------------------
	@classmethod
	def drawAll(cls):
		[entity.draw() for entity in cls._entities]

	# ------------------------------------------------------------------------------
	@classmethod
	def clearAll(cls):
		cls._entities = []
		cls._new_entities = []

# ------------------------------------------------------------------------------
class HouseEntity(Entity):
	_index = 1
	# ------------------------------------------------------------------------------
	def __init__(self, x, y, size, loot_amount=-1, loot_difficulty=1):
		col = random.randint(1, 4)
		super(HouseEntity, self).__init__('data/textures/house{0}-{1}.jpg'.format(col, size), x, y)
		self.name = 'house ' + str(HouseEntity._index)
		HouseEntity._index += 1
		self.chest = ChestEntity(float(x), float(y) + self.height * 0.2)
		loot_half_width = self.width / 8
		self.loot_rect = [self.x - loot_half_width, self.top() - 32,
						  self.x + loot_half_width, self.top()]
		self.loot_difficulty = loot_difficulty
		self.loot_timer = None
		if loot_amount > 0:
			self.loot_amount = int(loot_amount * (0.9 + random.random() * 0.2))		# 10% variance on specified
		else:
			self.loot_amount = random.randint(90, 110)

	# ------------------------------------------------------------------------------
	def lootingDuration(self):
		return LOOT_DURATION_SEC * self.loot_difficulty

	# ------------------------------------------------------------------------------
	def startLooting(self):
		if self.chest is None:
			return
		if self.loot_timer is None:
			self.loot_timer = self.lootingDuration()	# otherwise leave as previous adjusted value
		stats.setLootingCompletion(0)

	# ------------------------------------------------------------------------------
	def stopLooting(self):
		self.loot_timer = (self.loot_timer + self.lootingDuration()) / 2
		stats.stopLooting()

	# ------------------------------------------------------------------------------
	def updateLooting(self, dt):
		"Returns True iff looting was completed."
		self.loot_timer -= dt
		if self.loot_timer > 0:
			stats.setLootingCompletion(1 - self.loot_timer / self.lootingDuration())
			return False
		self.chest.startDying()
		self.chest = None
		stats.addLoot(self.loot_amount)
		self.loot_amount = 0
		return True

# ------------------------------------------------------------------------------
class ChestEntity(Entity):
	_index = 1
	# ------------------------------------------------------------------------------
	def __init__(self, x, y):
		super(ChestEntity, self).__init__('data/textures/chest.png', x, y)
		self.name = 'chest ' + str(ChestEntity._index)
		ChestEntity._index += 1

# ------------------------------------------------------------------------------
class SpinnerEntity(Entity):
	# ------------------------------------------------------------------------------
	def __init__(self):
		super(SpinnerEntity, self).__init__('placeholders/textures/arrow.png', 200, 200)
		self.name = 'spinner'
		self.rotation = 0
		self.sprite.scale = 1.5

	# ------------------------------------------------------------------------------
	def update(self, dt):
		self.rotation += 90 * dt
		super(SpinnerEntity, self).update(dt)

# ------------------------------------------------------------------------------
class DaggerEntity(Entity):
	_index = 1
	# ------------------------------------------------------------------------------
	def __init__(self, x, y, dir_x, dir_y):
		"dir_x and dir_y should be -1, 0, or +1 to give a direction for the dagger."
		super(DaggerEntity, self).__init__('data/textures/dagger.png', x, y)
		self.name = 'dagger ' + str(DaggerEntity._index)
		DaggerEntity._index += 1
		self.die_off_screen = True
		self.move_speed = 300
		self.vx = dir_x * self.move_speed
		self.vy = dir_y * self.move_speed
		if dir_x != 0 and dir_y != 0:
			self.vx *= COS_45_DEG
			self.vy *= COS_45_DEG

		self.setSpriteRotationFromDir(dir_x, dir_y)

	# ------------------------------------------------------------------------------
	def update(self, dt):
		super(DaggerEntity, self).update(dt)
		if self.dying:
			return

	# ------------------------------------------------------------------------------
	def checkCollisions(self, dt):
		# Check collision with everything (except itself)
		for entity in Entity._entities:
			if entity is self:
				continue
			if self.rectsCollide(entity):
				if isinstance(entity, GuardEntity):
					entity.damage(1)
					self.startDying()
				elif isinstance(entity, HouseEntity):
					self.startDying()
					events.trigger('dagger_hit_house', self, very_low_priority=True)
					self.alertGuardsInRange()

# ------------------------------------------------------------------------------
class LivingEntity(Entity):
	# ------------------------------------------------------------------------------
	def __init__(self, img_fname, x=0, y=0):
		super(LivingEntity, self).__init__(img_fname, x, y)
		self.hp = 1		# anything above zero
		self.damage_timer = 0

	# ------------------------------------------------------------------------------
	def damage(self, amount):
		self.hp -= amount
		if self.hp <= 0 and not self.dying:
			self.startDying()
		else:
			self.damage_timer = DAMAGE_ANIM_DURATION_SEC

	# ------------------------------------------------------------------------------
	def startDying(self):
		super(LivingEntity, self).startDying()
		events.removeAllEventsForEntity(self)
		self.damage_timer = 0
		self.sprite.color = (255, 64, 64)

	# ------------------------------------------------------------------------------
	def update(self, dt):
		# Test the movement
		super(LivingEntity, self).update(dt)
		if self.damage_timer > 0:
			# Go red, then fade back to white
			self.damage_timer = max(self.damage_timer - dt, 0)
			damaged_col = 255 - 192 * self.damage_timer / DAMAGE_ANIM_DURATION_SEC
			self.sprite.color = (255, damaged_col, damaged_col)

# ------------------------------------------------------------------------------
class GuardEntity(LivingEntity):

	DECISION_INTERVAL_SEC = 0.5
	ALERT_PAUSE_SEC = 2.5

	PATROL_STATE = 0
	CHASE_STATE = 1
	ALERT_STATE = 2

	# ------------------------------------------------------------------------------
	def __init__(self, img_name, start_x, start_y, *args):
		super(GuardEntity, self).__init__(img_name, start_x, start_y)
		self.chat_col = (192, 192, 192, 255)
		self.hp = 2
		self.force_on_screen = True
		self.seen_player = False
		self.view_dist = 150
		self.patrol_speed = 60
		self.chase_speed = 100
		self.attack_recharge_timer = None
		self.min_time_between_attacks = 2
		self.dir_x = +1
		self.dir_y = 0
		self.type_name = 'guard'
		self.close_range_vision_min_cos = 0

		self.patrol_points = [(start_x, start_y)]
		self.next_patrol_point = 1
		self.patrol_point_direction = +1
		self.patrol_is_loop = False
		prev_coord = None
		next_coord_is_dir = False
		for coord in args:
			if coord == 'dir':
				next_coord_is_dir = True
				continue
			coord = int(coord)
			if prev_coord is not None:
				if next_coord_is_dir:
					self.dir_x = misc.sign(prev_coord)
					self.dir_y = misc.sign(coord)
					next_coord_is_dir = False
				else:
					self.patrol_points.append((prev_coord, coord))
				prev_coord = None
			elif not next_coord_is_dir and coord < 0:
				self.patrol_is_loop = True
			else:
				prev_coord = coord

		self.alert_x = None
		self.alert_y = None
		self.alert_timer = None
		self.decision_timer = 0
		self.decided_dir_x = 0
		self.decided_dir_y = 0
		self.decided_move_diagonally = False
		self.state = GuardEntity.PATROL_STATE

		self.vision_img = misc.getImage('data/textures/vision.jpg')
		self.vision_sprite = pyglet.sprite.Sprite(self.vision_img, 0, 0,
								blend_src=pyglet.gl.GL_SRC_COLOR, blend_dest=pyglet.gl.GL_ONE_MINUS_SRC_COLOR)
		self.vision_sprite.base_col = (255, 255, 255)
		self.vision_half_width = self.vision_img.width / 2
		self.vision_half_height = self.vision_img.height / 2
		self.updateVisionDisplay()

	# ------------------------------------------------------------------------------
	def updateVisionDisplay(self):
		offset_dist = self.half_width + self.vision_half_width
		diagonal = self.dir_x != 0 and self.dir_y != 0
		if diagonal:
			offset_dist *= COS_45_DEG

		rotation = misc.getRotationFromDir(self.dir_x, self.dir_y)
		if rotation is not None:
			self.vision_sprite.rotation = rotation

		vision_centre_x = self.x + self.dir_x * offset_dist
		vision_centre_y = self.y + self.dir_y * offset_dist
		misc.setSpriteRenderPos(self.vision_sprite, vision_centre_x, vision_centre_y, self.vision_sprite.rotation)

	# ------------------------------------------------------------------------------
	def setState(self, new_state):
		if new_state == self.state:
			return
		self.state = new_state
		if new_state == GuardEntity.CHASE_STATE:
			self.vision_sprite.base_col = (255, 96, 96)
		elif new_state == GuardEntity.ALERT_STATE:
			self.vision_sprite.base_col = (255, 255, 96)
		else:
			self.vision_sprite.base_col = (255, 255, 255)
		self.vision_sprite.color = self.vision_sprite.base_col

		self.alert_timer = None

	# ------------------------------------------------------------------------------
	def damage(self, amount):
		prev_hp = self.hp
		super(GuardEntity, self).damage(amount)
		if self.hp > 0:
			events.trigger(self.type_name + '_hit', self)
			self.spotPlayer()
			self.alertGuardsInRange()
		elif prev_hp > 0:		# just died
			events.trigger(self.type_name + '_death', self)
			stats.addKill(self.name)

	# ------------------------------------------------------------------------------
	def lookForPlayer(self, dt):
		if self.seen_player:
			return
		player = PlayerEntity.instance

		# Look for the player
		dist_to_player = misc.circleSeparation(self.circle, player.circle)
		if dist_to_player >= self.view_dist:
			return

		# Check the angle to the player using a dot product of the offset to the player with the facing direction
		offset_to_player_x = player.x - self.x
		offset_to_player_y = player.y - self.y
		offset_dot_facing = misc.normalisedDotProduct(offset_to_player_x, offset_to_player_y, self.dir_x, self.dir_y)
		# Must be the same direction and within 45 degrees - or for very small distances, within 90 degrees
		min_cos_angle = self.close_range_vision_min_cos if dist_to_player < 50 else COS_45_DEG
		if offset_dot_facing < min_cos_angle:
			return

		# Check if the player is visible by intersecting the line joining the centres with all of
		# the obstacles
		for entity in Entity._entities:
			if isinstance(entity, HouseEntity):		# can see through everything other than houses
				if misc.lineIntersectsRect(player.x, player.y, self.x, self.y, entity.rect):
					#print self.name, "can't see the player because", entity.name, 'is in the way'
					return

		events.trigger(self.type_name + '_saw_player', self)
		self.spotPlayer()

	# ------------------------------------------------------------------------------
	def spotPlayer(self):
		PlayerEntity.instance.setSpotted(True)
		self.setState(GuardEntity.CHASE_STATE)
		self.seen_player = True

		self.alertGuardsInRange()

	# ------------------------------------------------------------------------------
	def think(self, dt):
		last_vx = self.vx
		last_vy = self.vy
		self.vx = 0
		self.vy = 0

		player = PlayerEntity.instance
		if player.dying or not player.alive:
			return

		self.lookForPlayer(dt)

		objective_x = None
		objective_y = None
		move_speed = self.patrol_speed
		chasing_player = False
		if self.seen_player:
			# Chase the player
			chasing_player = True
			objective_x = player.x
			objective_y = player.y
			move_speed = self.chase_speed
			range_to_reach = self.radius + player.radius + 5	# max attack distance
		elif self.alert_x is not None:
			objective_x = self.alert_x
			objective_y = self.alert_y
			range_to_reach = 20		# arbitrary
		else:
			# Patrol
			if self.next_patrol_point < len(self.patrol_points):
				point = self.patrol_points[self.next_patrol_point]
				objective_x = point[0]
				objective_y = point[1]
				range_to_reach = 5

		if self.decision_timer > 0:
			self.decision_timer -= dt

		if objective_x is not None:
			# Move towards the objective
			offset_to_objective_x = objective_x - self.x
			offset_to_objective_y = objective_y - self.y

			move_diagonally = False
			abs_offset_x = abs(offset_to_objective_x)
			abs_offset_y = abs(offset_to_objective_y)
			if abs_offset_x > 0 and abs_offset_y > 0:		# avoid division by zero
				# If the offsets are reasonably similar, go diagonally; otherwise, go straight
				ratio = abs_offset_x / abs_offset_y
				if ratio < 1:
					ratio = 1 / ratio
				if ratio < 1.1:
					move_diagonally = True

			desired_dir_x = -1 if offset_to_objective_x < 0 else +1
			desired_dir_y = -1 if offset_to_objective_y < 0 else +1

			# Go with the previous decision for a set time
			if self.decision_timer > 0:
				desired_dir_x = self.decided_dir_x
				desired_dir_y = self.decided_dir_y
				move_diagonally = self.decided_move_diagonally
			else:
				# Change the decision if we're stuck
				if self.stuck_duration > 0:
					if self.decision_timer <= 0:
						# Make a decision
						self.decision_timer = GuardEntity.DECISION_INTERVAL_SEC
						if abs_offset_x > 0 and abs_offset_y > 0:
							# Moving diagonally is an option.  If our decision was the same as the current idea,
							# try the opposite
							if move_diagonally == self.decided_move_diagonally:
								move_diagonally = not move_diagonally
							else:
								# We tried changing move_diagonally, or it isn't relevant.
								# Move directly in the direction with the smaller offset (but possibly the other way)
								if abs_offset_x > abs_offset_y:
									desired_dir_x = 0
									if desired_dir_y == 0:
										desired_dir_y = -1 if random.randint(0, 1) == 0 else +1
									elif random.randint(0, 3) == 0:
										desired_dir_y = -desired_dir_y	# randomly switch every now and then
								else:
									desired_dir_y = 0
									if desired_dir_x == 0:
										desired_dir_x = -1 if random.randint(0, 1) == 0 else +1
									elif random.randint(0, 3) == 0:
										desired_dir_x = -desired_dir_x	# randomly switch every now and then
				else:
					# Follow the basic target
					if not move_diagonally:
						# Zero one of the directions
						if abs_offset_x > abs_offset_y:
							desired_dir_y = 0
						else:
							desired_dir_x = 0

				self.decided_dir_x = desired_dir_x
				self.decided_dir_y = desired_dir_y
				self.decided_move_diagonally = move_diagonally

			if move_diagonally:
				move_speed *= COS_45_DEG
			self.vx = desired_dir_x * move_speed
			self.vy = desired_dir_y * move_speed

			# Check if in range of the objective
			if abs_offset_x * abs_offset_x + abs_offset_y * abs_offset_y < range_to_reach * range_to_reach:
				# In range
				if chasing_player:
					# Can hit the player
					if self.attack_recharge_timer is None:
						events.trigger(self.type_name + '_hit_player', self)
						player.damage(1)
						self.attack_recharge_timer = self.min_time_between_attacks
				elif self.alert_x is not None:
					# Reached the objective, but nothing happened
					# Pause for a bit, and then go back to it
					if self.alert_timer is None:
						self.alert_timer = GuardEntity.ALERT_PAUSE_SEC
					else:
						self.alert_timer -= dt
						if self.alert_timer <= 0:
							self.alert_x = None
							self.alert_y = None
							self.setState(GuardEntity.PATROL_STATE)
				else:
					# Reached a patrol point
					self.next_patrol_point += self.patrol_point_direction
					num_patrol_points = len(self.patrol_points)
					patrol_finished = False
					if self.next_patrol_point < 0:
						self.next_patrol_point = (num_patrol_points - 1) if self.patrol_is_loop else 1
						patrol_finished = True
					elif self.next_patrol_point >= num_patrol_points:
						self.next_patrol_point = 0 if self.patrol_is_loop else (num_patrol_points - 2)
						patrol_finished = True
					if patrol_finished and not self.patrol_is_loop:
						self.patrol_point_direction = -self.patrol_point_direction

			if self.vx != 0 or self.vy != 0:
				self.dir_x = misc.sign(self.vx)
				self.dir_y = misc.sign(self.vy)

	# ------------------------------------------------------------------------------
	def alertTo(self, x, y):
		# Make the guard go towards the location - unless they're already chasing the player
		if self.seen_player:
			return
		self.alert_x = x
		self.alert_y = y
		self.setState(GuardEntity.ALERT_STATE)

	# ------------------------------------------------------------------------------
	def update(self, dt):
		if not self.dying:
			self.think(dt)

		if self.attack_recharge_timer is not None:
			self.attack_recharge_timer -= dt
			if self.attack_recharge_timer <= 0:
				self.attack_recharge_timer = None

		super(GuardEntity, self).update(dt)
		self.updateVisionDisplay()

	# ------------------------------------------------------------------------------
	def draw(self):
		self.vision_sprite.draw()
		super(GuardEntity, self).draw()

# ------------------------------------------------------------------------------
class HumanGuardEntity(GuardEntity):
	# ------------------------------------------------------------------------------
	def __init__(self, name, start_x, start_y, *args):
		super(HumanGuardEntity, self).__init__('data/textures/guard.png', start_x, start_y, *args)
		self.name = name + ' the Guard'
		self.loot_amount = random.randint(5, 15)

		self.sprite_left = 4
		self.sprite_right = 29
		self.updateRadius()

# ------------------------------------------------------------------------------
class GuardDogEntity(GuardEntity):
	# ------------------------------------------------------------------------------
	def __init__(self, name, start_x, start_y, *args):
		super(GuardDogEntity, self).__init__('data/textures/dog.png', start_x, start_y, *args)
		self.name = name + ' the dog'
		self.type_name = 'dog'
		self.close_range_vision_min_cos = -COS_45_DEG	# dogs can see further at very close range (smell, etc)
		self.loot_amount = 0

		self.sprite_left = 3
		self.sprite_right = 30
		self.updateRadius()

# ------------------------------------------------------------------------------
class PlayerEntity(LivingEntity):
	instance = None

	LEFT_KEYS =    (key.LEFT, key.NUM_LEFT, key.NUM_4, key.NUM_HOME, key.NUM_7, key.NUM_END, key.NUM_1, key.HOME, key.END)
	RIGHT_KEYS =   (key.RIGHT, key.NUM_RIGHT, key.NUM_6, key.NUM_PAGE_UP, key.NUM_9, key.NUM_PAGE_DOWN, key.NUM_3, key.PAGEUP, key.PAGEDOWN)
	UP_KEYS =      (key.UP, key.NUM_UP, key.NUM_8, key.NUM_HOME, key.NUM_7, key.NUM_PAGE_UP, key.NUM_9, key.HOME, key.PAGEUP)
	DOWN_KEYS =    (key.DOWN, key.NUM_DOWN, key.NUM_2, key.NUM_END, key.NUM_1, key.NUM_PAGE_DOWN, key.NUM_3, key.END, key.PAGEDOWN)
	FIRE_KEYS =    (key.LCTRL, key.RCTRL)
	LOOT_KEY =      key.SPACE

	MAX_DIST_TO_PICKPOCKET = 10		# not including entity radii

	# ------------------------------------------------------------------------------
	def __init__(self, x, y, initial_daggers=10):
		PlayerEntity.instance = self
		super(PlayerEntity, self).__init__('data/textures/thief.png', x, y)
		self.name = 'Kane'
		self.chat_col = (80, 80, 80, 255)
		self.max_hp = 3
		self.hp = self.max_hp
		self.move_speed = 130
		self.dir_x = +1
		self.dir_y = 0
		self.firing = False
		self.force_on_screen = True
		self.loot_target = None
		self.ever_spotted = False

		self.sprite_left = 6
		self.sprite_right = 27
		self.updateRadius()

		stats.daggers = initial_daggers

	# ------------------------------------------------------------------------------
	def setSpotted(self, spotted):
		if self.ever_spotted == spotted:
			return
		self.ever_spotted = spotted
		if spotted:
			events.trigger('player_spotted', self, low_priority=True)

	# ------------------------------------------------------------------------------
	def damage(self, amount):
		prev_hp = self.hp
		super(PlayerEntity, self).damage(amount)
		stats.updateHealth(self.hp, self.max_hp)
		if prev_hp > 0 and (self.dying or not self.alive):
			if self.loot_target is not None:
				self.loot_target.stopLooting()
				self.loot_target = None
			stats.reportDeath()
			events.trigger('player_death', self, low_priority=True)

	# ------------------------------------------------------------------------------
	def processInput(self, dt):
		# Movement
		if misc.isAnyKeyHeld(PlayerEntity.LEFT_KEYS):
			self.vx -= self.move_speed
		if misc.isAnyKeyHeld(PlayerEntity.RIGHT_KEYS):
			self.vx += self.move_speed
		if misc.isAnyKeyHeld(PlayerEntity.UP_KEYS):
			self.vy += self.move_speed
		if misc.isAnyKeyHeld(PlayerEntity.DOWN_KEYS):
			self.vy -= self.move_speed
		# Scale diagonal movement so it's the same speed
		if self.vx != 0 and self.vy != 0:
			self.vx *= COS_45_DEG
			self.vy *= COS_45_DEG

		# Direction: only stored when moving in some direction
		if self.vx != 0 or self.vy != 0:
			if self.vx == 0:
				self.dir_x = 0
			else:
				self.dir_x = -1 if self.vx < 0 else +1
			if self.vy == 0:
				self.dir_y = 0
			else:
				self.dir_y = -1 if self.vy < 0 else +1

		# Firing
		if misc.isAnyKeyHeld(PlayerEntity.FIRE_KEYS):
			if not self.firing and stats.haveDagger():
				self.firing = True
				events.trigger('threw_dagger', self)
				stats.removeDagger()
				dagger = DaggerEntity(self.x, self.y, self.dir_x, self.dir_y)
				dagger.x += (self.half_width + dagger.half_width - 4) * self.dir_x	# -4 for width in texture
				dagger.y += (self.half_height + dagger.half_height) * self.dir_y
				dagger.updatePos()
		else:
			self.firing = False

		self.checkLooting(dt)

	# ------------------------------------------------------------------------------
	def checkLooting(self, dt):
		loot_target = None
		if misc.isKeyHeld(PlayerEntity.LOOT_KEY):
			for entity in Entity._entities:
				if isinstance(entity, HouseEntity):
					if entity.chest is not None and misc.rectsOverlap(self.rect, entity.loot_rect):
						loot_target = entity
				elif isinstance(entity, HumanGuardEntity):
					if entity.loot_amount > 0 and not entity.seen_player:
						dist_to_guard = misc.distBetween(self.x, self.y, entity.x, entity.y)
						dist_to_guard -= self.radius + entity.radius
						if dist_to_guard <= PlayerEntity.MAX_DIST_TO_PICKPOCKET:
							# Pickpocket the guard
							events.trigger('pickpocketed_guard', self)
							stats.addLoot(entity.loot_amount, pickpocket_target=entity)
							entity.loot_amount = 0
							return

		if loot_target != self.loot_target:
			# Loot target changed
			if self.loot_target is not None:
				self.loot_target.stopLooting()
			self.loot_target = loot_target
			if self.loot_target is not None:
				self.loot_target.startLooting()
		elif loot_target is not None:
			if self.loot_target.updateLooting(dt):
				# Checking only for chests on houses, as they may be fading out ("dying")
				won = not any((isinstance(entity, HouseEntity) and entity.chest is not None \
								for entity in Entity._entities))
				if won:
					events.trigger('won_level', self)
					stats.setWonLevel()
				else:
					events.trigger('looted_chest', self)
				self.loot_target = None

	# ------------------------------------------------------------------------------
	def update(self, dt):
		self.vx = 0
		self.vy = 0
		if not self.dying and self.alive:
			self.processInput(dt)

		super(PlayerEntity, self).update(dt)

# ------------------------------------------------------------------------------

updateAll = Entity.updateAll
drawAll = Entity.drawAll
clearAll = Entity.clearAll

# ------------------------------------------------------------------------------


import sqlite3
from pickle import loads, dumps

from unittest import TestCase
import unittest

class Tests(TestCase):
    def setUp(self):
        self.conn = sqlite3.connect('shelf.db')
        self.c = self.conn.cursor()
        self.c.execute(f'''CREATE TABLE IF NOT EXISTS books
           (id INTEGER PRIMARY KEY  AUTOINCREMENT,
           url           TEXT    NOT NULL UNIQUE,
           name          TEXT,
           selectors      BLOB,
           chapters       BLOB,
           queue          BLOB
           all_text      TEXT);''')

    def tearDown(self):
        self.conn.commit()
        self.conn.close()

    def test_select_not_found_book(self):
        self.c.execute("select * from books where name=?", ("NoThisBook", ))
        self.assertEqual(self.c.fetchall(), [])

    @unittest.skip
    def test_load_from_db(self):
        self.c.execute("select chapters, queue from books where name=?", ("hi",))
        chs , que = self.c.fetchone()
        chapters = loads(chs)
        queue = loads(que)

    @unittest.skip
    def test_insert_repeat(self):
        self.c.execute("insert into books (url) values(?)", ('repaeat',))
        self.conn.commit()
        try:
            self.c.execute("insert into books (url) values(?)", ('repaeat',))
            self.conn.commit()
        except Exception:
            pass

    @unittest.skip
    def test_pickle_binary(self):
        a = 1
        b = 2
        c = 3
        bdata = [sqlite3.Binary(dumps(x)) for x in [a, b, c]]
        self.c.execute("insert into books (url) values(?)", ('array', ))
        self.c.execute("update books set chapters=?, selectors=?, queue=?", [b for b in bdata])
        self.conn.commit()

    @unittest.skip
    def test_selector_config(self):
        self.c.execute(f'''CREATE TABLE IF NOT EXISTS sites
           (id INTEGER PRIMARY KEY  AUTOINCREMENT,
           url           TEXT    NOT NULL UNIQUE,
           selectors      BLOB,
           n_ignore_first   INTEGER);''')
        url = 'https://www.ibiquge.net/'
        selectors = sqlite3.Binary(dumps(('#list','#content')))
        n_ignore_first = 12
        self.c.execute("insert into sites (url, selectors, n_ignore_first) values(?,?,?)",
                       (url, selectors, n_ignore_first))
        self.conn.commit()

    def test_site_url(self):
        from urllib.parse import urlparse
        book_url = 'https://www.ibiquge.net/48_48106/'
        site_url = urlparse(book_url).netloc
        self.assertEqual(site_url, 'www.ibiquge.net')

    def test_find_no_site(self):
        site_url = 'www.ibiquge.net'

        self.c.execute("select * from sites where url=?", (site_url, ))
        print(self.c.fetchall())
        print(self.c.fetchall())

if __name__ == '__main__':
   unittest.main()


from selenium import webdriver
from selenium.webdriver.common.action_chains import ActionChains

import time
import os
import numpy as np

from solve_auntithication import get_chromedriver

def parse_the_cite(base_url):
    """
    Этот парсер работает напрямую с сайтом.
    Его задача - получить все нужные HTML документы из интернета.

    :param url: - url сайта, который мы парсим
    :return: - None. Все данные сохраняются в папку template

    """
    if not os.path.exists('templates'):
        os.makedirs("templates")

    i = 0 # Индекс стартового элемента для запроса
    pagination_size = 32 # Колличество выводимых элементов на сайт
    count_items = 141 # Всго страниц на сайте
    while i < count_items:
        # Уменьшение пагинации для последнего запроса
        if i + pagination_size > count_items:
            pagination_size -= i + pagination_size - count_items

        # Поптыка запроса
        try:
            # Аутентификация proxy
            driver = get_chromedriver(use_proxy=True)
            url = base_url+f'?sw1=sw-cache-me&webcat=men%7Cclothing%7Cmen-clothing-hoodies-sweatshirts&start={i}&sz={pagination_size}'
            # Заходим на сайт
            driver.get(url)

            # Листаем его до footer
            actions = ActionChains(driver)
            actions.move_to_element(driver.find_element(webdriver.common.by.By.CLASS_NAME,'footer-container')).perform()
            time.sleep(1)

            # Сохраняем страницу
            with open(f'templates/page{i}_{i + pagination_size}.html','w',encoding='utf-8') as file:
                file.write(driver.page_source)
            driver.close()

            # Подготовка к новому запросу
            i += pagination_size
            print(f'Готово {i} из {count_items}.')
            # Ожидание, чтобы не нагружать сайт
            if i < count_items:
                sleeping_time = round(0.5 + np.random.sample(), 2)
                print(f'Время ожидания до следующего запроса: {sleeping_time} мин')
                time.sleep(sleeping_time*60)

        except Exception as _ex: # Если сайт заподозрил, что-то неладное
            print('Попробуем ещё раз...')
            # Закрываем прошлую сессию
            driver.close()

            # Ожидаем, чтобы не нагружать сайт
            if i < count_items:
                sleeping_time = round(0.5 + np.random.sample(), 2)
                print(f'Время ожидания до следующего запроса: {sleeping_time} мин')
                time.sleep(sleeping_time * 60)

if __name__ == '__main__':
    parse_the_cite('https://www.ralphlauren.nl/en/men/clothing/hoodies-sweatshirts/10204')


from os import F_TEST
from nnet import Nnet
import numpy as np
import matplotlib.pyplot as plt
import read_MNIST
import random
import sys
import consts
import tqdm

def sigmoid(x):
    return 1 / (1 + np.exp(-x))

def dsigmoid(x):
    return sigmoid(x) * (1 - sigmoid(x))

def run(nnet :Nnet, test):
    ll_a = test # last layer a
    for i in range(1, len(nnet.Ws)):
        ll_a = sigmoid((nnet.Ws[i] @ ll_a) + nnet.bs[i])

    return ll_a

def get_mx_ind(arr :np.array):
    return np.argmax(arr)

def test_nn(nnet :Nnet, test_set):
    passed = 0
    mse = 0
    for it in range(len(test_set)):
        out, ans = run(nnet, test_set[it][0]), test_set[it][1]
        mse += (np.square(out - ans)).mean(axis=None)
        if get_mx_ind(out) == get_mx_ind(ans):
            passed += 1

    return mse / len(test_set), passed/len(test_set)

def deriv(nnet :Nnet, test, ans):
    zs = ['_']
    a_s = [test]
    for i in range(1, len(nnet.Ws)):
        z = (nnet.Ws[i] @ a_s[i-1]) + nnet.bs[i]
        a = sigmoid(z)

        zs.append(z)
        a_s.append(a)

    da = ['_' for i in range(len(nnet.Ws))]
    dW = ['_' for i in range(len(nnet.Ws))]
    db = ['_' for i in range(len(nnet.Ws))]
    da[-1] = (a_s[-1] - ans)

    for i in range(len(nnet.Ws)-1, 0, -1):
        v = dsigmoid(zs[i]) * da[i]
        dW[i] = v @ a_s[i-1].T
        db[i] = v
        da[i-1] = nnet.Ws[i].T @ v

    return dW, db

def learn(nnet :Nnet, batch):
    sdW = [np.zeros(nnet.Ws[i].shape) if i > 0 else '_' for i in range(len(nnet.Ws))]
    sdb = [np.zeros(nnet.bs[i].shape) if i > 0 else '_' for i in range(len(nnet.Ws))]
    for test in batch:
        dW, db = deriv(nnet, test[0], test[1])
        for i in range(1, len(nnet.Ws)):
            sdW[i] += dW[i]
            sdb[i] += db[i]

    for i in range(1, len(nnet.Ws)):
        nnet.Ws[i] -= consts.LEARNING_RATE * (sdW[i] / len(batch))
        nnet.bs[i] -= consts.LEARNING_RATE * (sdb[i] / len(batch))
    # print(nnet.bs[3])

def train(nnet :Nnet, train_set, test_set):
    mses = []
    accs = []
    axis = []

    def measure(id):
        mse, acc = test_nn(nnet, test_set)
        mses.append(mse)
        accs.append(acc)
        axis.append(id)

    for _ in range(consts.EPOCH_COUNT):
        print("epoch num: {}".format(_+1))
        measure(_)
        random.shuffle(train_set)
        for i in tqdm.tqdm(range(0, len(train_set), consts.BATCH_SIZE)):
            learn(nnet, train_set[i:min(i + consts.BATCH_SIZE, len(train_set))])
    measure(consts.EPOCH_COUNT)

    plt.plot(axis, mses)
    plt.show()
    plt.plot(axis, accs)
    plt.show()

if __name__ == '__main__':
    print("loading tests.")
    train_set, test_set = read_MNIST.load_tests("tests/")
    try:
        if sys.argv[1] == 'sh': # shift
            sh_count = int(sys.argv[2]) # for example: python main.py sh 4
            for i in range(len(test_set)):
                test_set[i] = (
                    np.concatenate((np.zeros((28*sh_count, 1)), test_set[i][0][:-28*sh_count, :])),
                    test_set[i][1]
                )
    except:
        pass

    nnet = Nnet()
    train(nnet, train_set, test_set)
    mse, acc = test_nn(nnet, test_set)
    print(mse, acc)


from tkinter import *
from PIL import ImageTk, Image

from Character import Character
from BattleMage import BattleMage
from Warrior import Warrior
from Mage import Mage
from Peasant import Peasant

def click():

    try:
        name = hero_name_textentry.get()
        life = life_textentry.get()
        mana = mana_textentry.get()

        if int(life) >= Character.LIFE_DELIMITER and int(mana) < Character.MANA_DELIMITER:
            character = Warrior(name, life, mana)
        elif int(life) >= Character.LIFE_DELIMITER and int(mana) >= Character.MANA_DELIMITER:
            character = BattleMage(name, life, mana)
        elif int(life) < Character.LIFE_DELIMITER and int(mana) >= Character.MANA_DELIMITER:
            character = Mage(name, life, mana)
        else:
            character = Peasant(name, life, mana)

        output.delete(0.0, END)
        output.insert(END, character)

    except:
        output.delete(0.0, END)
        output.insert(END, 'Wypełnij, proszę, wszystkie pola.')

def close_window():
    my_app.destroy()
    exit()

my_app = Tk()
my_app.title('CREATE YOUR OWN HERO!')
my_app.configure(background = 'black')

logo = ImageTk.PhotoImage(Image.open('photos\layout.jpg'))

Label(my_app, image = logo, bg = 'black').grid(sticky = N)
Label(my_app, text = "Hero's Name:", bg = 'black', fg = 'white', font = 'gothic 10').grid(row = 1, sticky = W)
Label(my_app, text = 'Life:', bg = 'black', fg = 'white', font = 'gothic 10', bd = 1).grid(row = 3, sticky = W)
Label(my_app, text = 'Mana:', bg = 'black', fg = 'white', font = 'gothic 10', bd = 1).grid(row = 5, sticky = W)

hero_name_textentry = Entry(my_app, width = 13, bg ='white')
hero_name_textentry.grid(row = 2, sticky = W)
life_textentry = Entry(my_app, width = 13, bg ='white')
life_textentry.grid(row = 4, sticky = W)
mana_textentry = Entry(my_app, width = 13, bg ='white')
mana_textentry.grid(row = 6, sticky = W)

Button(my_app, text = 'Create a hero!', width = 11, command = click).grid(row = 12, pady = 1, sticky = W)
Button(my_app, text = "I don't want to create another hero!", width = 29, command = close_window).grid(row = 12, sticky = E)

output = Text(my_app, width = 53, height = 6, wrap = WORD, bg = 'white')
output.grid(row = 11, pady = 10, columnspan = 2, sticky = N)

my_app.mainloop()


# -*- coding: utf-8 -*-
"""
Created on Wed May  2 11:54:15 2018

@author: Franziska Pleissner

System C: concrete example: Plot of the cooling process with a solar collector
"""

############
# Preamble #
############

# Import packages
from oemof.tools import economics
import oemof.solph as solph

import oemof.outputlib as outputlib
import oemof_visio as oev

import logging
import pandas as pd

# import oemof plots
try:
    import matplotlib.pyplot as plt
except ImportError:
    plt = None

#####################################
# Input and Creating of raw results #
#####################################

# file to Import:
file_imp = 'Oman_SS20181029-1624.oemof'

# Weighted average cost of capital
wacc = 0.07

# investcosts without chillers and cooling tower:
ic_pv = 650  # Euro/kW_p
ic_collector = 100  # Euro/m^2
ic_boiler = 100  # Euro/kW_heat
ic_cool = 40  # Euro/kWh
ic_heat = 20  # Euro/kWh
ic_elec_kWh = 165   # Euro/kWh
ic_elec_kW = 50  # Euro/kW

# lifetimes [years]:
lt_pv = 25
lt_collector = 25
lt_ACM = 18
lt_CCM = 15
lt_boiler = 20
lt_cool = 25
lt_heat = 25
lt_ct = 25
lt_elec_kWh = 20
lt_elec_kW = 20

# OPEX [% of ic]:
op_pv = 0.015
op_collector = 0.02
op_ACM = 0.015
op_CCM = 0.02
op_boiler = 0.02
op_cool = 0.02
op_heat = 0.02
op_ct = 0.02
op_elec_kWh = 0
op_elec_kW = 0

# Emission factors
H_gas = 10  # (kWh/m^3)
Em_CO2_el = 403.2  # g/kWh
Em_CO2_gas = 201.6  # g/kWh

# water price
p_water = 1  # Euro/m^3

# Plotting time
sp = start_of_plot = 5523
ep = end_of_plot = sp + 48

# restoring the system and creating results
energysystem = solph.EnergySystem()
energysystem.restore(dpath="Dumps",
                     filename = file_imp)

results_strings = outputlib.views.convert_keys_to_strings(energysystem.results['main'])
results_strings_param = outputlib.views.convert_keys_to_strings(energysystem.results['param'])

logging.info('results received')

# get data from the import
nv_pv = (results_strings_param['pv', 'elec']['scalars']['nominal_value'])
nv_collector = (results_strings_param['collector', 'heat']['scalars']
                ['nominal_value'])
nv_ACM = (results_strings_param['absorpion_chiller', 'cool']['scalars']
          ['nominal_value'])
nv_CCM = (results_strings_param['compression_chiller', 'cool']['scalars']
          ['nominal_value'])
nv_boiler = (results_strings_param['boiler', 'heat']['scalars']
             ['nominal_value'])
nv_ct = (results_strings_param['waste', 'cooling_tower']['scalars']
         ['nominal_value'])
nc_cool = (results_strings_param['storage_cool', 'None']['scalars']
           ['nominal_capacity'])
nc_heat = (results_strings_param['storage_heat', 'None']['scalars']
           ['nominal_capacity'])
nc_elec = (results_strings_param['storage_el', 'None']['scalars']
           ['nominal_capacity'])
p_elec_b = (results_strings_param['el_grid', 'elec']['scalars']
           ['variable_costs'])
p_elec_s = (results_strings_param['elec', 'el_output']['scalars']
           ['variable_costs'])
p_gas_b = (results_strings_param['naturalgas', 'gas']['scalars']
           ['variable_costs'])

#print(nv_pv)
#print(nv_collector)
#print(nv_ACM)
#print(nv_CCM)
#print(nv_boiler)
#print(nv_ct)
#print(nc_cool)
#print(nc_heat)
#print(nc_elec)
#print(p_elec_b)
#print(p_elec_s)
#print(p_gas_b)

if nv_CCM < 25:
    ic_CCM = 650
else:
    if nv_CCM < 50:
        ic_CCM = 600
    else:
        if nv_CCM < 75:
            ic_CCM = 400
        else:
            if nv_CCM < 100:
                ic_CCM = 325
            else:
                ic_CCM = 200

if nv_ACM < 25:
    ic_ACM = 1000
else:
    if nv_ACM < 50:
        ic_ACM = 900
    else:
        if nv_ACM < 75:
            ic_ACM = 625
        else:
            if nv_ACM < 100:
                ic_ACM = 550
            else:
                ic_ACM = 400

if nv_ct < 100:
    ic_ct = 55
else:
    if nv_ct < 200:
        ic_ct = 40
    else:
        if nv_ct < 400:
            ic_ct = 32.5
        else:
            ic_ct = 28

#print(ic_ACM)
#print(ic_CCM)
#print(ic_ct)

#########################
# Work with the results #
#########################

cool_bus = outputlib.views.node(energysystem.results['main'], 'cool')
heat_bus = outputlib.views.node(energysystem.results['main'], 'heat')
waste_bus = outputlib.views.node(energysystem.results['main'], 'waste')
el_bus = outputlib.views.node(energysystem.results['main'], 'elec')
gas_bus = outputlib.views.node(energysystem.results['main'], 'gas')
storage_cool_res = outputlib.views.node(energysystem.results['main'], 'storage_cool')

cool_seq = cool_bus['sequences']
heat_seq = heat_bus['sequences']
waste_seq = waste_bus['sequences']
el_seq = el_bus['sequences']
gas_seq = gas_bus['sequences']
storage_cool_seq = storage_cool_res['sequences']

### Calculations ###

# demand
cooling_demand = results_strings[('cool', 'demand')]['sequences'].sum()
covered_demand = (cooling_demand -
                  results_strings[('shortage', 'cool')]['sequences'].sum())
fraction_covered = covered_demand/cooling_demand

# chillers
results_CCM = results_strings[('compression_chiller', 'cool')]['sequences']
cooling_from_CCM = results_CCM.sum()
hoo_CCM = results_CCM[results_CCM['flow'] > 0.01].count()
results_ACM = results_strings[('absorpion_chiller', 'cool')]['sequences']
cooling_from_ACM = results_ACM.sum()
hoo_ACM = results_ACM[results_ACM['flow'] > 0.01].count()
if (cooling_from_CCM+cooling_from_ACM).all() == 0:
    fraction_CCM = 0
    fraction_ACM = 0
else:
    fraction_CCM = cooling_from_CCM/(cooling_from_CCM+cooling_from_ACM)
    fraction_ACM = cooling_from_ACM/(cooling_from_CCM+cooling_from_ACM)

# heat
heat_total_in = (results_strings[('boiler', 'heat')]['sequences'].sum() +
                 results_strings[('collector', 'heat')]['sequences'].sum())
if heat_total_in.all() == 0:
    fraction_boiler = 0
    fraction_collector = 0
else:
    fraction_boiler = ((results_strings[('boiler', 'heat')]
                       ['sequences'].sum()) / heat_total_in)
    fraction_collector = ((results_strings[('collector', 'heat')]
                          ['sequences'].sum()) / heat_total_in)

# gas
gas_Energy = results_strings[('naturalgas', 'gas')]['sequences'].sum()  # kWh
gas_vol = gas_Energy/H_gas  # m^3
gas_CO2 = gas_Energy*Em_CO2_gas  # g

# electricity
electricity_input = results_strings[('el_grid', 'elec')]['sequences'].sum()
electricity_CO2 = electricity_input*Em_CO2_el  # g
electricity_output = results_strings[('elec', 'el_output')]['sequences'].sum()
electricity_Diff = electricity_input-electricity_output
elec_total_in = (results_strings[('el_grid', 'elec')]['sequences'].sum() +
                 results_strings[('pv', 'elec')]['sequences'].sum())
if elec_total_in.all() == 0:
    fraction_el_grid = 0
    fraction_pv = 0
else:
    fraction_el_grid = ((results_strings[('el_grid', 'elec')]
                        ['sequences'].sum()) / elec_total_in)
    fraction_pv = ((results_strings[('pv', 'elec')]
                   ['sequences'].sum()) / elec_total_in)

# cooling tower
ct_energy = results_strings[('waste', 'cooling_tower')]['sequences'].sum()

## cost results

# invest annuities
ian_pv = economics.annuity(ic_pv, lt_pv, wacc)
ian_collector = economics.annuity(ic_collector, lt_collector, wacc)
ian_ACM = economics.annuity(ic_ACM, lt_ACM, wacc)
ian_CCM = economics.annuity(ic_CCM, lt_CCM, wacc)
ian_boiler = economics.annuity(ic_boiler, lt_boiler, wacc)
ian_cool = economics.annuity(ic_cool, lt_cool, wacc)
ian_heat = economics.annuity(ic_heat, lt_heat, wacc)
ian_ct = economics.annuity(ic_ct, lt_ct, wacc)
ian_elec_kWh = economics.annuity(ic_elec_kWh, lt_elec_kWh, wacc)
ian_elec_kW = economics.annuity(ic_elec_kW, lt_elec_kW, wacc)

# total annuities (invest + operational costs)
an_pv = ian_pv + op_pv * ic_pv
an_collector = ian_collector + op_collector * ic_collector
an_ACM = ian_ACM + op_ACM * ic_ACM
an_CCM = ian_CCM + op_CCM * ic_CCM
an_boiler = ian_boiler + op_boiler * ic_boiler
an_cool = ian_cool + op_cool * ic_cool
an_heat = ian_heat + op_heat * ic_heat
an_ct = ian_ct + op_ct * ic_ct
an_elec_kWh = ian_elec_kWh + op_elec_kWh * ic_elec_kWh
an_elec_kW = ian_elec_kW + op_elec_kW * ic_elec_kW

power_battery = max(el_seq[(('storage_el', 'elec'), 'flow')])

# calculating the costs
fixed_costs_pa = (nv_pv * an_pv + nv_collector * an_collector +
                  nv_ACM * an_ACM + nv_CCM * an_CCM + nv_boiler * an_boiler +
                  nv_ct * an_ct + nc_cool * an_cool + nc_heat * an_heat +
                  nc_elec * an_elec_kWh + an_elec_kW * power_battery)

total_invest_costs = (nv_pv * ic_pv + nv_collector * ic_collector + nv_ACM *
                      ic_ACM + nv_CCM * ic_CCM + nv_boiler * ic_boiler +
                      nv_ct * ic_ct + nc_cool * ic_cool + nc_heat * ic_heat +
                      nc_elec * ic_elec_kWh + ic_elec_kW * power_battery)

costs_gas = gas_Energy * p_gas_b
costs_el = electricity_input * p_elec_b
profits_el = electricity_output * p_elec_s
costs_water = ct_energy * 1.5 / 1000 * p_water  # kWh * l/kWh / l/m3 * Euro/m3 = Euro

total_costs_pa = (fixed_costs_pa + costs_gas + costs_el + profits_el
                  + costs_water)

lcoc = total_costs_pa / covered_demand  # levelised costs of cold

# utilization

lf_ACM = cooling_from_ACM / (nv_ACM * 8760)
lf_CCM = cooling_from_CCM / (nv_CCM * 8760)
max_load_ACM = results_ACM.max()
max_load_CCM = results_CCM.max()
stor_cool_max = results_strings[('storage_cool', 'None')]['sequences'].max()
stor_heat_max = results_strings[('storage_heat', 'None')]['sequences'].max()
stor_el_max = results_strings[('storage_el', 'None')]['sequences'].max()
stor_cool_max_fraction = stor_cool_max / nc_cool
stor_heat_max_fraction = stor_heat_max / nc_heat
stor_el_max_fraction = stor_el_max / nc_elec

if fraction_covered.all() > 0.99:
    is_covered = 'Yes'
else:
    is_covered = 'No'

### showing the results ###

print('')
print('*****************************************')
print('results No test')
print('')
print('*Demand*')
print("Cooling Demand: %.2f kWh" % cooling_demand)
# print("Demand covered: %.2f kWh" % covered_demand)
print("covered fraction of demand: %.2f %%" % (fraction_covered*100))

print("cooling energy from CCM: %.2f kWh" % cooling_from_CCM)
print("cooling energy from ACM: %.2f kWh" % cooling_from_ACM)
print("fraction covered from CCM: %.2f %%" % (fraction_CCM*100))
print("fraction covered from ACM: %.2f %%" % (fraction_ACM*100))
# print("hours of operation of the CCM: %i h" % hoo_CCM)
# print("hours of operation of the ACM: %i h" % hoo_ACM)
print('')
print('*Energy consumption*')
print("fraction of produced heat covered from collectors: %.2f %%" % (fraction_collector*100))
print("fraction of produced electricity covered from pv: %.2f %%" % (fraction_pv*100))

print("used natural gas: %.2f m^3" % gas_vol)
print("electrical input: %.2f kWh" % electricity_input)
print("electrical output: %.2f kWh" % electricity_output)
print("electrical difference: %.2f kWh" % electricity_Diff)
print('')
print('*Emissions*')
print("CO2 output: %.2f kg" % ((gas_CO2+electricity_CO2)/1000))

# cool_stor = results_strings[('storage_cool', 'None')]['sequences']
print('')
print('*costs*')
print("total invest costs: %i €" % total_invest_costs)
print("total costs per year: %.2f €/a" % total_costs_pa)
print("levelised costs of cold: %.4f €/kWh" % lcoc)
print('')
print('*utilization*')
print('load factor absorption chilller: %.2f' % lf_ACM)
print('load factor compression chiller: %.2f' % lf_CCM)
print('maximum load of absorption chilller: %.2f' % max_load_ACM)
print('maximum load of compression chilller: %.2f' % max_load_CCM)
# print('maximum charging level of cold storage: %.2f' % stor_cool_max)
# print('maximum charging level of heat storage: %.2f' % stor_heat_max)
# print('maximum charging level of electricity storage: %.2f' % stor_el_max)
print('used fraction of cold storage capacity: %.2f %%' % (stor_cool_max_fraction*100))
print('used fraction of heat storage capacity: %.2f %%' % (stor_heat_max_fraction*100))
print('used fraction of electricity storage capacity: %.2f %%' % (stor_el_max_fraction*100))
print('')
print('*****************************************')
print('*summary*')
print('')
print('Is the demand covered? %s' % is_covered)
print("levelised costs of cold: %.4f €/kWh" % lcoc)
print("CO2 output: %.2f kg" % ((gas_CO2+electricity_CO2)/1000))
print('*****************************************')
print('')

### Results into csv  ###

# cool_seq.to_csv('data_output/results.csv')

########################
# Plotting the results #
########################

cool_seq_resample = cool_seq.iloc[sp:ep]
heat_seq_resample = heat_seq.iloc[sp:ep]
waste_seq_resample = waste_seq.iloc[sp:ep]
el_seq_resample = el_seq.iloc[sp:ep]
gas_seq_resample = gas_seq.iloc[sp:ep]
storage_cool_seq_resample = storage_cool_seq.iloc[sp:ep]

def shape_legend(node, reverse=False, **kwargs):  # just copied
    handels = kwargs['handles']
    labels = kwargs['labels']
    axes = kwargs['ax']
    parameter = {}

    new_labels = []
    for label in labels:
        label = label.replace('(', '')
        label = label.replace('), flow)', '')
        label = label.replace('None', '')
        label = label.replace(')', '')
        label = label.replace('_'+str(node), '')
        label = label.replace(node, '')
        label = label.replace(',', '')
        label = label.replace(' ', '')
        label = label.replace('_el', '')
        label = label.replace('el_', '')
        label = label.replace('_', ' ')
        new_labels.append(label)
    labels = new_labels

    parameter['bbox_to_anchor'] = kwargs.get('bbox_to_anchor', (1, 1))
    parameter['loc'] = kwargs.get('loc', 'upper left')
    parameter['ncol'] = kwargs.get('ncol', 1)
    plotshare = kwargs.get('plotshare', 0.9)

    if reverse:
        handels = handels.reverse()
        labels = labels.reverse()

    box = axes.get_position()
    axes.set_position([box.x0, box.y0, box.width * plotshare, box.height])

    parameter['handles'] = handels
    parameter['labels'] = labels
    axes.legend(**parameter)
    return axes

cdict = {
    (('collector', 'heat'), 'flow'): '#ffde32',
    (('boiler', 'heat'), 'flow'): '#ff0000',
    (('heat', 'absorpion_chiller'), 'flow'): '#4682b4',
    (('heat', 'storage_heat'), 'flow'): '#9acd32',
    (('storage_heat', 'heat'), 'flow'): '#9acd32',
    (('el_grid', 'elec'), 'flow'): '#999999',
    (('pv', 'elec'), 'flow'): '#ffde32',
    (('storage_el', 'elec'), 'flow'): '#9acd32',
    (('elec', 'storage_el'), 'flow'): '#9acd32',
    (('elec', 'compression_chiller'), 'flow'): '#87ceeb',
    (('elec', 'el_output'), 'flow'): '#555555',
    (('elec', 'cooling_tower'), 'flow'): '#ff0000',
    (('compression_chiller', 'cool'), 'flow'): '#87ceeb',
    (('absorpion_chiller', 'cool'), 'flow'): '#4682b4',
    (('shortage', 'cool'), 'flow'): '#555555',
    (('cool', 'demand'), 'flow'): '#cd0000',
    (('cool', 'storage_cool'), 'flow'): '#9acd32',
    (('storage_cool', 'None'), 'capacity'): '#555555',
    (('storage_cool', 'cool'), 'flow'): '#9acd32',
    (('naturalgas', 'gas'), 'flow'): '#42c77a',
    (('gas', 'boiler'), 'flow'): '#42c77a',
    (('compression_chiller', 'waste'), 'flow'): '#87ceeb',
    (('absorpion_chiller', 'waste'), 'flow'): '#4682b4',
    (('waste', 'cool_tower'), 'flow'): '#42c77a'}

# define order of inputs and outputs
inorderstor = [(('cool', 'storage_cool'), 'flow')]
outorderstor = [(('storage_cool', 'cool'), 'flow'),
                (('storage_cool', 'None'), 'capacity')]
inordercool = [(('absorpion_chiller', 'cool'), 'flow'),
               (('compression_chiller', 'cool'), 'flow'),
               (('storage_cool', 'cool'), 'flow'),
               (('shortage', 'cool'), 'flow')]
outordercool = [(('cool', 'demand'), 'flow'),
                (('cool', 'storage_cool'), 'flow')]
inorderel = [(('pv', 'elec'), 'flow'),
             (('storage_el', 'elec'), 'flow'),
             (('el_grid', 'elec'), 'flow')]
outorderel = [(('elec', 'compression_chiller'), 'flow'),
              (('elec', 'storage_el'), 'flow'),
              (('elec', 'cooling_tower'), 'flow'),
              (('elec', 'el_output'), 'flow')]
inorderheat = [(('collector', 'heat'), 'flow'),
               (('storage_heat', 'heat'), 'flow'),
               (('boiler', 'heat'), 'flow')]
outorderheat = [(('heat', 'absorpion_chiller'), 'flow'),
                (('heat', 'storage_heat'), 'flow')]

fig = plt.figure(figsize=(15, 15))

# plot cooling energy
my_plot_cool = oev.plot.io_plot(
        'cool', cool_seq_resample, cdict=cdict,
        inorder=inordercool, outorder=outordercool,
        ax=fig.add_subplot(2, 2, 1), smooth=False)

ax_cool = shape_legend('cool', **my_plot_cool)
oev.plot.set_datetime_ticks(ax_cool, cool_seq_resample.index, tick_distance=14,
                            date_format='%d-%m-%H', offset=1)

ax_cool.set_ylabel('Power in kW')
ax_cool.set_xlabel('time')
ax_cool.set_title("cool")

# plot heat energy
my_plot_heat = oev.plot.io_plot(
        'heat', heat_seq_resample, cdict=cdict,
        inorder=inorderheat, outorder=outorderheat,
        ax=fig.add_subplot(2, 2, 2), smooth=False)

ax_heat = shape_legend('heat', **my_plot_heat)
oev.plot.set_datetime_ticks(ax_heat, heat_seq_resample.index, tick_distance=14,
                            date_format='%d-%m-%H', offset=1)

ax_heat.set_ylabel('Power in kW')
ax_heat.set_xlabel('time')
ax_heat.set_title("heat")

# plot electric energy
my_plot_el = oev.plot.io_plot(
        'elec', el_seq_resample, cdict=cdict,
        inorder=inorderel, outorder=outorderel,
        ax=fig.add_subplot(2, 2, 3), smooth=False)

ax_el = shape_legend('elec', **my_plot_el)
oev.plot.set_datetime_ticks(ax_el, el_seq_resample.index, tick_distance=14,
                            date_format='%d-%m-%H', offset=1)

ax_el.set_ylabel('Power in kW')
ax_el.set_xlabel('time')
ax_el.set_title("electricity")

def shape_legend_stor(node, reverse=False, **kwargs):  # just copied
    handels = kwargs['handles']
    labels = kwargs['labels']
    axes = kwargs['ax']
    parameter = {}

    new_labels = []
    for label in labels:
        label = label.replace('(', '')
        label = label.replace('), flow)', '')
        label = label.replace('None', '')
        label = label.replace(')', '')
        label = label.replace('_'+str(node), '')
        label = label.replace(node, '')
        label = label.replace(',', '')
        label = label.replace(' ', '')
        label = label.replace('cool', 'input/output')
        if label not in new_labels:
            new_labels.append(label)
    labels = new_labels

    parameter['bbox_to_anchor'] = kwargs.get('bbox_to_anchor', (1, 1))
    parameter['loc'] = kwargs.get('loc', 'upper left')
    parameter['ncol'] = kwargs.get('ncol', 1)
    plotshare = kwargs.get('plotshare', 0.9)

    if reverse:
        handels = handels.reverse()
        labels = labels.reverse()

    box = axes.get_position()
    axes.set_position([box.x0, box.y0, box.width * plotshare, box.height])

    parameter['handles'] = handels
    parameter['labels'] = labels
    axes.legend(**parameter)
    return axes

# plot storage capacity
my_plot_stor = oev.plot.io_plot(
        'storage_cool', storage_cool_seq_resample, cdict=cdict,
        inorder=inorderstor, outorder=outorderstor,
        ax=fig.add_subplot(2, 2, 4), smooth=False)

ax_stor = shape_legend_stor('storage_cool', **my_plot_stor)
oev.plot.set_datetime_ticks(ax_stor, storage_cool_seq_resample.index,
                            tick_distance=14,
                            date_format='%d-%m-%H', offset=1)

ax_stor.set_ylabel('Power in kW and capacity in kWh')
ax_stor.set_xlabel('time')
ax_stor.set_title("cooling storage")

'''
# plot gas energy
my_plot_gas = oev.plot.io_plot(
        'gas', gas_seq_resample, cdict=cdict,
        ax=fig.add_subplot(3, 2, 5), smooth=False)

ax_gas = shape_legend('gas', **my_plot_gas)
oev.plot.set_datetime_ticks(ax_gas, gas_seq_resample.index, tick_distance=148,
                            date_format='%d-%m-%H', offset=1)

ax_gas.set_ylabel('Power in kW')
ax_gas.set_xlabel('2017')
ax_gas.set_title("gas bus")

# plot waste heat energy
my_plot_waste = oev.plot.io_plot(
        'waste', waste_seq_resample, cdict=cdict,
        ax=fig.add_subplot(3, 2, 5), smooth=False)

ax_waste = shape_legend('waste', **my_plot_waste)
oev.plot.set_datetime_ticks(ax_waste,
                            waste_seq_resample.index,tick_distance=148,
                            date_format='%d-%m-%H', offset=1)

ax_waste.set_ylabel('Power in kW')
ax_waste.set_xlabel('2017')
ax_waste.set_title("waste heat bus")
'''

plt.subplots_adjust(right=0.87)
plt.subplots_adjust(wspace=0.6)
plt.show()


import re
from re import RegexFlag
from typing import Optional, Set

from django.conf import settings
from django.dispatch.dispatcher import receiver

from classification.enums import ValidationCode, SpecialEKeys
from classification.models import EvidenceKeyMap, PatchMeta
from classification.models.classification import Classification, \
    classification_validation_signal
from classification.models.classification_utils import ValidationMerger
from genes.hgvs import HGVSMatcher
from genes.models import NoTranscript
from snpdb.models import GenomeBuild, Variant, VariantCoordinate

VARIANT_VALIDATING_CODES = {ValidationCode.MATCHING_ERROR, ValidationCode.INCONSISTENT_VARIANT, ValidationCode.MATCHING_ERROR}

@receiver(classification_validation_signal, sender=Classification)
def validate_variant_fields(sender, **kwargs) -> Optional[ValidationMerger]:  # pylint: disable=unused-argument
    """
    Ensures hgvs fields are valid, and that they match each other and the variant coordinate
    """
    vm = ValidationMerger()

    if not settings.CLASSIFICATION_MATCH_VARIANTS:
        # while this technically isn't about matching a variant, much of the work in get_variant_tuple -> pyhgvs
        # requires
        return vm

    patch_meta: PatchMeta = kwargs.get('patch_meta')
    evidence_key_map: EvidenceKeyMap = kwargs.get('key_map')

    VARIANT_LINKING_KEYS_SET = set(SpecialEKeys.VARIANT_LINKING_KEYS)
    # Only do this validation if at least variant linking key value has changed
    variant_values = {}
    variant_map = {}

    build_name = patch_meta.get(SpecialEKeys.GENOME_BUILD)
    if not build_name:
        return None

    build_changed = patch_meta.is_modified(SpecialEKeys.GENOME_BUILD)
    variant_key_changed = patch_meta.intersection_modified(VARIANT_LINKING_KEYS_SET)
    if build_changed or variant_key_changed:
        for evidence_key in SpecialEKeys.VARIANT_LINKING_KEYS:
            value = patch_meta.get(evidence_key)
            if value:
                # only consider it if it doesn't have other outstanding validation messages
                variant_values[evidence_key] = value

    if len(variant_values) >= 1:
        vm.tested(
            keys=VARIANT_LINKING_KEYS_SET,
            codes=[ValidationCode.MATCHING_ERROR, ValidationCode.INCONSISTENT_VARIANT, ValidationCode.MATCHING_ERROR]
        )

        try:
            # Take values from patch (not saved yet)
            genome_build = GenomeBuild.get_name_or_alias(build_name)
        except GenomeBuild.DoesNotExist:
            genome_build = None

        if genome_build:
            hgvs_matcher = HGVSMatcher(genome_build)
            for evidence_key, variant_value in variant_values.items():
                try:
                    if evidence_key in SpecialEKeys.VARIANT_LINKING_HGVS_KEYS:

                        # Not unsupported transcript type is handled by Imported Allele validation (not classification)
                        # if evidence_key == SpecialEKeys.C_HGVS and not Classification.is_supported_transcript(variant_value):
                        #     vm.add_message(evidence_key, code=ValidationCode.MATCHING_ERROR, severity='error', message='Transcript type not yet supported')
                        #     continue

                        variant_coordinate = hgvs_matcher.get_variant_coordinate(variant_value)
                    elif evidence_key == SpecialEKeys.VARIANT_COORDINATE:
                        variant_coordinate = VariantCoordinate.from_string(variant_value)
                    else:
                        raise ValueError(f'Unexpected evidence_key for variant matching {evidence_key}')

                    variant_str = Variant.format_tuple(*variant_coordinate)
                    variant_map[variant_str] = variant_map.get(variant_str, []) + [evidence_key]
                except NoTranscript:
                    # this should find its way into Validation Matching flag
                    pass
                except:
                    # this should be handled by ImportedAlleleInfo
                    pass
                # except ValueError as ve:
                #     # shouldn't happen as other validation should have already done this
                #     vm.add_message(evidence_key, code=ValidationCode.MATCHING_ERROR, severity='error', message=f'Error attempting to parse variant coordinate ({ve})')
                # except:
                #     vm.add_message(evidence_key, code=ValidationCode.MATCHING_ERROR, severity='error', message='Error attempting to parse variant coordinate')

    # check for this because if there were parsing errors we wont have any variant_values
    if len(variant_map) > 1:
        # generate message like:
        # c HGVS resolves to 4:53454A>T but g HGVS resolves to 4:4:53454A>C
        resolutions = ['%s resolves to (%s)' % (' and '.join([evidence_key_map.get(key).pretty_label for key in keys]), variant_str) for variant_str, keys in variant_map.items()]
        message = ' but '.join(resolutions)

        vm.add_message(SpecialEKeys.VARIANT_COORDINATE, code=ValidationCode.INCONSISTENT_VARIANT, severity='warning', message=message)

    return vm

# SUB_VUS_1 = re.compile(r'this variant .{2,20}? classified .{0,5}?3(A|B|C).{0,10}?V(?:O)?US', RegexFlag.IGNORECASE)
# SUB_VUS_2 = re.compile(r'this variant .{2,20}? classified .{0,5}?V(?:O)?US.{0,10}?3(A|B|C)', RegexFlag.IGNORECASE)
SUB_CLIN_SIG = re.compile(r'this variant .{2,20}? classified .*?[.]', RegexFlag.IGNORECASE | RegexFlag.DOTALL)

@receiver(classification_validation_signal, sender=Classification)
def validate_clinical_significance(sender, **kwargs) -> Optional[ValidationMerger]:  # pylint: disable=unused-argument
    vm = ValidationMerger()
    vm.tested(
        keys=[SpecialEKeys.CLINICAL_SIGNIFICANCE],
        codes=['cs_mismatch']
    )

    patch_meta: PatchMeta = kwargs.get('patch_meta')

    if patch_meta.is_modified(SpecialEKeys.CLINICAL_SIGNIFICANCE) or patch_meta.is_modified(SpecialEKeys.INTERPRETATION_SUMMARY):
        cs: str = patch_meta.get(SpecialEKeys.CLINICAL_SIGNIFICANCE)
        inter_summary: str = patch_meta.get(SpecialEKeys.INTERPRETATION_SUMMARY)

        if cs and inter_summary:
            attempt_clin_sig = SUB_CLIN_SIG.search(inter_summary)
            if attempt_clin_sig:
                clin_text = attempt_clin_sig.group(0).lower()

                actual_cs_parts: Set[str] = set()

                if cs.startswith('VUS'):  # we don't complain about VUS A,B,C differences
                    actual_cs_parts.add('VUS')
                else:
                    if cs.startswith('L'):
                        actual_cs_parts.add('likely')
                    if cs.endswith('B'):
                        actual_cs_parts.add('benign')
                    elif cs.endswith('P'):
                        actual_cs_parts.add('pathogenic')
                    elif cs == 'R':
                        actual_cs_parts.add('risk factory')
                    elif cs == 'D':
                        actual_cs_parts.add('drug response')
                    elif cs == 'A':
                        actual_cs_parts.add('artefact')

                detected_cs_parts: Set[str] = set()
                if 'vous' in clin_text or 'vus' in clin_text or 'variant of uncertain' in clin_text:
                    detected_cs_parts.add('VUS')
                if 'like' in clin_text:
                    detected_cs_parts.add('likely')
                if 'benign' in clin_text:
                    detected_cs_parts.add('benign')
                if 'pathogenic' in clin_text:
                    detected_cs_parts.add('pathogenic')
                if 'risk factor' in clin_text:
                    detected_cs_parts.add('risk factor')
                if 'drug response' in clin_text:
                    detected_cs_parts.add('drug response')
                if 'artefact' in clin_text:
                    detected_cs_parts.add('artefact')

                if detected_cs_parts and actual_cs_parts != detected_cs_parts:
                    detected_text = ' '.join(detected_cs_parts)
                    vm.add_message(
                        key=SpecialEKeys.CLINICAL_SIGNIFICANCE,
                        code='cs_mismatch',
                        severity='warning',
                        message=f'Classified as {EvidenceKeyMap.cached_key(SpecialEKeys.CLINICAL_SIGNIFICANCE).pretty_value(cs)} but detected terms ({detected_text}) in {EvidenceKeyMap.cached_key(SpecialEKeys.INTERPRETATION_SUMMARY).pretty_label}'
                    )
    return vm


import streamlit as st
from sklearn.metrics.pairwise import cosine_similarity
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.metrics import confusion_matrix
from sklearn.preprocessing import LabelEncoder
import pandas as pd
import numpy as np
import time

def get_topic_val(modelname, topics):
    """
    Input: a) modelname: Name of the BERTopic() model used.
           b) topics: The list of topics generated by the model.

    Function: Takes in the input and returns a dictionary with topic names as the keys and the keyword's index values from the df as the values.
    """

    grouped_topics = {topic: [] for topic in set(topics)}

    for index, topic in enumerate(topics):
        grouped_topics[topic].append(index)

    return grouped_topics

def make_result_df(dictionary, topicsdict):
    ''''
    Input: a) Dictionary: The dictionary with results of dict of get_topic_val function. It has all the topics as the keys with their respective keywords as the index given by the model.
           b) topicsdict: The dictionary with the index of the corresponding keyword row in the dataframe as the key and their string keyword as the value.

    Function: Takes in the inputs, maps the index values of keywords with their actual keyword names and returns a result dataframe.
    '''

    key = []
    re_keywords = []
    val = dictionary.items()

    for i, value in val:
        key.append(i)
        val = [*map(topicsdict.get, value)]
        re_keywords.append(val)

    new_dict = {k: v for k, v in zip(key, re_keywords)}
    result_df = pd.DataFrame(new_dict.items(), columns=['Topic Nr', 'Present_Input_Keywords'])
    result_df = result_df.rename_axis('Index').reset_index()

    return result_df

def representativedocs(model, topics, docs, keywords):
    """
    Input: a) Model: Name of the model you want the results for.
           b) topics: topics extracted by the model
           c) docs: documents given as the input to the model. This is the different topic names that the model suggests. (Top n)
           d) Keywords: the input keywords given to the model

    Function: Takes in all the inputs and extracts the representative documents per topic.
    """
    model.get_topic_info()

    # extracting the topic names/numbers
    top_names = model.topic_names
    top_names = pd.DataFrame(top_names.items(), columns=[topics, docs])

    # extracting representative docs for all the topics
    rep_docs = model.representative_docs
    rep_docs = pd.DataFrame(rep_docs.items(), columns=[topics, keywords])

    # get topics with probability
    top_proba = model.get_topics()

    output = pd.merge(top_names,
                      rep_docs,
                      how='left',
                      left_on='topic_num',
                      right_on='topic_num')
    return output

def make_final_dataframe(model, representdocsdf):
    """
    Inputs: a) Model: name of the model
            b) dataframe1: This is the dataframe formed including the topics and their top n topic names for each
            c) representdocsdf: This is the resultant dataframe of the representative docs function

    Function: Returns the resultant dataframe with topic number, their top n names with c-tf-idf scores and all the keywords they contain.
    """
    dataframe1 = pd.DataFrame(model.topics.items(), columns=['Topic Nr', 'Possible Topic Names'])
    finaldfname = pd.merge(dataframe1, representdocsdf)

    return finaldfname

def get_similarity_score(model, topicnr, resultdf, threshold):
    '''
    Parameters:
        Inputs: a) model: the model used to train your topic modelling
                b) topicnr: the topic for which you want to see the similarity score. IMP: here the nr is the index of the row and not the topic nr so for topic -1 = topicnr is 0
                c) resultdf: the resultant df to merge to get combined results
                d) threshold: the threshold above which you want to get similar topics
        Ouput: A pandas dataframe with topicnr, topic names, keywords present (input) and the distance score for each.
    '''

    if model.topic_embeddings is not None:
        embeddings = np.array(model.topic_embeddings)
    else:
        embeddings = model.c_tf_idf

    distance_matrix = cosine_similarity(embeddings)
    data = distance_matrix[topicnr]
    score_df = pd.DataFrame(data=data, columns={'similarity_score'})
    score_df = score_df.rename_axis('Index').reset_index()

    # merging score with resultant dataframe
    df = pd.merge(score_df, resultdf, on='Index')

    df = df[df['similarity_score'] >= threshold]

    return df

def get_similarity_score(model):
    '''
    Parameters:
        Inputs: a) model: the model used to train your topic modelling
                b) topicnr: the topic for which you want to see the similarity score. IMP: here the nr is the index of the row and not the topic nr so for topic -1 = topicnr is 0
                c) resultdf: the resultant df to merge to get combined results
                d) threshold: the threshold above which you want to get similar topics
        Ouput: A pandas dataframe with topicnr, topic names, keywords present (input) and the distance score for each.
    '''

    topics = sorted(list(model.get_topics().keys()))

    # Extract topic words and their frequencies
    topic_list = sorted(topics)

    embeddings = model.c_tf_idf
    distance_matrix = cosine_similarity(embeddings)

    most_similar_ind = []
    most_similar_val = []

    for topic in topic_list:
        data = distance_matrix[topic]  # topic -1
        i = np.argsort(data, axis=0)[-2]
        most_similar_ind.append(i)  # ensure length and order for the list
        most_similar_val.append(data[i])

    similar_df = pd.DataFrame()
    similar_df['Topic Nr'] = topic_list
    similar_df['most_similar'] = most_similar_ind
    similar_df['similarity_score'] = most_similar_val
    return similar_df

def plot_confusion_matrix(model, true, predicted, xfig, yfig, title, topic):
    fig, ax = plt.subplots(figsize=(xfig, yfig))
    ax = sns.heatmap(confusion_matrix(predicted, true),
                     annot=True,
                     fmt='d',
                     cmap="PiYG",
                     xticklabels = topic,
                     yticklabels=topic)
    plt.ylabel('True Values')
    plt.xlabel('Predicted Values')
    plt.title(title)
    plt.xticks(rotation=90, ha='right')
    plt.yticks(rotation=360, ha='right')
    plt.show()
    st.pyplot(fig)

def _update_progress(text, progress):
    text = f"{text}"
    st.write(text, unsafe_allow_html=True)
    progress_bar = st.progress(0)
    progress_bar.progress(progress)

@st.cache
def convert_df(df):
     # IMPORTANT: Cache the conversion to prevent computation on every rerun
    return df.to_csv().encode('utf-8')

def return_top_n_pred_prob_df(n, model, docs, column_name):
    '''
    Function to predict the top n topics for a specific keyword with it's accuracy score
    Parameters:
      Input:
        a) n = Top n topic classes you want
        b) model = the model you have trained your dataset on
        c) docs = the keywords on which you want to predict the top n topics
        d) column_name = name of the column in the resultant df which takes in this as it's input for naming it

      Output: A dataframe with keywords and their corresponding topic names with its associated percentage accuracy.
    '''
    predictions = model.predict_proba(docs)
    preds_idx = np.argsort(-predictions, axis=1)
    top_n_preds = pd.DataFrame()

    for i in range(n):
        top_n_preds['keywords'] = docs
        top_n_preds[column_name + "_" + '{}'.format(i)] = [preds_idx[doc][i] for doc in range(len(docs))]
        top_n_preds[column_name + "_" + '{}_prob'.format(i)] = [predictions[doc][preds_idx[doc][i]] for doc in
                                                                range(len(docs))]

        top_n_preds = top_n_preds.rename(columns={'class_name': column_name + ''.format(i)})
        try:
            top_n_preds.drop(columns=['index', column_name + '_prediction_{}_num'.format(i)], inplace=True)
        except:
            pass
    return top_n_preds

from sklearn.metrics.pairwise import cosine_similarity

def get_class_similarity_score(model):
    '''
    Parameters:
        Inputs: a) model: the model used to train your topic modelling
                b) topicnr: the topic for which you want to see the similarity score. IMP: here the nr is the index of the row and not the topic nr so for topic -1 = topicnr is 0
                c) resultdf: the resultant df to merge to get combined results
                d) threshold: the threshold above which you want to get similar topics
        Ouput: A pandas dataframe with topicnr, topic names, keywords present (input) and the distance score for each.
    '''

    topics = sorted(list(model.get_topics().keys()))

    # Extract topic words and their frequencies
    topic_list = sorted(topics)

    embeddings = model.c_tf_idf
    distance_matrix = cosine_similarity(embeddings)

    most_similar_ind = []
    most_similar_val = []

    for topic in topic_list:
        data = distance_matrix[topic]  # topic -1
        i = np.argsort(data, axis=0)[-2]
        most_similar_ind.append(i)  # ensure length and order for the list
        most_similar_val.append(data[i])

    similar_df = pd.DataFrame()
    similar_df['Topic Nr'] = topic_list
    similar_df['most_similar'] = most_similar_ind
    similar_df['c_similarity_score'] = most_similar_val
    similar_df = similar_df.rename_axis('Index').reset_index()
    return similar_df


import shutil
from uuid import UUID, uuid4
from tempfile import SpooledTemporaryFile
from pathlib import Path

from fastapi import HTTPException
from fastapi.responses import StreamingResponse

from ucloud.settings import Config
from ucloud.services.files.base import FilesBase

class FilesLocal(FilesBase):
    _path = None

    async def write(self, data: SpooledTemporaryFile) -> dict:
        uid = uuid4()

        path = self._path / str(self.root)
        path.mkdir(parents=True, exist_ok=True)
        path = path / str(uid)

        with open(path, 'wb') as f:
            shutil.copyfileobj(data.file, f)

        return {
            'uid': uid,
            'root': self.root
        }

    async def read(self, uid: UUID) -> StreamingResponse:
        self._raise_404_if_not_exists(uid)

        path = self._path / str(self.root) / str(uid)

        def stream():
            with open(path, 'rb') as f:
                yield from f

        return StreamingResponse(stream())

    async def remove(self, uid: UUID):
        self._raise_404_if_not_exists(uid)
        path = self._path / str(self.root) / str(uid)
        path.unlink()

    @classmethod
    async def startup(cls, config: Config):
        cls._path = Path(config.UCLOUD_FILES_LOCAL_PATH)

    @classmethod
    async def shutdown(cls, config: Config):
        cls._path = None

    def _raise_404_if_not_exists(self, uid: str):
        path = self._path / str(self.root) / str(uid)
        if not path.exists():
            msg = f'Item {uid} from {self.root} not found in local'
            raise HTTPException(status_code=404, detail=msg)


# Python
import logging
import argparse
# Lib
import tensorflow as tf
from tensorflow.keras.applications.mobilenet_v2 import MobileNetV2
# App
from rpi_vision.dataset.flowers import FlowerDataset

logger = logging.getLogger(__name__)

AUTOTUNE = tf.data.experimental.AUTOTUNE
BATCH_SIZE = 24
INPUT_SHAPE = (192, 192, 3)
EPOCHS = 50
LOG_DIR = 'job-output/flowers'

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--epochs',
        type=int,
        default=EPOCHS,
        help='Number of iterations (training rounds) to run'
    )
    parser.add_argument(
        '--batch-size',
        type=int,
        default=BATCH_SIZE
    )
    parser.add_argument(
        '--restore',
        action='store_true',
        default=False,
        help='Restore from latest checkpoint in --job-dir'
    )
    parser.add_argument(
        '--save-checkpoint-steps',
        type=int,
        default=10,
        help='Maximum number of checkpoint steps to keep'
    )
    parser.add_argument(
        '--job-dir',
        default='./checkpoints',
        help='Directory where Tensorflow checkpoints will be written'
    )
    return parser.parse_args()

if __name__ == '__main__':

    args = parse_args()

    if args.restore:
        raise NotImplementedError

    model_base = MobileNetV2(
        include_top=False,
        input_shape=INPUT_SHAPE
    )
    model_base.trainable = False
    dataset = FlowerDataset()

    steps_per_epoch = tf.math.ceil(
        len(dataset.image_paths)/BATCH_SIZE).numpy()

    image_batch, label_batch = next(iter(dataset.image_label_ds))

    model = tf.keras.Sequential([
        model_base,
        tf.keras.layers.GlobalAveragePooling2D(),
        tf.keras.layers.Dense(len(dataset.label_names))])

    model.compile(optimizer=tf.keras.optimizers.Adam(),
                  loss='sparse_categorical_crossentropy',
                  metrics=['accuracy'])

    ds = dataset.image_label_ds.apply(
        tf.data.experimental.shuffle_and_repeat(buffer_size=len(dataset.image_paths)))
    ds = ds.batch(BATCH_SIZE)
    ds = ds.prefetch(buffer_size=AUTOTUNE)

    logger.info(model.summary())

    callbacks = [
        tf.keras.callbacks.TensorBoard(
            log_dir=LOG_DIR, write_images=True, write_graph=False,
            histogram_freq=1,
        ),
        tf.keras.callbacks.ModelCheckpoint(
            LOG_DIR + '/weights.{epoch:02d}.hdf5', monitor='val_loss',
            save_best_only=False, save_weights_only=False, mode='auto', save_freq=100)
    ]

    model.fit(ds, epochs=EPOCHS, steps_per_epoch=steps_per_epoch,
              callbacks=callbacks
              )


import argparse
import itertools
from tqdm import tqdm

import torch
import torch.nn as nn
from torchvision import datasets, transforms

from utils import Net, Normalize
from attacks import pgd_untargeted

parser = argparse.ArgumentParser()
parser.add_argument('--seed', type=int, default='42', help='seed')
parser.add_argument('--device', type=str, choices=['cpu', 'cuda'], default='cuda', help='device')
parser.add_argument('--defense', type=bool, default=False, help='defense')
parser.add_argument('--num_epochs', type=int, default=1, help='epochs')
args = parser.parse_args()

# Setting the random number generator
torch.manual_seed(args.seed)

# Setting up the Model
model = nn.Sequential(Normalize(), Net())
# Loading a previously trained one
model.load_state_dict(torch.load(f'./models/Net_{args.num_epochs}_{args.defense}'))
model.to(args.device)
model.eval()

# disable batches
train_dataset = datasets.MNIST('mnist_data/', train=True, download=True, transform=transforms.Compose([transforms.ToTensor()]))
train_loader = torch.utils.data.DataLoader(train_dataset, shuffle=True)

# number of samples we evaluate on
num_samples = 1000

xlist = [] # collect x tensors to use the same values in the next loop
ylist = [] # collect y tensors to use the same values in the next loop

# not attacked
###################################
# Evaluate on non-attacked images #
###################################

num_correct = 0
for _, (x, y) in enumerate(tqdm(itertools.islice(train_loader, num_samples))):
    xlist.append(x)
    ylist.append(y)
    x = x.to(args.device)
    y = y.to(args.device)
    out = model(x)
    pred = out.argmax(dim=1)
    num_correct += pred.eq(y).sum().item()

print('Accuracy %i samples original %.5lf' % (num_samples, num_correct / num_samples))

# attacked
###############################
# Evaluate on attacked images #
###############################

num_correct = 0
for x, y in tqdm(zip(xlist, ylist)):
    xperturbed = pgd_untargeted(model, x, y, k=10, eps=0.15, eps_step=0.08,
                   device=args.device, clip_min=0.0, clip_max=1.0)
    out = model(xperturbed)
    pred = out.argmax(dim=1)
    y = y.to(args.device)
    num_correct += pred.eq(y).sum().item()

print('Accuracy %i samples perturbed %.5lf' % (num_samples, num_correct / num_samples))


import cv2
import time
import HandTrackingModule as htm

# Camera
wCam, hCam = 640, 480

cap = cv2.VideoCapture(0, cv2.CAP_DSHOW)
cap.set(3, wCam)
cap.set(4, hCam)

pTime = 0

# HandTracking Module
detector = htm.HandDetector(maxHands=1, detectionCon=0.75)

tipIds = [4, 8, 12, 16, 20]

while cap.isOpened():
    success, img = cap.read()
    if not success:
        break
    img = detector.findHands(img)
    lmList, bbox = detector.findPosition(img, draw=False)
    if len(lmList) != 0:
        fingers = []

        # Thumb
        if lmList[tipIds[0]][1] < lmList[tipIds[0] - 1][1]:
            fingers.append(1)
        else:
            fingers.append(0)

        # 4 Fingers
        for id in range(1, 5):
            # Fingers Open
            if lmList[tipIds[id]][2] < lmList[tipIds[id] - 2][2]:
                fingers.append(1)
            else:
                fingers.append(0)

        totalFingers = fingers.count(1)

        cv2.rectangle(img, (20, 225), (170, 425), (0, 255, 0), cv2.FILLED)
        cv2.putText(img, str(totalFingers), (45, 375), cv2.FONT_HERSHEY_PLAIN,
                    10, (255, 0, 0), 25)

    # Compute Frames Per Second
    cTime = time.time()
    fps = 1 / (cTime - pTime)
    pTime = cTime

    # Display Frames Per Second
    cv2.putText(img, f'FPS: {int(fps)}', (40, 50), cv2.FONT_HERSHEY_COMPLEX,
                1, (255, 0, 0), 3)

    cv2.imshow("Image", img)
    cv2.waitKey(1)


import numpy as np
from datasets import load_dataset_wrapper

def initialize_parameters(sizes):
    """
    初试化参数
    weights的维度：(当前层神经元个数，前一层的神经元个数)
    biases的维度：(当前层神经元个数，1)

    输入：sizes
        如sizes=[2,3,1],表示一个输入为2维的2层神经网络，
        隐藏层的神经元个数分别为3,1

    返回：weights,biases
        weights = [[W1],W2,...] : W1,W2,...为矩阵
        biases = [[b1],[b2],...] ：b1,b2,...为向量
    """
    # 初试为均值为0，方差为1的高斯分布
    weights = [np.random.randn(y, x)
                        for x, y in zip(sizes[:-1], sizes[1:])]
    biases = [np.random.randn(y, 1) for y in sizes[1:]]
    return  weights,biases

def feedforward(x,W,b):
    """
    # 前项传播
    参数：
        A - 来自上一层（或输入数据）的激活，维度为(上一层的节点数量，示例的数量）
        W - 权重矩阵，numpy数组，维度为（当前图层的节点数量，前一图层的节点数量）
        b - 偏向量，numpy向量，维度为（当前图层节点数量，1）

    返回：
         A
    """
    activation = x  # 第0层的激活值A^[0]
    zs = [] # 存储所有的Z
    activations = [] # 存储所有的激活值，A^[0],A^[1],......

    activations.append(activation)

    for w,b in zip(W, b):
        z = np.dot(w, activation)+b
        zs.append(z)
        activation = sigmoid(z)
        activations.append(activation)
    return activations[-1]

def backprop(x, y,weights,biases,sizes):
        """
        反向传播算法
        返回偏导数
        """
        num_layers = len(sizes)

        m = len(x[0])  # 输入m组数据
        learning_rate = 0.1 # 学习率

        dW = [np.zeros(w.shape) for w in weights]
        db = [np.zeros(b.shape) for b in biases]

        # 前项传播
        activation = x  # 第0层的激活值A^[0]
        zs = [] # 存储所有的Z
        activations = [] # 存储所有的激活值，A^[0],A^[1],......

        activations.append(activation)

        for b, w in zip(biases, weights):
            z = np.dot(w, activation)+b
            zs.append(z)
            activation = sigmoid(z)
            activations.append(activation)
        # print("zs:",zs)
        # print("activations:",activations)

        # 反向传播
        dZ = cost_derivative(activations[-1], y) * sigmoid_derivative(zs[-1])
        dW[-1] = np.dot(dZ, activations[-2].transpose())
        db[-1] = dZ

        for l in range(2, num_layers):
            z = zs[-l]
            sp = sigmoid_derivative(z)
            dZ = np.dot(weights[-l+1].transpose(), dZ) * sp

            dW[-l] = np.dot(dZ, activations[-l-1].transpose())
            db[-l] = dZ

        #更新参数
        weights = [W_temp-(learning_rate/m)*dW_temp for W_temp,dW_temp in zip(weights,dW)]
        biases = [b_temp-(learning_rate/m)*db_temp for b_temp,db_temp in zip(biases,db)]

        return  weights,biases

def predict(x, y,weights,biases):
        # test_results = [(np.argmax(feedforward(x,weights,biases)), y)
        #                 for (x, y) in zip(x,y)]
        # return sum(int(x == y) for (x, y) in test_results)/len(x)

        y_hat=np.zeros(y.shape)
        A =feedforward(x,weights,biases)

        for i in range(len(A[0])):
            if(A[0][i]>0.5):
                y_hat[0][i] = 1
        # print(y)
        # print(y_hat)
        return (sum(int(y1 == y2) for (y1, y2) in zip(y_hat[0],y[0]))/len(y[0]))

def cost_derivative(output_activations, y):
    """
    返回第L层：dL(a,y)/dz
    """
    return (output_activations-y)

def sigmoid(z):
    """
    sigmoid 函数.
    """
    return 1.0/(1.0+np.exp(-z))

def sigmoid_derivative(z):
    """sigmoid 函数的导数."""
    return sigmoid(z)*(1-sigmoid(z))

def relu(z):
    """
    RELU 函数
    """
    return np.maximum(0,z)

# 定义神经网络
sizes = [12288,7,9,1]

#初始化参数
w,b = initialize_parameters(sizes)

# 加载数据
m_train, train_set_x, train_set_y, m_test, test_set_x, test_set_y = load_dataset_wrapper()

print("train_set_x:",train_set_x.shape)
print("train_set_y:",train_set_y.shape)
for i in range(1000):
    w ,b = backprop(test_set_x, test_set_y,w,b,sizes)
    if i%10==0:
        print("第",i,"轮：",predict(test_set_x, test_set_y,w,b))


from im2dhist import  im2dhist
import cv2
import numpy as np
from mpl_toolkits import mplot3d
import matplotlib.pyplot as plt

def plot2DHistogram(Hist2D, title):
    # plot 2D-Histogram
    [K, _] = Hist2D.shape
    x = np.outer(np.arange(0, K), np.ones(K))
    y = x.copy().T
    # ln-ing Hist2D makes its details more prominent.
    Hist2D_ln = Hist2D.copy()
    Hist2D_ln[np.where(Hist2D_ln<=0)] = 1e-15
    z = np.log(Hist2D_ln)
    fig = plt.figure()
    ax = plt.axes(projection='3d')
    ax.plot_surface(x, y, z,cmap='viridis', edgecolor='none')
    ax.set_title(title)
    plt.show()

def main():
    image_name = 'assets/Plane.jpg'
    image = cv2.imread(image_name)
    # convert rgb image to gray
    gray_image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)

    # 2*w_neighboring+1 is width of the square window around each pixel, when counting neiboring pixels
    # calculate 2 dimensional histogram of gray_image
    v_image_2DHisteq = im2dhist(gray_image, w_neighboring=6)
    #np.save(f'{image_name}-2D-Histogram', v_image_2DHisteq)
    # v_image_2DHisteq_cmpr = np.load(f'{image_name}-2D-Histogram.npy')
    plot2DHistogram(v_image_2DHisteq, title="2D Histogram of Plane.jpg")

if __name__ == '__main__': main()


"""Script to find phone numbers assoiated with an address(s)."""
import pprint
import re
import itertools
import time
import pandas
import requests
import phonenumbers
from googlesearch import search
from bs4 import BeautifulSoup
from requests import HTTPError

def search_timer(things, time_per_thing, overhead, min_time=3):
    """Estimates the seconds it may take to complete a task.

    Parameters
    ----------
    things : TYPE
        DESCRIPTION.
    time_per_thing : TYPE
        DESCRIPTION.
    overhead : TYPE
        DESCRIPTION.
    min_time : TYPE, optional
        DESCRIPTION. The default is 3.

    Returns
    -------
    None.

    """
    if (things*(time_per_thing+overhead)) < min_time:
        run_time = min_time
    else:
        run_time = REC_COUNT*(DELAY+RUN_FACT)
    return run_time

REGEX = r"\(?\b[2-9][0-9]{2}\)?[-. ]?[2-9][0-9]{2}[-. ]?[0-9]{4}\b"
# The secret sauce
results = {}
my_user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) \
                    AppleWebKit/537.36 (KHTML, like Gecko) \
                    Chrome/68.0.3440.106 Safari/537.36'
HEADERS = {
    'User-Agent': (my_user_agent)
}
REC_COUNT = 5
DELAY = 2
RUN_FACT = 1.7

# targets and parms || will be prompted for in future
doc_path = r"C:\Users\andrew.white\Documents\Lists\CPC\Wong Numbers Jan 2020"
file_name = "Wrong Number DB Jan 2020.xlsx"
full_path = r"{doc}\{file}".format(doc=doc_path, file=file_name)
ws_name = "Worksheet"
worksheet = pandas.read_excel(full_path, sheet_name=ws_name)
addresses_to_find = worksheet["Full Address"]

if REC_COUNT < 0:
    things_to_look_up = addresses_to_find
    REC_COUNT = len(addresses_to_find)
    print("Allons-y... should be about {run_time} secs"
          .format(run_time=search_timer(REC_COUNT, DELAY, RUN_FACT)))
else:
    things_to_look_up = itertools.islice(addresses_to_find, 0, REC_COUNT + 1)
    print("Allons-y... should be about {run_time} secs"
          .format(run_time=search_timer(REC_COUNT, DELAY, RUN_FACT)))

start_time = time.time()

for address in things_to_look_up:
    try:
        urls_found = search(address, pause=DELAY, stop=10,
                            user_agent=my_user_agent)
        entry = list(urls_found)
        count = len(entry)
        pprint.pprint("{count} urls for {address} added to results..."
                      .format(count=count, address=address))
        results[address] = entry
    except HTTPError as err:
        print("Oops... {error}".format(error=err))

s_count = 0
for s in results.values():
    s_count = s_count+len(s)

print("Search complete... {count} found".format(count=s_count))
print("--- Total seach time %s seconds ---" % (time.time() - start_time))
for k, r in results.items():
    for u in r:
        try:
            data_got = requests.get(u, HEADERS)
        except Exception:
            pass
        b = data_got.text
        soup = BeautifulSoup(b, 'html.parser')
        for node in soup.findAll('body'):
            matches = re.finditer(REGEX, str(node.findAll(text=True)),
                                  re.MULTILINE)
            phone_numbers_found = {}
            phone_numbers_found = set()
            for matchNum, match in enumerate(matches, start=1):
                # print("Match {matchNum} was found at {start}-{end}:{match}"
                #       .format(matchNum=matchNum, start=match.start(),
                #               end=match.end(), match=match.group()))
                if match.group() in phone_numbers_found:
                    print("Gotz it...")
                else:
                    phone_numbers_found.add(match.group())
                    print("{x} numbers for {address} found so far"
                          .format(x=len(phone_numbers_found), address=k))
                    results[k]

print(("Total of {things} address(s) searched for.\nRetrived {count} \
       urls.\nYeilding {numbers} phone numbers.")
      .format(things=len(results), count=s_count,
              numbers=len(phone_numbers_found)))
out = pandas.DataFrame.from_dict(results, orient="index")
out.to_csv("numbers.csv")