# myTeam.py
# ---------
# Licensing Information:  You are free to use or extend these projects for
# educational purposes provided that (1) you do not distribute or publish
# solutions, (2) you retain this notice, and (3) you provide clear
# attribution to UC Berkeley, including a link to http://ai.berkeley.edu.
#
# Attribution Information: The Pacman AI projects were developed at UC Berkeley.
# The core projects and autograders were primarily created by John DeNero
# (denero@cs.berkeley.edu) and Dan Klein (klein@cs.berkeley.edu).
# Student side autograding was added by Brad Miller, Nick Hay, and
# Pieter Abbeel (pabbeel@cs.berkeley.edu).
import copy

from captureAgents import CaptureAgent
import random, util

#################
# Team creation #
#################

def createTeam(firstIndex, secondIndex, isRed,
               first = 'DummyAgent', second = 'DummyAgent'):
  """
  This function should return a list of two agents that will form the
  team, initialized using firstIndex and secondIndex as their agent
  index numbers.  isRed is True if the red team is being created, and
  will be False if the blue team is being created.

  As a potentially helpful development aid, this function can take
  additional string-valued keyword arguments ("first" and "second" are
  such arguments in the case of this function), which will come from
  the --redOpts and --blueOpts command-line arguments to capture.py.
  For the nightly contest, however, your team will be created without
  any extra arguments, so you should make sure that the default
  behavior is what you want for the nightly contest.
  """

  # The following line is an example only; feel free to change it.
  return [eval(first)(firstIndex), eval(second)(secondIndex)]

##########
# Agents #
##########

targetFood = None
class DummyAgent(CaptureAgent):
  """
  A Dummy agent to serve as an example of the necessary agent structure.
  You should look at baselineTeam.py for more details about how to
  create an agent as this is the bare minimum.
  """

  def registerInitialState(self, gameState):
    """
    This method handles the initial setup of the
    agent to populate useful fields (such as what team
    we're on).

    A distanceCalculator instance caches the maze distances
    between each pair of positions, so your agents can use:
    self.distancer.getDistance(p1, p2)

    IMPORTANT: This method may run for at most 15 seconds.
    """

    '''
    Make sure you do not delete the following line. If you would like to
    use Manhattan distances instead of maze distances in order to save
    on initialization time, please take a look at
    CaptureAgent.registerInitialState in captureAgents.py.
    '''
    self.start = gameState.getAgentPosition(self.index)

    CaptureAgent.registerInitialState(self, gameState)

    '''
    Your initialization code goes here, if you need any.
    '''
    self.cost = 0
    self.food = self.getFood(gameState).asList()
    self.totalFoodNumber = len(self.food)
    self.foodLeft = self.totalFoodNumber
    self.carry = gameState.getAgentState(self.index).numCarrying
    self.myPosition = gameState.getAgentPosition(self.index)
    self.isDefense = False
    # maze distance to my position, it's a dictionary
    self.myStepTo = self.stepToMe(gameState)
    self.targetFood = random.choice(self.getFood(gameState).asList())
    self.initalTarget(gameState)
    self.safeFoddList = self.initialSafeFoodList(gameState)
    self.defenseTarget = self.start

    global targetFood
    targetFood = self.targetFood

    self.weights = util.Counter({"Food": 93, "FoodCarrying": 87, "Enemy": 100,
                                "Capsule": 87, "Teammate": 5,"Defense": 50, "Home": 500, "Bias": 55})

    # self.weights = util.Counter()
    # # read weights from file
    # file = open("weightUpdating.txt", "r+")
    # f = file.readlines()
    # print("Read file: ", f)
    # for line in f:
    #   line = line.split(":")
    #   weight = line[0].strip()
    #   value = float(line[1])
    #   self.weights[weight] = value
    # file.close()

  #################
  # derived funcs #
  #################

  # choose an action based on Q_value of that specific action
  # set up some conditions
  def chooseAction(self, gameState):
    global targetFood

    # if both enemies are not pacman, no need to be defense mode!
    if not self.getEnemy(gameState)[0][0] and self.getEnemy(gameState)[1][0]:
      self.isDefense = False

    # [condition] : avoid enemy, treat its surrounding as walls
    for isPacman, position, distance, isScared in self.getEnemy(gameState):
      if (not isPacman) and (not isScared) and (position is not None):
        x, y = position
        gameState.data.layout.walls[x][y] = True
        gameState.data.layout.walls[x + 1][y] = True
        gameState.data.layout.walls[x - 1][y] = True
        gameState.data.layout.walls[x][y + 1] = True
        gameState.data.layout.walls[x][y - 1] = True

    self.cost+=1
    self.food = self.getFood(gameState).asList()
    self.foodLeft = len(self.food)
    self.carry = gameState.getAgentState(self.index).numCarrying
    self.myPosition = gameState.getAgentPosition(self.index)
    self.myStepTo = self.stepToMe(gameState)

    # [condition] : if defense mode on! and agent is not pacman
    if (self.isDefense) and (not gameState.getAgentState(self.index).isPacman):
      # if agent still has capsule to eat, change to defense that capsule
      if len(self.getCapsulesYouAreDefending(gameState))>0 and self.defenseTarget == self.start:
        self.defenseTarget = self.getCapsulesYouAreDefending(gameState)[0]

    # [condition] : if this is not first move, which means agent has Previous State
    # agent need to check the different from previous state
    if not self.getPreviousObservation() is None:
      # capsules miss? change agent target
      if len(self.getCapsules(self.getPreviousObservation())) - len(self.getCapsules(gameState)) == 1:
        self.changeNearestFood(gameState)
      # food miss? change agent target
      if (not self.targetFood in self.food):
        self.changeNearestFood(gameState)
      # ate a food, change agent target
      if self.getPreviousObservation().getAgentState(self.index).numCarrying < self.carry:
        self.changeNearestFood(gameState)

      # previous agent is not pacman and now is pacman
      if (not self.getPreviousObservation().getAgentState(self.index).isPacman) and gameState.getAgentState(
              self.index).isPacman:
        self.changeNearestFood(gameState)
      # detect if any food was eaten, if any, change target to that position
      defendFood = self.getFoodYouAreDefending(gameState).asList()
      preDefendFood = self.getFoodYouAreDefending(self.getPreviousObservation()).asList()
      if len(preDefendFood) - len(defendFood) == 1:
        for food in preDefendFood:
          if not(food in defendFood):
            self.defenseTarget = food
            break
        print("food be eaten", self.defenseTarget)

    # [condition] : if food left (3-5), which means very close to end game,
    # one of agent can go back to defense
    if self.foodLeft <=5 and self.foodLeft >= 3:
      if self.carry == 0 and gameState.getAgentState((self.index+2)%4).isPacman:
        self.isDefense = True
      else:
        self.isDefense = False

    # [condition] : if my target is very close to my teammate's (10 steps ),
    # if yes, change my target far away from my teammates
    if self.getMazeDistance(self.targetFood, targetFood) <= 10 and \
      self.getMazeDistance(self.myPosition, self.targetFood)>= \
      self.getMazeDistance(gameState.getAgentPosition((self.index+2)%4), self.targetFood):
      self.changeTargetAwayToTeammate(gameState)

    print("Agent ", self.index)
    # print("Defense Mode: ", self.isDefense)
    # print("Cost ", self.cost)
    # print("current pos: ", self.myPosition)
    # print("target: ", self.targetFood)
    # print("Carrying: ", self.carry)
    # print("enemy: ", self.getEnemy(gameState))

    action = self.selectAction(gameState)
    # update target food to global variable, so teammate can detect my target
    targetFood = self.targetFood
    print("choose action:", action)

    # Doing updating
    # if gameState.getLegalActions(self.index) is not None:
    #   if len(gameState.getLegalActions(self.index)) != 0 and action != "Stop":
    #     self.updateWeights(gameState, action)

    print("------------------------------------")

    return action

  # calculate the features value based on a specific action which could trigger some events
  # using Decision Trees
  def getFeatures(self, gameState, action):
    successor = self.getSuccessor(gameState, action)
    features = util.Counter({"Food": 0, "FoodCarrying": 0, "Enemy": 0,
                                "Capsule": 0, "Teammate": 0, "Defense": 0, "Home": 0, "Bias": 1})

    # [condition] : agent is away from the enemy  , get point!
    # enemy(isPacman, position, distance, isScared)
    for i in range(len(self.getEnemy(gameState))):
      if (self.getEnemy(successor)[i][2] - self.getEnemy(gameState)[i][2]) > 1 and \
        successor.getAgentPosition(self.index) != self.start:
        features["Enemy"] = 100
        return features

    # [condition] : agent get a capsule while enemy is not scared! , get point!
    if len(self.getCapsules(gameState))-len(self.getCapsules(successor)) == 1:
      enemy1, enemy2 = self.getEnemy(gameState)
      if (not enemy1[3]) or (not enemy2[3]):
        features["Capsule"] = 200

    # [condition] : if teammate is close to me, lose point!
    if self.getMazeDistance(self.myPosition, gameState.getAgentPosition((self.index+2)%4)) <= 5:
      features["Teammate"] = -1

    """
    [Event] : capture enemy(pacman) -> if enemy is pacman , position can be observed, distance to me <=5
    and I'm ghost, not scared, (and no capsules in our area??!!)
    OR enemy is pacman, and I'm ghost and I'm defense
    """
    enemy = self.getEnemy(gameState)
    for i in range(len(self.getEnemy(gameState))):
      if (enemy[i][0] and (enemy[i][1] is not None) and enemy[i][2] < 5
          and (not gameState.getAgentState(self.index).isPacman)
          and gameState.getAgentState(self.index).scaredTimer ==0) and (not enemy[i][3]) \
        or (enemy[i][0] and self.isDefense and (not gameState.getAgentState(self.index).isPacman)):
        # more closer to enemy, get point!   ;  away from enemy, lose point!
        features["Enemy"] = enemy[i][2] - self.getEnemy(successor)[i][2]

        # if I have to save my teammate
        if self.isSaveTeammate(gameState):
          # there is a capsule I can eat (no barrier)
          if min(self.getCapsulesDistance(gameState))<9999:
            features["Enemy"] = 0
            features["Defense"] = 0
        # if in defense mode , and more closer to defense target, get point!
        elif self.isDefense:
          features["Defense"] = self.getMazeDistance(self.myPosition, self.defenseTarget) - \
                                self.getMazeDistance(successor.getAgentPosition(self.index), self.defenseTarget)
        # if enemy was captured (which means enemy reborn at start position)
        if features["Enemy"] < -1:
          features["Enemy"] = 100

        # if I'm scared, stay away from the enemy  (negative point of current features["Enemy])
        # no defense point
        if gameState.getAgentState(self.index).scaredTimer != 0:
          features["Enemy"] = -features["Enemy"]
          features["Defense"] = 0

        if self.isDefense:
          features["Bias"] = 0

        #TODO: if I can be the barrier on the path that enemy go back
        print("Capture Enemy ", action, features)
        return features
    """
    [Event] : Score!  -> if this successor back to the border and scored some points
    """
    # get point when this action can go back to score
    if self.backToScore(gameState, successor):
      features["FoodCarrying"] = abs(self.carry-successor.getAgentState(self.index).numCarrying)* 1000
      features["Home"] = self.getDistanceToHome(gameState) - self.getDistanceToHome(successor)
      print("Score! ", action, features)
      return features

    """
    [Event] : Escape! -> when I'm trying to eat (not safe food) food, if in below conditions, Escape!
    1. if target food is not safe or
    2. target food safe and teammate is pacman or
    3. no way home and no way to get capsule or
    4. there is a way to capsule
    """
    if not (self.targetFood in self.safeFoddList) or ((self.targetFood in self.safeFoddList) and
      gameState.getAgentState((self.index + 2) % 4).isPacman) or \
      (self.myStepTo[self.start] > 999 and min(self.getCapsulesDistance(gameState)) > 999) or \
            (min(self.getCapsulesDistance(gameState)) < 9999):

      # if enemy is 5 step close to me, not scared, and not pacman and I'm pacman, Escape!
      doEscape = False
      enemy = self.getEnemy(gameState)
      for i in range(len(self.getEnemy(gameState))):
        if gameState.getAgentState(self.index).isPacman and (not enemy[i][0]) and (not enemy[i][3]) and \
          (self.getEnemyDistance(gameState)[i] <= 5):
          doEscape = True
      # if no way home and no way to get capsule, Escape!
      if (self.myStepTo[self.start] > 999 and min(self.getCapsulesDistance(gameState)) > 999):
        doEscape = True
      # ok, do Escape!
      if (doEscape):

        # if there is still food, choose random food
        if len(self.food) != 0:
          self.targetFood = random.choice(self.getFood(gameState).asList())

        # if food left <= 2 and there is a way home(which means no barrier)
        # OR no way to get a capsule or no capsule anymore

        if (self.foodLeft <= 2 and self.getDistanceToHome(gameState) < 9999) or \
          (min(self.getCapsulesDistance(gameState)) > 999) or (len(self.getCapsules(gameState)) == 0):
          # lose point! when you get closer to teammate
          print("Event happened!")
          features["Teammate"] = self.getMazeDistance(successor.getAgentPosition(self.index),
                              successor.getAgentPosition((self.index + 2) % 4))- \
          self.getMazeDistance(gameState.getAgentPosition(self.index),
                               gameState.getAgentPosition((self.index + 2) % 4))

          # if I have a way home (no barrier)
          if self.getDistanceToHome(gameState) < 9999:
            features["Home"] = self.getDistanceToHome(gameState)- self.getDistanceToHome(successor)

          # if no way home
          else:
            features["Home"] = self.getMazeDistance(self.start, self.myPosition) - \
                               self.getMazeDistance(self.start, successor.getAgentPosition(self.index))
        else:

          # if still not get a capsule, add point if go closer to a capsule
          if features["Capsule"] != 200 :
            features["Capsule"] = min(self.getCapsulesDistance(gameState)) - min(self.getCapsulesDistance(successor))
        print("Escape! ", action, features)
        return features

    """
    [Event] : Go Home! -> if isGoHome condition is true, go home for some reason, still save teammate?!
    """
    if (self.isGoHome(gameState)):
      features["Home"] = self.getDistanceToHome(gameState) - self.getDistanceToHome(successor)
      # if I have to save my teammate
      if self.isSaveTeammate(gameState) and len(self.getCapsules(gameState)) != 0:
        if gameState.getAgentState((self.index + 2) % 4).isPacman:
          if self.getMazeDistance(self.myPosition, gameState.getAgentPosition((self.index+2)%4)) >= 3:
            features["Capsule"] = min(self.getCapsulesDistance(gameState)) - \
                                  min(self.getCapsulesDistance(successor))
            features["Home"] = 0
      # if teammate and I are both pacman or both ghost, keep distance to each other
      if gameState.getAgentState(self.index).isPacman == gameState.getAgentState((self.index + 2) % 4).isPacman:
        features["Teammate"] = self.getMazeDistance(successor.getAgentPosition(self.index),
                              successor.getAgentPosition((self.index + 2) % 4))- \
          self.getMazeDistance(gameState.getAgentPosition(self.index),
                               gameState.getAgentPosition((self.index + 2) % 4))

      # if there is no way home
      if self.getDistanceToHome(gameState) >999:
        features["Home"] = self.getMazeDistance(gameState.getAgentPosition(self.index), self.start) - \
                           self.getMazeDistance(successor.getAgentPosition(self.index), self.start)

      # if I'm ghost
      if (not gameState.getAgentState(self.index).isPacman):
        features["Home"] = 0
        features["Teammate"] = 0
        features["Bias"] = 0

      print("Go Home! ", action, features)
      return features

    """
    [Event] : Get Food! -> when I'm eating a (safe) food
    """
    if gameState.getAgentState(self.index).numCarrying - successor.getAgentState(self.index).numCarrying == -1:
      # if I certainly get 1 food, add point!
      if successor.getAgentPosition(self.index) == self.targetFood:
        features["Food"] = 2
        features["FoodCarrying"] = 1
      print("Get Food! ", action, features)
      return features

    """
    [Event] : Find Food! -> if above events are not triggered, do this event! (like a general action)
    """
    if gameState.getAgentState(self.index).numCarrying < self.totalFoodNumber - 2:

      features["Food"] = self.getDistanceToTarget(gameState) - self.getDistanceToTarget(successor)
      # if there is no way to target food(BFSMap), use maze distance
      if self.getDistanceToTarget(gameState) > 999:
        features["Food"] = self.getMazeDistance(self.myPosition, self.targetFood) - self.getMazeDistance(
          successor.getAgentPosition(self.index), self.targetFood)

      enemy = self.getEnemy(gameState)
      if features["Capsule"] != 200:
        features["Capsule"] = min(self.getCapsulesDistance(gameState)) - min(self.getCapsulesDistance(successor))
      # if enemies are scared, no need to eat capsules
      if (enemy[0][3]) and (enemy[1][3]):
        features["Capsule"] = 0
      # if there are barriers to get food, capsule,and I can observe an enemy, go home!
      if self.getDistanceToTarget(gameState) >999 and \
        min(self.getCapsulesDistance(gameState)) > 999 and min(self.getEnemyDistance(gameState)) <=2:
        features["Home"] = self.getDistanceToHome(gameState) - self.getDistanceToHome(successor)

      # if enemy and I are all around border! I can block them
      for i in range(len(self.getEnemy(gameState))):
        if (not gameState.getAgentState(self.index).isPacman) and (not enemy[i][0]) and \
          (enemy[i][1] is not None) and (self.getEnemyDistance(gameState)[i]<=3) and (not enemy[i][3]):
          if len(self.food) != 0 :
            self.targetFood = random.choice(self.food)
          features["Food"] = enemy[i][2]*0.1
          features["Capsule"] = 0

      # save teammate, and if this action can not eat a capsule
      if self.isSaveTeammate(gameState) and min(self.getCapsulesDistance(gameState))<9999 \
              and features["Capsule"] != 200:
        # if there is a capsule that I can find, try to get it
        features["Capsule"] = 2*(min(self.getCapsulesDistance(gameState)) - min(self.getCapsulesDistance(successor)))
        # if I actually got it
        if len(self.getCapsules(gameState))-len(self.getCapsules(successor)) == 1:
          features["Capsule"] = 200

      print("Find Food! ", action, features)
      return features

    return features

  #################
  # help function #
  #################

  # When to go home (to the border)?
  def isGoHome(self, gameState):
    # 1. if food left <= 2 which means we can end game, then go home and stop eating
    if self.foodLeft <= 2:
      if gameState.getAgentState(self.index).numCarrying == 0:
        self.isDefense = True
      return True
    # 2. if food left 3~5 , if teammate is more closer to food, I can go home
    if self.foodLeft >2 and self.foodLeft <=5:
      teammateMap = self.BFSMap(gameState, gameState.getAgentPosition((self.index + 2) % 4))
      teammateFoodList = []
      for food in self.food:
        if food in teammateMap:
          teammateFoodList.append(teammateMap[food])
      if len(teammateFoodList) > 0 and min(self.getDistanceOfFoods(gameState)) > max(teammateFoodList):
        return True
    # 3. if I carry much food, and my target food is far away to me, I can go home
    if gameState.getAgentState(self.index).numCarrying > 10 and \
      self.getMazeDistance(gameState.getAgentPosition(self.index), self.targetFood) > 20:
      return True
    # 4. if in the end game, go home and defense
    if self.cost >= 250 and gameState.getAgentState(self.index).numCarrying > 0:
      return True
    # 5. if no way to target food (enemy there?!), go home and reset target
    if self.targetFood in self.myStepTo and self.myStepTo[self.targetFood] > 999 and \
            (min(self.getDistanceOfFoods(gameState)) < 999):
      if len(self.food) != 0:
        self.targetFood = random.choice(self.getFood(gameState).asList())
      return True
    return False

  # set up initial target, which make 2 agents have different target - highest one and lowest one (Y-axis)
  def initalTarget(self, gameState):
    x, y = self.targetFood
    for food in self.food:
      x_food, y_food = food
      if gameState.isOnRedTeam(self.index) and (self.index == 0):
        if y_food < y:
          y = y_food
          self.targetFood = (x_food, y_food)
      elif (gameState.isOnRedTeam(self.index) and (self.index >= 2)):
        if y_food > y:
          y = y_food
          self.targetFood = (x_food, y_food)
      elif (self.index >=2):
        if y_food > y:
          y = y_food
          self.targetFood = (x_food, y_food)
      else:
        if y_food < y:
          y = y_food
          self.targetFood = (x_food, y_food)

  # return a list of enemy includes (isPacman, position, distance, isScared)
  def getEnemy(self, gameState):
    myPosition = gameState.getAgentPosition(self.index)
    enemy = self.getOpponents(gameState)
    enemyState = []
    for e in enemy:
      enemyPosition = gameState.getAgentPosition(e)
      isEnemyScared = gameState.getAgentState(e).scaredTimer > 0
      if enemyPosition == None:
        enemyDistance = gameState.getAgentDistances()[e]
        enemyState.append((gameState.getAgentState(e).isPacman, enemyPosition, enemyDistance, isEnemyScared))
      else:
        enemyDistance = self.getMazeDistance(enemyPosition, myPosition)
        enemyState.append((gameState.getAgentState(e).isPacman, enemyPosition, enemyDistance, isEnemyScared))
    return enemyState

  # change target food based on the lowest distance to foods (heuristic search)
  def changeNearestFood(self, gameState):
    distanceOfFoods = self.getDistanceOfFoods(gameState)
    foodList = self.getFood(gameState).asList()
    if len(foodList) > 1:
      self.targetFood = foodList[distanceOfFoods.index(min(distanceOfFoods))]
    if len(foodList) != 0:
      self.targetFood = foodList[0]

  # return a list of all foods distance based on Map,
  # if not in map which means enemy is 1 distance away there
  def getDistanceOfFoods(self, gameState):
    foodList = self.getFood(gameState).asList()
    foodDistance = []
    for food in foodList:
      if food in self.myStepTo:
        foodDistance.append(self.myStepTo[food])
      else:
        foodDistance.append(9999)
    return foodDistance

  # avoid to choose a target food too close to my teammate's
  def changeTargetAwayToTeammate(self, gameState):
    foodList = self.getFood(gameState).asList()
    if self.targetFood in foodList:
      for food in foodList:
        if self.getMazeDistance(food, targetFood)<=10:
          foodList.remove(food)
    if len(foodList) >0:
      self.targetFood = random.choice(foodList)

  # get successor from an action,
  # if there is a very close enemy, treat its surroundings as walls
  def getSuccessor(self, gameState, action):
    for isPacman,position,distance,isScared in self.getEnemy(gameState):
      if (not isPacman) and (position is not None) and distance<10 and (not isScared):
        x,y = position
        gameState.data.layout.walls[x][y] = True
        gameState.data.layout.walls[x+1][y] = True
        gameState.data.layout.walls[x-1][y] = True
        gameState.data.layout.walls[x][y+1] = True
        gameState.data.layout.walls[x][y-1] = True
    successor = gameState.generateSuccessor(self.index, action)
    return successor

  # calculate the "step" distance to my current position, using BFSMap algorithm
  def stepToMe(self, gameState):
    myPosition = gameState.getAgentPosition(self.index)
    return self.BFSMap(gameState, myPosition)

  # BFS algorithm, uses to scan the map and calculates the "step" distance for each node
  # (not including walls, and the surrounding of enemy)
  # (when distance == 9999, it means no way to that node)
  def BFSMap(self, gameState, position):
    myPosition = position
    validNodes = gameState.data.layout.walls.asList(False)
    step = {}
    closed = {}
    for node in validNodes:
      step[node] = 9999
    queue = util.PriorityQueue()
    queue.push(myPosition, 0)
    step[myPosition] = 0
    while (not queue.isEmpty()):
      node = queue.pop()
      if node in closed:
        continue
      closed[node] = True
      nodeStep = step[node]
      adjacentList = []
      x, y = node
      if not (gameState.data.layout.isWall((x + 1, y))):
        adjacentList.append((x + 1, y))
      if not (gameState.data.layout.isWall((x - 1, y))):
        adjacentList.append((x - 1, y))
      if not (gameState.data.layout.isWall((x, y + 1))):
        adjacentList.append((x, y + 1))
      if not (gameState.data.layout.isWall((x, y - 1))):
        adjacentList.append((x, y - 1))
      for adjNode in adjacentList:
        if not adjNode in step:
          continue
        old = step[adjNode]
        new = nodeStep + 1
        if new < old:
          step[adjNode] = new
          queue.push(adjNode, new)

    return step

  # produce a safe list of food
  def initialSafeFoodList(self, gameState):
    safeFoodList = []
    for x,y in self.food:
      initalState = gameState.deepCopy()
      initalState.data.layout.walls[x][y] = True
      BFSMap = self.BFSMap(initalState,self.start)
      count = 0
      if (x, y + 1) in BFSMap and BFSMap[(x, y + 1)] == 9999:
        count += 1
      if (x, y - 1) in BFSMap and BFSMap[(x, y - 1)] == 9999:
        count += 1
      if (x - 1, y) in BFSMap and BFSMap[(x - 1, y)] == 9999:
        count += 1
      if (x + 1, y) in BFSMap and BFSMap[(x + 1, y)] == 9999:
        count += 1
      if count < 3:
        safeFoodList.append((x,y))
    return safeFoodList

  # return true if I have to save my teammate
  def isSaveTeammate(self, gameState):
    # if teammate's target food is safe and 20 distance away to capsules, no need to save her
    if targetFood in self.safeFoddList and min(self.getCapsulesDistance(gameState)) >= 20:
      return False
    for enemy in self.getEnemy(gameState):
      if (not enemy[0]) and (not enemy[1] is None) and (not enemy[3]) \
        and self.getMazeDistance(enemy[1], gameState.getAgentPosition((self.index+2)%4)) < 5:
        return True
    return False

  # return a list of my position to capsules
  def getCapsulesDistance(self, gameState):
    capsulesList = self.getCapsules(gameState)
    myStepTo = self.stepToMe(gameState)
    if capsulesList != []:
      capsulesDistance = []
      for cap in capsulesList:
        if cap in myStepTo:
          capsulesDistance.append(myStepTo[cap])
        else:
          capsulesDistance.append(9999)
    else:
      capsulesDistance = [9999]
    return capsulesDistance

  # return a list of my position to enemies
  def getEnemyDistance(self, gameState):
    enemys = self.getOpponents(gameState)
    enemyDistance = []
    for enemy in enemys:
      if gameState.getAgentPosition(enemy) == None:
        enemyDistance.append(9999)
      else:
        enemyDistance.append(self.getMazeDistance(gameState.getAgentPosition(self.index),
                                                  gameState.getAgentPosition(enemy)))
    return enemyDistance

  # return true, if the successor crosses the border to home and get score!
  def backToScore(self, gamestate, successor):
    # if successor is alive (which means not in the initial position)
    if successor.getAgentPosition(self.index) != successor.getInitialAgentPosition(self.index):
      if (gamestate.getAgentState(self.index).numCarrying -
        successor.getAgentState(self.index).numCarrying) > 1:
        return True
      if self.foodLeft <=2 and gamestate.getAgentState(self.index).numCarrying - successor.getAgentState(
            self.index).numCarrying == 1:
        return True
    return False

  # calculate the distance to my target, and return (9999 means there is a barrier)
  def getDistanceToTarget(self, gameState):
    myPosition = gameState.getAgentPosition(self.index)
    # if not safe
    if not (self.targetFood in self.getFood(gameState).asList()):
      return 0
    # if I'm not reborn in start position
    if myPosition == self.myPosition:
      if self.targetFood in self.myStepTo:
        distance = self.myStepTo[self.targetFood]
      else:
        distance = 9999
    # I died
    else:
      # recalculate my map
      myStepTo = self.stepToMe(gameState)
      if self.targetFood in myStepTo:
        distance = myStepTo[self.targetFood]
      else:
        distance = 9999
    return distance

  # compute distance to home
  def getDistanceToHome(self, gameState):
    currentPosition = gameState.getAgentPosition(self.index)
    if (currentPosition == self.myPosition):
      distance = self.myStepTo[self.start]
    else:
      distance = self.stepToMe(gameState)[self.start]
    return distance

  # select an action based on the Q score it gets
  def selectAction(self, gameState):
    if gameState.getLegalActions(self.index) is not None:
      actions = gameState.getLegalActions(self.index)
    else:
      actions = ["Stop"]
    qScoreList = []
    for action in actions:
      score = self.getFeatures(gameState, action) * self.weights
      qScoreList.append(score)
    print(qScoreList)
    if len(qScoreList) == 0:
      action = "Stop"
    else:
      if max(qScoreList) == 0:
        action = "Stop"
      else:
        myActions = []
        for i in range(len(qScoreList)):
          if qScoreList[i] == max(qScoreList):
            myActions.append(actions[i])
        action = random.choice(myActions)
    return action

  # Update weight
  # Q(s,a) = (1- alpha) * Q(s,a) + alpha * (reward + estimated future value)
  # w(i) = w(i) + alpha((reward + discount*value(nextState)) - Q(s,a)) * f(i)(s,a)
  # def updateWeights(self, gameState, action):
  #   if gameState.getLegalActions(self.index) is not None:
  #     if len(gameState.getLegalActions(self.index)) != 0:
  #       successor = self.getSuccessor(gameState, action)
  #       if not(successor.data._win) :
  #         # update
  #         # learningValue = abs(10*self.getScore(successor))
  #         # update
  #         actions = successor.getLegalActions(self.index)
  #         futureValue = [self.getFeatures(successor, action) * self.weights for action in actions]
  #         learningValue = 0.01 * (0 + 0.8*max(futureValue) - self.getFeatures(gameState,action)*self.weights)
  #       else:
  #         learningValue = 0
  #       self.weightsCopy = util.Counter()
  #       for key, value in self.weights.items():
  #         updatedValue = value + learningValue * self.getFeatures(gameState, action).get(key)
  #         self.weightsCopy[key] = updatedValue
  #
  #       file = open("weightUpdating.txt", "w+")
  #       for key, value in self.weightsCopy.items():
  #         line = key + ": "+str(self.weightsCopy[key])+"\n"
  #         file.write(line)
  #       print("Update weights")
  #       file.close()


import decimal
import datetime
import json

from django.core import mail
from django.core.exceptions import ObjectDoesNotExist
from django.core.urlresolvers import reverse
from django.test import TestCase
from django.utils import timezone
from django.utils.http import urlquote
from model_mommy import mommy
from rest_framework.authtoken.models import Token
from rest_framework.test import APIClient

from main.main_models.translation import Translation
from main.main_models.bid import Status
from main.serializers import TranslationSerializer

class AdminDashboardViewsTestCase(TestCase):
    def setUp(self):
        self.client = APIClient()
        self.language = mommy.make('main.Language')
        self.items = mommy.make('main.Translation', make_m2m=True, _quantity=50)
        self.item = mommy.make('main.Translation', make_m2m=True)

        self.item_json = TranslationSerializer(self.item).data
        self.status = mommy.make('main.Status')

        self.admin_user = mommy.make('main.User')
        self.admin_user.is_staff = True
        self.admin_user.save()
        self.reg_user = mommy.make('main.User')

        # Generate token for token auth based on the user in the requests
        self.admin_token = Token()
        self.admin_token.key = self.admin_token.generate_key()
        self.admin_token.user_id = self.admin_user.id
        self.admin_token.save()
        self.reg_token = Token()
        self.reg_token.key = self.reg_token.generate_key()
        self.reg_token.user_id = self.reg_user.id
        self.reg_token.save()

    def test_admin_create_item_view(self):
        Translation.objects.get(id=self.translation.id).delete()

        self.client.force_authenticate(self.admin_user, self.admin_token)
        request = self.client.post(reverse('create_item'), data=self.item_json, format='json')
        item = Translation.objects.get(pk=52)

        self.assertEqual(request.status_code, 201)
        self.assertEqual(translation.name, self.item_json['name'])
        self.assertEqual(translation.description, self.item_json['description'])
        self.assertEqual(translation.weight, decimal.Decimal(self.item_json['weight']))
        self.assertEqual(translation.price, decimal.Decimal(self.item_json['price']))
        self.assertEqual(translation.languages.count(), len(self.item_json['languages']))

    def test_admin_create_view_as_reg_user(self):
        Translation.objects.get(id=self.translation.id).delete()

        self.client.force_authenticate(self.reg_user, self.reg_token)
        request = self.client.post(reverse('create_item'), data=self.item_json, format='json')

        self.assertEqual(request.status_code, 403)
        with self.assertRaises(ObjectDoesNotExist):
            Translation.objects.get(pk=52)

    def test_admin_item_list(self):
        self.client.force_authenticate(self.admin_user, self.admin_token)
        request = self.client.get(reverse('create_item'))
        data = json.loads(request.content)

        self.assertEqual(request.status_code, 200)
        self.assertEqual('count' in data, True)
        self.assertEqual('next' in data, True)
        self.assertEqual(len(data['results']), 25)

        for result in data['results']:
            self.assertEqual('id' in result, True)
            self.assertEqual('name' in result, True)
            self.assertEqual('description' in result, True)
            self.assertEqual('weight' in result, True)
            self.assertEqual('price' in result, True)
            self.assertEqual('languages' in result, True)
            self.assertEqual('tags' in result, True)

    def test_admin_item_list_as_reg_user(self):
        self.client.force_authenticate(self.reg_user, self.reg_token)
        request = self.client.get(reverse('create_item'))
        self.assertEqual(request.status_code, 403)

    def test_admin_language_list(self):
        self.client.force_authenticate(self.admin_user, self.admin_token)
        request = self.client.get(reverse('create_language'))
        data = json.loads(request.content)

        self.assertEqual(request.status_code, 200)
        self.assertEqual(data['count'], 256)
        self.assertEqual(data['results'][0]['name'], self.language.name)

    def test_admin_language_list_as_reg_user(self):
        self.client.force_authenticate(self.reg_user, self.reg_token)
        request = self.client.get(reverse('create_language'))
        self.assertEqual(request.status_code, 403)

    def test_admin_language_detail(self):
        self.client.force_authenticate(self.admin_user, self.admin_token)
        request = self.client.get(reverse('language_detail_update', args=(1,)))
        data = json.loads(request.content)

        self.assertEqual(request.status_code, 200)
        self.assertEqual(data['id'], self.language.id)
        self.assertEqual(data['name'], self.language.name)

    def test_admin_language_detail_as_reg_user(self):
        self.client.force_authenticate(self.reg_user, self.reg_token)
        request = self.client.get(reverse('language_detail_update', args=(1,)))
        self.assertEqual(request.status_code, 403)

    def test_admin_status_list(self):
        self.client.force_authenticate(self.admin_user, self.admin_token)
        request = self.client.get(reverse('create_status'))
        data = json.loads(request.content)

        self.assertEqual(request.status_code, 200)
        self.assertEqual(data['count'], 1)
        self.assertEqual(data['results'][0]['status'], self.status.status)
        self.assertEqual(data['results'][0]['description'], self.status.description)

    def test_admin_status_list_as_reg_user(self):
        self.client.force_authenticate(self.reg_user, self.reg_token)
        request = self.client.get(reverse('create_status'))
        self.assertEqual(request.status_code, 403)

    def test_admin_status_detail(self):
        self.client.force_authenticate(self.admin_user, self.admin_token)
        request = self.client.get(reverse('status_detail_update', args=(1,)))
        data = json.loads(request.content)

        self.assertEqual(request.status_code, 200)
        self.assertEqual(data['status'], self.status.status)
        self.assertEqual(data['id'], self.status.id)
        self.assertEqual(data['description'], self.status.description)

    def test_admin_status_detail_as_reg_user(self):
        self.client.force_authenticate(self.reg_user, self.reg_token)
        request = self.client.get(reverse('status_detail_update', args=(1,)))
        self.assertEqual(request.status_code, 403)

class TranslationTestCase(TestCase):
    def setUp(self):
        now = timezone.now()
        self.items = mommy.make('main.Translation', make_m2m=True, _quantity=25)
        self.single_item_no_sales = mommy.make('main.Translation', make_m2m=True)
        self.single_item_sold = mommy.make('main.Translation', make_m2m=True)
        self.single_item_sold_multiple_times = mommy.make('main.Translation', make_m2m=True)
        self.single_item_sold_old_orders = mommy.make('main.Translation', make_m2m=True)
        self.order_with_sold_item = mommy.make('main.Bid', items=[self.single_item_sold], make_m2m=True, placed=True,
                                               date_placed=now)
        self.orders_with_single_item = mommy.make('main.Bid', items=[self.single_item_sold_multiple_times],
                                                  make_m2m=True, _quantity=25, placed=True, date_placed=now)

        old_date = timezone.now() - datetime.timedelta(days=60)
        self.old_orders = mommy.make('main.Bid', items=[self.single_item_sold_old_orders], make_m2m=True,
                                     _quantity=25, placed=True)
        for order in self.old_orders:
            bid.date_placed = old_date
            bid.save()

    def test_to_unicode(self):
        self.assertEqual(self.single_item_sold.name, str(self.single_item_sold))

    def test_number_sold(self):
        self.assertEqual(self.single_item_sold.number_sold, 1)
        self.assertEqual(self.single_item_sold_multiple_times.number_sold, 25)
        self.assertEqual(self.single_item_sold_old_orders.number_sold, 25)

    def test_best_selling(self):
        self.assertListEqual(list(Translation.get_best_selling()), [self.single_item_sold_multiple_times,
                                                             self.single_item_sold_old_orders, self.single_item_sold])
        self.assertEqual(Translation.get_best_selling()[0], self.single_item_sold_multiple_times)
        self.assertEqual(Translation.get_best_selling()[1], self.single_item_sold_old_orders)

    def test_best_selling_recently(self):
        self.assertListEqual(list(Translation.get_best_selling_recently()), [self.single_item_sold_multiple_times,
                                                                      self.single_item_sold])

class TranslationViewsTestCase(TestCase):
    def setUp(self):
        self.language1 = mommy.make('main.Language', make_m2m=True)
        self.language2 = mommy.make('main.Language', make_m2m=True)
        self.item1 = mommy.make('main.Translation', name='Translation 1',
                                description='item 1 description', make_m2m=True)

        self.item2 = mommy.make('main.Translation', name='Translation 2',
                                description='item 1 description', make_m2m=True)

        self.item1.languages.clear()
        self.item1.languages.add(self.language1)

        self.client = APIClient()

    def test_item_search_with_null_search_term(self):
        request = self.client.get(reverse('items_search'))
        data = json.loads(request.content)

        self.assertEqual(request.status_code, 200)
        self.assertEqual(data['count'], 0)

    # def test_item_search_by_language_name(self):
    #     print self.language1.name
    #     request = self.client.get(reverse('items_search'), {'search': self.language1.name})
    #     data = json.loads(request.content)
    #
    #     self.assertEqual(request.status_code, 200)
    #     self.assertEqual(data['count'], 1)
    #     self.assertEqual(data['results'][0]['id'], 1)

    def test_item_search_by_item_name(self):
        request = self.client.get(reverse('items_search'), {'search': 'Translation 1'})
        data = json.loads(request.content)

        self.assertEqual(request.status_code, 200)
        self.assertEqual(data['count'], 2)
        self.assertEqual(data['results'][0]['name'], "Translation 1")
        self.assertEqual(data['results'][1]['name'], "Translation 2")

    def test_item_search_by_item_description(self):
        request = self.client.get(reverse('items_search'), {'search': 'description'})
        data = json.loads(request.content)

        self.assertEqual(request.status_code, 200)
        self.assertEqual(data['count'], 2)
        self.assertEqual(data['results'][0]['name'], 'Translation 1')
        self.assertEqual(data['results'][1]['name'], 'Translation 2')

    def test_item_list(self):
        request = self.client.get(reverse('items_list'))
        data = json.loads(request.content)

        self.assertEqual(request.status_code, 200)
        self.assertEqual(data['count'], 2)
        self.assertEqual(data['results'][0]['name'], self.item1.name)
        self.assertEqual(decimal.Decimal(data['results'][0]['weight']), self.item1.weight)
        self.assertEqual(decimal.Decimal(data['results'][0]['price']), self.item1.price)
        self.assertEqual(data['results'][0]['languages'][0]['id'], self.language1.id)
        self.assertEqual(data['results'][0]['languages'][0]['name'], self.language1.name)
        self.assertEqual(data['results'][0]['description'], self.item1.description)

    def test_item_detail(self):
        request = self.client.get(reverse('item_detail', args=(1,)))
        data = json.loads(request.content)

        self.assertEqual(request.status_code, 200)
        self.assertEqual(data['name'], self.item1.name)
        self.assertEqual(decimal.Decimal(data['weight']), self.item1.weight)
        self.assertEqual(decimal.Decimal(data['price']), self.item1.price)
        self.assertEqual(data['description'], self.item1.description)
        # self.assertEqual(data['language']['id'], self.language1.id)

class LanguageTestCase(TestCase):
    def setUp(self):
        self.language = mommy.make('main.Language')

    def test_to_unicode(self):
        self.assertEqual(str(self.language), self.language.name)

class BidTestCase(TestCase):
    def setUp(self):
        now = timezone.now()
        self.status = mommy.make('main.Status')
        self.orders = mommy.make('main.Bid', _quantity=20, make_m2m=True, placed=True, date_placed=now)
        self.old_order = mommy.make('main.Bid', make_m2m=True)
        self.old_bid.date_placed = timezone.now() - datetime.timedelta(days=45)
        self.single_order = mommy.make('main.Bid', make_m2m=True, _current_status=self.status)

    def test_to_unicode(self):
        self.assertEqual(str(self.single_order), str(self.single_bid.id))

    # def test_weight_set_automatically(self):
    #     for order in self.orders:
    #         self.assertEqual(type(bid.weight), decimal.Decimal)
    #         self.assertNotEqual(bid.weight, None)
    #         self.assertNotEqual(bid.weight, 0.00)
    #
    #         total_weight = decimal.Decimal()
    #         items = Bid.objects.get(id=bid.id).items.all()
    #         for item in items:
    #             total_weight += translation.weight
    #
    #         self.assertEqual(total_weight, bid.weight)

    # def test_total_price_set_automatically(self):
    #     for order in self.orders:
    #         self.assertEqual(type(bid._total_price), decimal.Decimal)
    #         self.assertNotEqual(bid.total_price, None)
    #         self.assertNotEqual(bid.total_price, 0.00)
    #
    #         total_price = decimal.Decimal()
    #         items = Bid.objects.get(id=bid.id).items.all()
    #         for item in items:
    #             total_price += translation.price
    #
    #         self.assertEqual(total_price, bid.total_price)

    def test_recently_placed(self):
        for order in self.orders:
            self.assertEqual(bid.recently_placed(), True)

    def test_recently_placed_with_old_order(self):
        self.assertEqual(self.old_bid.recently_placed(), False)

    def test_setters_throw_exception(self):
        with self.assertRaises(AttributeError):
            self.old_bid.total_price = decimal.Decimal(5.00)

        with self.assertRaises(AttributeError):
            self.old_bid.last_modified = timezone.now()

        with self.assertRaises(AttributeError):
            self.old_bid.weight = decimal.Decimal(1.25)

    def test_current_status_property(self):
        self.assertIsInstance(self.single_bid.current_status, Status)

    def test_current_status_property_setter(self):
        old_last_modified = self.single_bid.last_modified
        status = Status.objects.get(pk=1)
        self.single_bid.current_status = status
        self.assertNotEqual(self.single_bid.last_modified, old_last_modified)
        self.assertEqual(self.single_bid.current_status, status)

class UserTestCase(TestCase):
    def setUp(self):
        self.shop_user = mommy.make('main.User', first_name='Alice', last_name='Smith')

    def test_get_full_name(self):
        self.assertEqual(self.shop_user.get_full_name(),
                         str(self.shop_user.first_name + ' ' + self.shop_user.last_name))

    def test_get_short_name(self):
        self.assertEqual(self.shop_user.get_short_name(), self.shop_user.first_name)

    def test_get_absolute_url(self):
        self.assertEqual(self.shop_user.get_absolute_url(), "/users/{}/".format(urlquote(self.shop_user.email)))

    def test_email_user(self):
        subject = "Email Test"
        message = "Message for email test"
        from_email = "unit_test@localhost"
        self.shop_user.email_user(subject, message, from_email)
        self.assertEqual(len(mail.outbox), 1)
        email = mail.outbox[0]
        self.assertEqual(email.subject, subject)

class StatusTestCase(TestCase):
    def setUp(self):
        self.status = mommy.make('main.Status')

    def test_to_unicode(self):
        self.assertEqual(str(self.status), self.status.status)

class BillingAddressTestCase(TestCase):
    def setUp(self):
        self.address = mommy.make('main.Address', type='Billing')

    def test_to_unicode(self):
        self.assertEqual(str(self.address), self.address.name)

class ShopTaggedTranslationTestCase(TestCase):
    def setUp(self):
        self.shop_tagged_item = mommy.make('main.Shop_Tagged_Translation', make_m2m=True)

    def test_to_unicode(self):
        self.assertEqual(str(self.shop_tagged_translation.tag), self.shop_tagged_translation.tag.name)


import argparse
import cma
import matplotlib.pyplot as plt
import numpy as np
import os
import pickle
import shutil
import subprocess
import yaml
from multiprocessing import Pool
import time

def is_float(test_string):
    # Check for float string
    try:
        float(test_string)
        res = True
    except:
        res = False
    return res

def get_mAP(file_path):

    with open(file_path, 'r') as f:
        for line in f:
            if 'mAP' in line:
                for s in line.split(":"):
                    if is_float(s):
                        return float(s)

def ssd(ips):
    x = ips[0]
    itr = ips[1]
    gene_no = ips[2]

    key = 'Itr_' + str(itr) + '_gene' + str(gene_no)

    print('current i/p',x)
    anchor_scales = [(300 * anc).item() for anc in x[:7]]
    min_sizes = anchor_scales[:6]
    max_sizes = anchor_scales[1:7]

    print(type(min_sizes[0]))
    # anchor_ratios = [[x[7]], [x[7], x[8]], [x[7], x[8]], [x[7], x[8]], [x[7]], [x[7]]]

    # Opening the cfg template and writing the config template
    stream = open('templates/cfg_template.yml', 'r')
    cfg_data = yaml.safe_load(stream)
    cfg_data['MODEL']['PRIORS']['MIN_SIZES'] = min_sizes
    cfg_data['MODEL']['PRIORS']['MAX_SIZES'] = max_sizes
    # cfg_data['MODEL']['PRIORS']['ASPECT_RATIOS'] = anchor_ratios
    cfg_data['OUTPUT_DIR'] = 'outputs/{}_vgg_ssd300_voc07'.format(key)

    with open(
            'configs/' + '{}_vgg_ssd300_voc07'.format(key) + '.yaml',
            'w'
    ) as outfile:
        yaml.dump(cfg_data, outfile, default_flow_style=False)

    shutil.copy(
        'templates/{}'.format('bash_file_template.sh'),
        'bash_files/{}'.format(key) +  '.sh'
    )

    job_file = 'bash_files/{}'.format(key) +  '.sh'

    with open(job_file, "a") as myfile:
        myfile.write(
            "\nsrun "
            "--ntasks 1 "
            "--job-name job_trial "
            "--mem-per-cpu 15g "
            "--cpus-per-task 1 "
            "--partition gpu_titan "
            "--gres gpu:1 "
            "python train.py "
            "--config-file configs/{}_vgg_ssd300_voc07.yaml".format(key)
        )

    # Running the job file
    os.system('sbatch -W %s' % job_file)

    result_file = 'outputs/{}_vgg_ssd300_voc07/inference/voc_2007_test/result_final.txt'.format(key)
    mAP = get_mAP(result_file)
    print("map is", mAP)
    return 1 - mAP

def CMA_ES():

    previous_cmaes_path = None
    run_from_previous = False

    max_evaluations = 18
    initial_guess = [0.1, 0.2, 0.37, 0.54, 0.71, 0.88, 1.05]
    sigma = 0.1
    bounds = [
        [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
        [1.06, 1.06, 1.06, 1.06, 1.06, 1.06, 1.06]
    ]

    cma_obj_path = 'objects/'

    if run_from_previous:
        es = pickle.load(open('_saved-cma-object.pkl', 'rb'))
    else:
        es = cma.CMAEvolutionStrategy(initial_guess, sigma, {'bounds': bounds})

    itr = 0
    max_itr = int(max_evaluations / es.popsize)
    print('Population size is:', es.popsize)

    solution_dict = {i: [] for i in range(max_itr)}
    results_dict = {i: [] for i in range(max_itr)}

    best_gene = initial_guess
    best_value = float("inf")

    while itr < max_itr:

        print('Iteration {} is Running:'.format(itr+1) )
        # getting the genes for evaluation
        solutions = es.ask()

        # evaluating the genes
        with Pool(es.popsize + 1 ) as p:
            fn_results = p.map(ssd, [[s,itr,i] for i,s in enumerate(solutions)])
        # fn_results = [frosenbrock(s) for s in solutions]

        print('Solutions: ', solutions)
        print('Results: ',fn_results)
        # optimising based on the results
        es.tell(solutions, fn_results)

        # logging the data for results
        es.logger.disp_header()
        es.disp()
        es.logger.add()

        # Appendind the datas
        solution_dict[itr].append(solutions)
        results_dict[itr].append(fn_results)

        # checking for best gene
        if min(fn_results) < best_value:
            best_value = np.min(fn_results)
            best_gene = solutions[np.argmin(fn_results)]
        itr = itr + 1

        pickle.dump(es, open('{}saved-cma-object.pkl'.format(cma_obj_path), 'wb'))
        pickle.dump(solution_dict, open('{}solution_dict.pkl'.format(cma_obj_path), 'wb'))
        pickle.dump(results_dict, open('{}results_dict.pkl'.format(cma_obj_path), 'wb'))

CMA_ES()


# -*- coding: utf-8 -*-
"""
Created on Fri Mar 16 09:33:03 2018

@author: Evgeny

Functions to load and process the data
"""

import collections as cl
import csv
import itertools as it
import os
import pickle
import nltk
import sklearn as sk
from collections import namedtuple

from config import DataSetConfig, ROOT_PATH
from load_embeddings import load_embedding

Text2EndWord = namedtuple('Text2EndWord', [
    'input_sentences',
    'pos',
    'neg',
])

Text2EndInd = namedtuple('Text2EndInd', [
    'input_sentences_merged',
    'input_sentences',
    'pos',
    'neg',
])

Text2End  = namedtuple('Text2End', [
    'word',
    'ind',
])

class InputData:

    @staticmethod
    def load_data():
        data_path = os.path.join(ROOT_PATH, 'Data')
        return InputData(data_path, vocabulary_size=20000)

    __TRAIN_PICKLE_KEY = 'TRAIN'
    __EVAL_PICKLE_KEY = 'EVAL'
    __TRAIN_SENTACES_PICKLE_KEY = 'TRAIN_SENTACES'
    __EVAL_SENTACES_PICKLE_KEY = 'EVAL_SENTACES'
    __ALL_2END_PICKLE_KEY = 'ALL_2END'
    __ALL_2END_VALID_PICKLE_KEY = 'ALL_2END_VALID'
    __ALL_2END_TEST_PICKLE_KEY = 'ALL_2END_TEST'
    __VOCABULARY_PICKLE_KEY = 'VOCABULARY'
    __WORD2VEC_PICKLE_KEY = 'WORD2VEC'

    PAD = '<pad>'
    EOS = '<eos>'
    BOS = '<bos>'
    UNK = '<unk>'

    def __init__(self, data_path, vocabulary_size):
        self.all_1end_path = os.path.join(data_path, "train_stories.csv")
        self.all_2end_path = os.path.join(data_path, "cloze_test_val__spring2016 - cloze_test_ALL_val.csv")
        self.all_2end_valid_path = os.path.join(data_path, "cloze_test_spring2016-test.csv")
        self.all_2end_test_path = os.path.join(data_path, "test_nlu18_utf-8.csv")
        self.word2vec_path = os.path.join(data_path, "GoogleNews-vectors-negative300.bin")
        self.pickle_path = os.path.join(data_path, "data.pk")
        self.vocabulary_size = vocabulary_size
        self.dataset_cfg = DataSetConfig(
            dict_size=vocabulary_size,
        )

        self.__create_data_pickle_if_not_exist()

        with open(self.pickle_path, 'rb') as pfile:
            data_map = pickle.load(pfile)
            self.train_data = data_map[InputData.__TRAIN_PICKLE_KEY]
            self.eval_data = data_map[InputData.__EVAL_PICKLE_KEY]
            self.train_data_sentences = data_map[InputData.__TRAIN_SENTACES_PICKLE_KEY]
            self.eval_data_sentences = data_map[InputData.__EVAL_SENTACES_PICKLE_KEY]
            self.all_2end_data = data_map[InputData.__ALL_2END_PICKLE_KEY]
            self.all_2end_valid_data = data_map[InputData.__ALL_2END_VALID_PICKLE_KEY]
            self.all_2end_test_data = data_map[InputData.__ALL_2END_TEST_PICKLE_KEY]
            self.vocabulary = data_map[InputData.__VOCABULARY_PICKLE_KEY]
            self.word2vec = data_map[InputData.__WORD2VEC_PICKLE_KEY]

        self.unk_ind, self.word_to_ind = InputData.__compute_word_to_ind(self.vocabulary)
        self.ind_to_word = self.__build_ind_to_word()
        self.pad_ind = self.word_to_ind(InputData.PAD)
        self.eos_ind = self.word_to_ind(InputData.EOS)
        self.bos_ind = self.word_to_ind(InputData.BOS)

    def __build_vocabulary(self, train_data):
        join = it.chain.from_iterable
        words = join(join(train_data))
        special_words = [InputData.PAD, InputData.EOS, InputData.BOS]
        # self.vocabulary_size-1 becouse we also have unkown word
        frequencies = nltk.FreqDist(words).most_common(self.vocabulary_size - len(special_words) - 1)
        top_words = list(map(lambda p: p[0], frequencies)) + special_words
        return dict(map(lambda p: (p[1], p[0]), enumerate(top_words)))

    def __compute_word_to_ind(vocabulary):
        unk_ind = len(vocabulary)

        def word_to_ind(word):
            return vocabulary.get(word, unk_ind)

        return unk_ind, word_to_ind

    def __load_1end_data(self, tokenize_fun):
        with open(self.all_1end_path, 'r') as file:
            txts = list(csv.reader(file, delimiter=',', quotechar='"'))[1:]
            txts = map(lambda s: s[2:], txts)
            txts = list(list(map(lambda s: list(map(tokenize_fun, s)), txts)))
            assert all(map(lambda s: len(s) == 5, txts))
            return txts

    def __load_2end_data(self, tokenize_fun, file_path):
        with open(file_path, 'r') as file:
            txts = list(csv.reader(file, delimiter=',', quotechar='"'))[1:]
            txts = map(lambda s: s[1:], txts)
            txts = list(list(map(lambda s: list(map(tokenize_fun, s)), txts)))

            def buidl_text_2end(txt):
                assert (len(txt) == 7)
                assert (len(txt[6]) == 1)
                corect_ind = int(txt[6][0])
                assert (corect_ind in [1,2])
                is_first_correct = (corect_ind == 1)
                res = Text2EndWord(
                    input_sentences =txt[:4],
                    pos = txt[4] if is_first_correct else txt[5],
                    neg = txt[5] if is_first_correct else txt[4],
                )
                assert (len(res.input_sentences) == 4)
                return res

            return list(map(buidl_text_2end, txts))

    #pos is the first ending, neg is the second ending
    def __load_2end_test_data(self, tokenize_fun, file_path):
        with open(file_path, 'r') as file:
            txts = list(csv.reader(file, delimiter=',', quotechar='"'))
            txts = list(list(map(lambda s: list(map(tokenize_fun, s)), txts)))

            def buidl_text_2end(txt):
                assert (len(txt) == 6)
                res = Text2EndWord(
                    input_sentences =txt[:4],
                    pos = txt[4],
                    neg = txt[5],
                )
                assert (len(res.input_sentences) == 4)
                return res

            return list(map(buidl_text_2end, txts))

    def __preproc_sentence(self, s):
        return [InputData.BOS] + s + [InputData.EOS]

    def __preproc_1end_data_sentaces(self, data, word_to_ind):
        return list(map(lambda t: list(map( lambda s: list(map(word_to_ind, s)), map(self.__preproc_sentence, t))), data))

    def __preproc_1end_data(self, data, word_to_ind):
        join = it.chain.from_iterable
        data = map(lambda t: list(join(map(self.__preproc_sentence, t))), data)
        return list(map(lambda s: list(map(word_to_ind, s)), data))

    def __preproc_2end_data(self, data, word_to_ind):
        def build_text2end(text_2_end_word):
            def to_ind(s):
                return list(map(word_to_ind, s))

            input_sentences = list(map(self.__preproc_sentence, text_2_end_word.input_sentences))
            input_sentences_merged = list(it.chain.from_iterable(input_sentences))
            input_sentences = list(map(to_ind, input_sentences))
            input_sentences_merged = to_ind(input_sentences_merged)
            pos = to_ind(self.__preproc_sentence(text_2_end_word.pos))
            neg = to_ind(self.__preproc_sentence(text_2_end_word.neg))
            text_2_end_ind = Text2EndInd(
                input_sentences_merged=input_sentences_merged,
                input_sentences= input_sentences,
                pos=pos,
                neg=neg)
            res = Text2End(word=text_2_end_word, ind=text_2_end_ind)
            return res

        return list(map(build_text2end, data))

    def __create_data_pickle_if_not_exist(self):
        if os.path.exists(self.pickle_path): return
        tokenizer = nltk.WordPunctTokenizer()

        def tokenize_fun(text): return tokenizer.tokenize(text.lower())

        all_1end_data = self.__load_1end_data(tokenize_fun)
        train_1end_data, eval_1end_data = sk.model_selection.train_test_split(all_1end_data, test_size=0.1, random_state=42)
        vocabulary = self.__build_vocabulary(train_1end_data)
        unk_ind, word_to_ind = InputData.__compute_word_to_ind(vocabulary)
        train_data = self.__preproc_1end_data(train_1end_data, word_to_ind)
        eval_data = self.__preproc_1end_data(eval_1end_data, word_to_ind)
        train_data_sentaces = self.__preproc_1end_data_sentaces(train_1end_data, word_to_ind)
        eval_data_sentaces = self.__preproc_1end_data_sentaces(eval_1end_data, word_to_ind)

        all_2end_data = self.__load_2end_data(tokenize_fun, self.all_2end_path)
        all_2end_data = self.__preproc_2end_data(all_2end_data, word_to_ind)

        all_2end_valid_data = self.__load_2end_data(tokenize_fun, self.all_2end_valid_path)
        all_2end_valid_data = self.__preproc_2end_data(all_2end_valid_data, word_to_ind)

        all_2end_test_data = self.__load_2end_test_data(tokenize_fun, self.all_2end_test_path)
        all_2end_test_data = self.__preproc_2end_data(all_2end_test_data, word_to_ind)

        vocabulary_with_unk = vocabulary.copy()
        vocabulary_with_unk[InputData.UNK] = unk_ind

        word2vec = load_embedding(vocabulary_with_unk, self.word2vec_path) if os.path.exists(self.word2vec_path) else None

        with open(self.pickle_path, 'wb') as pfile:
            data_map = {
                InputData.__TRAIN_PICKLE_KEY: train_data,
                InputData.__EVAL_PICKLE_KEY: eval_data,
                InputData.__TRAIN_SENTACES_PICKLE_KEY: train_data_sentaces,
                InputData.__EVAL_SENTACES_PICKLE_KEY: eval_data_sentaces,
                InputData.__ALL_2END_PICKLE_KEY: all_2end_data,
                InputData.__ALL_2END_VALID_PICKLE_KEY: all_2end_valid_data,
                InputData.__ALL_2END_TEST_PICKLE_KEY: all_2end_test_data,
                InputData.__VOCABULARY_PICKLE_KEY: vocabulary,
                InputData.__WORD2VEC_PICKLE_KEY: word2vec,
            }
            pickle.dump(data_map, pfile, protocol=pickle.HIGHEST_PROTOCOL)

    def __build_ind_to_word(self):
        ind_to_word_dic = {v: k for k, v in self.vocabulary.items()}
        self.unk_ind = len(self.vocabulary)
        ind_to_word_dic[self.unk_ind] = InputData.UNK

        def ind_to_word(ind):
            return ind_to_word_dic[ind]

        return ind_to_word

if __name__ == '__main__':
    data_loader = InputData.load_data()

    train_data = data_loader.train_data
    eval_data = data_loader.eval_data
    all_2end_data = data_loader.all_2end_data
    vocabulary = data_loader.vocabulary
    ind_to_word = data_loader.ind_to_word
    word_to_ind = data_loader.word_to_ind
    word2vec = data_loader.word2vec

    print("OK")


from sofabase3 import get_data_from_directory, save_data_item_to_directory
from controllers import SceneController, SceneRunningController
from logic_object import LogicObjects
from sofabase3 import AlexaDevice, Activate, Directive, ErrorResponse
import uuid

import logging
logger = logging.getLogger("sofabase3")

class LogicScenes(LogicObjects):

    @property
    def data_type(self):
        return "scene"

    def get_id_from_endpoint_id(self, endpoint_id):
        for id in self.data:
            if self.data[id]['endpointId'] == endpoint_id:
                return id
        return None

    async def add_object_device(self, device_id, nativeObject):
        nativeObject['running'] = False
        device = AlexaDevice('logic/%s/%s' % (self.data_type, device_id), device_id, displayCategories=["SCENE_TRIGGER"], description="Sofa Logic Scene", adapter=self.logic_adapter)
        device.SceneController = SceneController(device=device)
        device.SceneRunningController = SceneRunningController(name='Running', device=device, friendlyNames=["Running"], nonControllable=True)
        return self.dataset.add_device(device)

    def get_cached_property(self, endpoint_id, property_name, deep_value = False):
        if endpoint_id not in self.dataset.state_cache:
            return None

        device_properties = self.dataset.state_cache[endpoint_id].get('properties', [])
        for prop in device_properties:
            if prop['name'] == property_name:
                value = prop.get('value', None)
                if deep_value and type(value)==dict and 'value' in value:
                    return value['value']
                return value

        # TODO - should we try to get the uncached version?
        return None

    def calculate_scene_light_brightness(self, scene, light_endpointId):

        # Compare the power on state and brightness level from the scene definition and determine
        # whether the light is bright or not

        scene_light = self.data[scene]['children'][light_endpointId]
        scene_light_brightness = 0

        if 'powerState' in scene_light and scene_light['powerState']=='OFF':
            scene_light_brightness = 0
        elif 'brightness' in scene_light:
            scene_light_brightness = scene_light['brightness']
        else:
            scene_light_brightness = 100

        return scene_light_brightness

    def calculate_state_light_brightness(self, light_id):

        # Compare the power on state and brightness level from the state in dataset
        # whether the light is bright or not
        connectivity = self.get_cached_property(light_id, "connectivity", deep_value=True)
        power_state = self.get_cached_property(light_id, "powerState")
        brightness = self.get_cached_property(light_id, "brightness")
        #logger.info(f'.. state {light_id} - {connectivity} {power_state} {brightness}')

        if connectivity == "UNREACHABLE" or power_state == "OFF":
            return 0

        if brightness:
            return brightness

        if power_state == "ON":
            return 100

        return 0

    def calculate_scene_score(self, endpoint_id):

        scene_id = self.get_id_from_endpoint_id(endpoint_id)

        if scene_id not in self.data:
            logger.warning(f'!! scene {scene_id} does not exist in scene definition file. This data error should be fixed manually.')
            logger.info(f'self data: {self.data}')
            return 0

        scene = self.data[scene_id]
        scene_score = 0
        brightness_score = 0
        scene_lights_on = []
        state_lights_on = []

        for light_endpointId in scene['children']:
            scene_light_brightness = self.calculate_scene_light_brightness(scene_id, light_endpointId)
            if scene_light_brightness: scene_lights_on.append(light_endpointId)

            device_light_brightness = self.calculate_state_light_brightness(light_endpointId)
            if device_light_brightness: state_lights_on.append(light_endpointId)

            scene_score += (50 - abs(device_light_brightness - scene_light_brightness))

        brightness_score = round(scene_score / len(scene['children']))
        lights_match = len( set(scene_lights_on) & set(state_lights_on) )
        scene_score = brightness_score + (lights_match * .1)
        #logger.info(f'.. scene score for {scene_id} is {scene_score}: {brightness_score} + {lights_match}')
        return scene_score

    async def calculate_area_scene(self, area_id):

        if area_id not in self.dataset.nativeDevices['area']:
            logger.warning(f'!! warning area {area_id} does not exist')
            return None

        current_scene = self.dataset.nativeDevices['area'][area_id].get('scene', "")

        if area_id in self.logic_adapter.areas.areas_calc_pending:
            logger.info(f'.. area calc is still pending {area_id}')
            return ''

        if self.logic_adapter.busy:
            if area_id not in self.logic_adapter.areas.areas_calc_deferred:
                self.logic_adapter.areas.areas_calc_deferred.append(area_id)
            return ''

        area_scenes=[]
        for item in self.logic_adapter.areas.data[area_id]['children']:
            if item in self.dataset.localDevices and 'SCENE_TRIGGER' in self.dataset.localDevices[item].displayCategories:
                if item not in area_scenes:
                    area_scenes.append(item)

        if len(area_scenes)<1: # no local scenes in area
            return ""

        self.logic_adapter.areas.areas_calc_pending.append(area_id)

        devstate_cache={}
        high_score=0
        best_scene = ""

        for endpoint_id in area_scenes:
            #scene=child.split(':')[2] # TODO stop using names combined with endpointId's!
            scene_score = self.calculate_scene_score(endpoint_id)
            if scene_score > high_score:
                best_scene = endpoint_id
                high_score = scene_score

        if area_id in self.logic_adapter.areas.areas_calc_pending:
            self.logic_adapter.areas.areas_calc_pending.remove(area_id)
        if area_id in self.logic_adapter.areas.areas_calc_deferred:
            self.logic_adapter.areas.areas_calc_deferred.remove(area_id)

        if best_scene and best_scene != current_scene:
            logger.info('.. %s scene change %s to %s' % (area_id, current_scene, best_scene))
            changes = await self.dataset.ingest({ "area" : { area_id: {'scene': best_scene }}})

        return best_scene

    async def run_scene(self, id):

        # We use scenes to represent groups of light objects only and the shorthand requires some manipulation of the powerState
        # in conjunction with the other settings to match the scene json.

        try:
            await self.dataset.ingest({"scene": { id: { "running": True }}})
            self.logic_adapter.busy=True
            scene = self.data[id]
            actions=[]
            for light in scene['children']:
                if 'powerState' in scene['children'][light] and scene['children'][light]['powerState']=='OFF':
                    actions.append(Directive("Alexa.PowerController", "TurnOff", light))

                elif 'brightness' in scene['children'][light] and int(scene['children'][light]['brightness'])==0 and 'powerState' not in scene['children'][light]:
                    actions.append(Directive("Alexa.PowerController", "TurnOff", light))

                else:
                    if 'hue' in scene['children'][light]:
                        payload={
                                "color": {
                                    "brightness": scene['children'][light]['brightness']/100,
                                    "saturation": scene['children'][light]['saturation'],
                                    "hue": scene['children'][light]['hue']
                                }
                        }
                        actions.append(Directive("Alexa.ColorController", "SetColor", light, payload=payload))
                    else:
                        actions.append(Directive("Alexa.PowerController", "TurnOn", light))
                        if 'brightness' in scene['children'][light]:
                            actions.append(Directive("Alexa.BrightnessController", "SetBrightness", light, payload={ "brightness": int(scene['children'][light]['brightness']) } ))

            results = await self.send_batch_directives(actions, id=str(uuid.uuid1()) )
            logger.debug('.. scene %s result: %s' % (id, results))
            self.logic_adapter.busy=False
        except:
            logger.error('!! Error executing Scene %s' % id, exc_info=True)
        await self.dataset.ingest({"scene": { id: { "running": False }}})

    async def virtualAddDevice(self, deviceId, change):

        try:
            for area in self.areas:
                if deviceId in self.areas[area]['children']:
                    #logger.info('Area %s' % (self.dataset.nativeDevices['area'][area]))
                    bestscene=await self.calculateAreaLevel(area)
        except AttributeError:
            # areas not ready yet
            pass
            #logger.error('Error in virtual add handler: %s %s' % (deviceId, change), exc_info=True)

        except:
            logger.error('Error in virtual add handler: %s %s' % (deviceId, change), exc_info=True)

    def scene_list_item(self, scene_id):
        # gets a scene item for the web list interface
        if scene_id in self.data:
            return self.data[scene_id]
        if scene_id.split(':')[2] in self.data:
            return self.data[scene_id.split(':')[2]]
        logger.warning('!! list handler - invalid scene: %s' % scene_id)
        return None


# install py_stringmatching lib
# get_ipython().system(' pip3 install py_stringmatching')

import numpy as np

from scipy.spatial import distance

import py_stringmatching as sm
from py_stringmatching import OverlapCoefficient
from py_stringmatching import Jaccard
from py_stringmatching import MongeElkan
from py_stringmatching import Cosine
from py_stringmatching import Dice
from py_stringmatching import SoftTfIdf
from py_stringmatching import TverskyIndex
from py_stringmatching import Affine

# # Numerical

def cosine (o1,o2):
    """
    cosine similarity function
    input:
    - o1: first object (List)
    - o2: Second object (List)
    output:
    - cosine between 01 and o2
    """
#     return np.arccos(np.dot(o1,o2)/(np.linalg.norm(o1)*np.linalg.norm(o2)))
    o1, o2 = list(o1) , list(o2)
    return Cosine().get_sim_score(o1,o2)

def Euclidean_distance (o1,o2):
    """
    distance similarity function
    input:
    - o1: first object (List)
    - o2: Second object (List)
    output:
    - distance between 01 and o2 float
    """
    o1 , o2= np.array(o1) , np.array(o2)
    return abs(o1-o2).sum()**(0.5)

def wieghted_euclidean (o1,o2,w=0.1):
    """
    wieghted euclidean similarity function
    input:
    - o1: first object (List)
    - o2: Second object (List)
    output:
    - wieghted euclidean distance between 01 and o2 (float)
    """
    o1 , o2= np.array(o1) , np.array(o2)
    return distance.wminkowski(o1,o2,1,w)

def mahalanobis (o1,o2):
    """
    Mahalanobis similarity function
    input:
    - o1: first object (List)
    - o2: Second object (List)
    output:
    - Mahalanobis distance between 01 and o2 (float)
    """
    o1 , o2= np.array(o1) , np.array(o2)
    inv_cov_matrix=np.eye(len(o1))
    return distance.mahalanobis(o1,o2,inv_cov_matrix)

def minkowski (o1,o2):
    """
    minkowski similarity function
    input:
    - o1: first object (List)
    - o2: Second object (List)
    output:
    - minkowski distance between 01 and o2 (float)
    """
    return distance.minkowski(o1,o2)

# # Categorical

def base_line (o1,o2):
    """
    base_line similarity function
    input:
    - o1: first object (List)
    - o2: Second object (List)
    output:
    - 1 if o1=02, 0 otherwise
    """
    if o1==o2:
        return 1
    else:
        return 0

def overlap (o1,o2):
    """
     overlap similarity function
    input:
    - o1: first object (List)
    - o2: Second object (List)
    output:
    -  measure that measures the overlap between two sets
    """
    o1, o2 = list(o1) , list(o2)
    for i in range (len(o1)):
        o1[i]=str(o1[i])
    for i in range (len(o2)):
        o2[i]=str(o2[i])
    return OverlapCoefficient().get_sim_score(o1,o2)

def Jaccard_Distance (o1,o2):
    """
     Jaccard similarity function
    input:
    - o1: first object (List)
    - o2: Second object (List)
    output:
    -  Computes Jaccard measure.
    """
    o1, o2 = list(o1) , list(o2)
    for i in range (len(o1)):
        o1[i]=str(o1[i])
    for i in range (len(o2)):
        o2[i]=str(o2[i])
    return Jaccard().get_sim_score(o1,o2)

def Monge_Elkan(o1,o2):
    """
     Monge Elkan similarity function
    input:
    - o1: first object (List)
    - o2: Second object (List)
    output:
    -  Computes Monge Elkan measure.
    """
    o1, o2 = list(o1) , list(o2)
    for i in range (len(o1)):
        o1[i]=str(o1[i])
    for i in range (len(o2)):
        o2[i]=str(o2[i])
    return MongeElkan().get_raw_score(o1,o2)

def dice (o1,o2):
    """
     Dice similarity function
    input:
    - o1: first object (List)
    - o2: Second object (List)
    output:
    -  Computes Dice measure.
    """
    o1, o2 = list(o1) , list(o2)
    for i in range (len(o1)):
        o1[i]=str(o1[i])
    for i in range (len(o2)):
        o2[i]=str(o2[i])
    return Dice().get_sim_score(o1,o2)

def soft_TF_IDF (o1,o2,threshold=0.9):
    """
     soft TF IDF similarity function
    input:
    - o1: first object (List)
    - o2: Second object (List)
    output:
    -  Computes Dice measure.
    """
    o1, o2 = list(o1) , list(o2)
    for i in range (len(o1)):
        o1[i]=str(o1[i])
    for i in range (len(o2)):
        o2[i]=str(o2[i])
    a=SoftTfIdf(sim_func=Affine().get_raw_score, threshold=threshold)
    return a.get_raw_score(o1,o2)

def tversky_Index (o1,o2):
    """
    tversky_Index  similarity function
    input:
    - o1: first object (List)
    - o2: Second object (List)
    output:
    -  Computes Tversky index similarity.
    """
    o1, o2 = list(o1) , list(o2)
    for i in range (len(o1)):
        o1[i]=str(o1[i])
    for i in range (len(o2)):
        o2[i]=str(o2[i])
    return TverskyIndex().get_sim_score(o1,o2)

def voting_similarity (o1,o2, numerical=False):
    """
     weight similarity function
    input:
    - o1: first object
    - o2: Second object
    - numerical (pool) True if the data is numerical
    - categorical (pool) True if the data is categorical
    output:
    -  Computes overall score.
    """

    if numerical:
        w=1/4
        score=w * (cosine(o1,o2) + Euclidean_distance(o1,o2) + wieghted_euclidean(o1,o2) + mahalanobis(o1,o2))
    else:
        w=1/7
        score=w * (cosine(o1,o2) + overlap(o1,o2) + Jaccard_Distance(o1,o2) + Monge_Elkan(o1,o2) + dice(o1,o2) + soft_TF_IDF(o1,o2) + tversky_Index(o1,o2))
    return score

numerical_similarity_fun= {'cosine':cosine ,  'Euclidean_distance' : Euclidean_distance,
                          'wieghted_euclidean':wieghted_euclidean , 'mahalanobis':mahalanobis , 'minkowski':minkowski}
categorical_similarity_fun = { 'cosine':cosine ,  'overlap':overlap , 'Jaccard_Distance':Jaccard_Distance , 'Monge_Elkan':Monge_Elkan ,
                              'dice':dice , 'soft_TF_IDF':soft_TF_IDF , 'tversky_Index':tversky_Index }


#!/bin/env python3

import shutil
import subprocess

def assembleHeaders(path, tab):
    head = ""
    for t in tab:
        head = head + " -i " + path + t
    return head

gen = shutil.which("hpp2plantuml") #checking if command exist

if gen is None:
    subprocess.run(["pip3", "install", "--user", "hpp2plantuml"]) #installing command if does not exist

sharedHeadersPath = "./CommonHeaders/"
sharedHeaders = ["api.hpp", "IModule.hpp", "ModuleManager.hpp"]

sharedTotal = assembleHeaders(sharedHeadersPath, sharedHeaders)

#server gen
subprocess.run(["hpp2plantuml " + sharedTotal[1:] + " -o " + "./UML/sharedHeaders.puml"], shell=True)


import os
import pandas as pd
import numpy as np

# ---------------------------------------------------------------------
# Question #1
# ---------------------------------------------------------------------

def get_assignment_names(grades):
    '''
    get_assignment_names takes in a dataframe like grades and returns
    a dictionary with the following structure:

    The keys are the general areas of the syllabus: lab, project,
    midterm, final, disc, checkpoint

    The values are lists that contain the assignment names of that type.
    For example the lab assignments all have names of the form labXX where XX
    is a zero-padded two digit number. See the doctests for more details.

    :Example:
    >>> grades_fp = os.path.join('data', 'grades.csv')
    >>> grades = pd.read_csv(grades_fp)
    >>> names = get_assignment_names(grades)
    >>> set(names.keys()) == {'lab', 'project', 'midterm', 'final', 'disc', 'checkpoint'}
    True
    >>> names['final'] == ['Final']
    True
    >>> 'project02' in names['project']
    True
    '''

    labs = []
    projects = []
    discs = []
    checkpoints = []

    for column_name in grades.columns:
        if column_name[:3] == 'lab' and column_name[-2:].isnumeric():
            labs.append(column_name)
        elif column_name[:7] == 'project' and column_name[-2:].isnumeric():
            if column_name[10:20] == 'checkpoint':
                checkpoints.append(column_name)
            else:
                projects.append(column_name)
        elif column_name[:4] == 'disc' and column_name[-2:].isnumeric():
            discs.append(column_name)

    names = {
        'lab' : labs,
        'project': projects,
        'midterm': ['Midterm'],
        'final': ['Final'],
        'disc' : discs,
        'checkpoint' : checkpoints
    }

    return names

# ---------------------------------------------------------------------
# Question #2
# ---------------------------------------------------------------------

def projects_total(grades):
    '''
    projects_total that takes in grades and computes the total project grade
    for the quarter according to the syllabus.
    The output Series should contain values between 0 and 1.

    :Example:
    >>> grades_fp = os.path.join('data', 'grades.csv')
    >>> grades = pd.read_csv(grades_fp)
    >>> out = projects_total(grades)
    >>> np.all((0 <= out) & (out <= 1))
    True
    >>> 0.7 < out.mean() < 0.9
    True
    '''
    proj_names = get_assignment_names(grades)['project']
    proj_grades_sum = np.array([float(0) for _ in range(grades.shape[0])])
    grades = grades.fillna(0)

    for proj in proj_names:
        points = grades[proj]
        max_points = grades[f'{proj} - Max Points']
        if f'{proj}_free_response' in grades.columns:
            points += grades[f'{proj}_free_response']
            max_points += grades[f'{proj}_free_response - Max Points']

        proj_grade = points/max_points
        proj_grades_sum += proj_grade

    proj_grades = proj_grades_sum/len(proj_names)
    return proj_grades

# ---------------------------------------------------------------------
# Question # 3
# ---------------------------------------------------------------------

def last_minute_submissions(grades):
    """
    last_minute_submissions takes in the dataframe
    grades and a Series indexed by lab assignment that
    contains the number of submissions that were turned
    in on time by the student, yet marked 'late' by Gradescope.

    :Example:
    >>> fp = os.path.join('data', 'grades.csv')
    >>> grades = pd.read_csv(fp)
    >>> out = last_minute_submissions(grades)
    >>> isinstance(out, pd.Series)
    True
    >>> np.all(out.index == ['lab0%d' % d for d in range(1,10)])
    True
    >>> (out > 0).sum()
    8
    """

    assignment_names = get_assignment_names(grades)

    false_late_count = []
    for l in assignment_names['lab']:
        lateness = f'{l} - Lateness (H:M:S)'
        false_late_count.append(grades[lateness][grades[lateness] < '07:00:00'][grades[lateness] > '00:00:00'].count())

    last_minutes = pd.Series(false_late_count, assignment_names['lab'])

    return last_minutes

# ---------------------------------------------------------------------
# Question #4
# ---------------------------------------------------------------------

def lateness_penalty(col):
    """
    lateness_penalty takes in a 'lateness' column and returns
    a column of penalties according to the syllabus.

    :Example:
    >>> fp = os.path.join('data', 'grades.csv')
    >>> col = pd.read_csv(fp)['lab01 - Lateness (H:M:S)']
    >>> out = lateness_penalty(col)
    >>> isinstance(out, pd.Series)
    True
    >>> set(out.unique()) <= {1.0, 0.9, 0.8, 0.5}
    True
    """

    penalty = col.apply(lambda x: 1.0 if x < '07:00:00' else 0.9 if x < '168:00:00' else 0.8 if x < '336:00:00' else 0.5)

    return penalty

# ---------------------------------------------------------------------
# Question #5
# ---------------------------------------------------------------------

def process_labs(grades):
    """
    process_labs that takes in a dataframe like grades and returns
    a dataframe of processed lab scores. The output should:
      * share the same index as grades,
      * have columns given by the lab assignment names (e.g. lab01,...lab10)
      * have values representing the lab grades for each assignment,
        adjusted for Lateness and scaled to a score between 0 and 1.

    :Example:
    >>> fp = os.path.join('data', 'grades.csv')
    >>> grades = pd.read_csv(fp)
    >>> out = process_labs(grades)
    >>> out.columns.tolist() == ['lab%02d' % x for x in range(1,10)]
    True
    >>> np.all((0.65 <= out.mean()) & (out.mean() <= 0.90))
    True
    """

    lab_names = get_assignment_names(grades)['lab']
    labdf = grades[lab_names]

    for l in lab_names:
        #print(labdf[l] / grades[f'{l} - Max Points'])
        #print(lateness_penalty(grades[f'{l} - Lateness (H:M:S)']))
        labdf.loc[:,l] = (labdf[l] * lateness_penalty(grades[f'{l} - Lateness (H:M:S)'])) / grades[f'{l} - Max Points']

    return labdf

# ---------------------------------------------------------------------
# Question #6
# ---------------------------------------------------------------------

def lab_total(processed):
    """
    lab_total takes in dataframe of processed assignments (like the output of
    Question 5) and computes the total lab grade for each student according to
    the syllabus (returning a Series).

    Your answers should be proportions between 0 and 1.

    :Example:
    >>> cols = 'lab01 lab02 lab03'.split()
    >>> processed = pd.DataFrame([[0.2, 0.90, 1.0]], index=[0], columns=cols)
    >>> np.isclose(lab_total(processed), 0.95).all()
    True
    """

    tot = (processed.sum(axis=1) - processed.min(axis=1)) / (processed.shape[1] - 1)
    return tot

# ---------------------------------------------------------------------
# Question # 7
# ---------------------------------------------------------------------

def total_points(grades):
    """
    total_points takes in grades and returns the final
    course grades according to the syllabus. Course grades
    should be proportions between zero and one.

    :Example:
    >>> fp = os.path.join('data', 'grades.csv')
    >>> grades = pd.read_csv(fp)
    >>> out = total_points(grades)
    >>> np.all((0 <= out) & (out <= 1))
    True
    >>> 0.7 < out.mean() < 0.9
    True
    """

    assgn_names = get_assignment_names(grades)

    proj_tot = projects_total(grades)
    lab_tot = lab_total(process_labs(grades)).fillna(0)

    checkpoint_names = assgn_names['checkpoint']
    checkpoint_grades_sum = [float(0) for _ in range(grades.shape[0])]
    for checkpoint in checkpoint_names:
        checkpoint_grade = grades[checkpoint] / grades[f'{checkpoint} - Max Points']
        checkpoint_grade = checkpoint_grade.fillna(0)
        checkpoint_grades_sum += checkpoint_grade
    checkpoint_tot = checkpoint_grades_sum / len(checkpoint_names)

    disc_names = assgn_names['disc']
    disc_grades_sum = [float(0) for _ in range(grades.shape[0])]
    for disc in disc_names:
        disc_grade = grades[disc] / grades[f'{disc} - Max Points']
        disc_grade = disc_grade.fillna(0)
        disc_grades_sum += disc_grade
    disc_tot = disc_grades_sum / len(disc_names)

    mid = grades['Midterm'] / grades['Midterm - Max Points']
    mid = mid.fillna(0)

    fin = grades['Final'] / grades['Final - Max Points']
    fin = fin.fillna(0)

    total = (0.20 * lab_tot) + (0.30 * proj_tot) + (0.025 * checkpoint_tot) + (0.025 * disc_tot) + (0.15 * mid) + (0.30 * fin)

    return total

def final_grades(total):
    """
    final_grades takes in the final course grades
    as above and returns a Series of letter grades
    given by the standard cutoffs.

    :Example:
    >>> out = final_grades(pd.Series([0.92, 0.81, 0.41]))
    >>> np.all(out == ['A', 'B', 'F'])
    True
    """

    letters = total.apply(lambda x: 'A' if x >= 0.9 else 'B' if x >= 0.8 else 'C' if x >= 0.7 else 'D' if x >= 0.6 else 'F')
    return letters

def letter_proportions(grades):
    """
    letter_proportions takes in the dataframe grades
    and outputs a Series that contains the proportion
    of the class that received each grade.

    :Example:
    >>> fp = os.path.join('data', 'grades.csv')
    >>> grades = pd.read_csv(fp)
    >>> out = letter_proportions(grades)
    >>> np.all(out.index == ['B', 'C', 'A', 'D', 'F'])
    True
    >>> out.sum() == 1.0
    True
    """

    letter_grades = (final_grades(total_points(grades)))
    proportions = letter_grades.value_counts(normalize=True)
    return proportions

# ---------------------------------------------------------------------
# Question # 8
# ---------------------------------------------------------------------

def simulate_pval(grades, N):
    """
    simulate_pval takes in the number of
    simulations N and grades and returns
    the likelihood that the grade of sophomores
    was no better on average than the class
    as a whole (i.e. calculate the p-value).

    :Example:
    >>> fp = os.path.join('data', 'grades.csv')
    >>> grades = pd.read_csv(fp)
    >>> out = simulate_pval(grades, 100)
    >>> 0 <= out <= 0.1
    True
    """

    soph_not_better = []

    for i in range (N):
        samp = grades.sample(frac = 1, replace = True)
        avg_all = total_points(samp).mean()
        avg_so = total_points(samp.loc[grades['Level'] == 'SO']).mean()

        if avg_so <= avg_all:
            soph_not_better.append(1)
        else:
            soph_not_better.append(0)
    p_val = np.count_nonzero(soph_not_better) / len(soph_not_better)
    return p_val

# ---------------------------------------------------------------------
# Question # 9
# ---------------------------------------------------------------------

def total_points_with_noise(grades):
    """
    total_points_with_noise takes in a dataframe like grades,
    adds noise to the assignments as described in notebook, and returns
    the total scores of each student calculated with noisy grades.

    :Example:
    >>> fp = os.path.join('data', 'grades.csv')
    >>> grades = pd.read_csv(fp)
    >>> out = total_points_with_noise(grades)
    >>> np.all((0 <= out) & (out <= 1))
    True
    >>> 0.7 < out.mean() < 0.9
    True
    """

    assgn_names = get_assignment_names(grades)
    grades = grades.fillna(0)
    num_rows = grades.shape[0]

    proj_names = get_assignment_names(grades)['project']
    proj_grades_sum = np.array([float(0) for _ in range(grades.shape[0])])

    for proj in proj_names:
        points = grades[proj]
        max_points = grades[f'{proj} - Max Points']
        if f'{proj}_free_response' in grades.columns:
            points += grades[f'{proj}_free_response']
            max_points += grades[f'{proj}_free_response - Max Points']

        proj_grade = (points / max_points) + np.random.normal(0, 0.02, size=(num_rows))
        proj_grade = np.clip(proj_grade,0,1)
        proj_grades_sum += proj_grade

    proj_tot = proj_grades_sum / len(proj_names)

    lab_names = get_assignment_names(grades)['lab']
    labdf = grades[lab_names]

    for l in lab_names:
        labdf.loc[:, l] = np.clip(\
                                ((labdf[l] / grades[f'{l} - Max Points'])\
                                  + np.random.normal(0, 0.02, size=(num_rows)))\
                                * lateness_penalty(grades[f'{l} - Lateness (H:M:S)'])\
                        ,0,1)

    lab_tot = lab_total(labdf).fillna(0)

    checkpoint_names = assgn_names['checkpoint']
    checkpoint_grades_sum = [float(0) for _ in range(grades.shape[0])]
    for checkpoint in checkpoint_names:
        checkpoint_grade = grades[checkpoint] / grades[f'{checkpoint} - Max Points']
        checkpoint_grade = checkpoint_grade.fillna(0) + np.random.normal(0, 0.02, size=(num_rows))
        checkpoint_grade = np.clip(checkpoint_grade,0,1)
        checkpoint_grades_sum += checkpoint_grade
    checkpoint_tot = checkpoint_grades_sum / len(checkpoint_names)

    disc_names = assgn_names['disc']
    disc_grades_sum = [float(0) for _ in range(grades.shape[0])]
    for disc in disc_names:
        disc_grade = grades[disc] / grades[f'{disc} - Max Points']
        disc_grade = disc_grade.fillna(0) + np.random.normal(0, 0.02, size=(num_rows))
        disc_grade = np.clip(disc_grade,0,1)
        disc_grades_sum += disc_grade
    disc_tot = disc_grades_sum / len(disc_names)

    mid = grades['Midterm'] / grades['Midterm - Max Points']
    mid = mid.fillna(0) + np.random.normal(0, 0.02, size=(num_rows))
    mid = np.clip(mid,0,1)

    fin = grades['Final'] / grades['Final - Max Points']
    fin = fin.fillna(0) + np.random.normal(0, 0.02, size=(num_rows))
    fin = np.clip(fin,0,1)

    total = (0.20 * lab_tot) + (0.30 * proj_tot) + (0.025 * checkpoint_tot) + (0.025 * disc_tot) + (0.15 * mid) + (
                0.30 * fin)

    return total

# ---------------------------------------------------------------------
# Question #10
# ---------------------------------------------------------------------

def short_answer():
    """
    short_answer returns (hard-coded) answers to the
    questions listed in the notebook. The answers should be
    given in a list with the same order as questions.

    :Example:
    >>> out = short_answer()
    >>> len(out) == 5
    True
    >>> len(out[2]) == 2
    True
    >>> 50 < out[2][0] < 100
    True
    >>> 0 < out[3] < 1
    True
    >>> isinstance(out[4], bool)
    True
    """

    return [0.0001, 0.85, [79,86], 0.06, True]

# ---------------------------------------------------------------------
# DO NOT TOUCH BELOW THIS LINE
# IT'S FOR YOUR OWN BENEFIT!
# ---------------------------------------------------------------------

# Graded functions names! DO NOT CHANGE!
# This dictionary provides your doctests with
# a check that all of the questions being graded
# exist in your code!

GRADED_FUNCTIONS = {
    'q01': ['get_assignment_names'],
    'q02': ['projects_total'],
    'q03': ['last_minute_submissions'],
    'q04': ['lateness_penalty'],
    'q05': ['process_labs'],
    'q06': ['lab_total'],
    'q07': ['total_points', 'final_grades', 'letter_proportions'],
    'q08': ['simulate_pval'],
    'q09': ['total_points_with_noise'],
    'q10': ['short_answer']
}

def check_for_graded_elements():
    """
    >>> check_for_graded_elements()
    True
    """

    for q, elts in GRADED_FUNCTIONS.items():
        for elt in elts:
            if elt not in globals():
                stmt = "YOU CHANGED A QUESTION THAT SHOULDN'T CHANGE! \
                In %s, part %s is missing" %(q, elt)
                raise Exception(stmt)

    return True