import importlib
import os
import types
from tensorflow.python.platform import tf_logging as logging

_TENSORFLOW_LAZY_LOADER_PREFIX = "_tfll"

class LazyLoader(types.ModuleType):
  """Lazily import a module, mainly to avoid pulling in large dependencies.

  `contrib`, and `ffmpeg` are examples of modules that are large and not always
  needed, and this allows them to only be loaded when they are used.
  """

  # The lint error here is incorrect.
  def __init__(self, local_name, parent_module_globals, name, warning=None):
    self._tfll_local_name = local_name
    self._tfll_parent_module_globals = parent_module_globals
    self._tfll_warning = warning

    # These members allows doctest correctly process this module member without
    # triggering self._load(). self._load() mutates parant_module_globals and
    # triggers a dict mutated during iteration error from doctest.py.
    # - for from_module()
    super().__setattr__("__module__", name.rsplit(".", 1)[0])
    # - for is_routine()
    super().__setattr__("__wrapped__", None)

    super().__init__(name)

  def _load(self):
    """Load the module and insert it into the parent's globals."""
    # Import the target module and insert it into the parent's namespace
    module = importlib.import_module(self.__name__)
    self._tfll_parent_module_globals[self._tfll_local_name] = module

    # Emit a warning if one was specified
    if self._tfll_warning:
      logging.warning(self._tfll_warning)
      # Make sure to only warn once.
      self._tfll_warning = None

    # Update this object's dict so that if someone keeps a reference to the
    #   LazyLoader, lookups are efficient (__getattr__ is only called on lookups
    #   that fail).
    self.__dict__.update(module.__dict__)

    return module

  def __getattr__(self, name):
    module = self._load()
    return getattr(module, name)

  def __setattr__(self, name, value):
    if name.startswith(_TENSORFLOW_LAZY_LOADER_PREFIX):
      super().__setattr__(name, value)
    else:
      module = self._load()
      setattr(module, name, value)
      self.__dict__[name] = value
      try:
        # check if the module has __all__
        if name not in self.__all__ and name != "__all__":
          self.__all__.append(name)
      except AttributeError:
        pass

  def __delattr__(self, name):
    if name.startswith(_TENSORFLOW_LAZY_LOADER_PREFIX):
      super().__delattr__(name)
    else:
      module = self._load()
      delattr(module, name)
      self.__dict__.pop(name)
      try:
        # check if the module has __all__
        if name in self.__all__:
          self.__all__.remove(name)
      except AttributeError:
        pass

  def __repr__(self):
    # Carefully to not trigger _load, since repr may be called in very
    # sensitive places.
    return f"<LazyLoader {self.__name__} as {self._tfll_local_name}>"

  def __dir__(self):
    module = self._load()
    return dir(module)

  def __reduce__(self):
    return importlib.import_module, (self.__name__,)

class KerasLazyLoader(LazyLoader):
  """LazyLoader that handles routing to different Keras version."""

  def __init__(  # pylint: disable=super-init-not-called
      self, parent_module_globals, mode=None, submodule=None, name="keras"):
    self._tfll_parent_module_globals = parent_module_globals
    self._tfll_mode = mode
    self._tfll_submodule = submodule
    self._tfll_name = name
    self._tfll_initialized = False

  def _initialize(self):
    """Resolve the Keras version to use and initialize the loader."""
    self._tfll_initialized = True
    package_name = None
    keras_version = None
    if os.environ.get("TF_USE_LEGACY_KERAS", None) in ("true", "True", "1"):
      try:
        import tf_keras  # pylint: disable=g-import-not-at-top,unused-import

        keras_version = "tf_keras"
        if self._tfll_mode == "v1":
          package_name = "tf_keras.api._v1.keras"
        else:
          package_name = "tf_keras.api._v2.keras"
      except ImportError:
        logging.warning(
            "Your environment has TF_USE_LEGACY_KERAS set to True, but you "
            "do not have the tf_keras package installed. You must install it "
            "in order to use the legacy tf.keras. Install it via: "
            "`pip install tf_keras`"
        )
    else:
      try:
        import keras  # pylint: disable=g-import-not-at-top

        if keras.__version__.startswith("3."):
          # This is the Keras 3.x case.
          keras_version = "keras_3"
          package_name = "keras._tf_keras.keras"
        else:
          # This is the Keras 2.x case.
          keras_version = "keras_2"
          if self._tfll_mode == "v1":
            package_name = "keras.api._v1.keras"
          else:
            package_name = "keras.api._v2.keras"
      except ImportError:
        raise ImportError(  # pylint: disable=raise-missing-from
            "Keras cannot be imported. Check that it is installed."
        )

    self._tfll_keras_version = keras_version
    if keras_version is not None:
      if self._tfll_submodule is not None:
        package_name += "." + self._tfll_submodule
      super().__init__(
          self._tfll_name, self._tfll_parent_module_globals, package_name
      )
    else:
      raise ImportError(  # pylint: disable=raise-missing-from
          "Keras cannot be imported. Check that it is installed."
      )

  def __getattr__(self, item):
    if item in ("_tfll_mode", "_tfll_initialized", "_tfll_name"):
      return super(types.ModuleType, self).__getattribute__(item)
    if not self._tfll_initialized:
      self._initialize()
    if self._tfll_keras_version == "keras_3":
      if (
          self._tfll_mode == "v1"
          and not self._tfll_submodule
          and item.startswith("compat.v1.")
      ):
        raise AttributeError(
            "`tf.compat.v1.keras` is not available with Keras 3. Keras 3 has "
            "no support for TF 1 APIs. You can install the `tf_keras` package "
            "as an alternative, and set the environment variable "
            "`TF_USE_LEGACY_KERAS=True` to configure TensorFlow to route "
            "`tf.compat.v1.keras` to `tf_keras`."
        )
      elif (
          self._tfll_mode == "v2"
          and not self._tfll_submodule
          and item.startswith("compat.v2.")
      ):
        raise AttributeError(
            "`tf.compat.v2.keras` is not available with Keras 3. Just use "
            "`import keras` instead."
        )
      elif self._tfll_submodule and self._tfll_submodule.startswith(
          "__internal__.legacy."
      ):
        raise AttributeError(
            f"`{item}` is not available with Keras 3."
        )
    module = self._load()
    return getattr(module, item)

  def __repr__(self):
    if self._tfll_initialized:
      return (
          f"<KerasLazyLoader ({self._tfll_keras_version}) "
          f"{self.__name__} as {self._tfll_local_name} mode={self._tfll_mode}>"
      )
    return "<KerasLazyLoader>"

  def __dir__(self):
    if not self._tfll_initialized:
      self._initialize()
    return super().__dir__()


from sqlalchemy import create_engine, exc
from sqlalchemy.orm import sessionmaker

from database import DBUpdate, DBBase, Game, User, GameKill, Item, GameMod, TeamStats, \
        GameMap, GameType, EndCause, Score, Model, HModel, Say, Tell, SayTeam, \
        Pk3Files, GameParameter, Team, Config

class DBHandler(object):

    engine = None
    game_id = None
    user_dict = {}
    team_dict = {}
    team_leader_dict = {}
    user_team_dict = {}
    item_dict = {}
    game_param_dict = [
        'timelimit',
        'fraglimit',
        'capturelimit',
        'bot_minplayer'
    ]

    const_blue = 2
    const_red = 1

    def __init__(self):
        pass

    def renew_db_connection(self):
        if self.engine:
            # maybe a better soluton exists for this?
            try:
                self.create_session()
                if self.game_id:
                    self.game_now = self.session.query(Game).filter(Game.id==self.game_id).first()
                    self.game_id = self.game_now.id
                return 0
            except exc.SQLAlchemyError as ex:
                log.error("SQLAlchemyError: %s" % ex)
                return 1
        else:
            return 0

    def get_config(self, key):
        config = self.session.query(Config).filter(key=key).first()
        if not config:
            return ""
        return config.value

    def set_config(self, key, value):
        config = Config(parameter=key,
                    value=value
        )
        self.session.add(config)

    def get_db_version(self):
        db_version = self.get_config('database_version')

        if not db_version:
            db_version = 1

        return db_version

    def set_db_version(version):
        set_config('database_version', version)

    def db_update(self):
        db = DBUpdate()
        version = self.get_db_version

        while version < db.max_version:
            new_version = version + 1

            if hasattr(db, version + '_to_' + new_version):
                getattr(db, version + '_to_' + new_version)
            else:
                log.error("missing update Rule from Databaseversion %d to Version %d" % (version, new_version))
                return

            version = self.get_db_version

    def get_team_id(self, teamname):
        #if key not in dictionary,
        #try to load from database,
        #otherwise write in database,
        #now load into dictionary for future use in game
        if teamname not in self.team_dict:
            if not self.session.query(Team).filter(Team.name==teamname).count():
                new_team = Team(name=teamname)
                self.session.add(new_team)
                self.session.commit()
            team = self.session.query(Team).filter(Team.name==teamname).first()
            self.team_dict[teamname] = team.id

        return self.team_dict[teamname]

    def get_user_id(self, username):
        #if key not in dictionary,
        #try to load from database,
        #otherwise write in database,
        #now load into dictionary for future use in game
        if username not in self.user_dict:
            if not self.session.query(User).filter(User.username==username).count():
                new_user = User(username=username)
                self.session.add(new_user)
                self.session.commit()
            user = self.session.query(User).filter(User.username==username).first()
            self.user_dict[username] = user.id

        return self.user_dict[username]

    def get_item_id(self, item, weapon=False):
        #if key not in dictionary,
        #try to load from database,
        #otherwise write in database,
        #now load into dictionary for future use in game
        if item not in self.item_dict:
            if not self.session.query(Item).filter(Item.item==item).count():
                new_item = Item(item=item, weapon=weapon)
                self.session.add(new_item)
                self.session.commit()
            dict_item = self.session.query(Item).filter(Item.item==item).first()
            self.item_dict[item] = [dict_item.id, dict_item.weapon]

        # if killed once with this item an enemy, mark as weapon
        if weapon and self.item_dict[item][1] != weapon:
            item_type_change = self.session.query(Item).filter(Item.item==item).first()
            item_type_change.weapon = weapon
            del self.item_dict[item]
            self.item_dict[item] = [dict_item.id, dict_item.weapon]

        return self.item_dict[item][0]

    def set_team_leader(self, teamid, username):
        self.team_leader_dict[teamid] = username

    def get_team_leader(self, teamid):
        if teamid in self.team_leader_dict:
            return self.team_leader_dict[teamid]
        return ''

    def set_user_teamid(self, username, teamid):
        self.user_team_dict[username] = teamid

    def get_user_teamid(self, username):
        if username not in self.user_team_dict:
            return ''
        return self.user_team_dict[username]

    def connect(self, db, debug=False):
        try:
            self.engine = create_engine(db, echo=debug)
            return True
        except ImportError as ex:
            log.error(ex.message + "\n -> maybe missed pip installation of database game_modul like 'pymysql'?\n maybe need the Package like 'python-mysqldb' from your Distribution also?")
            self.db_help()
            return False

    def db_create_models(self):
        DBBase.metadata.create_all(self.engine)

    def set_engine(self, engine):
        self.engine = engine

    def get_engine(self):
        return self.engine

    def create_session(self):
        session = sessionmaker()
        session.configure(bind=self.engine)
        self.session = session()
        self.db_update()

    def commit_session(self):
        self.session.commit()

    def close_session(self):
        try:
            self.session.close()
        except:
            pass

    def db_help(self):
        print ('sqlite:////absolute/path/to/sqlite.db')
        print ('mysql://username:password@hostname/mydatabase')
        print ('postgresql://username:password@localhost/mydatabase')
        print ('mssql+pymssql://username:password@hostname:port/dbname')
        print ('oracle://username:password@hostname:1521/sidname')

    def init_game(self, data):
        #{'protocol': '68', 'sv_privateClients': '0', 'sv_maxRate': '0', 'bot_minplayers': '4', 'g_maxGameClients': '0', 'version': 'Q3 1.32c linux-i386 May  8 2006', 'game_mod_name': 'baseq3', 'sv_minPing': '0', '.Admin': 'Mr.T', 'timestamp': '1402165340', 'g_needpass': '1', 'sv_punkbuster': '0', 'sv_allowDownload': '1', 'sv_maxclients': '8', 'sv_hostname': 'Wotans Daddelbude', 'fraglimit': '20', 'capturelimit': '6', 'dmflags': '0', 'game_type': '3', 'timelimit': '10', 'time': '2059:48', 'map_name': 'japandm', 'sv_maxPing': '0', 'runtime': 0, 'sv_floodProtect': '1'}

        if not self.session.query(GameMod).filter(GameMod.name==data['mod_name']).count():
            new_game_mod = GameMod(name=data['mod_name'])
            self.session.add(new_game_mod)
            self.session.commit()

        if not self.session.query(GameMap).filter(GameMap.name==data['map_name']).count():
            new_map = GameMap(name=data['map_name'])
            self.session.add(new_map)
            self.session.commit()

        if not self.session.query(GameType).filter(GameType.type==data['game_type']).count():
            new_game_type = GameType(type=data['game_type'])
            self.session.add(new_game_type)
            self.session.commit()

        game_mod = self.session.query(GameMod).filter(GameMod.name==data['mod_name']).first()
        game_map = self.session.query(GameMap).filter(GameMap.name==data['map_name']).first()
        game_type = self.session.query(GameType).filter(GameType.type==data['game_type']).first()

        game = Game(starttime=data['timestamp'],
                            game_mod_id=game_mod.id,
                            game_map_id=game_map.id,
                            game_type_id=game_type.id,
        )
        self.session.add(game)
        self.session.commit()

        #remember for actual game
        self.game_now = self.session.query(Game).filter(Game.starttime==data['timestamp']).first()
        self.game_id = self.game_now.id

        # save additional game parameters
        for key in data.iterkeys():
            if key in self.game_param_dict:
                game_param = GameParameter(game_id=self.game_now.id,
                                        game_key=key,
                                        game_value=data[key],
                )
                self.session.add(game_param)

    def exit(self, data):
        # {'endcause': 'Fraglimit', 'timestamp': 1402184024, 'runtime': 520, 'time': '2055:50'}
        if not self.session.query(EndCause).filter(EndCause.reason==data['endcause']).count():
            new_end_reason = EndCause(reason=data['endcause'])
            self.session.add(new_end_reason)
            self.session.commit()

        reason = self.session.query(EndCause).filter(EndCause.reason==data['endcause']).first()
        self.game_now.endtime = data['timestamp']
        self.game_now.end_cause_id = reason.id

    def shutdown_game(self, data):
        self.close_session()

    def score(self, data):
        # {'username': 'Xaero', 'timestamp': 1402403195, 'ping': '0', 'score': '3', 'time': '2055:50', 'runtime': 520, 'id': '3'}
        team_id = self.get_user_teamid(data['username'])

        if self.get_team_leader(team_id) == data['username']:
            team_leader=True
        else:
            team_leader=False

        user_id = self.get_user_id(data['username'])
        user_score = Score(user_id=user_id,
                            game_id=self.game_now.id,
                            count=data['score'],
                            team_id=team_id,
                            team_leader=team_leader,
        )
        self.session.add(user_score)

    def red(self, data):
        #{'blue_score': '19', 'red_score': '20', 'runtime': 297, 'timestamp': 1402180740, 'time': '1089:59'}

        blue_team_id = self.get_team_id(self.const_blue)
        red_team_id = self.get_team_id(self.const_red)

        team_blue = TeamStats(team_id=blue_team_id,
                            score=data['blue_score'],
                            game_id=self.game_now.id,
        )
        team_red = TeamStats(team_id=red_team_id,
                            score=data['red_score'],
                            game_id=self.game_now.id,
        )
        self.session.add(team_blue)
        self.session.add(team_red)

    def blue(self, data):
        self.red(data)

    def client_connect(self, data):
        pass

    def client_userinfo_changed(self, data):
        #{'hmodel': 'crash', 'skill': '    3.00', 'timestamp': 1402689747, 'team_task': '0', 'looses': '0', 'health_count': '90', 'n': 'Crash', 'team_leader': '0', 'wins': '0', 'time': '14856:4', 'color2': '5', 'model': 'crash', 'runtime': -36, 'color1': '4', 'id': '0', 'team': '2'}

        if not self.session.query(Model).filter(Model.name==data['model']).count():
            new_model = Model(name=data['model'])
            self.session.add(new_model)
            self.session.commit()
        if not self.session.query(HModel).filter(HModel.name==data['hmodel']).count():
            new_model = HModel(name=data['hmodel'])
            self.session.add(new_model)
            self.session.commit()
        model = self.session.query(Model).filter(Model.name==data['model']).first()
        hmodel = self.session.query(HModel).filter(HModel.name==data['hmodel']).first()

        user_id = self.get_user_id(data['n'])
        user = self.session.query(User).filter(User.id==user_id).first()

        if 'skill' in data:
            user.bot = True
            skill_value, sub_value = data['skill'].strip(' \t').split('.')
            self.game_now.bot_skill = int(skill_value)
        else:
            user.bot = False

        user.model_id = model.id
        user.hmodel_id = hmodel.id
        team_id = self.get_team_id(data['team'])
        self.set_user_teamid(data['n'], team_id)

        if data['team_leader']:
            self.set_team_leader(team_id, data['n'])

    def client_begin(self, data):
        pass

    def client_disconnect(self, data):
        pass

    def item(self, data):
        # {'item': 'item_armor_shard', 'runtime': 13, 'id': '0', 'timestamp': 1403357083, 'time': '0:13'}
        item_id = self.get_item_id(data['item'])

    def kill(self, data):
        #{'timestamp': 1402174290, 'weapon': 'MOD_PLASMA', 'killer': 'Visor', 'id_2': '8', 'id_1': '1', 'time': '4965:24', 'runtime': 406, 'killee': 'Major', 'id': '4'}

        killer_id = self.get_user_id(data['killer'])
        killee_id = self.get_user_id(data['killee'])
        weapon_id = self.get_item_id(data['weapon'], weapon=True)

        kill_action = GameKill(time=data['runtime'],
                            killer_id=killer_id,
                            killee_id=killee_id,
                            weapon_id=weapon_id,
                            game_id=self.game_now.id
        )
        self.session.add(kill_action)

    def say(self, data):
        #{'timestamp': 1403358308, 'message': 'team blue', 'runtime': 7061, 'user': 'Eye', 'time': '1085:01'}
        talker_id = self.get_user_id(data['user'])

        say_action = Say(time=data['runtime'],
                            talker_id=talker_id,
                            game_id=self.game_now.id,
                            message=data['message']
        )
        self.session.add(say_action)

    def tell(self, data):
        #{'timestamp': 1403351164, 'listener': 'Uriel', 'talker': 'Razor', 'time': '969:59', 'message': "it's agreed. I'll follow you around uriel.", 'runtime': 159}
        talker_id = self.get_user_id(data['talker'])
        listener_id = self.get_user_id(data['listener'])

        tell_action = Tell(time=data['runtime'],
                            talker_id=talker_id,
                            game_id=self.game_now.id,
                            listener_id=listener_id,
                            message=data['message']
        )
        self.session.add(tell_action)

    def sayteam(self, data):
        #{'timestamp': 1403351037, 'message': 'Who is the team leader', 'runtime': 35, 'user': 'Keel', 'time': '963:35'}

        sayteam_action = SayTeam(time=data['runtime'],
                            talker_id=self.get_user_id(data['user']),
                            game_id=self.game_now.id,
                            team_id=self.get_user_teamid(data['user']),
                            message=data['message']
        )
        self.session.add(sayteam_action)

    def empty(self, data):
        pass


import io
import sys
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import time

sys.stdout=io.TextIOWrapper(sys.stdout.detach(),encoding='utf-8')
sys.stderr=io.TextIOWrapper(sys.stderr.detach(),encoding='utf-8')

class NcafeWriteAtt:
    #초기화 실행(webdriver 설정)
    def __init__(self):
        chrome_options=Options()
        chrome_options.add_argument("--headless")
        self.driver=webdriver.Chrome(chrome_options=chrome_options,executable_path=r'D:/atom_py/section3/webdriver/chrome/chromedriver')
        self.driver.implicitly_wait(5)

    def writeAttendCheck(self):
        self.driver.get('https://nid.naver.com/nidlogin.login')
        self.driver.find_element_by_name('id').send_keys('516smile')
        self.driver.find_element_by_name('pw').send_keys('12345!')
        self.driver.find_elements_by_xpath('//*[@id="log.login"]').click()
        self.driver.implicitly_wait(5)


from copy import copy
from itertools import cycle
from random import randint, shuffle, choice
import pygame
from pygame import Color

scale = 2

class Langtonsant:
    def __init__(self, screen):
        self.ruleslist = None
        self.direction = None
        self.rules = None
        self.position = None
        self.grid = None
        self.index = 0
        self.screen = screen
        self.canvas = pygame.Surface((screen.get_width(), screen.get_height()))
        self.colorlist = ['gray50', 'white', 'red', 'green', 'blue', 'magenta', 'cyan', 'yellow', 'orange', 'teal',
                          'violet', 'aqua']
        self.colornumberlist = {0: 'gray50', 1: 'white', 2: 'red', 3: 'green', 4: 'blue', 5: 'magenta', 6: 'cyan',
                                7: 'yellow', 8: 'orange', 9: 'teal', 10: 'violet', 11: 'aqua'}
        self.colors = cycle(self.colorlist)
        self.directionslist = ['North', 'East', 'South', 'West']
        self.directions = cycle(['North', 'East', 'South', 'West'])
        self.actions = ['L', 'R']
        self.extendedactions = ['F', 'B']

        self.reinit()

    def step(self):

        currentcolor = self.grid[self.position[0]][self.position[1]]

        temp = cycle(iter(self.rules))
        for key in temp:
            if key == currentcolor:
                self.grid[self.position[0]][self.position[1]] = next(temp)
                break

        if self.rules[currentcolor] == 'L':
            # for i in range(len(self.colorlist) - 2):
            next(self.directions)
            next(self.directions)
            self.direction = next(self.directions)
        elif self.rules[currentcolor] == 'R':
            self.direction = next(self.directions)
        else:
            next(self.directions)
            self.direction = next(self.directions)

        if self.direction == 'North':
            self.position = (self.position[0], self.position[1] - 1)
        elif self.direction == 'East':
            self.position = (self.position[0] + 1, self.position[1])
        elif self.direction == 'South':
            self.position = (self.position[0], self.position[1] + 1)
        elif self.direction == 'West':
            self.position = (self.position[0] - 1, self.position[1])

        if self.position[0] >= self.screen.get_width() / scale:
            self.position = (self.position[0] - self.screen.get_width(), self.position[1])
        if self.position[1] >= self.screen.get_height() / scale:
            self.position = (self.position[0], self.position[1] - self.screen.get_height())
        if self.position[0] < 0:
            self.position = (self.screen.get_width() - 1 + self.position[0], self.position[1])
        if self.position[1] < 0:
            self.position = (self.position[0], self.screen.get_height() - 1 + self.position[1])

        pygame.draw.rect(self.canvas, Color(currentcolor), (self.position[0] * scale, self.position[1] * scale, 1*scale, 1*scale))

        self.screen.blit(self.canvas, (0, 0))

    def generaterules(self, defaultcolor):  # (color, action)
        length = randint(2, len(self.colorlist) - 1)
        tmp = copy(self.colorlist)
        tmp.remove(defaultcolor)
        shuffle(tmp)
        rules = [(tmp[i], self.actions[randint(0, len(self.actions) - 1)]) for i in range(length)]
        rules[0] = (defaultcolor, self.actions[randint(0, len(self.actions) - 1)])

        rpresent = False
        lpresent = False

        for rule in rules:
            if 'R' in rule:
                rpresent = True

        for rule in rules:
            if 'L' in rule:
                lpresent = True

        if not rpresent:
            rules[0] = (rules[0][0], 'R')

        if not lpresent:
            rules[0] = (rules[0][0], 'L')

        self.ruleslist = []
        for rule in rules:
            self.ruleslist.append(rule[0])
        self.ruleslist = cycle(self.ruleslist)
        print(rules)
        return dict(rules)

    def reinit(self):
        defaultcolor = choice(self.colorlist)
        self.grid = [[defaultcolor for _ in range(self.screen.get_height())] for _ in range(self.screen.get_width())]
        self.position = (int(self.screen.get_width() / 2 / scale), int(self.screen.get_height() / 2 / scale))
        self.rules = self.generaterules(defaultcolor)
        self.direction = self.directionslist[0]
        self.canvas = pygame.Surface((self.screen.get_width(), self.screen.get_height()))
        self.index = 0


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import sys
import re
import os
import time
import logging
from . import config
from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler
import watchdog.events as events
from threading import Thread
from .cleepapi import CleepApi
from .file import File
from collections import deque

class ActionFileSync():
    def __init__(self, module=None):
        self.module = module

class ActionRestart():
    def __init__(self, frontend=False):
        self.frontend = frontend

class ActionsExecutor(Thread):
    """
    Actions executor
    Implements queue to execute sequentially actions
    """
    def __init__(self, rpc_url):
        Thread.__init__(self)
        Thread.daemon = True

        self.logger = logging.getLogger(self.__class__.__name__)
        self.running = True
        self.__queue = deque(maxlen=200)
        self.file = File()
        self.cleep = CleepApi(rpc_url)

    def stop(self):
        """
        Stop process
        """
        self.running = False

    def add_action(self, action):
        """
        Add an action to execute
        """
        self.__queue.appendleft(action)

    def __execute_action(self, action):
        """
        Execute specified action

        Args:
            action (dict): action to execute
        """
        if isinstance(action, ActionFileSync):
            if action.module is None:
                self.file.core_sync()
            else:
                self.file.module_sync(action.module)

        elif isinstance(action, ActionRestart):
            if action.frontend:
                self.cleep.restart_frontend()
            else:
                self.cleep.restart_backend()

    def run(self):
        """
        Main process: unqueue action and process it
        """
        while self.running:
            try:
                action = self.__queue.pop()
                self.__execute_action(action)

            except IndexError:
                #no action available
                time.sleep(0.25)

            except:
                #error occured during action execution
                self.logger.exception('Error during action execution:')

class CleepHandler(FileSystemEventHandler):
    """
    Watchdog handler for Cleep
    """
    REJECTED_FILENAMES = [
        '4913', #vim furtive temp file to check user permissions
        '.gitignore'
    ]
    REJECTED_EXTENSIONS = [
        '.swp', #vim
        '.swpx', #vim
        '.swx', #vim
        '.tmp', #generic?
        '.offset', #pygtail
        '.pyc', #python compiled
        '.log' #log file
    ]
    REJECTED_PREFIXES = [
        '.',
        '~'
    ]
    REJECTED_SUFFIXES = [
        '~'
    ]
    REJECTED_DIRS = [
        '.git',
        '.vscode',
        '.editor',
        'backend/__pycache__',
    ]
    DEBOUNCE = 0.25 #seconds

    def __init__(self, actions_executor):
        self.__last_times = {
            'backend': 0,
            'frontend': 0
        }
        self.logger = logging.getLogger(self.__class__.__name__)
        self.actions_executor = actions_executor

    def __is_event_dropped(self, event):
        """
        Analyse event and return True if event must be dropped
        Return:
            bool: True if event must be dropped
        """
        #drop dir modified event or created file (that always comes with modified file)
        if type(event) in (events.DirModifiedEvent, events.FileCreatedEvent):
            self.logger.debug('Filtered: DirModifiedEvent or FileCreatedEvent')
            return True

        #filter invalid event
        if not event:
            self.logger.debug('Filtered: invalid event')
            return True

        #filter event on current script
        if event.src_path == '.%s' % __file__:
            self.logger.debug('Filtered: event on current script')
            return True

        #filter root event
        if event.src_path == '.':
            self.logger.debug('Filtered: root event')
            return True

        #filter invalid extension
        src_ext = os.path.splitext(event.src_path)[1]
        if src_ext in self.REJECTED_EXTENSIONS:
            self.logger.debug('Filtered: invalid extension')
            return True

        #filter by prefix
        for prefix in self.REJECTED_PREFIXES:
            if event.src_path.startswith(prefix):
                self.logger.debug('Filtered: source prefix')
                return True
            if getattr(event, 'dest_path', None) and event.dest_path.startswith(prefix):
                self.logger.debug('Filtered: destination prefix')
                return True

        #filter by suffix
        for suffix in self.REJECTED_SUFFIXES:
            if event.src_path.endswith(suffix):
                self.logger.debug('Filtered: source suffix')
                return True
            if getattr(event, 'dest_path', None) and event.dest_path.endswith(prefix):
                self.logger.debug('Filtered: destination suffix')
                return True

        #filter by filename
        for filename in self.REJECTED_FILENAMES:
            if event.src_path.endswith(filename):
                self.logger.debug('Filtered: filename')
                return True

        #filter by dir
        parts = event.src_path.split(os.path.sep)
        for folder in self.REJECTED_DIRS:
            if folder in parts:
                self.logger.debug('Filtered: directory')
                return True

        return False

    def __debounce(self, now, last_time):
        if now<(last_time+self.DEBOUNCE):
            return True
        return False

    def __change_on_module(self, event):
        """
        Change on module

        Args:
            event: watchdog event

        Returns:
            module name (string) or None if nothing found
        """
        pattern = '^%s/(.*?)/(?:frontend|backend|scripts)/.*$' % (config.MODULES_SRC)
        #self.logger.debug('Module pattern: %s' % pattern)

        if hasattr(event, 'src_path'):
            res = re.findall(pattern, event.src_path)
            if res and len(res)>0:
                return res[0]

        if hasattr(event, 'dest_path'):
            res = re.findall(pattern, event.dest_path)
            if res and len(res)>0:
                return res[0]

        return None

    def __change_on_core(self, event):
        """
        Change on core

        Args:
            event: watchdog event

        Returns:
            True if change occurs in core, False otherwise
        """
        pattern_core = '^(%s/(?!tests|modules)(.*\.py)|%s/.*)$' % (config.CORE_SRC, config.HTML_SRC)
        pattern_bin = '^%s/cleep' % (config.BIN_SRC)
        #self.logger.debug('Core pattern: %s' % pattern_core)
        #self.logger.debug('Bin pattern: %s' % pattern_bin)

        if hasattr(event, 'src_path'):
            if re.search(pattern_core, event.src_path):
                return True
            if re.search(pattern_bin, event.src_path):
                return True

        if hasattr(event, 'dest_path'):
            if re.search(pattern_core, event.dest_path):
                return True

        return False

    def __change_on_frontend(self, event):
        """
        Detect change on frontend

        Args:
            event: watchdog event

        Returns:
            True if change on frontend, False otherwise
        """
        pattern = '^(%s/|%s/(?:.*?)/frontend/).*$' % (config.HTML_SRC, config.MODULES_SRC)
        #self.logger.debug('Frontend pattern: %s' % pattern)

        if hasattr(event, 'src_path'):
            res = re.search(pattern, event.src_path)
            if res:
                return True

        if hasattr(event, 'dest_path'):
            res = re.search(pattern, event.dest_path)
            if res:
                return True

        return False

    def on_any_event(self, event):
        try:
            now = time.time()
            self.logger.debug('Event: %s' % event)
            if not self.__is_event_dropped(event):
                self.__last_action_time = now

                #is event on backend or frontend ?
                frontend = self.__change_on_frontend(event)

                #debounce ?
                if self.__debounce(now, self.__last_times[('frontend' if frontend else 'backend')]):
                    self.logger.debug('  Dropped by debounce')
                    return

                #is event on module ?
                change_on_core = False
                change_on_module = self.__change_on_module(event)
                if change_on_module is None:
                    change_on_core = self.__change_on_core(event)

                #push action and update last action time
                if change_on_module:
                    self.logger.debug('  Change on module "%s"' % change_on_module)
                    self.actions_executor.add_action(ActionFileSync(change_on_module))
                elif change_on_core:
                    self.logger.debug('  Change on core')
                    self.actions_executor.add_action(ActionFileSync())
                else:
                    self.logger.debug('  Dropped due to useless change')
                    return
                if frontend:
                    self.logger.debug('  Change on frontend')
                    self.actions_executor.add_action(ActionRestart(True))
                    self.__last_times['frontend'] = now
                else:
                    self.logger.debug('  Change on backend')
                    self.actions_executor.add_action(ActionRestart())
                    self.__last_times['backend'] = now

        except:
            self.logger.exception('Error occured during watcher event processing:')

class CleepWatchdog():
    """
    Cleep core and module watchdog
    """
    def __init__(self, rpc_url):
        """
        Constructor

        Args:
            rpc_url (str): Cleep RPC url
        """
        self.logger = logging.getLogger(self.__class__.__name__)
        self.actions_executor = ActionsExecutor(rpc_url)

    def watch(self):
        """
        Start watchdog that monitors filesystem changes on Cleep sources
        """
        path = '%s/' % config.REPO_DIR
        self.actions_executor.start()

        if not os.path.exists(path):
            self.logger.error('Please get Cleep sources using "cleep-cli getcore" command')
            return False

        event_handler = CleepHandler(self.actions_executor)
        observer = Observer()
        observer.schedule(event_handler, path, recursive=True)
        self.logger.info('Cleep watchdog is running on "%s" (CTRL-C to stop)' % path)
        observer.start()
        try:
            while True:
                time.sleep(0.25)
        except KeyboardInterrupt:
            observer.stop()
            self.actions_executor.stop()
        observer.join()

        return True


from datetime import datetime

from pytrends.request import TrendReq
from PersistenceLayer.ExternalAPIsORM.ExternalAPIsORMFacade import ExternalAPIsORMFacade

class GoogleTrendsManager:
    def __init__(self):
        self.pytrends = TrendReq(hl='en-US', tz=360, retries=3)
        self.ExternalOrm = ExternalAPIsORMFacade()
        self.trends = {}
        self.all_trends = self.ExternalOrm.get_all_trends()
        self.connect()

    def connect(self):
        try:
            self.trends = {}
            date = datetime.today().date()
            trends = self.pytrends.trending_searches().values.tolist()
            for trend in trends:
                if trend[0] not in self.all_trends.keys():
                    t_id = self.ExternalOrm.add_trend(trend[0], str(date))
                    self.trends[t_id] = {'keywords': trend[0]} # was trend instead of trend[0]
                    self.all_trends[trend[0]] = self.ExternalOrm.get_trend(t_id).copy()
                else:
                    flag = False
                    for trend_dict in self.all_trends[trend[0]]:
                        # print(trend_dict['date'])
                        # if type(trend_dict) == str:
                        #     trend_date = trend_dict
                        if type(trend_dict) == list:
                            trend_dict = trend_dict[0]
                            trend_date = trend_dict['date']
                        elif type(trend_dict) != dict:
                            trend_date = trend_dict
                            print(f"at GoogleTrendsManager.connect, the type of the trends dict is {type(trend_dict)}")

                        else:
                            trend_date = trend_dict['date']
                        if self.compare_dates(trend_date, date) == 0:
                        # if trend_dict['date'] == str(date):
                        #     print(f"at connect, trend_dict['id']= {trend_dict['id']}")
                        #     print(f"at connect, self.trends.keys()= {self.trends.keys()}")
                        #     print(f"flag = {flag}")
                            if trend_dict['id'] not in self.trends.keys() and not flag:
                                self.trends[trend_dict['id']] = {'keywords': trend[0]}
                            flag = True

                    if not flag:
                        t_id = self.ExternalOrm.add_trend(trend[0], str(date))
                        self.trends[t_id] = {'keywords': trend[0]}
                        self.all_trends[trend[0]].append(self.ExternalOrm.get_trend(t_id).copy())
        except:
            raise Exception("Connection problems")

    def get_trends(self):
        trends = {}
        for trend_name in self.all_trends:
            for trend in self.all_trends[trend_name]:
                stop = False
                if type(trend)==list:
                    if len(trend)>0:
                        print(f"at get_trends, trend type = {type(trend)}")
                        print(f"trends= {trends}")
                        trend=trend[0]
                    else:
                        stop=True
                        print(f"problem at get_trends, trend= {trend}")
                if not stop and (trend['date'] == str(datetime.today().date()) or trend['date'] == str(datetime.today())):
                    trends[self.all_trends[trend_name][0]['id']] = {'keywords': trend_name}
        for t_id in self.trends:
            if t_id not in trends.keys():
                trends[t_id] = self.trends[t_id]
        return trends

    def compare_dates(self, date1, date2):
        if type(date1) is str:
            if '-' in date1:
                date1 = datetime(int(date1[:4]), int(date1[5:7]), int(date1[8:])).date()
            else:
                date1 = datetime(int("20"+date1[6:]), int(date1[3:5]), int(date1[:2])).date()
        if type(date1) != type(datetime.today().date()) or type(date2) != type(datetime.today().date()):
            print(f"at googleTrendsManager.compare_dates     type of date1= {type(date1)},  type of date2= {type(date2)}")
            print(f"at googleTrendsManager.compare_dates     date1= {date1},  date2= {date2}")
        if date1.year != date2.year:
            if date1.year > date2.year:
                return 1
            else:
                return -1
        elif date1.month != date2.month:
            if date1.month > date2.month:
                return 1
            else:
                return -1
        elif date1.day == date2.day:
            return 0
        elif date1.day > date2.day:
            return 1
        else:
            return -1


from dash import Dash, dcc, html, Input, Output,ctx
import plotly.express as px
import pandas as pd
import os
import json

app = Dash(__name__)

#directory to data
directory = 'data/norm_data/'
files = os.listdir(directory)

jsonname = 'timestamps.json'
val_jsonname = 'val.json'

#get column names from a random file
defaultfile = files[0]
columns = pd.read_csv(directory+defaultfile).columns

#create a json file if one doesnt exist
if not os.path.exists(jsonname):
	with open(jsonname, 'w') as outfile:
				json.dump({'0':''}, outfile,indent=4)

#create a json file if one doesnt exist
if not os.path.exists(val_jsonname):
	with open(val_jsonname, 'w') as outfile:
				json.dump({'0':''}, outfile,indent=4)

app.layout = html.Div(
	children = [
		dcc.Dropdown(
			id='file',
			options=files,
			value=defaultfile,
			),
		dcc.Dropdown(
			id='tags',
			options=columns,
			multi=True,
			value=[],
			),
		dcc.Graph(
			id="vars",
			figure={}
			),

		html.Button('Save Training', id='save', n_clicks=0),
		html.Button('Save Validation', id='save_val', n_clicks=0),
		html.Div(id='save_msg',children=[])
		]
	)

#graph and drop down callbacks
@app.callback(
    Output(component_id="vars",component_property='figure'),
    [Input(component_id="tags",component_property='value'),
	Input(component_id="file",component_property='value'),]
	)
def update_line_chart(tags,file):
	data = pd.read_csv('data/norm_data/'+file)
	graph_df = pd.concat((data['TimeStamp'],data[tags]),axis=1)
	graph_df['TimeStamp'] = pd.to_datetime(data['TimeStamp'])
	fig = px.line(
		graph_df,
		x='TimeStamp',
		y=tags,
		height = 700,
		width = 1800,
		template = "plotly_dark",)

	return fig

#save function callback
@app.callback(
	Output('save_msg', 'children'),
	[Input('save', 'n_clicks'),
  	Input('save_val', 'n_clicks'),
	Input('vars', 'relayoutData'),
	Input(component_id="file",component_property='value')]
	)

def savefile(n_clicks1,n_clicks2,relayoutData,file):
	msg = "Save plot data"
	if 'save' == ctx.triggered_id:
		#open json
		with open(jsonname, 'r') as savefile:
			savedict = json.load(savefile)

		#get next record
		records = list(savedict.keys())
		lastrecord = int(records[-1])

		xmin = relayoutData['xaxis.range[0]']
		xmax = relayoutData['xaxis.range[1]']
		savedict[str(lastrecord+1)] = {
			'file': file,
			'xmin': xmin,
			'xmax': xmax
			}

		#delete the 0th entry because its blank
		if '0' in savedict:
			del savedict['0']

		#save file
		with open(jsonname, 'w') as outfile:
			json.dump(savedict, outfile,indent=4)

		msg = 'Training record saved ' + str(file) + '  Range: ' + xmin + ' to ' + xmax

	elif 'save_val' == ctx.triggered_id:

		#open json
		with open(val_jsonname, 'r') as savefile:
			savedict = json.load(savefile)

		#get next record
		records = list(savedict.keys())
		lastrecord = int(records[-1])

		xmin = relayoutData['xaxis.range[0]']
		xmax = relayoutData['xaxis.range[1]']
		savedict[str(lastrecord+1)] = {
			'file': file,
			'xmin': xmin,
			'xmax': xmax
			}

		#delete the 0th entry because its blank
		if '0' in savedict:
			del savedict['0']

		#save file
		with open(val_jsonname, 'w') as outfile:
			json.dump(savedict, outfile,indent=4)

		msg = 'Validation record saved ' + str(file) + '  Range: ' + xmin + ' to ' + xmax

	return html.Div(msg)

if __name__ == '__main__':
	app.run_server(debug=True)


"""
Layerwise Optimizer Intraprocessing Algorithm.
"""
import copy
import logging
import math

import numpy as np
import torch
from models import load_model
from skopt import gbrt_minimize
from skopt.space import Real
from utils import get_best_thresh, get_test_objective, get_valid_objective

logger = logging.getLogger("Debiasing")

def layerwiseOpt_debiasing(model_state_dict, data, config, device):
    logger.info('Training layerwiseOpt model.')
    base_model = load_model(data.num_features, config.get('hyperparameters', {}))
    base_model.load_state_dict(model_state_dict)
    base_model.to(device)
    best_state_dict, best_obj, best_thresh = None, math.inf, -1

    total_params = len(list(base_model.parameters()))
    for index, param in enumerate(base_model.parameters()):
        if index < total_params - config['layerwiseOpt']['num_layers']:
            continue
        logger.info(f'Evaluating param number {index} of {total_params}')
        param_copy = copy.deepcopy(param)

        def objective(new_param, return_thresh=False):
            param.data[indices] = torch.tensor(new_param)
            base_model.eval()
            with torch.no_grad():
                scores = base_model(data.X_valid_gpu)[:, 0].reshape(-1).numpy()
            best_thresh, best_obj = get_best_thresh(scores, np.linspace(0, 1, 501), data, config, valid=False, margin=config['layerwiseOpt']['margin'])
            print(f'Evaluating param number {index} of {total_params}')
            if return_thresh:
                return -float(best_obj), float(best_thresh)
            return -float(best_obj)

        mean = param.flatten().cpu().detach().numpy().mean()
        std = param.flatten().cpu().detach().numpy().std()
        num_elems = param.size().numel()
        ratio = min(1., config['layerwiseOpt']['max_sparsity'] / num_elems)
        indices = torch.rand(param.size()) < ratio
        space = [Real(float(x.cpu().detach()) - 2.2*std, float(x.cpu().detach()) + 2.2*std) for x in param[indices]]

        # std = param.flatten().cpu().detach().numpy().std()
        # num_elems = param.size().numel()
        # ratio = min(1., config['layerwiseOpt']['max_sparsity'] / num_elems)
        # indices = torch.rand(param.size()) < ratio
        logger.info(f'Number of sparse indices: {indices.sum().item()}')
        res_gbrt = gbrt_minimize(
            objective,
            space,
            n_calls=config['layerwiseOpt']['n_calls'],
            verbose=True
        )

        if res_gbrt.fun < best_obj:
            param.data[indices] = torch.tensor(res_gbrt.x)
            best_state_dict = base_model.state_dict()
            best_obj, best_thresh = objective(res_gbrt.x, return_thresh=True)
            best_obj = -best_obj
        param.data = param_copy.data

    best_model = load_model(data.num_features, config.get('hyperparameters', {}))
    best_model.to(device)
    best_model.load_state_dict(best_state_dict)
    best_model.eval()
    with torch.no_grad():
        y_pred = (best_model(data.X_valid_gpu)[:, 0] > best_thresh).reshape(-1).numpy()
    results_valid = get_valid_objective(y_pred, data, config)

    best_model.eval()
    with torch.no_grad():
        y_pred = (best_model(data.X_test_gpu)[:, 0] > best_thresh).reshape(-1).numpy()
    results_test = get_test_objective(y_pred, data, config)

    return results_valid, results_test


import argparse
from pathlib import Path
import typing

import numpy as np
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
from PIL import Image

import torch
import torch.nn as nn
import torch.optim as optim
import torchvision
import torchvision.transforms.functional as F
from torchinfo import summary

from omegaconf import DictConfig, OmegaConf

import hdrpy

from deepy.data.dataset import SelfSupervisedDataset
from deepy.data.vision.visiondataset import UnorganizedImageFolder
import deepy.data.transform
import deepy.data.vision.transform
from deepy.train.trainer import RegressorTrainer
from deepy.train.extension import (
    IntervalTrigger,
    MinValueTrigger,
    MaxValueTrigger,
    ModelSaver,
    HistorySaver
)

import transform as mytransform
import util as myutil
import train_id as tid

def get_transform(cfg: DictConfig):
    pre_transforms = None

    dict_cfg_crop = cfg.dataset.transforms.paired_random_resized_crop.param
    if dict_cfg_crop.interpolation == 'Image.BICUBIC':
        interpolation = Image.BICUBIC

    transforms = deepy.data.transform.PairedCompose([
        deepy.data.transform.ToPairedTransform(
            torchvision.transforms.ToTensor()),
        deepy.data.vision.transform.PairedPowerTwoResize(interpolation=interpolation),
        deepy.data.transform.SeparatedTransform(
            transform=mytransform.ReinhardTMO(),
            target_transform=mytransform.RandomEilertsenTMO()),
    ])

    return pre_transforms, transforms

def get_dataset(cfg: DictConfig):
    cwd = Path.cwd()
    p = Path(cwd) / cfg.dataset.path

    testset = SelfSupervisedDataset(
        UnorganizedImageFolder(
            str(p / "test"),
            pre_load=False,
            loader=hdrpy.io.read, is_valid_file=myutil.is_valid_file)
    )

    return testset

def predict(path, dataset, net, device):
    p = Path(path)
    (p / 'imgs').mkdir(parents=True, exist_ok=True)
    transforms = deepy.data.transform.Compose(
        [torchvision.transforms.ToTensor(),
         mytransform.ResizeToMultiple(divisor=16, interpolation=Image.BICUBIC),
         mytransform.RandomEilertsenTMO()])

    post_transforms = mytransform.KinoshitaITMO()
    target_transforms = deepy.data.transform.Compose(
        [torchvision.transforms.ToTensor(),
         mytransform.ReinhardTMO()])

    with torch.no_grad():
        for i in range(len(dataset)):
            q = dataset.dataset.samples[i]
            q = Path(q)

            print('{:04d}/{:04d}: File Name {}'.format(i, len(dataset), q.name))

            sample, hdr_target = dataset[i]
            sample = transforms(sample).unsqueeze(0).float().to(device)
            ldr_predict = net(sample).to('cpu').clone().detach().squeeze(0)
            hdr_predict = post_transforms(torch.clamp(ldr_predict, 0, 1))
            height, width, _ = hdr_target.shape
            sample = sample.to('cpu').clone().detach().squeeze(0)
            sample = F.resize(sample, (height, width), Image.BICUBIC)
            ldr_predict = F.resize(ldr_predict, (height, width), Image.BICUBIC)
            hdr_predict = F.resize(hdr_predict, (height, width), Image.BICUBIC)
            ldr_target = target_transforms(hdr_target)

            hdrpy.io.write(
                p / 'imgs' / ('{:04d}'.format(i) + '_' + q.stem + '_input.jpg'),
                sample.clone().detach().numpy().transpose((1, 2, 0)))
            hdrpy.io.write(
                p / 'imgs' / ('{:04d}'.format(i) + '_' + q.stem + '_output.jpg'),
                ldr_predict.clone().detach().numpy().transpose((1, 2, 0)))
            hdrpy.io.write(
                p / 'imgs' / ('{:04d}'.format(i) + '_' + q.stem + '_prediction.pfm'),
                hdr_predict.clone().detach().numpy().transpose((1, 2, 0)))
            hdrpy.io.write(
                p / 'imgs' / ('{:04d}'.format(i) + '_' + q.stem + '_reinhard.jpg'),
                ldr_target.clone().detach().numpy().transpose((1, 2, 0)))
            hdrpy.io.write(
                p / 'imgs' / ('{:04d}'.format(i) + '_' + q.stem + '_target.pfm'),
                hdr_target)

def main(cfg: DictConfig, train_id:str) -> None:
    cwd = Path.cwd()
    myutil.print_config(cfg)
    # Setting seed
    myutil.set_random_seed(0)

    model_file_name = "{}_best.pth".format(cfg.model.name)

    # Checking history directory
    history_dir = cwd / 'history' / train_id
    if (history_dir / model_file_name).exists():
        pass
    else:
        return

    # Setting result directory
    # All outputs will be written into (p / 'result' / train_id).
    if not (cwd / 'result').exists():
        (cwd / 'result').mkdir(parents=True)
    result_dir = cwd / 'result' / train_id
    if result_dir.exists():
        # removing files in result_dir?
        pass
    else:
        result_dir.mkdir(parents=True, exist_ok=True)

    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

    # Testing
    testset = get_dataset(cfg)
    net = myutil.get_model(cfg)
    net.module.load_state_dict(torch.load(str(history_dir / model_file_name)))
    predict(result_dir, testset, net, device)

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('--history_dir', type=str,
                        default='./history',
                        help='Directory path for searching trained models')
    args = parser.parse_args()
    p = Path.cwd() / args.history_dir
    for q in p.glob('**/config.yaml'):
        cfg = OmegaConf.load(str(q))
        cfg.loader.num_workers = 0
        train_id = q.parent.name
        main(cfg, train_id)


#!/usr/bin/python
# -*- coding: utf-8 -*-
'''
@date: 2012-04-26
@author: shell.xu
'''
import logging

logger = logging.getLogger('http')

BUFSIZE = 8192
CODE_NOBODY = [100, 101, 204, 304]
DEFAULT_PAGES = {
    100:('Continue', 'Request received, please continue'),
    101:('Switching Protocols',
          'Switching to new protocol; obey Upgrade header'),

    200:('OK', ''),
    201:('Created', 'Document created, URL follows'),
    202:('Accepted', 'Request accepted, processing continues off-line'),
    203:('Non-Authoritative Information', 'Request fulfilled from cache'),
    204:('No Content', 'Request fulfilled, nothing follows'),
    205:('Reset Content', 'Clear input form for further input.'),
    206:('Partial Content', 'Partial content follows.'),

    300:('Multiple Choices', 'Object has several resources -- see URI list'),
    301:('Moved Permanently', 'Object moved permanently -- see URI list'),
    302:('Found', 'Object moved temporarily -- see URI list'),
    303:('See Other', 'Object moved -- see Method and URL list'),
    304:('Not Modified', 'Document has not changed since given time'),
    305:('Use Proxy',
          'You must use proxy specified in Location to access this resource.'),
    307:('Temporary Redirect', 'Object moved temporarily -- see URI list'),

    400:('Bad Request', 'Bad request syntax or unsupported method'),
    401:('Unauthorized', 'No permission -- see authorization schemes'),
    402:('Payment Required', 'No payment -- see charging schemes'),
    403:('Forbidden', 'Request forbidden -- authorization will not help'),
    404:('Not Found', 'Nothing matches the given URI'),
    405:('Method Not Allowed', 'Specified method is invalid for this server.'),
    406:('Not Acceptable', 'URI not available in preferred format.'),
    407:('Proxy Authentication Required',
          'You must authenticate with this proxy before proceeding.'),
    408:('Request Timeout', 'Request timed out; try again later.'),
    409:('Conflict', 'Request conflict.'),
    410:('Gone', 'URI no longer exists and has been permanently removed.'),
    411:('Length Required', 'Client must specify Content-Length.'),
    412:('Precondition Failed', 'Precondition in headers is false.'),
    413:('Request Entity Too Large', 'Entity is too large.'),
    414:('Request-URI Too Long', 'URI is too long.'),
    415:('Unsupported Media Type', 'Entity body in unsupported format.'),
    416:('Requested Range Not Satisfiable', 'Cannot satisfy request range.'),
    417:('Expectation Failed', 'Expect condition could not be satisfied.'),

    500:('Internal Server Error', 'Server got itself in trouble'),
    501:('Not Implemented', 'Server does not support this operation'),
    502:('Bad Gateway', 'Invalid responses from another server/proxy.'),
    503:('Service Unavailable',
          'The server cannot process the request due to a high load'),
    504:('Gateway Timeout',
          'The gateway server did not receive a timely response'),
    505:('HTTP Version Not Supported', 'Cannot fulfill request.'),
}

def dummy_write(d): return

def capitalize_httptitle(k):
    return '-'.join([t.capitalize() for t in k.split('-')])

class HttpMessage(object):
    def __init__(self): self.headers, self.body = [], None

    def add_header(self, k, v):
        self.headers.append([k, v])

    def set_header(self, k, v):
        for h in self.headers:
            if h[0] == k:
                h[1] = v
                return
        self.add_header(k, v)

    def get_header(self, k, v=None):
        for ks, vs in self.headers:
            if ks == k: return vs
        return v

    def get_headers(self, k):
        return [vs for ks, vs in self.headers if ks == k]

    def has_header(self, k): return self.get_header(k) is not None

    def send_header(self, stream):
        stream.write(self.get_startline() + '\r\n')
        for k, l in self.headers: stream.write("%s: %s\r\n" % (k, l))
        stream.write('\r\n')

    def recv_header(self, stream):
        while True:
            line = stream.readline()
            if not line: raise EOFError()
            line = line.strip()
            if not line: break
            if line[0] not in (' ', '\t'):
                h, v = line.split(':', 1)
                self.add_header(h.strip(), v.strip())
            else: self.add_header(h.strip(), line.strip())

    def read_chunk(self, stream, hasbody=False, raw=False):
        if self.get_header('Transfer-Encoding', 'identity') != 'identity':
            logger.debug('recv body on chunk mode')
            chunk_size = 1
            while chunk_size:
                line = stream.readline()
                chunk = line.split(';')
                chunk_size = int(chunk[0], 16)
                if raw: yield line + stream.read(chunk_size + 2)
                else: yield stream.read(chunk_size + 2)[:-2]
        elif self.has_header('Content-Length'):
            length = int(self.get_header('Content-Length'))
            logger.debug('recv body on length mode, size: %s' % length)
            for i in xrange(0, length, BUFSIZE):
                yield stream.read(min(length - i, BUFSIZE))
        elif hasbody:
            logger.debug('recv body on close mode')
            d = stream.read(BUFSIZE)
            while d:
                yield d
                d = stream.read(BUFSIZE)

    def read_body(self, hasbody=False, raw=False):
        return ''.join(self.read_chunk(self.stream, hasbody, raw))

    def read_form(self):
        return dict([i.split('=', 1) for i in self.read_body().split('&')])

    def sendto(self, stream, *p, **kw):
        self.send_header(stream)
        if self.body is None: return
        elif callable(self.body):
            for d in self.body(*p, **kw): stream.write(d)
        else: stream.write(self.body)

    def debug(self):
        logger.debug(self.d + self.get_startline())
        for k, v in self.headers: logger.debug('%s%s: %s' % (self.d, k, v))

class HttpRequest(HttpMessage):
    d = '> '

    def __init__(self, method, uri, version):
        HttpMessage.__init__(self)
        self.method, self.uri, self.version = method, uri, version

    def get_startline(self):
        return ' '.join((self.method, self.uri, self.version))

class HttpResponse(HttpMessage):
    d = '< '

    def __init__(self, version, code, phrase):
        HttpMessage.__init__(self)
        self.version, self.code, self.phrase = version, int(code), phrase
        self.connection, self.cache = False, 0

    def __nonzero__(self): return self.connection

    def get_startline(self):
        return ' '.join((self.version, str(self.code), self.phrase))

def recv_msg(stream, cls):
    line = stream.readline().strip()
    if len(line) == 0: raise EOFError()
    r = line.split(' ', 2)
    if len(r) < 2: raise Exception('unknown format')
    if len(r) < 3: r.append(DEFAULT_PAGES[int(r[1])][0])
    msg = cls(*r)
    msg.stream = stream
    msg.recv_header(stream)
    return msg

def request_http(uri, method=None, version=None, headers=None, data=None):
    if not method: method = 'GET' if data is None else 'POST'
    if not version: version = 'HTTP/1.1'
    if not headers: headers = []
    req = HttpRequest(method, uri, version)
    req.headers, req.body = headers, data
    if req.body and isinstance(req.body, basestring):
        req.set_header('Content-Length', str(len(req.body)))
    return req

def response_http(code, phrase=None, version=None, headers=None,
                  cache=0, body=None):
    if not phrase: phrase = DEFAULT_PAGES[code][0]
    if not version: version = 'HTTP/1.1'
    res = HttpResponse(version, code, phrase)
    if body and isinstance(body, basestring):
        res.set_header('Content-Length', str(len(body)))
    if headers:
        for k, v in headers: res.set_header(k, v)
    res.cache, res.body = cache, body
    return res

def http_client(req, addr, creator):
    sock = creator()
    sock.connect(addr)
    try:
        stream = sock.makefile()
        req.sendto(stream)
        stream.flush()
        return recv_msg(stream, HttpResponse)
    finally: sock.close()


"""Functions for reviewing HITs"""

from hit_info import *
from hit_results import *
import log
import create
import save
import email_worker
import logging
import pandas as pd
import boto3
import xmltodict
import random

logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)

def get_pending_hit_ids(annotation_log, dataset_filter=None):
    """Returns a list of all HIT ids whose results have not been logged

    Parameters
    ----------
    annotation_log : str
        Path to annotations log file
    dataset_filter : str
        String of dataset that only its pending HITs will be fetched

    Returns
    -------
    list
        The list of pairs of pending HIT ids, and their type
    """
    # return HIT id of HITs without results yet
    pending_hits = []
    df = pd.read_csv(annotation_log)
    df = df.set_index('annotation_id')
    pending_gen_df = df[~df['hitid_gen'].isnull() & df['decomposition'].isnull() & df['decomposition_accepted'].isnull()]
    pending_val_df = df[(~df['hitid_validation'].isnull()) & (~df['hitid_validation'].isin([log.MANUAL_VALIDATION, log.PENDING_MANUAL_VALIDATION])) & \
                        (df['validation_annotation_1'].isnull() | df['validation_annotation_2'].isnull())]

    if dataset_filter != None:
        # filter only pending HITs of specified dataset
        pending_gen_df = pending_gen_df[pending_gen_df['question_id'].str.contains(dataset_filter)]
        pending_val_df = pending_val_df[pending_val_df['question_id'].str.contains(dataset_filter)]
    gen_hit_ids = pending_gen_df['hitid_gen'].values.tolist()
    val_hit_ids = pending_val_df['hitid_validation'].values.tolist()
    for hit_id in gen_hit_ids:
        pending_hits += [(hit_id, 'gen')]
    for hit_id in val_hit_ids:
        pending_hits += [(hit_id, 'val')]
    # randomly shuffle HITs for manual review process
    random.shuffle(pending_hits)
    return pending_hits

def review_pending_hits(manual_review, client, annotation_log, bank_annotations, save_results_to_json=False, json_dir=None):
    """Choose to automatically or manually review all pending HIT results.
        Either automatically return all pending HIT results and update the annotation log files,
        or manually review HIT assignment results and accept/reject and save them.

    Parameters
    ----------
    manual_review : bool
        Flag whether the HIT results will be manually reviewed, or results saved automatically
    client : MTurk.Client
        A boto3 client for MTurk
    annotation_log : str
        Path to annotations log file
    bank_annotations : str
        Path to bank annotations log files for validation HIT tests
    save_results_to_json : bool
        Flag indicating whether to save HITResults of pending HITs to json files
    json_dir : str
        Path to directory where published HIT json files are to be saved

    Returns
    -------
    bool
        returns True if successfully terminated
    """
    # get all pending HIT ids
    dataset = None ####'HOTPOT_' ####None # review only HOTPOT HITs
    pending_hits = get_pending_hit_ids(annotation_log, dataset)
    hits_results_saved = 0
    # automatic review - save all HIT results and finish
    if not manual_review:
        # go over all HITs
        for hit in pending_hits:
            # get HIT information
            hit_id, hit_type = hit
            hit_info = create.get_hit_info(annotation_log=annotation_log, bank_annotations=bank_annotations,\
                                    annotation_id=None, hit_type=hit_type, hit_id=hit_id)
            # get HIT results, if any
            hit_results_list = get_hit_results(client, hit_info.type, hit_info.hit_id)
            if len(hit_results_list) > 0:
                for hit_result in hit_results_list:
                    res = save.save_hit_results(hit_info, hit_result, annotation_log, save_results_to_json, json_dir)
                    assert(res == True)
                    hits_results_saved += 1
        logger.info(f'* Reviewed (automatically) and saved {hits_results_saved} HIT results')
    # manual review - review each pending HIT
    else:
        continue_review = None
        review_batch_size = 50
        batch_iteration = 0
        while continue_review != False:
            pending_hit_info_list = []
            # create a btach of HITs, watch for the end of the list
            limit = min((batch_iteration+1)*review_batch_size, len(pending_hits)-1)
            if limit == len(pending_hits)-1:
                # final batch, quit review afterwards
                continue_review = False
            for hit in pending_hits[batch_iteration*review_batch_size:limit]:
                # get HIT information
                hit_id, hit_type = hit
                hit_info = create.get_hit_info(annotation_log=annotation_log, bank_annotations=bank_annotations,\
                                        annotation_id=None, hit_type=hit_type, hit_id=hit_id)
                pending_hit_info_list += [hit_info]
            # increase batch iteration
            batch_iteration += 1
            # review a batch of HITs
            for hit_info in pending_hit_info_list:
                hit_results_list = get_hit_results(client, hit_info.type, hit_info.hit_id)
                if len(hit_results_list) > 0:
                    # retreive HIT results
                    for hit_result in hit_results_list:
                        result = review_hit_results(client, annotation_log, hit_info, hit_result, json_dir)
                        assert(result != None)
                        hits_results_saved += 1
                    # ask whether to continue
                    user_input = input('Continue manual review of pending HITs? [y/n]').strip().lower()
                    if user_input in ['y', 'n']:
                        continue_review = user_input == 'y'
                    else:
                        print('Please type either "y" or "n".')
                    if not continue_review:
                        break
        logger.info(f'* Reviewed (manually) and saved {hits_results_saved} HIT results')
    return True

def get_hit_results(client, hit_type, hit_id):
    """Returns the results of the specified HIT

    Parameters
    ----------
    client : MTurk.Client
        A boto3 client for MTurk
    hit_type : str
        String indicating if HIT is of type generation or validation
    hit_id : str
        Id of the relevant HIT

    Returns
    -------
    list
        list of GenerationResults/ValidationResults objects representing the relevant HIT results
    """
    assert(hit_type in ['gen', 'val'])
    assignments_results = []
    worker_results = client.list_assignments_for_hit(HITId=hit_id, AssignmentStatuses=['Submitted'])
    num_results = worker_results['NumResults']
    if num_results == 0:
        # If no results returned for 'Submitted' status, check if there are any for 'Approved' status
        worker_results = client.list_assignments_for_hit(HITId=hit_id, AssignmentStatuses=['Approved'])
        num_results = worker_results['NumResults']
    if num_results > 0:
        for assignment in worker_results['Assignments']:
            result = None
            # general assignment info
            assignment_id = assignment['AssignmentId']
            worker_id = assignment['WorkerId']
            submit_time = assignment['SubmitTime']
            # assignment answer
            decomposition = annotation_validation = bank_validation = None
            xml_doc = xmltodict.parse(assignment['Answer'])
            logger.info(f'* Returning results for HIT {hit_id}: found {num_results} results.')
            # Multiple fields in HIT layout
            if type(xml_doc['QuestionFormAnswers']['Answer']) is list:
                for answer_field in xml_doc['QuestionFormAnswers']['Answer']:
                    if hit_type == 'gen':
                        # generation HIT results
                        if answer_field['QuestionIdentifier'] == 'surveycode':
                            decomposition = answer_field['FreeText']
                    else:
                        # validation HIT results
                        if answer_field['QuestionIdentifier'] == 'ant':
                            annotation_validation_str = answer_field['FreeText'].lower().strip()
                            annotation_validation = 1 if (annotation_validation_str == 'yes') else 0
                        elif answer_field['QuestionIdentifier'] == 'bnk':
                            bank_validation_str = answer_field['FreeText'].lower().strip()
                            bank_validation = 1 if (bank_validation_str == 'yes') else 0
            # One field found in HIT layout
            else:
                raise Exception('HIT results should have more than one result field!')
            # Build assignment result
            if hit_type == 'gen':
                result = GenerationResults(hit_id, assignment_id, worker_id, submit_time, decomposition)
                logger.info(f'* Returning generation results for HIT: {hit_id}, decomposition: {decomposition}.')
            else:
                result = ValidationResults(hit_id, assignment_id, worker_id, submit_time, annotation_validation, bank_validation)
                logger.info(f'* Returning validation results for HIT: {hit_id}, annotation val: {annotation_validation}, bank val: {bank_validation}.')
            assignments_results += [result]
    else:
        logger.info(f'* Returning results for HIT {hit_id}: no results were found.')
    return assignments_results

def review_hit_results(client, annotation_log, hit_info, hit_results, json_dir):
    """Manually review HIT assignment results, accept/reject and save them.

    Parameters
    ----------
    client : MTurk.Client
        A boto3 client for MTurk
    annotation_log : str
        Path to annotations log file
    hit_info : GenerationHITInfo / ValidationHITInfo
        HITInfo object storing information regarding the HIT
    hit_results : GenerationResults / ValidationResults
        HIT results object storing the results of the relevant HIT
    json_dir : str
        Path to directory where published HIT json files are to be saved

    Returns
    -------
    GenerationResults / ValidationResults
        returns the HIT results possibly manually validated (if generation HIT)
    """
    assignment_id = hit_results.assignment_id
    hit_type = hit_results.type
    hit_id = hit_results.hit_id
    annotation_id = hit_info.annotation_id
    assert(hit_type in ['gen', 'val'])
    # generation HIT
    if hit_type == 'gen':
        display_generation_hit_results(hit_info, hit_results)
        # would you like to manaully validate the HIT?
        validate = None
        while validate is None:
            user_input = input('Manually validate? [y/n]').strip().lower()
            if user_input in ['y', 'n']:
                validate = user_input == 'y'
            else:
                print('Please type either "y" or "n".')
        if validate:
            logger.info(f'Manually validating (HIT id: {hit_id}, Assignment id: {assignment_id}).')
            hit_results = validate_annotation(hit_results)
        else:
            logger.info(f'Did not manually validate (HIT id: {hit_id}, Assignment id: {assignment_id}).'
                        f'Please make sure to publish a corresponding validation HIT in the future.')
    # validation HIT
    else:
        display_validation_hit_results(hit_info, hit_results)
    # Would you like to accept/reject the HIT?
    accept = None
    while accept is None:
        user_input = input('Manually accept HIT assignment? [accept/reject]').strip().lower()
        if user_input in ['accept', 'reject']:
            accept = user_input == 'accept'
        else:
            print('Please type either "accept" or "reject".')
    if accept:
        hit_results.accept()
        accept_hit(client, assignment_id, "Accepted HIT.")
        logger.info(f'Manually accepted (HIT id: {hit_id}, Assignment id: {assignment_id}).')
    else:
        hit_results.reject()
        reject_hit(client, assignment_id, "Worker results did not comply with the assignment instructions.")
        logger.info(f'Manually rejected (HIT id: {hit_id}, Assignment id: {assignment_id}).')
    # Would you like to save HIT results in a separate json file?
    save_to_json = True ##### None
    while save_to_json is None:
        user_input = input('Save HIT results to JSON file? [y/n]').strip().lower()
        if user_input in ['y', 'n']:
            save_to_json = user_input == 'y'
        else:
            print('Please type either "y" or "n".')
    if save_to_json:
        logger.info(f'Saving HIT results to: {json_dir}/{hit_id}_{assignment_id}.json (HIT id: {hit_id}, Assignment id: {assignment_id}).')
    # Update logs files
    logger.info(f'Updating annotaion & hit log files (HIT id: {hit_id}, Assignment id: {assignment_id}, Annotation id: {annotation_id}).')
    save.save_hit_results(hit_info, hit_results, annotation_log, create_json=save_to_json, json_dir=json_dir)
    # Would you like to email worker regarding the results?
    send_email_worker = None
    while send_email_worker is None:
        user_input = input('Email worker? [y/n]').strip().lower()
        if user_input in ['y', 'n']:
            send_email_worker = user_input == 'y'
        else:
            print('Please type either "y" or "n".')
    if send_email_worker:
        message = format_hit_review_email(hit_info, hit_results)
        print(message)
        user_input = input('Enter additional email review:').strip()
        message += "\n"+user_input
        subject = "Review feedback on HIT id: " + hit_id
        worker_id = hit_results.worker_id
        workers = [worker_id]
        email_worker.email_mturk_workers(client, subject, message, workers)
        logger.info(f'Emailed worker {worker_id} on HIT results (HIT id: {hit_id}, Assignment id: {assignment_id}).')
    return hit_results

def format_hit_review_email(hit_info, hit_results):
    """Returns formatted string to be sent as a review to the worker

    Parameters
    ----------
    hit_info : GenerationHITInfo / ValidationHITInfo
        HITInfo object storing information regarding the HIT
    hit_results : GenerationResults / ValidationResults
        HIT results object storing the results of the relevant HIT

    Returns
    -------
    str
        returns the formatted review string
    """
    review = "Dear worker, \nPlease read the following review of your results on HIT: " + hit_results.hit_id
    if hit_results.type == 'gen':
        review += "\n\nOriginal question: " + "[" + hit_info.question_text + "]"
        review += "\n\nYour decomposition: "
        decomposition = hit_results.decomposition
        formatted_dec = format_decomposition_string(decomposition)
        review += formatted_dec
    else:
        # for the bank decomposition
        review += "\n\nYour answers: "
        bank_dec = hit_info.bank_annotation.decomposition
        formatted_bank_dec = format_decomposition_string(bank_dec)
        bank_valid = hit_results.bank_validation
        bank_verdict = "correct" if bank_valid else "incorrect"
        first_val = "\n 1. Marked the following decomposition as :" + bank_verdict + formatted_bank_dec
        review += first_val
        # for the annotated decomposition
        annotated_dec = hit_info.decomposition
        formatted_annot_dec = format_decomposition_string(annotated_dec)
        annotated_dec_valid = hit_results.annotation_validation
        annot_verdict = "correct" if annotated_dec_valid else "incorrect"
        second_val = "\n 2. Marked the following decomposition as :" + annot_verdict + formatted_annot_dec
        review += second_val
    return review

def display_generation_hit_results(hit_info, hit_results):
    """Displays the results of a generation HIT

    Parameters
    ----------
    hit_info : GenerationHITInfo
        HITInfo object storing information regarding the HIT
    hit_results : GenerationResults
        HIT results object storing the results of the relevant HIT

    Returns
    -------
    bool
        returns True
    """
    dec_string = format_decomposition_string(hit_results.decomposition)
    print(
        'HIT ID: {hit_id}'
        '\nAssignment ID: {assignment_id}'
        '\nHIT Type: Generation'
        '\n'
        '\nResults'
        '\n======='
        '\nAnnotation ID: {annotation_id}'
        '\nQuestion ID: {question_id}'
        '\nQuestion Text: {question_text}'
        '\nDecomposition: {decomposition}'.format(
                            hit_id=hit_results.hit_id,
                            assignment_id=hit_results.assignment_id,
                            annotation_id=hit_info.annotation_id,
                            question_id=hit_info.question_id,
                            question_text=hit_info.question_text,
                            decomposition=dec_string))
    return True

def display_validation_hit_results(hit_info, hit_results):
    """Displays the results of a validation HIT

    Parameters
    ----------
    hit_info : ValidationHITInfo
        HITInfo object storing information regarding the HIT
    hit_results : ValidationResults
        HIT results object storing the results of the relevant HIT

    Returns
    -------
    bool
        returns True
    """
    bank_annotation = hit_info.bank_annotation
    dec_string_annotation = format_decomposition_string(hit_info.decomposition)
    dec_string_bank = format_decomposition_string(bank_annotation.decomposition)
    print(
        'HIT ID: {hit_id}'
        '\nAssignment ID: {assignment_id}'
        '\nHIT Type: Validation'
        '\n'
        '\nResults'
        '\n======='
        '\nAnnotation ID: {annotation_id}'
        '\nQuestion ID: {question_id}'
        '\nQuestion Text: {question_text}'
        '\nDecomposition: {decomposition}'
        '\nValidator marked as correct: {annotation_validation}'
        '\n======='
        '\nBank Annotation ID: {bank_annotation_id}'
        '\nBank ID: {bank_question_id}'
        '\nBank Question Text: {bank_question_text}'
        '\nBank Decomposition: {bank_decomposition}'
        '\nIs correct: {decomposition_correct}'
        '\nValidator marked as correct: {bank_validation}'.format(
                            hit_id=hit_results.hit_id,
                            assignment_id=hit_results.assignment_id,
                            annotation_id=hit_info.annotation_id,
                            question_id=hit_info.question_id,
                            question_text=hit_info.question_text,
                            decomposition=dec_string_annotation,
                            annotation_validation=hit_results.annotation_validation,
                            bank_annotation_id=bank_annotation.annotation_id,
                            bank_question_id=bank_annotation.question_id,
                            bank_question_text=bank_annotation.question_text,
                            bank_decomposition=dec_string_bank,
                            decomposition_correct=bank_annotation.decomposition_correct,
                            bank_validation=hit_results.bank_validation))
    return True

def accept_hit(client, assignment_id, feedback):
    """Accepts a HIT assignment on MTurk

    Parameters
    ----------
    client : MTurk.Client
        A boto3 client for MTurk
    assignment_id : string
        The ID of the assignment. The assignment must correspond to a HIT created by the Requester.
    feedback: string
        A message for the Worker, which the Worker can see in the Status section of the web site.

    Returns
    -------
    bool
        returns whether the operation was successful
    """
    response = client.approve_assignment(
        AssignmentId=assignment_id,
        RequesterFeedback=feedback)
    return response=={}

def reject_hit(client, assignment_id, feedback):
    """Rejects a HIT assignment on MTurk

    Parameters
    ----------
    client : MTurk.Client
        A boto3 client for MTurk
    assignment_id : string
        The ID of the assignment. The assignment must correspond to a HIT created by the Requester.
    feedback: string
        A message for the Worker, which the Worker can see in the Status section of the web site.

    Returns
    -------
    bool
        returns whether the operation was successful
    """
    response = client.reject_assignment(
        AssignmentId=assignment_id,
        RequesterFeedback=feedback)
    return response=={}

def validate_annotation(generation_hit_results):
    """Manually validate a generation HIT annotation

    Parameters
    ----------
    generation_hit_results : GenerationResults
        results of the generation HIT

    Returns
    -------
    GenerationResults
        Returns manually validated results
    """
    valid_annotation = None
    while valid_annotation is None:
        user_input = input('Annotation valid? [y/n]').strip().lower()
        if user_input in ['y', 'n']:
            valid_annotation = user_input == 'y'
        else:
            print('Please type either "y" or "n".')
    generation_hit_results.validate(valid_annotation)
    logger.info(f'* Manually validated generation HIT to {valid_annotation} (HIT id: {generation_hit_results.hit_id}, Assignment id: {generation_hit_results.assignment_id})')
    return generation_hit_results

def format_decomposition_string(decomposition_string, delimiter=";"):
    """Returns a formatted text string representation of a query decomposition
        for manually reviewing HIT results.

    Parameters
    ----------
    decomposition_string : str
        decompositiong string with delimited steps
    delimiter : str
        delimiter at the end of decomposition steps

    Returns
    -------
    str
        Returns formatted text string, numbered steps, each step in a new line
    """
    text_decomposition = ""
    # remove last delimiter
    if decomposition_string[-1] == delimiter:
        decomposition_string = decomposition_string[:-1]
    # break into steps
    steps = decomposition_string.split(delimiter)
    for i in range(len(steps)):
        text_decomposition += '\n\t' + str(i+1) + '. ' + steps[i]
    return text_decomposition


import sys
import os
import yaml
import re
import urllib.request

from .drawio_parse.parse import xml2yaml
from .utils.get_yaml import get_yaml

DEBUG_FLAG = True

LEFT = 0
CENTER = 1
RIGHT = 2

class widget_configure:
    def __init__(self):
        self.x_center = 0
        self.y_center = 0
        self.x_left = 0
        self.x_right = 0
        self.y_top = 0
        self.width = 50
        self.height = 50
        self.text = ""
        self.items = []
        self.max = 100
        self.min = 0
        self.step = 1
        self.default = 0
        self.stylesheet_str = ""
        self.font = "Arial"
        self.font_size = 10

class label_configure(widget_configure):
    def is_url(self, path: str) -> bool:
        pattern = "https?://[\w/:%#\$&\?\(\)~\.=\+\-]+"
        if re.match(pattern, path):
            return True
        else:
            return False

    def __init__(self, yaml_abs_path: str, target_key: str) -> None:
        super().__init__()
        self.stylesheet_str = str()
        self.yaml_abs_path_file = yaml_abs_path
        self.target_key = target_key

        self.position = LEFT

        self.debug = DEBUG_FLAG

        self.yaml_data = get_yaml(self.yaml_abs_path_file)

        # exit when no yaml data
        if self.yaml_data is None:
            print("no yaml data")
            sys.exit()
        self.yaml_data = self.yaml_data[self.target_key]

    # include yaml from yaml ---------------------------------------------------
        if "include" in self.yaml_data:
            path = self.yaml_data["include"]["path"]
            key = self.yaml_data["include"]["key"]
            # is url or path
            if self.is_url(path):
                # is url -> save to ~/.cache/pyamlside2/yaml/***.yaml and load it
                # exists ~/.cache/pyamlside2/yaml/***.yaml ?
                if not os.path.exists(os.path.expanduser("~/.cache/pyamlside2/yaml/" + os.path.basename(path))):
                    os.makedirs(os.path.expanduser(
                        "~/.cache/pyamlside2/yaml"), exist_ok=True)
                    urllib.request.urlretrieve(path, os.path.expanduser(
                        "~/.cache/pyamlside2/yaml/") + os.path.basename(path))
                    print("download yaml file: " + path)
                else:
                    print("yaml file is already downloaded (Please delete ~/.cache/pyamlside2/yaml/" +
                          os.path.basename(path) + " to download again)")
                path = os.path.expanduser(
                    "~/.cache/pyamlside2/yaml/") + os.path.basename(path)

            with open(path, 'r') as f:
                self.yaml_data = yaml.load(f, Loader=yaml.FullLoader)
            self.yaml_data = label_configure(path, key).yaml_data
            print("yaml_data: " + str(self.yaml_data))
            # return force
            return

    # Using common tag --------------------------------------------------------

        if "type" in self.yaml_data:
            self.type = self.yaml_data["type"]
        else:
            print("type is undefined.")
            sys.exit()

        # switching position 1 ------------------
        if "x_center" in self.yaml_data:
            self.position = CENTER
            self.x_center = int(float(self.yaml_data["x_center"]))
        if "y_center" in self.yaml_data:
            self.y_center = int(float(self.yaml_data["y_center"]))
        # switching position 2 ------------------
        if "x_left" in self.yaml_data:
            self.position = LEFT
            self.x_left = int(float(self.yaml_data["x_left"]))
        # switching position 3 ------------------
        if "x_right" in self.yaml_data:
            self.position = RIGHT
            self.x_right = int(float(self.yaml_data["x_right"]))

        if "y_top" in self.yaml_data:
            self.y_top = int(float(self.yaml_data["y_top"]))

        if "width" in self.yaml_data:
            self.width = int(float(self.yaml_data["width"]))

        if "height" in self.yaml_data:
            self.height = int(float(self.yaml_data["height"]))

        if "text" in self.yaml_data:
            self.text = self.yaml_data["text"]

        if "items" in self.yaml_data:
            self.items = self.yaml_data["items"]

        if "max" in self.yaml_data:
            self.max = int(float(self.yaml_data["max"]))

        if "min" in self.yaml_data:
            self.min = int(float(self.yaml_data["min"]))

        if "step" in self.yaml_data:
            self.step = int(float(self.yaml_data["step"]))

        if "default" in self.yaml_data:
            self.default = int(float(self.yaml_data["default"]))

        if "style" in self.yaml_data:
            if "font" in self.yaml_data["style"]:
                font_list = self.yaml_data["style"]["font"].split(" ")

                for fl_s in font_list:
                    if fl_s.endswith("px") :
                        self.font_size = int(float(fl_s.replace("px", "")))

            if "font-family" in self.yaml_data["style"]:
                self.font = self.yaml_data["style"]["font-family"]

    # StyleSheet ---------------------------------------------------------------
        if "style" in self.yaml_data:
            # if include key
            if "include" in self.yaml_data["style"]:
                path = self.yaml_data["style"]["include"]["path"]
                key = self.yaml_data["style"]["include"]["key"]
                # is url or path
                print("path: " + path)
                if self.is_url(path):
                    # is url -> save to ~/.cache/pyamlside2/yaml/***.yaml and load it
                    # exists ~/.cache/pyamlside2/yaml/***.yaml ?
                    if not os.path.exists(os.path.expanduser("~/.cache/pyamlside2/yaml/" + os.path.basename(path))):
                        os.makedirs(os.path.expanduser(
                            "~/.cache/pyamlside2/yaml"), exist_ok=True)
                        urllib.request.urlretrieve(path, os.path.expanduser(
                            "~/.cache/pyamlside2/yaml/") + os.path.basename(path))
                        print("download yaml file: " + path)
                    else:
                        print("yaml file is already downloaded (Please delete ~/.cache/pyamlside2/yaml/" +
                              os.path.basename(path) + " to download again)")
                    print("yaml_data: " + str(self.yaml_data))
                    path = os.path.expanduser(
                        "~/.cache/pyamlside2/yaml/") + os.path.basename(path)

                with open(path, 'r') as f:
                    self.stylesheet_str = yaml.load(f, Loader=yaml.FullLoader)
                self.stylesheet_str = label_configure(
                    path, key).stylesheet_str
                # print("stylesheet_str: " + str(self.stylesheet_str))

            else:
                for key, value in self.yaml_data["style"].items():
                    # configure stylesheet
                    if (key == "background-image"):
                        # check path "./" or "../" ... (relative path)
                        # image_path : url(./image/back-image.png) -> ./image/back-image.png
                        image_path = value.replace("url(", "").replace(")", "")
                        if (image_path[0] == "." or image_path[0] == ".."):
                            basepath = os.path.dirname(self.yaml_abs_path_file)
                            image_path = os.path.join(basepath, image_path)
                            print("------------------")
                            print("image_path: " + image_path)
                            print("------------------")
                        self.stylesheet_str += key + \
                            ": url(" + image_path + ");"
                    else:
                        self.stylesheet_str += key + ": " + str(value) + "; "
            # print(self.stylesheet_str)

    # Label Rect ---------------------------------------------------------------
        if "rect" in self.yaml_data:
            if "include" in self.yaml_data["rect"]:
                path = self.yaml_data["rect"]["include"]["path"]
                key = self.yaml_data["rect"]["include"]["key"]
                if self.is_url(path):
                    # is url -> save to ~/.cache/pyamlside2/yaml/***.yaml and load it
                    # exists ~/.cache/pyamlside2/yaml/***.yaml ?
                    if not os.path.exists(os.path.expanduser("~/.cache/pyamlside2/yaml/" + os.path.basename(path))):
                        os.makedirs(os.path.expanduser(
                            "~/.cache/pyamlside2/yaml"), exist_ok=True)
                        urllib.request.urlretrieve(path, os.path.expanduser(
                            "~/.cache/pyamlside2/yaml/") + os.path.basename(path))
                        print("download yaml file: " + path)
                    else:
                        print("yaml file is already downloaded (Please delete ~/.cache/pyamlside2/yaml/" +
                              os.path.basename(path) + " to download again)")
                    path = os.path.expanduser(
                        "~/.cache/pyamlside2/yaml/") + os.path.basename(path)

                with open(path, 'r') as f:
                    self.rect_width = yaml.load(f, Loader=yaml.FullLoader)
                self.rect_width = label_configure(path, key).rect_width
                self.rect_height = label_configure(path, key).rect_height
            else:
                self.rect_width = self.yaml_data["rect"]["width"]
                self.rect_height = self.yaml_data["rect"]["height"]
        else:
            pass
            self.rect_width = self.width
            self.rect_height = self.height

        self.width = self.rect_width
        self.height = self.rect_height

    # debug ---------------------------------------------------------------
        # if "debug" in self.yaml_data:
        #     self.debug = self.yaml_data["debug"]
        # else:
        #     self.debug = False

        if self.position == LEFT:
            self.x = self.x_left
            self.y = self.y_top
        elif self.position == CENTER:
            self.x = self.x_center - self.width // 2
            self.y = self.y_center - self.height // 2
        elif self.position == RIGHT:
            self.x = self.x_right - self.width
            self.y = self.y_top - self.height