from typing import List, Optional, Tuple, Dict
import re
from functools import lru_cache
from action import ALL_ACTIONS

class FightingFeaturesExtractor:
    """
    Extracts Fighting ICE features.
    Features can be:
      - dot-separated strings, in which case they will be looked up recursively (e.g., 'P1.attack.hit_area.bottom')
      - special strings that compute functions over the available data (e.g., 'players_distance()')
    """

    def __init__(self, features: List[str]):
        self.features = features
        self.max_hp = None # type: Optional[Tuple[int, int]]
        self.character_names = None # type: Optional[Tuple[str, str]]
        self.stage_size = None # type: Optional[Dict[str, int]]
        self.frame_data = None
        self.discretize = True

    def set_game_data(self, game_data):
        if isinstance(game_data, dict):
            self.max_hp = game_data['max_hp']
            self.character_names = game_data['character_names']
            self.stage_size = game_data['stage_size']
        else:
            # max_hp info not available in game_data - will grab it from the first frameData
            self.character_names = {'P1': game_data.getPlayerOneCharacterName(), 'P2': game_data.getPlayerTwoCharacterName()}
            self.stage_size = {'x': game_data.getStageXMax(), 'y': game_data.getStageYMax()}

    @staticmethod
    def get_hp(frame_data, player):
        """
        Returns a given player's HP.
        :param frame_data: Frame data (can be a recursive dict or a java stub to a FrameData instance)
        :param player: Identifier for the player ('P1' or 'P2')
        :return: The given player's HP.
        """
        if isinstance(frame_data, dict):
            return frame_data[player]['hp']
        else:
            pl_data = frame_data.getP1() if player == 'P1' else frame_data.getP2()
            return FightingFeaturesExtractor.get_player_field(pl_data, 'hp')

    @staticmethod
    def get_action(frame_data, player):
        """
        Returns a given player's last action.
        :param frame_data: Frame data (can be a recursive dict or a java stub to a FrameData instance)
        :param player: Identifier for the player ('P1' or 'P2')
        :return: The given player's last action.
        """
        if isinstance(frame_data, dict):
            return frame_data[player]['action']
        else:
            pl_data = frame_data.getP1() if player == 'P1' else frame_data.getP2()
            return FightingFeaturesExtractor.get_player_field(pl_data, 'action')

    @staticmethod
    def get_energy(frame_data, player):
        """
        Returns a given player's energy.
        :param frame_data: Frame data (can be a recursive dict or a java stub to a FrameData instance)
        :param player: Identifier for the player ('P1' or 'P2')
        :return: The given player's energy.
        """
        if isinstance(frame_data, dict):
            return frame_data[player]['energy']
        else:
            pl_data = frame_data.getP1() if player == 'P1' else frame_data.getP2()
            return FightingFeaturesExtractor.get_player_field(pl_data, 'energy')

    @staticmethod
    def get_state(frame_data, player):
        """
        Returns a given player's state.
        :param frame_data: Frame data (can be a recursive dict or a java stub to a FrameData instance)
        :param player: Identifier for the player ('P1' or 'P2')
        :return: The given player's state.
        """
        if isinstance(frame_data, dict):
            return frame_data[player]['state']
        else:
            pl_data = frame_data.getP1() if player == 'P1' else frame_data.getP2()
            return FightingFeaturesExtractor.get_player_field(pl_data, 'state')

    @staticmethod
    def get_player_field(pl_data, field):
        if field == 'remaining_frames': return pl_data.getRemainingFrame()
        if field == 'action': return pl_data.getAction().toString()
        if field == 'action_id': return pl_data.getAction().ordinal()
        if field == 'state': return pl_data.getAction().toString()
        if field == 'state_id': return pl_data.getState().ordinal()
        if field == 'hp': return pl_data.getHp()
        if field == 'energy': return pl_data.getEnergy()
        if field == 'x': return pl_data.getX()
        if field == 'y': return pl_data.getY()
        if field == 'speed_x': return pl_data.getSpeedX()
        if field == 'speed_y': return pl_data.getSpeedY()
        if field == 'left': return pl_data.getLeft()
        if field == 'right': return pl_data.getRight()
        if field == 'top': return pl_data.getTop()
        if field == 'bottom': return pl_data.getBottom()
        raise ValueError("Unknown player field: %s" % field)

    @staticmethod
    def get_attack_field(att_data, field):
        if field == 'speed_x': return att_data.getSpeedX()
        if field == 'speed_y': return att_data.getSpeedY()
        if field == 'hit_damage': return att_data.getHitDamage()
        if field == 'guard_damage': return att_data.getGuardDamage()
        if field == 'start_add_energy': return att_data.getStartAddEnergy()
        if field == 'hit_add_energy': return att_data.getHitAddEnergy()
        if field == 'guard_add_energy': return att_data.getGuardAddEnergy()
        if field == 'give_energy': return att_data.getGiveEnergy()
        if field == 'give_guard_recov': return att_data.getGiveGuardRecov()
        if field == 'attack_type': return {1: 'HIGH', 2: 'MIDDLE', 3: 'LOW', 4: 'THROW'}[att_data.getAttackType()]
        if field == 'attack_type_id': return att_data.getAttackType()
        if field == 'impact_x': return att_data.getImpactX()
        if field == 'impact_y': return att_data.getImpactY()
        raise ValueError("Unknown attack field: %s" % field)

    @staticmethod
    def get_hit_area_field(hit_area_data, field):
        if field == 'top': return hit_area_data.getT()
        if field == 'bottom': return hit_area_data.getB()
        if field == 'left': return hit_area_data.getL()
        if field == 'right': return hit_area_data.getR()
        raise ValueError("Unknown hit area field: %s" % field)

    @lru_cache(maxsize=None)
    def get_feature(self, feature):
        fd = self.frame_data # shorthand

        if self.max_hp is None and not isinstance(fd, dict):
            self.max_hp = {'P1': fd.getP1().getMaxHp(), 'P2': fd.getP2().getMaxHp()}

        # feature of a player
        match = re.match(
            r'^(P1|P2).(remaining_frames|action|action_id|state|state_id|hp|energy|x|y|speed_x|speed_y|left|right|top|bottom)$',
            feature
        )
        if match:
            player = match.group(1)
            field = match.group(2)

            if isinstance(fd, dict):
                return fd[player][field]
            else:
                pl_data = fd.getP1() if player == 'P1' else fd.getP2()
                return FightingFeaturesExtractor.get_player_field(pl_data, field)

        # feature of an attack
        match = re.match(
            r'^(P1|P2).attack.(speed_x|speed_y|hit_damage|guard_damage|start_add_energy|hit_add_energy|'
            r'guard_add_energy|give_energy|give_guard_recov|attack_type|attack_type_id|impact_x|impact_y)$',
            feature
        )
        if match:
            player = match.group(1)
            field = match.group(2)
            if isinstance(fd, dict):
                if 'attack' in fd[player]:
                    return fd[player]['attack'][field]
                else:
                    return None
            else:
                pl_data = fd.getP1() if player == 'P1' else fd.getP2()
                att_data = pl_data.getAttack()
                return FightingFeaturesExtractor.get_attack_field(att_data, field)

        # feature of an attack hit area
        match = re.match(
            r'^(P1|P2).attack.hit_area.(bottom|top|left|right)$',
            feature
        )
        if match:
            player = match.group(1)
            field = match.group(2)
            if isinstance(fd, dict):
                if 'attack' in fd[player]:
                    return fd[player]['attack']['hit_area'][field]
                else:
                    return None
            else:
                pl_data = fd.getP1() if player == 'P1' else fd.getP2()
                att_data = pl_data.getAttack()
                hit_area_data = att_data.getHitAreaNow()
                return FightingFeaturesExtractor.get_hit_area_field(hit_area_data, field)

        # a player's projectiles count
        match = re.match(
            r'^(P1|P2).projectiles.count$',
            feature
        )
        if match:
            player = match.group(1)
            if isinstance(fd, dict):
                return len(fd[player]['projectiles'])
            else:
                projectiles = fd.getProjectilesByP1() if player == 'P2' else fd.getProjectilesByP2()
                return len(projectiles)

        match = re.match(
            r'^(P1|P2).projectiles\[([0-9]+)\].(speed_x|speed_y|hit_damage|guard_damage|start_add_energy|'
            r'hit_add_energy|guard_add_energy|give_energy|give_guard_recov|attack_type|attack_type_id|'
            r'impact_x|impact_y)$',
            feature
        )
        if match:
            player = match.group(1)
            projectile_index = int(match.group(2))
            field = match.group(3)
            if isinstance(fd, dict):
                if projectile_index < len(fd[player]['projectiles']):
                    return fd[player]['projectiles'][projectile_index][field]
                else:
                    return None
            else:
                projectiles = fd.getProjectilesByP1() if player == 'P2' else fd.getProjectilesByP2()
                return FightingFeaturesExtractor.get_attack_field(projectiles[projectile_index], field)

        match = re.match(
            r'^(P1|P2).projectiles\[([0-9]+)\].hit_area.(bottom|top|left|right)$',
            feature
        )
        if match:
            player = match.group(1)
            projectile_index = int(match.group(2))
            field = match.group(3)
            if isinstance(fd, dict):
                return fd[player]['projectiles'][projectile_index]['hit_area'][field]
            else:
                projectiles = fd.getProjectilesByP1() if player == 'P2' else fd.getProjectilesByP2()
                return FightingFeaturesExtractor.get_hit_area_field(projectiles[projectile_index].getHitAreaNow(), field)

        raise ValueError("Unknown feature: %s" % feature)

    def get_special(self, special, player):

        opponent = 'P2' if player == 'P1' else 'P1'

        if special == 'players_x_distance()':
            player_l = self.get_feature('%s.left' % player)
            player_r = self.get_feature('%s.right' % player)
            opponent_l = self.get_feature('%s.left' % opponent)
            opponent_r = self.get_feature('%s.right' % opponent)

            if player_l > opponent_r or player_r < opponent_l:
                value = min(abs(player_r - opponent_l), abs(player_l - opponent_r))
                if value == 0:
                    return None  # bounding boxes have overlapping y coordinates (map to special value None)
                else:
                    return discretize_intervals(value, thresholds=[100, 300]) if self.discretize else value
            else:
                return None # bounding boxes have overlapping x coordinates (map to special value None)

        elif special == 'players_x_diff()':
            player_x = self.get_feature('%s.x' % player)
            opponent_x = self.get_feature('%s.x' % opponent)
            value = abs(player_x - opponent_x)
            return discretize_intervals(value, thresholds=[90, 150, 300]) if self.discretize else value

        elif special == 'players_y_diff()':
            player_y = self.get_feature('%s.y' % player)
            opponent_y = self.get_feature('%s.y' % opponent)
            value = player_y - opponent_y
            if value >= 0:
                return discretize_intervals(value, thresholds=[30, 90, 150]) if self.discretize else value
            else:
                return -discretize_intervals(-value, thresholds=[30, 90, 150]) if self.discretize else value

        elif special == 'players_y_distance()':
            player_t = self.get_feature('%s.top' % player)
            player_b = self.get_feature('%s.bottom' % player)

            opponent_t = self.get_feature('%s.top' % opponent)
            opponent_b = self.get_feature('%s.bottom' % opponent)

            player_above_by = opponent_t - player_b
            opponent_above_by = player_t - opponent_b

            if player_above_by > 0:
                return discretize_intervals(player_above_by, thresholds=[150]) if self.discretize else player_above_by
            elif opponent_above_by > 0:
                return discretize_intervals(-opponent_above_by, thresholds=[150]) if self.discretize else -opponent_above_by
            else:
                return None  # bounding boxes have overlapping y coordinates (map to special value None)

        elif special == 'player_is_falling()':
            player_speed_y = self.get_feature('%s.speed_y' % player)
            return sign(player_speed_y)

        elif special == 'opponent_is_falling()':
            opponent_speed_y = self.get_feature('%s.speed_y' % opponent)
            return sign(opponent_speed_y)

        elif special == 'opponent_is_approaching()':
            player_x = self.get_feature('%s.x' % player)
            opponent_x = self.get_feature('%s.x' % opponent)
            opponent_speed_x = self.get_feature('%s.speed_x' % opponent)
            return sign(player_x - opponent_x) == sign(opponent_speed_x)

        elif special == 'opponent_is_attacking()':
            att_type = self.get_feature('%s.attack.attack_type_id' % opponent)
            return att_type is not None and att_type != 0

        elif special == 'closest_threat_x_distance()':
            player_l = self.get_feature('%s.left' % player)
            player_r = self.get_feature('%s.right' % player)
            player_t = self.get_feature('%s.top' % player)
            player_b = self.get_feature('%s.bottom' % player)
            threats_distance = [] # type: List[int]

            att_type = self.get_feature('%s.attack.attack_type_id' % opponent)
            if att_type is not None and att_type != 0:
                att_l = self.get_feature('%s.attack.hit_area.left' % opponent)
                att_r = self.get_feature('%s.attack.hit_area.right' % opponent)
                att_t = self.get_feature('%s.attack.hit_area.top' % opponent)
                att_b = self.get_feature('%s.attack.hit_area.bottom' % opponent)
                if player_t <= att_b <= player_b or player_t <= att_t <= player_b:
                    threats_distance.append(min(abs(player_r - att_l), abs(player_l - att_r)))

            n_proj = self.get_feature('%s.projectiles.count' % opponent)
            for proj_i in range(n_proj):
                proj_l = self.get_feature('%s.projectiles[%d].hit_area.left' % (opponent, proj_i))
                proj_r = self.get_feature('%s.projectiles[%d].hit_area.right' % (opponent, proj_i))
                proj_t = self.get_feature('%s.projectiles[%d].hit_area.top' % (opponent, proj_i))
                proj_b = self.get_feature('%s.projectiles[%d].hit_area.bottom' % (opponent, proj_i))
                if player_t <= proj_b <= player_b or player_t <= proj_t <= player_b:
                    threats_distance.append(min(abs(player_r - proj_l), abs(player_l - proj_r)))

            if threats_distance:
                value = min(threats_distance) + 1
                if value == 0:
                    return 0 # threat is overlapping (map to closest interval, 0)
                else:
                    return discretize_intervals(value, thresholds=[100, 150]) if self.discretize else value
            else:
                return None # there are no threats (map to special value None)

        elif special == 'attack_x_distance()':
            player_l = self.get_feature('%s.left' % player)
            player_r = self.get_feature('%s.right' % player)
            player_t = self.get_feature('%s.top' % player)
            player_b = self.get_feature('%s.bottom' % player)

            att_type = self.get_feature('%s.attack.attack_type_id' % opponent)
            if att_type is not None and att_type != 0:
                att_l = self.get_feature('%s.attack.hit_area.left' % opponent)
                att_r = self.get_feature('%s.attack.hit_area.right' % opponent)
                att_t = self.get_feature('%s.attack.hit_area.top' % opponent)
                att_b = self.get_feature('%s.attack.hit_area.bottom' % opponent)
                if player_t <= att_b <= player_b or player_t <= att_t <= player_b:
                    value = min(abs(player_r - att_l), abs(player_l - att_r)) + 1
                    if value == 0:
                        return 0 # threat is overlapping (map to closest interval, 0)
                    else:
                        return discretize_intervals(value, thresholds=[100, 150]) if self.discretize else value
            else:
                return None # there are no threats (map to special value None)

        elif special == 'closest_projectile_x_distance()':
            player_l = self.get_feature('%s.left' % player)
            player_r = self.get_feature('%s.right' % player)
            player_t = self.get_feature('%s.top' % player)
            player_b = self.get_feature('%s.bottom' % player)
            projectiles_distance = [] # type: List[int]

            n_proj = self.get_feature('%s.projectiles.count' % opponent)
            for proj_i in range(n_proj):
                proj_l = self.get_feature('%s.projectiles[%d].hit_area.left' % (opponent, proj_i))
                proj_r = self.get_feature('%s.projectiles[%d].hit_area.right' % (opponent, proj_i))
                proj_t = self.get_feature('%s.projectiles[%d].hit_area.top' % (opponent, proj_i))
                proj_b = self.get_feature('%s.projectiles[%d].hit_area.bottom' % (opponent, proj_i))
                if player_t <= proj_b <= player_b or player_t <= proj_t <= player_b:
                    projectiles_distance.append(min(abs(player_r - proj_l), abs(player_l - proj_r)))

            if projectiles_distance:
                value = min(projectiles_distance) + 1
                if value == 0:
                    return 0 # threat is overlapping (map to closest interval, 0)
                else:
                    return discretize_intervals(value, thresholds=[100, 150]) if self.discretize else value
            else:
                return None # there are no threats (map to special value None)

        elif special == 'opponent_is_busy()':
            opp_rem_fr = self.get_feature('%s.remaining_frames' % opponent)
            return opp_rem_fr > 0

        else:
            raise ValueError("Unknown special feature: %s" % special)

    def clear_cache(self):
        self.get_feature.cache_clear()

    def set_frame_data(self, frame_data):
        self.clear_cache()
        self.frame_data = frame_data

    def get_features(self, frame_data, player):
        """
        Returns all the features setup in __init__ for a given player.
        :param frame_data: Frame data (can be a recursive dict or a java object)
        :param player: Identifier for the player ('P1' or 'P2')
        :return: All features setup in __init__ for the given player.
        """
        self.set_frame_data(frame_data)

        values = {}
        for feature in self.features:
            if feature.endswith('()'):
                values[feature] = self.get_special(feature, player)
            else:
                if feature.startswith('self'):
                    values[feature] = self.get_feature(feature.replace('self', player, 1))
                elif feature.startswith('opponent'):
                    values[feature] = self.get_feature(feature.replace('opponent', 'P2' if player == 'P1' else 'P1', 1))
                else:
                    values[feature] = self.get_feature(feature)
                if self.discretize and isinstance(values[feature], int):
                    values[feature] = discretize_intervals(values[feature], thresholds=[5, 50, 300])
        return values

    def get_features_info(self, features: List[str]):
        """
        Extracts all the wanted features from the game state. Features list is
        set in the constructor.
        :param features: desired features
        """
        return {f: self.get_feature_info(f) for f in features}

    def get_feature_info(self, feature):

        match = re.match(
            r'^(?:P1|P2|self|opponent).(remaining_frames|action|action_id|state|state_id|hp|energy|x|y|speed_x|speed_y|left|right|top|bottom)$',
            feature
        )
        if match:
            player_feature = match.group(1)
            if player_feature == 'remaining_frames':
                return {'iterable': False, 'nullable': False, 'type': int, 'min': 0, 'max': 3615}
            elif player_feature == 'action':
                return {'iterable': False, 'nullable': False, 'type': 'enum', 'possible_values': ALL_ACTIONS}
            elif player_feature == 'action_id':
                return {'iterable': False, 'nullable': False, 'type': 'enum', 'possible_values': list(range(len(ALL_ACTIONS)))}
            elif player_feature == 'state':
                return {'iterable': False, 'nullable': False, 'type': 'enum', 'possible_values': ['STAND', 'CROUCH', 'AIR', 'DOWN']}
            elif player_feature == 'state_id':
                return {'iterable': False, 'nullable': False, 'type': 'enum', 'possible_values': list(range(4))}
            elif player_feature == 'hp':
                max_hp = max(self.max_hp)
                return {'iterable': False, 'nullable': False, 'type': int, 'min': 0 if max_hp > 0 else -1000, 'max': max_hp if max_hp > 0 else 0}  # FIXME this is a guess
            elif player_feature == 'energy':
                return {'iterable': False, 'nullable': False, 'type': int, 'min': 0, 'max': 1000} # FIXME this is a guess
            elif player_feature == 'x':
                return {'iterable': False, 'nullable': False, 'type': int, 'min': 0, 'max': self.stage_size['x']}
            elif player_feature == 'y':
                return {'iterable': False, 'nullable': False, 'type': int, 'min': 0, 'max': self.stage_size['y'] * 1.5}
            elif player_feature == 'speed_x':
                return {'iterable': False, 'nullable': False, 'type': int, 'min': 0, 'max': 500} # FIXME this is a guess
            elif player_feature == 'speed_y':
                return {'iterable': False, 'nullable': False, 'type': int, 'min': 0, 'max': 500} # FIXME this is a guess
            elif player_feature == 'left':
                return {'iterable': False, 'nullable': False, 'type': int, 'min': 0, 'max': self.stage_size['x']}
            elif player_feature == 'right':
                return {'iterable': False, 'nullable': False, 'type': int, 'min': 0, 'max': self.stage_size['x']}
            elif player_feature == 'top':
                return {'iterable': False, 'nullable': False, 'type': int, 'min': 0, 'max': self.stage_size['y'] * 1.5}
            elif player_feature == 'bottom':
                return {'iterable': False, 'nullable': False, 'type': int, 'min': 0, 'max': self.stage_size['y'] * 1.5}
            else:
                raise ValueError("Unknown feature: %s" % feature)

        match1 = re.match(
            r'^(?:P1|P2|self|opponent).attack.(speed_x|speed_y|hit_damage|guard_damage|start_add_energy|hit_add_energy|'
            r'guard_add_energy|give_energy|give_guard_recov|attack_type|attack_type_id|impact_x|impact_y)$',
            feature
        )
        match2 = re.match(
            r'^(?:P1|P2|self|opponent).projectiles\[[0-9]+\].(speed_x|speed_y|hit_damage|guard_damage|start_add_energy|'
            r'hit_add_energy|guard_add_energy|give_energy|give_guard_recov|attack_type|attack_type_id|'
            r'impact_x|impact_y)$',
            feature
        )
        if match1 or match2:
            attack_feature = match1.group(1) if match1 else match2.group(1)
            if attack_feature == 'speed_x':
                return {'iterable': False, 'nullable': bool(match1), 'type': int, 'min': 0, 'max': 500} # FIXME this is a guess
            elif attack_feature == 'speed_y':
                return {'iterable': False, 'nullable': bool(match1), 'type': int, 'min': 0, 'max': 500} # FIXME this is a guess
            elif attack_feature == 'hit_damage':
                return {'iterable': False, 'nullable': bool(match1), 'type': int, 'min': 0, 'max': 300}
            elif attack_feature == 'guard_damage':
                return {'iterable': False, 'nullable': bool(match1), 'type': int, 'min': 0, 'max': 100}
            elif attack_feature == 'start_add_energy':
                return {'iterable': False, 'nullable': bool(match1), 'type': int, 'min': -300, 'max': 0}
            elif attack_feature == 'hit_add_energy':
                return {'iterable': False, 'nullable': bool(match1), 'type': int, 'min': 0, 'max': 50}
            elif attack_feature == 'guard_add_energy':
                return {'iterable': False, 'nullable': bool(match1), 'type': int, 'min': 0, 'max': 30}
            elif attack_feature == 'give_energy':
                return {'iterable': False, 'nullable': bool(match1), 'type': int, 'min': 0, 'max': 60}
            elif attack_feature == 'give_guard_recov':
                return {'iterable': False, 'nullable': bool(match1), 'type': int, 'min': 0, 'max': 30}
            elif attack_feature == 'attack_type':
                return {'iterable': False, 'nullable': bool(match1), 'type': 'enum', 'possible_values': list(range(4))}
            elif attack_feature == 'impact_x':
                return {'iterable': False, 'nullable': bool(match1), 'type': int, 'min': 0, 'max': 30}
            elif attack_feature == 'impact_y':
                return {'iterable': False, 'nullable': bool(match1), 'type': int, 'min': -20, 'max': 0}
            else:
                raise ValueError("Unknown feature: %s" % feature)

        match1 = re.match(
            r'^(?:P1|P2|self|opponent).attack.hit_area.(bottom|top|left|right)$',
            feature
        )
        match2 = re.match(
            r'^(?:P1|P2|self|opponent).projectiles\[[0-9]+\].hit_area.(bottom|top|left|right)$',
            feature
        )
        if match1 or match2:
            hit_area_feature = match1.group(1) if match1 else match2.group(1)
            if hit_area_feature == 'left':
                return {'iterable': False, 'nullable': bool(match1), 'type': int, 'min': 0, 'max': self.stage_size['x']}
            elif hit_area_feature == 'right':
                return {'iterable': False, 'nullable': bool(match1), 'type': int, 'min': 0, 'max': self.stage_size['x']}
            elif hit_area_feature == 'top':
                return {'iterable': False, 'nullable': bool(match1), 'type': int, 'min': 0, 'max': self.stage_size['y'] * 1.5}
            elif hit_area_feature == 'bottom':
                return {'iterable': False, 'nullable': bool(match1), 'type': int, 'min': 0, 'max': self.stage_size['y'] * 1.5}

        match = re.match(
            r'^(?:P1|P2|self|opponent).projectiles.count$',
            feature
        )
        if match:
            return {'iterable': False, 'nullable': False, 'type': int, 'min': 0, 'max': 10}

        if feature == 'players_x_distance()':
            return {'iterable': False, 'nullable': False, 'type': int, 'min': 0, 'max': self.stage_size['x']}
        if feature == 'players_x_diff()':
            return {'iterable': False, 'nullable': False, 'type': int, 'min': 0, 'max': self.stage_size['x']}
        if feature == 'players_y_distance()':
            return {'iterable': False, 'nullable': False, 'type': int, 'min': -self.stage_size['y'] * 1.5, 'max': self.stage_size['y'] * 1.5}
        if feature == 'players_y_diff()':
            return {'iterable': False, 'nullable': False, 'type': int, 'min': -self.stage_size['y'] * 1.5, 'max': self.stage_size['y'] * 1.5}
        if feature == 'player_is_falling()':
            return {'iterable': False, 'nullable': False, 'type': 'enum', 'possible_values': [-1, 0, +1]}
        if feature == 'opponent_is_falling()':
            return {'iterable': False, 'nullable': False, 'type': 'enum', 'possible_values': [-1, 0, +1]}
        if feature == 'opponent_is_approaching()':
            return {'iterable': False, 'nullable': False, 'type': 'enum', 'possible_values': [True, False]}
        if feature == 'opponent_is_attacking()':
            return {'iterable': False, 'nullable': False, 'type': 'enum', 'possible_values': [True, False]}
        if feature == 'closest_threat_x_distance()':
            return {'iterable': False, 'nullable': True, 'type': int, 'min': 0, 'max': self.stage_size['x']}
        if feature == 'attack_x_distance()':
            return {'iterable': False, 'nullable': True, 'type': int, 'min': 0, 'max': self.stage_size['x']}
        if feature == 'closest_projectile_x_distance()':
            return {'iterable': False, 'nullable': True, 'type': int, 'min': 0, 'max': self.stage_size['x']}
        if feature == 'opponent_is_busy()':
            return {'iterable': False, 'nullable': False, 'type': 'enum', 'possible_values': [True, False]}

        raise ValueError("Unknown feature %s" % feature)

def discretize_intervals(value: Optional[float], thresholds: List[float]):
    """
    Discretizes a value using a list of thresholds. The thresholds should be of increasing value.
    This function returns:
    - None if value is None
    - 0 if value == 0
    - +len(thresholds) if value is greater than all thresholds and value > 0
    - -len(thresholds) if abs(value) is greater than all thresholds and value < 0
    - +i+1 where i is the smallest index such that thresholds[i] > value if value > 0
    - -i-1 where i is the smallest index such that thresholds[i] > abs(value) if value < 0

    :param value: The value to discretize
    :param thresholds: The thresholds to use for discretization
    :return: The bin containing value
    """
    if value is None:
        return None
    elif value == 0:
        return 0

    found_i = len(thresholds)
    for i, threshold in enumerate(thresholds):
        if threshold > value:
            found_i = i + 1
            break
    return found_i * sign(value)

def sign(value):
    if value > 0:
        return +1
    elif value < 0:
        return -1
    else:
        return 0

python/Feature Extractor in Python/features_extractor.py


from django.contrib.auth import views as auth_views
from views import leave, employee
from django.conf.urls import url
from django.core.urlresolvers import reverse

urlpatterns = [
    url(
        r'^login/$',
        auth_views.login,
        name='login',
        kwargs={'template_name': 'login.html'}
    ),
    url(
        r'^logout/$',
        auth_views.logout,
        name='logout',
        kwargs={'next_page': '/leave/login/'}
    ),
    url(
        r'^apply/$',
        leave,
        name='leave',
    ),
]


import os

os.environ["CUDA_VISIBLE_DEVICES"] = "-1"

import jax
import jax.numpy as np
import numpy as onp
import scipy.spatial.distance
from numpy.testing import assert_almost_equal, assert_array_almost_equal

from znnl.distance_metrics.mahalanobis_distance import MahalanobisDistance

class TestMahalanobisDistance:
    """
    Class to test the cosine distance measure module.
    """

    @classmethod
    def setup_class(cls):
        """
        Prepare the test suite.
        """
        cls.key = jax.random.PRNGKey(0)

    def test_mahalanobis_distance(self):
        """
        Test the Mahalanobis distance on functionality by comparing results a
        test Mahalanobis distance from scipy.

        Returns
        -------
        Assert if the Mahalanobis distance returns true values for sample set of
        random normal distributed points in two dimensions.
        """
        metric = MahalanobisDistance()

        # Create sample set
        point_1, point_2 = self.create_sample_set()

        # Calculate results from distance metric
        metric_results = metric(np.array(point_1), np.array(point_2))

        # Calculate test results from numpy distance metric
        test_metric_results = []
        self.calculate_numpy_mahalanobis_distance(point_1, point_2, test_metric_results)

        # Assert results
        assert_almost_equal(metric_results, test_metric_results, decimal=1)

    def test_identity(self):
        """
        Test the identity criterion of a metric, based on a randomly produced sample
        set (used to create the covariance matrix).

        Returns
        -------
        Asserts if the distance of the last point of point_1 and point_2 is equal to 0
        """
        # Create Sample set
        point_1, point_2 = self.create_sample_set()

        # Add point of interest
        point_of_interest = np.array([[7.0, 3.0]])
        point_1 = np.concatenate([np.array(point_1), point_of_interest], axis=0)
        point_2 = np.concatenate([np.array(point_2), point_of_interest], axis=0)

        # Assert identity
        metric = MahalanobisDistance()
        assert_array_almost_equal(metric(point_1, point_2)[-1], 0)

    def test_symmetry(self):
        """
        Test the symmetry criterion of a metric, based on a randomly produced sample
        set (used to create the covariance matrix).

        Returns
        -------
        Asserts if the distances of the last two points of point_1 and point_2 are
        identical.
        """
        # Create Sample set
        point_1, point_2 = self.create_sample_set()

        # Add point of interest
        point_1_of_interest = np.array([[-2.0, 5.0], [7.0, 3.0]])
        point_2_of_interest = np.array([[7.0, 3.0], [-2.0, 5.0]])
        point_1 = np.concatenate(
            [np.array(point_1), point_1_of_interest],
            axis=0,
        )
        point_2 = np.concatenate(
            [np.array(point_2), point_2_of_interest],
            axis=0,
        )

        # Assert identity
        metric = MahalanobisDistance()
        assert_array_almost_equal(
            metric(point_1, point_2)[-1], (metric(point_1, point_2)[-2])
        )

    @staticmethod
    def create_sample_set():
        """

        Returns
        -------
        Creates a random normal distributed sample set
        """
        point_1 = np.array(
            [onp.random.normal(0, 10, 100), onp.random.normal(0, 20, 100)]
        ).T
        point_2 = np.array(
            [onp.random.normal(0, 10, 100), onp.random.normal(0, 20, 100)]
        ).T
        return point_1, point_2

    @staticmethod
    def calculate_numpy_mahalanobis_distance(
        point_1: np.ndarray, point_2: np.ndarray, result_list: list
    ):
        """
        Calculates the Mahalanobis distance based on a scipy integration.

        Parameters
        ----------
        point_1 : np.ndarray
                Set of points in the distance calculation.
        point_2 : np.ndarray
                Set of points in the distance calculation.
        result_list : list
                Results for each point are appended to this list.

        Returns
        -------
        Appends all calculated distances to the result_list.
        """
        inv_cov = np.linalg.inv(np.cov(point_1.T))
        for index in range(len(point_1.T[0, :])):
            result_list.append(
                scipy.spatial.distance.mahalanobis(
                    point_1[index], point_2[index], inv_cov
                )
            )


import argparse, urllib.parse, hashlib
from datetime import timedelta, datetime
from pymongo import MongoClient
from selenium.webdriver.common.keys import Keys
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from time import sleep
from bs4 import BeautifulSoup

def changePage(browser,page):
    try:
        browser.get(page)
    except:
        return None
    return browser.title

def getPosts(browser,pagina):
    soup = BeautifulSoup(browser.find_element_by_tag_name('body').get_attribute("innerHTML"), 'html.parser')
    anchors = soup.find_all("a")
    for anchor in anchors:
        anchor_text = anchor.get_text()
        if len(anchor_text.split()) > 1:
            collectedUTC = datetime.utcnow().strftime("%d/%m/%Y-%H:%M:%S")
            # md5 = hashlib.md5(anchor_text.encode())
            # md5.hexdigest()
            if anchor.has_attr('title'):
                news.insert_one({
                    "parent":       pagina,
                    "collectedUTC": collectedUTC,
                    "href":         anchor.get('href'),
                    "title":        anchor.get('title'),
                    "text":         anchor_text,
                    "innerHTML":    anchor.decode_contents(formatter="html")
                })
            else:
                news.insert_one({
                    "parent":       pagina,
                    "collectedUTC": collectedUTC,
                    "href":         anchor.get('href'),
                    "text":         anchor_text,
                    "innerHTML":    anchor.decode_contents(formatter="html")
                })

if __name__ == "__main__":
    # parser = argparse.ArgumentParser(description='Global News Scraper')
    # parser.add_argument('delay', help='Delay')
    # args = parser.parse_args()
    client = MongoClient("mongodb://localhost:27017")
    db = client['news']
    sources = db['sources']
    news = db['news']

    chrome_options = webdriver.ChromeOptions()
    prefs = {"profile.default_content_setting_values.notifications" : 2}
    chrome_options.add_experimental_option("prefs",prefs)
    chrome_options.add_argument('log-level=3')
    # chrome_options.add_argument('--proxy-server=46.102.106.37:13228') # PROXY

    browser = webdriver.Chrome(chrome_options=chrome_options)

    fontes = sources.find({})
    for fonte in fontes:
        pagina = fonte['url']
        if changePage(browser,pagina):
            sources.find_and_modify(query={'url':pagina}, update={"$inc": {'views': 1}}, upsert=False, full_response= True)
            getPosts(browser,pagina)
        else:
            sources.find_and_modify(query={'url':pagina}, update={"$set": {'status': "unreachable"}}, upsert=False, full_response= True)

    browser.stop_client()
    browser.quit()


from tkinter import ttk
import tkinter as tk
import json

class LeaderBoard(ttk.Frame):
    def __init__(self, master=None, next_screen=lambda:None):
        ttk.Frame.__init__(self, master)
        try:
            with open('leaderboard.json') as file:
                score_data = json.load(file)
        except (FileNotFoundError, json.JSONDecodeError):
            score_data = []
        score_data.sort(reverse=True)
        style = ttk.Style(self)
        style.configure('Lboard.TButton', font=('Helvetica', 30),
                        foreground='red', background='black')
        ttk.Button(self, text='Back to Main Menu', style='Lboard.TButton',
                   command=next_screen).grid(row=0, columnspan=4)
        ttk.Label(self, text='Score').grid(row=1, column=1)
        ttk.Label(self, text='Username').grid(row=1, column=2)

        for i, (score, username) in enumerate(score_data[:5]):
            ttk.Label(self, text=str(score)).grid(row=i + 2, column=1)
            ttk.Label(self, text=username).grid(row=i + 2, column=2)


import pandas as pd
import re
import time
from functools import wraps
import os
import numpy as np
import plotly.graph_objects as go
from scipy.interpolate import griddata
import matplotlib.pyplot as plt

folder_path = r'forecast_files'
contour_path = r'Solving\PSATCMG_CAMARGOS.bln'

def read_contour_file(file_path: str) -> pd.DataFrame:
    line_split_comp = re.compile(r'\s*,')

    with open(file_path, 'r') as f:
        raw_file = f.readlines()

    l_raw_lines = [line_split_comp.split(raw_file_line.strip()) for raw_file_line in raw_file]
    l_raw_lines = list(filter(lambda item: bool(item[0]), l_raw_lines))
    float_raw_lines = [list(map(float, raw_line))[:2] for raw_line in l_raw_lines]
    header_line = float_raw_lines.pop(0)
    assert len(float_raw_lines) == int(header_line[0])
    return pd.DataFrame(float_raw_lines, columns=['lat', 'long'])

def read_data_file(file_path: str) -> pd.DataFrame:
    with open(file_path, 'r') as f:
        raw_file = f.readlines()

    list_dados = [line.split() for line in raw_file]
    float_raw_lines = [list(map(float, raw_line)) for raw_line in list_dados]
    return pd.DataFrame(float_raw_lines, columns=['lat', 'long', 'data_value'])

def main() -> None:
    dataframes = []

    for file_name in os.listdir(folder_path):
        file_path = os.path.join(folder_path, file_name)
        df_temp = read_data_file(file_path)
        dataframes.append(df_temp)

    combined_df = pd.concat(dataframes, ignore_index=True)

    df_acumulado_sum = combined_df.groupby(['lat', 'long'])['data_value'].sum()
    df_acumulado_sum = df_acumulado_sum.reset_index()
    contour_df: pd.DataFrame = read_contour_file(contour_path)
    filtered_df = df_acumulado_sum.query('-43.75 >= lat >= -45 & -21 >= long >= -22.6')

    latitudes = filtered_df['lat'].values
    longitudes = filtered_df['long'].values
    data_values = filtered_df['data_value'].values

    grid_x, grid_y = np.mgrid[min(latitudes):max(latitudes):100j, min(longitudes):max(longitudes):100j]
    grid_z = griddata((latitudes, longitudes), data_values, (grid_x, grid_y), method='cubic')
    plt.contourf(grid_x, grid_y, grid_z, cmap='viridis')

    plt.xlabel('Latitude')
    plt.ylabel('Longitude')
    plt.colorbar(label='Vol. Precipitação Acumulada')
    plt.plot(contour_df['lat'], contour_df['long'], linewidth=2, color='black')

    fill_spec = filtered_df.query('-43.9 >= lat >= -44.8 & -21.2 >= long >= -22.5 & data_value != 126.39999999999999')
    for index, row in fill_spec.iterrows():
        plt.text(row['lat'], row['long'], str(row['data_value']), fontsize=10, color='white', ha='center', va='center', weight='bold')

    total_precipitation = np.sum(fill_spec['data_value'])
    plt.figtext(0.45, 0.2, f'Vol. Total: {total_precipitation:.2f}', fontsize=10, color='white', weight='bold')
    plt.savefig('Contour_chart_Matplot.png')

#------------------------------###-------------------------------------------
#Graph2

    latitudes = filtered_df['lat'].values
    longitudes = filtered_df['long'].values
    data_values = filtered_df['data_value'].values

    grid_x, grid_y = np.mgrid[min(longitudes):max(longitudes):100j, min(latitudes):max(latitudes):100j]
    grid_z = griddata((longitudes, latitudes), data_values, (grid_x, grid_y), method='cubic')

    latitudes = filtered_df['lat'].values
    longitudes = filtered_df['long'].values
    data_values = filtered_df['data_value'].values

    grid_x, grid_y = np.mgrid[min(longitudes):max(longitudes):100j, min(latitudes):max(latitudes):100j]
    grid_z = griddata((longitudes, latitudes), data_values, (grid_x, grid_y), method='cubic')

    heatmap = go.Figure(data=go.Heatmap(
        x=grid_y[0, :],
        y=grid_x[:, 0],
        z=grid_z.T,
        colorscale='Viridis',
        transpose=True
    ))
    heatmap.update_layout(
        title=dict(text='<b>Vol Precipitação Acumulada | Camargos - Bacia do Grande<b>',  font=dict(size=17,)),
        xaxis_title=dict(text='<b>Latitude<b>',  font=dict(size=16,)),
        yaxis_title=dict(text='<b>Longitude<b>',  font=dict(size=16,)),
        width=600,
        height=700
    )
    contorno_trace = go.Scatter(
        x=contour_df['lat'],
        y=contour_df['long'],
        mode='lines',
        line=dict(color='black', width=2.5)
    )
    for index, row in fill_spec.iterrows():
        heatmap.add_annotation(
            x=row['lat'],
            y=row['long'],
            text=str(row['data_value']),
            showarrow=False,
            arrowhead=2,
            ax=-40,
            ay=-40,
            font=dict(color='white', size=12.5)
        )
    heatmap.add_annotation(
        x=-44,
        y=-22.4,
        text=f'Vol. Total: {total_precipitation:.2f}',
        font=dict(color='white', size=13),
        showarrow=False
    )

    heatmap.add_trace(contorno_trace)
    heatmap.write_html('Contour_Chart_Softened_Plotly.html')

if __name__ == '__main__':
    main()


import argparse
import logging
import pandas as pd

from utils import read_json, write_json
import data_mapper
from data_mapper import conversion
import utils

def parse_args() -> argparse.Namespace:
    """Function to parse command line arguments"""

    parser = argparse.ArgumentParser()

    parser.add_argument('hist_data_path', type=str, help='Path to info.json')
    parser.add_argument('info_path', type=str, help='Path to info.json')
    parser.add_argument('mapper_path', type=str, help='Path to Mapper')
    parser.add_argument('config_path', type=str, help='Path to config.json')
    parser.add_argument('--output_path', type=str, required = False, help='Path to save output', default = "./Output")

    args = parser.parse_args()

    return args

def main():

    args = parse_args()

    formatstr = '%(asctime)s: %(levelname)s: %(funcName)s Line: %(lineno)d %(message)s'
    datestr = '%m/%d/%Y %H:%M:%S'
    logging.basicConfig(
        level=logging.INFO,
        format=formatstr,
        datefmt=datestr,
        handlers=[
            logging.FileHandler('config_update.log'),
            logging.StreamHandler()
            ]
        )

    # load in data
    logging.info("Loading files")
    hist_df = pd.read_csv(args.hist_data_path)
    info = utils.read_json(args.info_path)
    config = read_json(args.config_path)
    mapper = pd.read_excel(args.mapper_path, config['mapper_sheet'])

    mh = data_mapper.MapperHandler(*config['data_mapper_columns'])

    live_format, sub, name_to_id, id_to_name = conversion.convert_to_ld(hist_df, mapper, mh, config['property_ids'])
    live_format['Date'] = pd.to_datetime(live_format['Date'])
    live_format = live_format.set_index('Date')

    all_object_ids = [*info.values()]
    missing_tags = [91453]
    all_object_names = ['___'.join(id_to_name[object_id, -6]) for object_id in all_object_ids if object_id not in missing_tags]

    for i in range(5):
            data_mapper.put_data(live_format.loc[:, all_object_names].iloc[[i], :], args.output_path, name_to_id)

    # data_mapper.put_data(live_format.loc[1:3, all_object_names].iloc[[-3], :], './', name_to_id)

    return

if __name__ == '__main__':
    main()


import os
import json
from datetime import datetime
import akshare as ak

today = str(datetime.now())[:10]
stock_zh_a_spot_em_df = ak.stock_zh_a_spot_em()

if len(stock_zh_a_spot_em_df) > 100:
    obj = json.loads(stock_zh_a_spot_em_df.to_json(orient='records'))
    output = json.dumps(obj, indent=4, ensure_ascii=False)
    root = 'market_daily'
    os.makedirs(root, exist_ok=True)
    output_path = os.path.join(root, today + '.json')
    with open(output_path, 'w') as fp:
        fp.write(output)


import json
import pandas as pd
import pickle
import streamlit as st
import requests

# load pipeline
pipe = pickle.load(open("model/preprocessor.pkl", "rb"))

st.title("Aplikasi Prediksi Churn")

gender = st.selectbox("Customer Gender", ["Male", "Female"])

seniorCitizen = st.selectbox("Whether the customer is a senior citizen or not", ["No", "Yes"] )

partner = st.selectbox("Whether the customer has a partner or not", ["No", "Yes"])

dependent = st.selectbox(" Whether the customer has dependents or not", ["No", "Yes"])

tenure = st.number_input("Number of months the customer has stayed with the company? (Tenure)", min_value=1)

phoneService = st.selectbox("Phone service ", ["No", "Yes"])

multipleLines = st.selectbox("Multiple Lines ", ["No", "Yes","No phone service"])

internetService = st.selectbox("Internet Service Provider ", ["No", "DSL", "Fiber optic"])

onlineSecurity = st.selectbox("Online Security ", ["No", "Yes","No internet service"])

onlineBackup = st.selectbox("Online Backup ", ["No", "Yes","No internet service"])

deviceProtection = st.selectbox("Device Protection", ["No", "Yes","No internet service"])

techSupport = st.selectbox("Tech Support ", ["No", "Yes","No internet service"])

streamingTV = st.selectbox("Streaming TV ", ["No", "Yes","No internet service"])

streamingMovies = st.selectbox("Streaming Movies ", ["No", "Yes","No internet service"])

contract = st.selectbox("Contract ", ["Month-to-month", "One year", "Two year"])

paperlessBilling = st.selectbox("Paperless Billing ", ["Yes", "No"])

paymentMethod = st.selectbox("Payment Method ",
            [
                "Electronic check",
                "Mailed check",
                "Bank transfer (automatic)",
                "Credit card (automatic)",
            ],
        )

monthlyCharges = st.number_input("Monthly Charges", value=18.0,min_value=18.0)

totalCharges = st.number_input("Total Charges ",value=18.0,min_value=18.0)

# Create dictionary with all customer information
new_data = {
        "gender": gender,
        "SeniorCitizen": seniorCitizen,
        "Partner": partner,
        "Dependents": dependent,
        "tenure": tenure,
        "PhoneService": phoneService,
        "MultipleLines": multipleLines,
        "InternetService": internetService,
        "OnlineSecurity": onlineSecurity,
        "OnlineBackup": onlineBackup,
        "DeviceProtection": deviceProtection,
        "TechSupport": techSupport,
        "StreamingTV": streamingTV,
        "StreamingMovies": streamingMovies,
        "Contract": contract,
        "PaperlessBilling": paperlessBilling,
        "PaymentMethod": paymentMethod,
        "MonthlyCharges": monthlyCharges,
        "TotalCharges": totalCharges,
    }

new_data = pd.DataFrame([new_data])

# build feature
new_data = pipe.transform(new_data)
new_data = new_data.tolist()

# inference
URL = "https://tf-serving-xyla-backend.herokuapp.com/v1/models/churn_model:predict"
param = json.dumps({
        "signature_name":"serving_default",
        "instances":new_data
    })
r = requests.post(URL, data=param)

if r.status_code == 200:
    res = r.json()
    if res['predictions'][0][0] > 0.5:
        st.title("Churn")
    else:
        st.title("Not Churn")
else:
    st.title("Unexpected Error")


import cv2

#Part-1-----------------------------------------------------------------------------------------------------------------
"""img=cv2.imread("HW8_1.jpg")
img = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
cv2.imshow('original image',img)
r=10 #diameter of the disk
kernel= cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(r,r))
opening = cv2.morphologyEx(img, cv2.MORPH_OPEN, kernel)
cv2.imshow('after opening image',opening)
closing=cv2.morphologyEx(opening, cv2.MORPH_CLOSE, kernel)
cv2.imshow('after closing image',closing)
#Part-2-----------------------------------------------------------------------------------------------------------------
img=cv2.imread("HW8_2.jpg")
img = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
cv2.imshow('original image',img)
kernel= cv2.getStructuringElement(cv2.MORPH_RECT,(3,3))
dilation = cv2.dilate(img,kernel,iterations = 1)
cv2.imshow('after dilation image',dilation)
erosion = cv2.erode(img,kernel,iterations = 1)
cv2.imshow('after erosion image',erosion)
# subtract the images
gradien = cv2.subtract(dilation, erosion)
cv2.imshow(' gradien image',gradien)
#Part-3-----------------------------------------------------------------------------------------------------------------
img=cv2.imread("HW8_3.jpg")
img = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
cv2.imshow('original image',img)
# Otsu's thresholding
ret,th = cv2.threshold(img,0,255,cv2.THRESH_BINARY+cv2.THRESH_OTSU)
cv2.imshow('after Otsu image',th)
r=80 #diameter of the disk
kernel= cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(r,r))
opening = cv2.morphologyEx(img, cv2.MORPH_OPEN, kernel)
cv2.imshow('after opening image',opening)
tophat = cv2.subtract(img, opening)
cv2.imshow('after tophat image',tophat)
ret2,th2 = cv2.threshold(tophat,0,255,cv2.THRESH_BINARY+cv2.THRESH_OTSU)
cv2.imshow('after tophat OTSU image',th2)"""
#Part-4-----------------------------------------------------------------------------------------------------------------
img=cv2.imread("HW8_4.jpg")
img = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
cv2.imshow('original image',img)
r=60 #diameter of the disk
kernel= cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(r,r))
closing=cv2.morphologyEx(img, cv2.MORPH_CLOSE, kernel)
cv2.imshow('after closing image',closing)
r2=120 #diameter of the disk
kernel2=kernel= cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(r2,r2))
opening = cv2.morphologyEx(closing, cv2.MORPH_OPEN, kernel)
cv2.imshow('after opening image',opening)
kernel3=cv2.getStructuringElement(cv2.MORPH_RECT,(3,3))
gradient = cv2.morphologyEx(opening, cv2.MORPH_GRADIENT, kernel3)
cv2.imshow(' gradien image',gradient)
final=cv2.add(gradient,img)
cv2.imshow(' final image',final)
# waitKey() waits for a key press to close the window and 0 specifies indefinite loop
cv2.waitKey(0)

# cv2.destroyAllWindows() simply destroys all the windows we created.
cv2.destroyAllWindows()


import random, datetime
from customer import Customer

cust = Customer(id)

# Method Menu
def menu():

    while True:
        print("""
-----------------------------------------
| Selamat Datang di ATM Helmi x Progate |
|                                       |
| 1. Cek Saldo                          |
| 2. Tarik Tunai                        |
| 3. Setor Tunai                        |
| 4. Ubah Pin                           |
| 5. Keluar                             |
-----------------------------------------
""")

        pilih_menu = int(input("\nSilahkan Pilih Menu: "))

        if pilih_menu == 1:
            print("\nsaldo anda: Rp." + str(cust.cek_saldo()))

        elif pilih_menu == 2:
            nominal = float(input("Masukkan nominal: "))
            ver_tarik = input("Apakah anda yakin ingin tarik uang sebanyak: Rp." + str(nominal) + "? (y/n): ")
            if ver_tarik == "y":
                print("\nSaldo awal anda adalah Rp." + str(cust.cek_saldo()))
                if nominal < int(cust.cek_saldo()):
                    cust.tarik_tunai(nominal)
                    print("Penarikan Berhasil, saldo anda saat ini menjadi Rp." + str(cust.cek_saldo()))
                else:
                    print("Maaf saldo anda kurang")
            else:
                break

        elif pilih_menu == 3:
            nominal = float(input("Masukkan nominal: "))
            ver_setor = input("Apakah anda yakin ingin setor uang sebanyak: Rp" + str(nominal) + "? (y/n): ")
            if ver_setor == "y":
                print("\nSaldo awal anda adalah Rp." + str(cust.cek_saldo()))
                cust.setor_tunai(nominal)
                print("Setor tunai berhasil, saldo anda saat ini menjadi Rp." + str(cust.cek_saldo()))
            else:
                break

        elif pilih_menu == 4:

            while True:
                pin_awal = int(input("\nMasukkan Pin Anda: "))
                kesempatan = 1

                while pin_awal != cust.cek_pin() and kesempatan < 3:
                    pin_awal = int(input("Pin yang anda masukkan salah, coba lagi: "))
                    kesempatan += 1
                    if kesempatan == 3:
                        print("\nError. Anda sudah memasukkan pin yang salah sebanyak 3x.")
                        print("Ambil Kartu dan silahkan coba lagi")
                        exit()

                while pin_awal == cust.cek_pin():
                    pin_baru = int(input("\nMasukkan pin baru: "))
                    konfirmasi_pin_baru = int(input("Masukkan lagi pin baru: "))
                    if pin_baru == konfirmasi_pin_baru:
                        cust.update_pin(pin_baru)
                        print("\nPin berhasil di ubah, silahkan masukkan kembali pin anda")
                        verifikasi_pin()
                    else:
                        print("\nPin baru yang anda masukkan tidak sama, silahkan masukkan kembali pin baru anda")

        elif pilih_menu == 5:
            print("\nResi tercetak otomatis saat anda keluar. \nHarap simpan tanda terima ini sebagai bukti transaksi anda.")
            print("\nNo. Rekord: ", random.randint(100000, 1000000))
            print("Tanggal: ", datetime.datetime.now())
            print("Saldo akhir: ", cust.cek_saldo())
            print("Terima kasih telah menggunakan ATM Helmi X Progate!")
            exit()

        else:
            print("\nMenu yang anda pilih salah! ")

# Program Utama
def verifikasi_pin():
    while True:
        ver_pin = int(input("\nMasukkan Pin Anda: "))
        kesempatan = 1
        while ver_pin != cust.cek_pin() and kesempatan < 3:
            ver_pin = int(input("Pin yang anda masukkan salah, coba lagi: "))
            kesempatan += 1
            if kesempatan == 3:
                print("\nError. Anda sudah memasukkan pin yang salah sebanyak 3x.")
                print("Ambil Kartu dan silahkan coba lagi")
                exit()
        while True:
            menu()

verifikasi_pin()


# Add a custom field to vulnerabilities via a CSV file.
#
# This script requires one command line parameter: the input CSV file name.
# The CSV file contains the custom field defintion parameters, the custom field value,
# and the vulnerability ID to update.

import os
import sys
import csv
import json
import requests

# Dump JSON in a readable format.
def print_json(json_obj):
    print(json.dumps(json_obj, sort_keys=True, indent=2))

def print_http_error(error_string, http_status_code, resp_json):
    print(f"{error_string} {http_status_code}: {resp_json['message']}, {resp_json['error']}")
    return

# Create a dictionary of custom field defintions keyed by custom field defintion name.
def get_custom_field_definitions(base_url, headers):
    # Dictionary of custom field definitions.
    custom_field_defs = {}

    list_custom_field_defs_url = f"{base_url}/custom_field_definitions"

    # List the the custom fields.  Only the first 100 with the code.
    response = requests.get(list_custom_field_defs_url, headers=headers)
    resp_json = response.json()
    if response.status_code != 200:
        print_http_error("List Risk Meters Error: ", response.status_code, resp_json)
        sys.exit(1)

    custom_field_defs_resp = resp_json['custom_field_definitions']

    # Create a custom field definition record from the API response.
    for cdf in custom_field_defs_resp:
        custom_field_def = {}
        custom_field_def['name'] = cdf['name']
        custom_field_def['id'] = cdf['id']
        custom_field_def['data_type'] = cdf['data_type']
        custom_field_def['entity_type'] = cdf['definition_entity_type']
        custom_field_def['facet'] = cdf['facet']

        # Add custom field definition records to custom field defintions keyed by name.
        custom_field_defs[cdf['name']] = custom_field_def

    return custom_field_defs

# Compare current custom field defintion obtained via the API, to the custom field defintion
# in the CSV file.
def compare_custom_field_defs(curr_cfd, csv_cfd):
    if curr_cfd['data_type'] != csv_cfd['data_type']:
        print(f"Data_types do not agree.  {curr_cfd['data_type']} != CVS {csv_cfd['data_type']}")
        sys.exit(1)
    if curr_cfd['entity_type'] != csv_cfd['entity_type']:
        print(f"Entity_types do not agree.  {curr_cfd['entity_type']} != CVS {csv_cfd['entity_type']}")
        sys.exit(1)

    return curr_cfd['id']

# Process the CSV input file custom field defintion dictionary.
# Return a dictionary of custom field defintions with custom field defintion parameters,
# custom field values, and vulnerability ID.
def process_input_file(csv_input_file_name):
    cdfs = {}

    try:
        with open(csv_input_file_name, newline='', mode='r', encoding='utf-8-sig') as input_file:
            reader = csv.DictReader(input_file, delimiter=',', skipinitialspace=True)
            for row in reader:
                cdf_name = row['name']
                cdfs[cdf_name] = row

    except FileNotFoundError:
        print(f"ERROR: CSV input file, {csv_input_file_name} not found.")
        sys.exit(1)

    return cdfs

# Create a custom field defintion.
def create_custom_field_definition(base_url, headers, cfd_dict):
    create_url = f"{base_url}/custom_field_definitions"

    payload = {}
    payload['name'] = cfd_dict['name']
    payload['description'] = cfd_dict['description']
    payload['data_type'] = cfd_dict['data_type']
    payload['definition_entity_type'] = cfd_dict['entity_type']
    payload['facet'] = cfd_dict['facet']
    if 'dropdown_options' in cfd_dict:
        payload['drop_down_options'] = cfd_dict['dropdown_options']

    response = requests.post(create_url, headers=headers, data=json.dumps(payload))
    resp_json = response.json()
    if response.status_code != 201:
        print_http_error("Create Custom Field Definition Error: ", response.status_code, resp_json)
        sys.exit(1)

    custom_field_def = resp_json['custom_field_definition']
    return custom_field_def['id']

# Update a custom field in a vulnerability.
def update_vuln(base_url, headers, vuln_id, custom_field_def_id, custom_field_value):
    vuln_id = vuln_id.strip()

    update_url = f"{base_url}/vulnerabilities/{vuln_id}"
    update_custom_field_id = f"{custom_field_def_id}"
    update_data = {
        "vulnerability": {
            "custom_fields": {
                update_custom_field_id: custom_field_value
            }
        }
    }
    # print_json(update_data)

    # Invoke the update vulnerability endpoint.
    response = requests.put(update_url, headers=headers, data=json.dumps(update_data))
    resp_json = response.json()

    if response.status_code == 202:
        if f"{resp_json['id']}" == vuln_id:
            print(f"Updating vulnerability ID {vuln_id} in the background.")
        else:
            print(f"Vulnerability IDs don't match: response ID = '{resp_json['id']}', API ID = '{vuln_id}'")
            sys.exit(1)
    elif response.status_code == 204:
            print(f"Updated vulnerability ID {vuln_id}.")
    else:
        print_http_error("Vulnerability Update API Error: ", response.status_code, resp_json)
        sys.exit(1)

    return

if __name__ == "__main__":
    if len(sys.argv) < 2:
       print(f"{sys.argv[0]} csv_input_file")
       sys.exit(1)

    # Get the command line parameters.
    csv_input_file = sys.argv[1]

    print("Add Custom Field Definitions")
    api_key = os.getenv('KENNA_API_KEY')
    if api_key is None:
        print("KENNA API key is non-existent")
        sys.exit(1)

    headers = {'X-Risk-Token': api_key,
               'Accept': 'application/json',
               'Content-Type': 'application/json; charset=utf-8',
               'User-Agent': 'add_custom_field_definitions/1.0.0 (Kenna Security)'}

    v2_base_url = "https://api.kennasecurity.com/v2"

    custom_field_defs = get_custom_field_definitions(v2_base_url, headers)
    print(f"{len(custom_field_defs)} custom field definitions exist.")

    # Holds all the custom field definitions from the CSV file.
    csv_custom_field_defs = process_input_file(csv_input_file)

    # Obtain all the rows in CSV file.  If custom field definition exists, then fetch the ID,
    # else create new custom field defintion.
    for key, csv_custom_field_def in csv_custom_field_defs.items():
        print(f"{key} -> {csv_custom_field_def}")
        if csv_custom_field_def['data_type'] == "attachment":
            print(f"{key} -> Attachment data_type is not supported.")
            continue

        if key in custom_field_defs:
            cfd_id = compare_custom_field_defs(custom_field_defs[key], csv_custom_field_def)
        else:
            cfd_id = create_custom_field_definition(v2_base_url, headers, csv_custom_field_def)
            print(f"{cfd_id} ID created for custom field definition {key}")
            csv_custom_field_def['id'] = cfd_id
            custom_field_defs[key] = csv_custom_field_def

        custom_value = csv_custom_field_def['value']
        vuln_id = csv_custom_field_def['vuln_id']

        update_vuln(v2_base_url, headers, vuln_id, cfd_id, custom_value)

    print("All pau")


"""auth_server model (database) API."""
from datetime import timedelta
import time
import flask
import mct_checkin
import sqlite3
import pathlib

def dict_factory(cursor, row):
    """Convert database row objects to a dictionary keyed on column name.

    This is useful for building dictionaries which are then used to render a
    template.  Note that this would be inefficient for large queries.
    """
    return {col[0]: row[idx] for idx, col in enumerate(cursor.description)}

def get_db():
    """Open a new database connection.

    Flask docs:
    https://flask.palletsprojects.com/en/1.0.x/appcontext/#storing-data
    """
    if 'sqlite_db' not in flask.g:
        db_filename = mct_checkin.app.config['DATABASE_FILENAME']
        flask.g.sqlite_db = sqlite3.connect(str(db_filename))
        flask.g.sqlite_db.row_factory = dict_factory
        # Foreign keys have to be enabled per-connection.  This is an sqlite3
        # backwards compatibility thing.
        flask.g.sqlite_db.execute("PRAGMA foreign_keys = ON")
    return flask.g.sqlite_db

@mct_checkin.app.teardown_appcontext
def close_db(error):
    """Close the database at the end of a request.

    Flask docs:
    https://flask.palletsprojects.com/en/1.0.x/appcontext/#storing-data
    """
    assert error or not error  # Needed to avoid superfluous style error
    sqlite_db = flask.g.pop('sqlite_db', None)
    if sqlite_db is not None:
        sqlite_db.commit()
        sqlite_db.close()

@mct_checkin.app.before_request
def make_session_permanent():
    flask.session.permanent = True
    mct_checkin.app.permanent_session_lifetime = timedelta(minutes=5)

def check_session():
    """Check if logname exists in session."""
    if 'logname' not in flask.session:
        return False
    return flask.session['logname']

def insert_attendance(logname):
    """commit attendance to db."""

    database = get_db()
    cur = database.execute(
        "INSERT INTO participants "
        "(uniqname)"
        "VALUES(?)",
        (logname,)
    )
    cur.fetchone()

def get_attendance_history():
    """fetch all distinct attendance history."""

    database = get_db()
    cur = database.execute(
        "SELECT * "
        "FROM participants "
        "ORDER BY ts"
    )
    attendance = cur.fetchall()
    return attendance

import datetime

def round_minutes(dt, direction, resolution):
    new_minute = (dt.minute // resolution + (1 if direction == 'up' else 0)) * resolution

    return dt + datetime.timedelta(minutes=new_minute - dt.minute), abs(new_minute - dt.minute)

def round_nearest(ts: datetime.datetime, resolution=30):
    f = '%Y-%m-%d %H:%M:%S'

    dt = datetime.datetime.strptime(ts, f)

    min_delta = 1e6
    new_time = None
    for direction in 'up', 'down':
        new_t, delta = round_minutes(dt, direction, resolution)
        if delta < min_delta:
            new_time = new_t
            min_delta = delta

    return new_time


import codecs

from setuptools import find_packages, setup

with codecs.open('requirements.txt', 'r') as f:
    requirements = f.read().strip().split('\n')

    if '-i https://pypi.org/simple' in requirements[0]:
        requirements = requirements[1:]

setup(
    name='HyP3 Process',
    version='0.1dev',
    packages=find_packages('src'),
    package_dir={'': 'src'},
    license="GPLv3+",
    description=("Module for wrapping hyp3 functionality "
                 "around processing scripts"),
    install_requires=requirements,
    entry_points={
        'console_scripts': [
            'hyp3_process = hyp3_process.__main__:main'
        ]
    }
)


import pandas as pd
import streamlit as st
import numpy as np
import pymysql
import plotly.graph_objects as go
import plotly.express as px

from sqlquery import mysql_connection
from sqlquery import mysql_query_df

# configuracion de página
st.set_page_config(page_title='Home',
                    page_icon=':bar_chart:',
                    layout='wide',
                    initial_sidebar_state='collapsed'
                    )

a,b,c=st.columns(3)
with b:
    st.image('src/qxm_logo.png', use_column_width='always')

st.write('''## Otras métricas y graficos''')

# conexion a mysql
cursor = mysql_connection()

# VARIACION GLOBAL PROMEDIO DE LOS BUDGETS

# Extraigo tablas projects, categories y projects_budgets y le cambio el nombre a columnas id de las primeras dos tablas para el merge

projects=mysql_query_df('projects',cursor)
projects.rename(columns={'id': 'project_id'}, inplace=True)
categories=mysql_query_df('categories',cursor)
categories.rename(columns={'id': 'category_id'}, inplace=True)
project_budgets=mysql_query_df('project_budgets',cursor)

#Hago el merge

df = project_budgets.merge(projects, on='project_id', how='inner')
df2 = df.merge(categories, on='category_id', how='inner')
df2 = df2[["project_id","price","title"]]

# Calcula el promedio de price para cada project_id
df2['average_price'] = df2.groupby('project_id')['price'].transform('mean')

# Redondea la columna 'average_price' a dos decimales
df2['average_price'] = df2['average_price'].round(2)

# Agrego columna con variación relativa de cada cotización"

df2["variacion_relativa"] = abs(df2["price"]-df2["average_price"])/df2["average_price"]

# Agrego una columna donde calcula la variación relativa promedio de cada project

df2["var_rel_prom"] = ((df2.groupby('project_id')['variacion_relativa'].transform('mean'))*100).round(2)

# agrupo por project

df3 = df2.groupby(["project_id", "title"])["var_rel_prom"].mean().reset_index()

# agrupo para tener cada project con su variación relativa

grouped_df = df3.groupby("title")["var_rel_prom"].mean().reset_index()

    # dar un formato al gráfico

fig = px.bar(grouped_df, x="var_rel_prom", y="title", orientation='h', title="Media Global de las variaciones en Budgets")

fig.update_xaxes(title_text="Variación relativa (%)")
fig.update_yaxes(title_text="Categoría")

        # plotear
st.plotly_chart(fig)

st.write('''#### Con estas métricas intento calcular cuanto varían en promedio los budgets para cada categoría. Para eso saqué una variación relativa de los budgets de cada proyecto, luego los agrupé por categoría y saqué un promedio de esas variaciones. ''')

# PORCENTAJE DE PROJECTS CON Y SIN CONTRACTS

projects_view=mysql_query_df('projects_view',cursor)

# Obtén las categorías únicas del DataFrame projects_view
categories = projects_view["category_title"].unique()

# Agrega un checkbox para desactivar el filtro
reset_filter = st.sidebar.checkbox("Desactivar filtro")

if reset_filter:
    # Si el checkbox está marcado, no se aplica ningún filtro
    filtered_data = projects_view
else:
    # Si el checkbox no está marcado, muestra el selectbox y filtra los datos
    selected_category = st.sidebar.selectbox("Seleccione una categoría", categories)
    filtered_data = projects_view[projects_view["category_title"] == selected_category]

# Contar la cantidad de proyectos con y sin contrato
count_has_contract = filtered_data[filtered_data["has_contract"] == "Si"]["project_id"].count()
count_no_contract = filtered_data[filtered_data["has_contract"] == "No"]["project_id"].count()

# Crear un DataFrame para el gráfico
data = {
    "Contrato": ["Con contrato", "Sin contrato"],
    "Cantidad": [count_has_contract, count_no_contract]
}
df_pie = pd.DataFrame(data)

left, right = st.columns(2)

with left:
    # Crear el gráfico de torta
    # Definir el título del gráfico
    if reset_filter:
        title = "Porcentaje de proyectos con/sin contrato"
    else:
        title = f"Porcentaje de proyectos con/sin contrato {selected_category}"

    fig = px.pie(df_pie, values="Cantidad", names="Contrato", title=title,width=600,height=600)
    st.plotly_chart(fig)

# PIE CON PORCENTAJE DE PROJECTS CON REBUDGET

# Extraigo tablas projects, categories y projects_budgets y le cambio el nombre a columnas id de las primeras dos tablas para el merge

projects=mysql_query_df('projects',cursor)
projects.rename(columns={'id': 'project_id'}, inplace=True)
categories=mysql_query_df('categories',cursor)
categories.rename(columns={'id': 'category_id'}, inplace=True)
project_rebudgets=mysql_query_df('project_rebudgets',cursor)
project_rebudgets.rename(columns={'price': 'rebudget'}, inplace=True)

#Hago el merge

df = pd.merge(projects, categories, on='category_id', how='left')
df = df[["project_id","title"]]

# Crear una columna "rebudget" con valores por defecto "No"
df["rebudget"] = "No"

# Filtrar los project_ids que están en la tabla projects_rebudgets
rebudget_project_ids = set(project_rebudgets["project_id"].unique())

# Actualizar la columna "rebudget" a "Si" para los project_ids que están en la tabla projects_rebudgets
df.loc[df["project_id"].isin(rebudget_project_ids), "rebudget"] = "Si"

# Agrupar por título y rebudget y contar la cantidad de proyectos en cada grupo
grouped = df.groupby(["title", "rebudget"])["project_id"].count().reset_index()
grouped.columns = ["title", "rebudget", "project_count"]

if reset_filter:
    filtered_data = grouped
else:
    filtered_data = grouped[grouped["title"] == selected_category]

# Calcular la cantidad de proyectos con y sin rebudget
rebudget_counts = filtered_data["rebudget"].value_counts()

# Definir el título del gráfico
if reset_filter:
        title = "Porcentaje de proyectos con/sin rebudget"
else:
        title = f"Porcentaje de proyectos con/sin rebudget {selected_category}"

# Crear el gráfico de pastel

fig = px.pie(rebudget_counts, values=rebudget_counts.values, names=rebudget_counts.index,
             title=title,width=600,height=600)
with right:
    # Mostrar el gráfico
    st.plotly_chart(fig)

# CANTIDAD DE DIAS ENTRE PROJECT Y CONTRACT

# Extraigo tablas projects, categories y contracts, modifico nombres de columnas y hago merge para obtener project_id, title y las fechas de creación del project y del contract

projects=mysql_query_df('projects',cursor)
projects.rename(columns={'id': 'project_id'}, inplace=True)
projects.rename(columns={'created_at': 'created_at_project'}, inplace=True)
categories=mysql_query_df('categories',cursor)
categories.rename(columns={'id': 'category_id'}, inplace=True)
categories = categories[["category_id","title"]]
contracts=mysql_query_df('contracts',cursor)
contracts.rename(columns={'created_at': 'created_at_contract'}, inplace=True)
contracts = contracts[["project_id","created_at_contract"]]

#Hago el merge

df = pd.merge(projects, categories, on='category_id', how='left')
df = df[["project_id","title","created_at_project"]]
df2 = pd.merge(contracts, df, on='project_id', how='inner')

#calculo cantidad de días entre project y contract

df2["dias"] = (df2["created_at_contract"] - df2["created_at_project"]).dt.days

# agrupo por categoría de acuerdo al promedio de días entre contract y project
grouped_days = df2.groupby(["title"])["dias"].mean().reset_index()
grouped_days.columns = ["title", "dias"]

fig = px.bar(grouped_days, x="dias", y="title", orientation='h', title="Cantidad de días entre project y contract")

fig.update_xaxes(title_text="días")
fig.update_yaxes(title_text="Categoría")

        # plotear
st.plotly_chart(fig)


from asgiref.sync import async_to_sync
from channels.layers import get_channel_layer
import re
import json
import logging

logger = logging.getLogger(__name__)

# set of message types to be used when invoking send_notification
# important for client to demultiplex messages
TRANSFER_CONFIRMATION = 'TRANSFER_CONFIRMATION'
SWAP_CONFIRMATION = 'SWAP_CONFIRMATION'
SWAP_CANCELLATION = 'SWAP_CANCELLATION'
SWAP_FINALIZATION = 'SWAP_FINALIZATION'
INCOMING_SWAP = 'INCOMING_SWAP'
MATCHED_SWAP = 'MATCHED_SWAP'
CANCELLED_SWAP = 'CANCELLED_SWAP'
REGISTERED_WALLET = 'REGISTERED_WALLET'
CONFIRMED_DEPOSIT = 'CONFIRMED_DEPOSIT'
REQUESTED_WITHDRAWAL = 'REQUESTED_WITHDRAWAL'
CONFIRMED_WITHDRAWAL = 'CONFIRMED_WITHDRAWAL'
CHECKPOINT_CREATED = 'CHECKPOINT_CREATED'

# subscription stream types and formats
STREAM_TYPES = {
    'wallet': re.compile(r'^[a-f0-9]{40}\/[a-f0-9]{40}$'),
    'tokenpair': re.compile(r'^[a-f0-9]{40}\/[a-f0-9]{40}$'),
}

# allowed group names
group_regex = re.compile('^[a-zA-Z0-9-.]+$')

channel_layer = get_channel_layer()

def check_structure(struct, conf):
    if isinstance(struct, dict) and isinstance(conf, dict):
        # struct is a dict of types or other dicts
        return all(k in conf and check_structure(struct[k], conf[k]) for k in struct)
    if isinstance(struct, list) and isinstance(conf, list):
        # struct is list in the form [type or dict]
        return all(check_structure(struct[0], c) for c in conf)
    elif isinstance(struct, type):
        # struct is the type of conf
        return isinstance(conf, struct)
    elif isinstance(struct, tuple) and isinstance(struct[0], type):
        # struct is the type of conf
        return isinstance(conf, struct[0]) and re.compile(struct[1]).match(conf.lower())
    else:
        # struct is neither a dict, nor list, not type
        return False

# check if stream is valid
# returns group name of that stream
def is_valid_stream(name):
    parts = name.split('/', 1)

    if len(parts) < 2:
        return False

    stream_type, stream_id = parts[0].lower(), parts[1].lower()

    if stream_type not in STREAM_TYPES:
        return False

    if not STREAM_TYPES.get(stream_type).match(stream_id):
        return False

    stream_id = stream_id.replace("/", ".")

    if not group_regex.match(stream_type) or not group_regex.match(stream_id):
        return False

    return '{}.{}'.format(stream_type, stream_id)

# send websocket notification
async def send_notification_async(stream_prefix, stream_id, event_name, data):
    stream_name = '{}/{}'.format(stream_prefix, stream_id)
    group_name = is_valid_stream(stream_name)

    message = {
        'type': 'notification',
        'data': {
            'type': stream_name,
            'data': {
                'type': event_name,
                'data': data
            }
        }
    }

    # if failed for some reason, log error to aid debugging
    try:
        await channel_layer.group_send(
            group_name,
            {
                'type': 'ws.forward',
                'message': message
            }
        )
    except Exception as e:
        logger.warning('failed to send notification, {}'.format(e))

def send_notification(stream_prefix, stream_id, event_name, data):
    async_to_sync(send_notification_async)(
        stream_prefix, stream_id, event_name, data)

# send websocket response
async def send_response(channel_name, resource, data):
    message = {
        'type': 'response',
        'data': {
            'type': resource,
            'data': data
        }
    }

    # if failed for some reason, log error to aid debugging
    try:
        await channel_layer.send(
            channel_name,
            {
                'type': 'ws.forward',
                'message': message
            }
        )
    except Exception as e:
        logger.warning('failed to send response, {}'.format(e))

# send websocket error
async def send_error(channel_name, error, cause=None):
    message = {
        'type': 'error',
        'data': {
            'message': error,
            'cause': cause
        }
    }

    # if failed for some reason, log error to aid debugging
    try:
        await channel_layer.send(
            channel_name,
            {
                'type': 'ws.forward',
                'message': message
            }
        )
    except Exception as e:
        logger.warning('failed to send error, {}'.format(e))


import requests
import time
from countries.country_provider import Country
from web_crawler.cache import Cache
from web_crawler.restaurant import Restaurant
from web_crawler.processors import MichelinPageProcessor, RestaurantProcessor

MICHELIN_BASE_URL = "https://guide.michelin.com"

class MichelinCrawler:

	def __init__(self, country: Country):
		self.country = country
		self.url = f"{MICHELIN_BASE_URL}/{country.shortcode}/en/selection/{country.cname}/restaurants/page"
		self.cache = Cache(country)
		self.page_processors = []

	def fetch_pages(self):
		page_number = 1
		url = f"{self.url}/1"
		while True:
			page_file_name = f"page_{page_number}.html"
			page = self.cache.get_or_load(
				page_file_name,
				lambda: self.make_request_and_get_content(url))
			page_processor = MichelinPageProcessor(page)
			self.page_processors.append(page_processor)

			uri = page_processor.get_next_page_uri()
			if uri is not None:
				url = f"{MICHELIN_BASE_URL}{uri}"
				page_number = page_number + 1
			else:
				break

	def fetch_cards(self):
		cards = []
		for page_processor in self.page_processors:
			cards.extend(page_processor.get_card_processors())
		self.card_processors = cards

	def get_restaurant_objects(self):
		restaurants = []
		for card_processor in self.card_processors:
			uri = card_processor.get_restaurant_uri()
			url = f"{MICHELIN_BASE_URL}{uri}"
			file_name = f"restaurants/{uri.split('/')[-1]}.html"
			restaurant_page = self.cache.get_or_load(
				file_name,
				lambda: self.make_request_and_get_content(url))
			restaurant_processor = RestaurantProcessor(restaurant_page)
			restaurants.append(Restaurant(card_processor, restaurant_processor))
		self.restaurants = restaurants
		return restaurants

	def make_request_and_get_content(self, url):
		# Too lazy to use a real rate limiter, but this will at least help with getting throttled.
		# If the data is cached, this wont matter anyway.
		print(f"Sleeping for 1 seconds to avoid throttling. Then fetching {url}")
		time.sleep(1)
		return requests.get(url).content


#!/usr/bin/env python3
"""
This module contains a class, Cache
"""
import redis
import uuid
from typing import Union, Optional, Callable
import functools

def count_calls(method: Callable) -> Callable:
    """
    This function counts how many times a
    function has been called. This function
    will then be used as a decorator in
    the store method of Cache class
    Args:
        method (Callable): the method which has been called
    Return:
        increment(Callable): a wrapper function that
        handles the counting
    """
    @functools.wraps(method)
    def increment(self, *args):
        """The increment wrapper function"""
        name = method.__qualname__
        result = self._redis.incr(name, amount=1)
        return result
    return increment

class Cache:
    """
    This is the class definition of the Cache class
    It has a constructor and a store method
    """
    def __init__(self):
        """
        This method stores an instance of the Redis client
        as a private variable named _redis
        """
        self._redis = redis.Redis()
        self._redis.flushdb()

    @count_calls
    def store(self, data: Union[str, bytes, int, float]) -> str:
        """
        This method takes a data argument and returns a string.
        The method generates a random key, store the input data
        in Redis using the random key
        Args:
            data: the data to be stored
        Return:
             key(str): the key that holds the value of data
        """
        key = str(uuid.uuid4())  # generate a random key
        value = data
        self._redis.set(key, value)
        return key

    def get(self, key: str,
            fn: Optional[Callable] = None) -> Union[str, bytes, int, float]:
        """
        This method takes a key string argument and an optional
        Callable argument named fn
        Args:
            key: the key that holds data
            fn: a callable used to convert the data back to the desired format
        Return:
             value(str, bytes, int, float): the value
        """
        value = self._redis.get(key)

        if fn:
            value = fn(value)
        return value

    def get_str(self, key: str) -> str:
        """
        This method automatically parameterizes Cache.get
        with a string conversion method
        Args:
            key: the key that holds data
        Return:
             data(int) as a string decoded using utf-8
        """
        value = self._redis.get(key)
        return value.decode('utf-8')

    def get_int(self, key: str) -> int:
        """
        This method automatically parameterizes Cache.get
        with an int conversion method
        Args:
            key: the key that holds data
        Return:
            data as an int decoded using utf-8
        """
        value = self._redis.get(key)
        try:
            value = int(value.decode('utf-8'))
        except Exception:
            value = 0
        return value


import torch
from torchvision.datasets import CIFAR10
DATASET_PATH='/data/home/acw676/cifar10/'
from torchvision import transforms
import pytorch_lightning as pl
import torch.utils.data as data

test_transform = transforms.Compose([transforms.ToTensor(),
                                     transforms.Normalize([0.49139968, 0.48215841, 0.44653091], [0.24703223, 0.24348513, 0.26158784])
                                     ])
# For training, we add some augmentation. Networks are too powerful and would overfit.
train_transform = transforms.Compose([transforms.RandomHorizontalFlip(),
                                      transforms.RandomResizedCrop((32,32),scale=(0.8,1.0),ratio=(0.9,1.1)),
                                      transforms.ToTensor(),
                                      transforms.Normalize([0.49139968, 0.48215841, 0.44653091], [0.24703223, 0.24348513, 0.26158784])
                                     ])
# Loading the training dataset. We need to split it into a training and validation part
# We need to do a little trick because the validation set should not use the augmentation.
train_dataset = CIFAR10(root=DATASET_PATH, train=True, transform=train_transform, download=True)
val_dataset = CIFAR10(root=DATASET_PATH, train=True, transform=test_transform, download=True)
pl.seed_everything(42)
train_set, _ = torch.utils.data.random_split(train_dataset, [45000, 5000])
pl.seed_everything(42)
_, val_set = torch.utils.data.random_split(val_dataset, [45000, 5000])

# Loading the test set
test_set = CIFAR10(root=DATASET_PATH, train=False, transform=test_transform, download=True)

# We define a set of data loaders that we can use for various purposes later.
train_loader = data.DataLoader(train_set, batch_size=128, shuffle=True, drop_last=True, pin_memory=True, num_workers=4)
val_loader = data.DataLoader(val_set, batch_size=128, shuffle=False, drop_last=False, num_workers=4)
test_loader = data.DataLoader(test_set, batch_size=128, shuffle=False, drop_last=False, num_workers=4)


# -*- coding: utf-8 -*-

import sys
import spotipy
import spotipy.util as util
import spotipy.client
import vk_api
from models import VkAudio, SpotifyAudio

try:
    input = raw_input
except NameError:
    pass

class VkFyer:
    SPOTIFY_SCOPE = "user-library-modify user-library-read playlist-modify-public playlist-modify-private " + \
                    "playlist-read-collaborative playlist-read-private"

    SPOTIFY_PLAYLIST_NAME = "Imported from VK.com"
    SPOTIFY_PLAYLIST_LINK = "https://open.spotify.com/user/{login}/playlist/{id}"

    _spotify_login = None
    _spotify_client = None

    _vk_token = None
    _vk_client = None

    def __init__(self, spotify_login, vk_token):
        self._spotify_login = spotify_login
        self._vk_token = vk_token

    def authorize_clients(self):
        return self._authorize_spotify() and self._authorize_vkontakte()

    def _authorize_spotify(self):
        token = util.prompt_for_user_token(self._spotify_login, self.SPOTIFY_SCOPE)

        if token is not None:
            self._spotify_client = spotipy.Spotify(auth=token)
            return True
        else:
            return False

    def _authorize_vkontakte(self):
        self._vk_client = vk_api.VkApi(token=self._vk_token)

        try:
            api = self._vk_client.get_api()
            api.users.get()
        except vk_api.ApiError as error_msg:
            print(error_msg)
            return False

        return True

    def get_vk_audio(self, owner_id=None, offset=0):
        result = []
        api = self._vk_client.get_api()

        # TODO: improve search when > 5k
        try:
            audio = api.audio.get(owner_id=owner_id, count=5000, offset=offset)
        except vk_api.ApiError as e:
            print(e.error.get("error_msg"))
            exit()

        for track in audio["items"]:
            result.append(VkAudio(track))

        return result

    def find_spotify_track(self, audio):
        search_query = "artist:%s %s" % (audio.artist, audio.title)

        try:
            search_items = self._spotify_client.search(q=search_query, type="track").get("tracks").get("items")
        except spotipy.client.SpotifyException:
            return None

        if len(search_items) > 0:
            return SpotifyAudio(search_items[0])

        return None

    def create_playlist(self):
        playlist = self._spotify_client.user_playlist_create(self._spotify_login, self.SPOTIFY_PLAYLIST_NAME)
        playlist_link = self.SPOTIFY_PLAYLIST_LINK.format(login=self._spotify_login, id=playlist.get('id'))

        return playlist, playlist_link

    def add_tracks_to_playlist(self, playlist_id, tracks):
        imported = self._spotify_client.user_playlist_add_tracks(self._spotify_login, playlist_id, tracks)

        if 'snapshot_id' in imported:
            return True

        return False

    @staticmethod
    def draw_progress(status, total, count):
        bar_len = 60
        filled_len = int(round(bar_len * count / float(total)))

        percents = round(100.0 * count / float(total), 1)
        bar = '=' * filled_len + '-' * (bar_len - filled_len)

        sys.stdout.write('%s .. [%s] %s%s\r' % (status, bar, percents, '%'))
        sys.stdout.flush()