import io
import os
from typing import Any, Dict, List, Optional, Set

import numpy as np
from PIL import Image
from pythomata.impl.symbolic import PropositionalInterpretation

from multinav.envs import sapientino_defs
from multinav.envs.base import AbstractFluents
from multinav.envs.temporal_goals import SapientinoGoal, SapientinoOfficeGoal
from multinav.helpers.general import classproperty
from multinav.helpers.gym import (
    Action,
    MyDiscreteEnv,
    Probability,
    State,
    Transition,
    Transitions,
    from_discrete_env_to_graphviz,
    iter_space,
)
from multinav.wrappers.temprl import FlattenAutomataStates, MyTemporalGoalWrapper

class AbstractSapientino(MyDiscreteEnv):
    """Abstract Sapientino environment."""

    def __init__(self, nb_colors: int, failure_probability: float = 0.1):
        """
        Initialize the environment.

        :param nb_colors: the number of color to consider.
        """
        self._nb_colors = nb_colors
        self._failure_probability = failure_probability
        model = self._make_transitions()
        isd = np.zeros(self.nb_states)
        isd[self.initial_state] = 1.0
        super().__init__(self.nb_states, self.nb_actions, model, isd)

    @property
    def nb_colors(self) -> int:
        """Get the number of colors."""
        return self._nb_colors

    @property
    def fail_prob(self) -> Probability:
        """Get the failure probability."""
        return self._failure_probability

    @property
    def initial_state(self) -> int:
        """Get the initial state."""
        return 0

    @property
    def nb_states(self) -> int:
        """
        Get the number of states.

        That is:
        - one state for the corridor
        - one state for each color
        NOTE: Some parts of this module assume that the corridor, or "blank"
        state is number 0. This is also the same convention of gym-sapientino.

        :return: the number of states.
        """
        return self.nb_colors + 1

    @property
    def nb_actions(self) -> int:
        """
        Get the number of actions.

        It includes:
        - go to to each color (+nb_colors)
        - go back to the corridor (+1)
        - visit color (+1)

        :return: the number of actions.
        """
        return self.nb_colors + 1 + 1

    def _is_legal_color(self, color_id: int):
        """Check that it is a legal color."""
        assert 0 <= color_id < self.nb_colors, f"{color_id} is not a legal color."

    def state_from_color(self, color_id: int):
        """
        Get the state from the color.

        Sum +1 because 0 is the initial state.
        """
        self._is_legal_color(color_id)
        return color_id + 1

    def action_goto_color_from_color(self, color_id: int) -> Action:
        """Get the action "goto color" from the color id."""
        assert 0 <= color_id < self.nb_colors, f"{color_id} is not a legal color."
        return color_id + 2

    @classproperty
    def goto_corridor(cls) -> Action:  # pylint: disable=no-self-argument
        """Get the action "goto corridor"."""
        return 0

    @classproperty
    def visit_color(cls) -> Action:  # pylint: disable=no-self-argument
        """Get the action "visit_color"."""
        return 1

    def _make_transitions(self) -> Transitions:
        """
        Make the model.

        :return: the transitions.
        """
        model: Transitions = {}

        for color_id in range(self.nb_colors):
            color_state = self.state_from_color(color_id)

            # from the corridor, you can go to any color
            goto_color_action = self.action_goto_color_from_color(color_id)
            new_transition = (1.0 - self.fail_prob, color_state, 0.0, False)
            fail_transition = (self.fail_prob, self.initial_state, 0.0, False)
            model.setdefault(self.initial_state, {}).setdefault(
                goto_color_action, []
            ).extend([new_transition, fail_transition])

            # if you visit a color, you remain in the same state.
            # NOTE: these two transitions are equivalent to a single transition
            #   it may be useful later, though.
            new_transition = (1.0 - self.fail_prob, color_state, 0.0, False)
            fail_transition = (self.fail_prob, color_state, 0.0, False)
            model.setdefault(color_state, {}).setdefault(self.visit_color, []).extend(
                [new_transition, fail_transition]
            )

            # from any color, you can go back to the corridor.
            new_transition = (1.0 - self.fail_prob, self.initial_state, 0.0, False)
            fail_transition = (self.fail_prob, color_state, 0.0, False)
            model.setdefault(color_state, {}).setdefault(self.goto_corridor, []).extend(
                [new_transition, fail_transition]
            )

            # TODO: experiments in AbstractSapientinoOffice require the
            # possibility to travel between pairs of colors (out and in of a
            # room). Adapted graph for this purpose.
            related_color = color_id + 1 if color_id % 2 == 0 else color_id - 1
            related_state = self.state_from_color(related_color)
            new_transition = (1.0 - self.fail_prob, related_state, 0.0, False)
            fail_transition = (self.fail_prob, color_state, 0.0, False)
            model.setdefault(color_state, {}).setdefault(
                self.action_goto_color_from_color(related_color), []
            ).extend([new_transition, fail_transition])

        return model

    def _action_to_string(self, action: Action):
        """From action to string."""
        self._is_legal_action(action)
        if action == self.goto_corridor:
            return "goto_corridor"
        if action == self.visit_color:
            return "visit"
        return f"goto_{action}"

    def _state_to_string(self, state: State):
        """From state to string.

        All sapientino environments interpret IDs as the same color.
        """
        self._is_legal_state(state)
        if state == 0:
            return "corridor"
        else:
            return sapientino_defs.int2color[state + 1]

    def render(self, mode="human"):
        """Render the environment (only rgb_array mode)."""
        assert mode == "rgb_array"
        graph = from_discrete_env_to_graphviz(
            self, state2str=self._state_to_string, action2str=self._action_to_string
        )
        image_file = io.BytesIO(graph.pipe(format="png"))
        array = np.array(Image.open(image_file))
        return array

class AbstractSapientinoTemporalGoal(MyDiscreteEnv):
    """
    Abstract Sapientino with Temporal Goals.

    We need this instead of using the temporal goal wrapper because we
    need to build an explicit model.
    """

    def __init__(
        self,
        *,
        tg_reward: float = 1.0,
        save_to: Optional[str] = None,
        **sapientino_kwargs,
    ):
        """Initialize the environment.

        :param tg_reward: reward supplied when the temporal goal is reached.
        :param save_to: path where the automaton temporal goal should be saved.
        """
        # Make AbstractSapientino
        self.sapientino_env = AbstractSapientino(**sapientino_kwargs)

        # Make a temporal goal
        nb_colors = sapientino_kwargs["nb_colors"]
        first_color = 2
        color_sequence = [
            sapientino_defs.int2color[i] for i in range(first_color, nb_colors + first_color)]
        self.fluents = Fluents(nb_colors=nb_colors)
        self.temporal_goal = SapientinoGoal(
            colors=color_sequence,
            fluents=self.fluents,
            reward=tg_reward,
            save_to=save_to,
        )

        # Build env with temporal goal
        self.temporal_env = FlattenAutomataStates(
            MyTemporalGoalWrapper(self.sapientino_env, [self.temporal_goal])
        )

        # compute model
        model: Transitions = {}
        initial_state = self.temporal_env.reset()  # because deterministic
        self._generate_transitions(model, initial_state)

        # Set discrete env
        nb_states = self.temporal_env.observation_space.nvec.prod()
        nb_actions = self.sapientino_env.nb_actions
        isd = np.zeros(nb_states)
        isd[0] = 1.0
        MyDiscreteEnv.__init__(self, nS=nb_states, nA=nb_actions, P=model, isd=isd)

        # Update observation space
        self.observation_space = self.temporal_env.observation_space

    def _generate_transitions(
        self,
        model: Transitions,
        state: State,
    ):
        """Recusively visit states and generate transitions."""
        if state in model:
            return

        # Transition function from state
        automaton = self.temporal_goal.automaton
        sapientino_tf = self.sapientino_env.P[state[0]]

        # For all actions
        model[state] = {}
        for action in sapientino_tf:
            new_transitions: List[Transition] = []
            model[state][action] = new_transitions

            # For all nondeterministic transitions
            for transition in sapientino_tf[action]:
                p, sap_state, sap_reward, _sap_done = transition

                # Possible evaluations
                fluents_list = self.fluents.evaluations_prob(sap_state, action)
                for fluents, fluents_p in fluents_list:

                    automaton_state = automaton.get_successor(state[1], fluents)

                    # Compose state
                    goal_reached = automaton_state in automaton.accepting_states
                    new_reward = sap_reward + (
                        0.0 if goal_reached else -1.0
                    )
                    new_done = goal_reached
                    new_state = (sap_state, automaton_state)

                    new_transitions.append(
                        (p * fluents_p, new_state, new_reward, new_done))

                    # Recurse
                    self._generate_transitions(model, new_state)

    def render(self, mode="human"):
        """Render with temporal goal; mode is ignored."""
        return self.temporal_env.render(mode="rgb_array")

    def reset(self):
        """Reset the environment."""
        return self.temporal_env.reset()

    def step(self, action):
        """Do a step on the environment."""
        return self.temporal_env.step(action)

class AbstractSapientinoOffice(AbstractSapientinoTemporalGoal):
    """AbstractSapientino with the office scenario temporal goal."""

    def __init__(
        self,
        *,
        nb_rooms: int,
        tg_reward: float,
        saved_automaton: str,
        save_to: Optional[str] = None,
        seed: int,
        **sapientino_kwargs,
    ):
        """Initialize the environment.

        :param nb_rooms: number of rooms to visit and get inside.
        :param tg_reward: reward supplied when the temporal goal is reached.
        :param saved_automaton: path to a saved DFA that represents the
            temporal goal. Transitions must be interpretations of fluents.
        :param save_to: path where the automaton temporal goal should be saved.
        """
        # Define temporal goal
        self.fluents = OfficeFluents(n_rooms=nb_rooms, seed=seed)
        self.temporal_goal = SapientinoOfficeGoal(
            n_rooms=nb_rooms,
            fluents=self.fluents,
            saved_automaton=saved_automaton,
            reward=tg_reward,
            save_to=save_to,
        )

        # Make AbstractSapientino
        sapientino_kwargs["nb_colors"] = nb_rooms * 2
        self.sapientino_env = AbstractSapientino(**sapientino_kwargs)

        # Build env with temporal goal
        self.temporal_env = FlattenAutomataStates(
            MyTemporalGoalWrapper(self.sapientino_env, [self.temporal_goal])
        )

        # compute model
        model: Transitions = {}
        initial_state = self.temporal_env.reset()  # because deterministic
        self._generate_transitions(model, initial_state)

        # Set discrete env
        nb_states = len(model)
        nb_actions = self.sapientino_env.nb_actions
        isd = np.zeros(nb_states)
        isd[0] = 1.0
        MyDiscreteEnv.__init__(self, nS=nb_states, nA=nb_actions, P=model, isd=isd)

        # Update observation space
        self.observation_space = self.temporal_env.observation_space

class Fluents(AbstractFluents):
    """Define the propositions for `AbstractSapientino`."""

    def __init__(self, nb_colors: int):
        """Initialize.

        :param nb_colors: The number of colors/rooms in the environment.
        """
        base_id = 2  # the firsts are for corridor and wall
        self.fluents = {
            sapientino_defs.int2color[i] for i in range(base_id, base_id + nb_colors)
        }

    def evaluate(self, obs: int, action: int) -> PropositionalInterpretation:
        """Respects AbstractFluents.evaluate.

        :param obs: assuming that the observation comes from an
            `AbstractSapientino` environment.
        :param action: the last action.
        """
        if action == AbstractSapientino.visit_color:
            fluents = {sapientino_defs.int2color[obs]}
            if obs == 0:  # blank/corridor
                fluents = set()
        else:
            fluents = set()
        return {f: f in fluents for f in self.fluents}

class OfficeFluents(AbstractFluents):
    """Define propositions for AbstractSapientino with Office goal."""

    _colors_to_room = {
        "red": "at1",
        "green": "in1",
        "blue": "at2",
        "yellow": "in2",
        "pink": "at3",
        "brown": "in3",
        "gray": "at4",
        "orange": "in4",
    }

    def __init__(self, n_rooms: int, seed: int):
        """Initialize.

        :param nb_rooms: number of rooms to navigate.
        """
        assert n_rooms < 5, "Can't support more than four rooms"

        # Define the propositional symbols
        self.fluents = {"bip", "person", "closed"}
        for i in range(1, n_rooms + 1):
            at_room, in_room = f"at{i}", f"in{i}"
            self.fluents.add(at_room)
            self.fluents.add(in_room)

        self._rng = np.random.default_rng(seed)
        self._n_rooms = n_rooms

    def evaluate(self, obs: int, action: int) -> Set[str]:
        """Respects.AbstractFluents.evaluate.

        :param obs: assuming that the observation comes from an
            `AbstractSapientino` environment.
        :param action: the last action.
        """
        fluents = set()
        if obs != 0:
            color = sapientino_defs.int2color[obs + 1]
            fluents.add(self._colors_to_room[color])
        if action == AbstractSapientino.visit_color:
            fluents.add("bip")

        # Doors and people are not in the observation. Valuate at random
        samples = self._rng.integers(0, 2, size=2)
        if samples[0] == 1:
            fluents.add("closed")
        if samples[1] == 1:
            fluents.add("person")

        return {f: f in fluents for f in self.fluents}

    def evaluations_prob(self, obs, action):
        fluents = self.evaluate(obs, action)
        fluents["closed"] = False
        fluents["person"] = False

        p = 1.0 / 4
        values = [(dict(fluents), p) for i in range(4)]
        values[1][0].update({"closed": True}), p
        values[2][0].update({"person": True}), p
        values[3][0].update({"closed": True, "person": True}), p
        return values

def make(params: Dict[str, Any], log_dir: Optional[str] = None):
    """Make the sapientino abstract state environment (agent teleports).

    :param params: a dictionary of parameters; see in this function the
        only ones that are used.
    :param log_dir: directory where logs can be saved.
    :return: an object that respects the gym.Env interface.
    """
    # Check
    if params["shaping"]:
        raise ValueError("Can't shape rewards in the most abstract environment.")

    # Build
    env = AbstractSapientinoOffice(
        nb_rooms=params["nb_rooms"],
        tg_reward=params["tg_reward"],
        saved_automaton=params["tg_automaton"],
        save_to=os.path.join(log_dir, "reward-dfa.dot") if log_dir else None,
        failure_probability=params["sapientino_fail_p"],
        seed=params["seed"],
    )
    return env


"""MangaBuddy site downloader"""

import re
from typing import Iterable, Set

from bs4 import BeautifulSoup

from .base import Chapter, ChapterImage, Language, Manga, Site

class MangaBuddy(Site):
    @property
    def name(self) -> str:
        return "MangaBuddy"

    @property
    def url(self) -> str:
        return "https://mangabuddy.com"

    @property
    def supported_languages(self) -> Set[Language]:
        return {Language.en}

    def search(self, query: str, lang: Language = None) -> Iterable[Manga]:
        with self.session.get(f"{self.url}/search", params={"q": query}) as resp:
            resp.raise_for_status()
            soup = BeautifulSoup(resp.text, "html.parser")
        for card in soup("div", {"class": "book-item"}):
            anchor = card.a
            if anchor is None:
                continue
            yield Manga(
                url=f'{self.url}{anchor["href"].strip()}',
                name=anchor["title"].strip(),
                cover=card.findNext("img")["data-src"].strip(),
            )

    def get_chapters(self, manga: Manga) -> Iterable[Chapter]:
        url = f"https://mangabuddy.com/api/manga{manga.url[len(self.url):]}/chapters?source=detail"
        with self.session.get(url) as resp:
            resp.raise_for_status()
            soup = BeautifulSoup(resp.text, "html.parser")
        soup = soup.find("ul", {"id": "chapter-list"})
        for item in soup("li"):
            item = item.find("a")
            name = item.findNext("strong", {"class": "chapter-title"}).text.strip()
            yield Chapter(name=name, url=f'{self.url}{item["href"]}')

    def get_images(self, chapter: Chapter) -> Iterable[ChapterImage]:
        with self.session.get(chapter.url) as resp:
            resp.raise_for_status()
            imgs = re.findall(r"var chapImages = '(.*)'", resp.text)[0].split(",")
        for img in imgs:
            if not img.startswith(("https://", "http://")):
                img = f"https://s1.mbcdnv1.xyz/file/img-mbuddy/manga/{img}"
            yield ChapterImage(url=img)

    def download_image(self, image: ChapterImage) -> bytes:
        with self.session.get(image.url, headers={"referer": self.url}) as resp:
            resp.raise_for_status()
            return resp.content

    def download_cover(self, manga: Manga) -> bytes:
        assert manga.cover
        with self.session.get(manga.cover, headers={"referer": self.url}) as resp:
            resp.raise_for_status()
            return resp.content


import os
from datetime import datetime
import time
import tqdm
import pickle
import pandas as pd
import random
from sklearn.preprocessing import LabelEncoder
import numpy as np
import torch
import math
import copy
import random
from multiprocessing import Pool
import multiprocessing

class Preprocess:
    def __init__(self,args):
        self.args = args
        self.train_data = None
        self.test_data = None

    def get_train_data(self):
        return self.train_data

    def get_test_data(self):
        return self.test_data

    def split_data(self, data, ratio=0.7, shuffle=True, seed=0):
        """
        split data into two parts with a given ratio.
        """
        if self.args.cv_strategy :
            train_user_path = os.path.join(self.args.data_dir,"cv_train_index.pickle")
            valid_user_path = os.path.join(self.args.data_dir,"cv_valid_index.pickle")
            with open(train_user_path,"rb") as file :
                train_idx = pickle.load(file)
            with open(valid_user_path,"rb") as file :
                valid_idx = pickle.load(file)
            data_1 = data[data['userID'].isin(train_idx)]
            data_2 = data[data['userID'].isin(valid_idx)]
        else :
            idx_list = list(set(data['userID']))
            size = int(len(idx_list) * ratio)
            train_idx = random.sample(idx_list,size)
            data_1 = data[data['userID'].isin(train_idx)]
            data_2 = data[~data['userID'].isin(train_idx)]

        return data_1, data_2

    def __save_labels(self, encoder, name):
        le_path = os.path.join(self.args.asset_dir, name + '_classes.npy')
        np.save(le_path, encoder.classes_)

    def __preprocessing(self, df, is_train = True):

        # Encoding the Categorical Embedding
        cols = list(set(self.args.cate_col + self.args.temp_col)) # not to encode twice

        if not os.path.exists(self.args.asset_dir):
            os.makedirs(self.args.asset_dir)
        other = []
        for col in cols:
            le = LabelEncoder()
            if col in ['assessmentItemID', 'testId', 'KnowledgeTag', 'grade', 'last_problem', 'problem_number'] :
                if is_train:
                    #For UNKNOWN class
                    a = df[col].unique().tolist()
                    a = sorted(a) if str(type(a[0])) == "<class 'int'>" else a
                    a = a + ['unknown']
                    le.fit(a)
                    self.__save_labels(le, col)
                else:
                    label_path = os.path.join(self.args.asset_dir,col+'_classes.npy')
                    le.classes_ = np.load(label_path)
                    df[col]= df[col].astype(str)
                    df[col] = df[col].apply(lambda x: x if x in le.classes_ else 'unknown')
                            #모든 컬럼이 범주형이라고 가정
                df[col]= df[col].astype(str)
                test = le.transform(df[col])
                df[col] = test
            else :
                if is_train :
                    unq = df[col].unique().tolist()
                    unq = sorted(unq) if str(type(a[0])) == "<class 'int'>" else unq
                    other += list(map(lambda x : col+'_'+str(x),unq))
                    df[col] = df[col].apply(lambda x : col+'_'+str(x))
                else :
                    label_path = os.path.join(self.args.asset_dir,'other_classes.npy')
                    le.classes_ = np.load(label_path)
                    df[col]= df[col].astype(str)
                    df[col] = df[col].apply(lambda x : col+'_'+x)
                    df[col] = df[col].apply(lambda x: x if x in le.classes_ else 'unknown')

        if other :
            other += ['unknown']
            le = LabelEncoder()
            le.fit(other)
            self.__save_labels(le, 'other')

        label_path = os.path.join(self.args.asset_dir,'other_classes.npy')
        if os.path.exists(label_path):
            le.classes_ = np.load(label_path)

            for col in cols:
                if col in ['assessmentItemID', 'testId', 'KnowledgeTag', 'grade', 'last_problem', 'problem_number'] :
                    continue
                else :
                    df[col]= df[col].astype(str)
                    test = le.transform(df[col])
                    df[col] = test

        if not is_train and self.args.sep_grade:
            ddf = df[df['answerCode']==-1]
            df = df[df.set_index(['userID','grade']).index.isin(ddf.set_index(['userID','grade']).index)]

        return df

    def __feature_engineering(self, df,file_name):

        data_path = os.path.join(self.args.asset_dir,f"{file_name[:-4]}_FE.csv") # .csv빼고
        if os.path.exists(data_path):
            df = pd.read_csv(data_path)

        else :
            df.sort_values(by=['userID','Timestamp'], inplace=True)

            df['hour'] = df['Timestamp'].dt.hour
            df['dow'] = df['Timestamp'].dt.dayofweek

            diff = df.loc[:, ['userID','Timestamp']].groupby('userID').diff().fillna(pd.Timedelta(seconds=0))
            diff = diff.fillna(pd.Timedelta(seconds=0))
            diff = diff['Timestamp'].apply(lambda x: x.total_seconds())

            # 푸는 시간
            df['elapsed'] = diff
            df['elapsed'] = df['elapsed'].apply(lambda x : x if x <650 and x >=0 else 0)

            df['grade']=df['testId'].apply(lambda x : int(x[1:4])//10)
            df['mid'] = df['testId'].apply(lambda x : int(x[-3:]))
            df['problem_number'] = df['assessmentItemID'].apply(lambda x : int(x[-3:]))

            correct_t = df.groupby(['testId'])['answerCode'].agg(['mean', 'sum'])
            correct_t.columns = ["test_mean", 'test_sum']
            correct_k = df.groupby(['KnowledgeTag'])['answerCode'].agg(['mean', 'sum'])
            correct_k.columns = ["tag_mean", 'tag_sum']
            correct_a = df.groupby(['assessmentItemID'])['answerCode'].agg(['mean', 'sum'])
            correct_a.columns = ["ass_mean", 'ass_sum']
            correct_p = df.groupby(['problem_number'])['answerCode'].agg(['mean', 'sum'])
            correct_p.columns = ["prb_mean", 'prb_sum']
            correct_h = df.groupby(['hour'])['answerCode'].agg(['mean', 'sum'])
            correct_h.columns = ["hour_mean", 'hour_sum']
            correct_d = df.groupby(['dow'])['answerCode'].agg(['mean', 'sum'])
            correct_d.columns = ["dow_mean", 'dow_sum']

            df = pd.merge(df, correct_t, on=['testId'], how="left")
            df = pd.merge(df, correct_k, on=['KnowledgeTag'], how="left")
            df = pd.merge(df, correct_a, on=['assessmentItemID'], how="left")
            df = pd.merge(df, correct_p, on=['problem_number'], how="left")
            df = pd.merge(df, correct_h, on=['hour'], how="left")
            df = pd.merge(df, correct_d, on=['dow'], how="left")

            o_df = df[df['answerCode']==1]
            x_df = df[df['answerCode']==0]

            elp_k = df.groupby(['KnowledgeTag'])['elapsed'].agg('mean').reset_index()
            elp_k.columns = ['KnowledgeTag',"tag_elp"]
            elp_k_o = o_df.groupby(['KnowledgeTag'])['elapsed'].agg('mean').reset_index()
            elp_k_o.columns = ['KnowledgeTag', "tag_elp_o"]
            elp_k_x = x_df.groupby(['KnowledgeTag'])['elapsed'].agg('mean').reset_index()
            elp_k_x.columns = ['KnowledgeTag', "tag_elp_x"]

            df = pd.merge(df, elp_k, on=['KnowledgeTag'], how="left")
            df = pd.merge(df, elp_k_o, on=['KnowledgeTag'], how="left")
            df = pd.merge(df, elp_k_x, on=['KnowledgeTag'], how="left")

            ass_k = df.groupby(['assessmentItemID'])['elapsed'].agg('mean').reset_index()
            ass_k.columns = ['assessmentItemID',"ass_elp"]
            ass_k_o = o_df.groupby(['assessmentItemID'])['elapsed'].agg('mean').reset_index()
            ass_k_o.columns = ['assessmentItemID',"ass_elp_o"]
            ass_k_x = x_df.groupby(['assessmentItemID'])['elapsed'].agg('mean').reset_index()
            ass_k_x.columns = ['assessmentItemID',"ass_elp_x"]

            df = pd.merge(df, ass_k, on=['assessmentItemID'], how="left")
            df = pd.merge(df, ass_k_o, on=['assessmentItemID'], how="left")
            df = pd.merge(df, ass_k_x, on=['assessmentItemID'], how="left")

            prb_k = df.groupby(['problem_number'])['elapsed'].agg('mean').reset_index()
            prb_k.columns = ['problem_number',"prb_elp"]
            prb_k_o = o_df.groupby(['problem_number'])['elapsed'].agg('mean').reset_index()
            prb_k_o.columns = ['problem_number',"prb_elp_o"]
            prb_k_x = x_df.groupby(['problem_number'])['elapsed'].agg('mean').reset_index()
            prb_k_x.columns = ['problem_number',"prb_elp_x"]

            df = pd.merge(df, prb_k, on=['problem_number'], how="left")
            df = pd.merge(df, prb_k_o, on=['problem_number'], how="left")
            df = pd.merge(df, prb_k_x, on=['problem_number'], how="left")

            df['user_correct_answer'] = df.groupby('userID')['answerCode'].transform(lambda x: x.cumsum().shift(1))
            df['user_total_answer'] = df.groupby('userID')['answerCode'].cumcount()
            df['user_acc'] = df['user_correct_answer']/df['user_total_answer']
            df['Grade_o'] = df.groupby(['userID','grade'])['answerCode'].transform(lambda x: x.cumsum().shift(1))
            df['GradeCount'] = df.groupby(['userID','grade']).cumcount()
            df['GradeAcc'] = df['Grade_o']/df['GradeCount']
            df['GradeElp'] = df.groupby(['userID','grade'])['elapsed'].transform(lambda x: x.cumsum().shift(1))
            df['GradeMElp'] = df['GradeElp']/df['GradeCount']

            f = lambda x : len(set(x))
            test_df = df.groupby(['testId']).agg({
                'problem_number':'max',
                'KnowledgeTag':f
            })
            test_df.reset_index(inplace=True)

            test_df.columns = ['testId','problem_count',"tag_count"]

            df = pd.merge(df,test_df,on='testId',how='left')

            gdf = df[['userID','testId','problem_number','grade','Timestamp']].sort_values(by=['userID','grade','Timestamp'])
            gdf['buserID'] = gdf['userID'] != gdf['userID'].shift(1)
            gdf['bgrade'] = gdf['grade'] != gdf['grade'].shift(1)
            gdf['first'] = gdf[['buserID','bgrade']].any(axis=1).apply(lambda x : 1- int(x))
            gdf['RepeatedTime'] = gdf['Timestamp'].diff().fillna(pd.Timedelta(seconds=0))
            gdf['RepeatedTime'] = gdf['RepeatedTime'].apply(lambda x: x.total_seconds()) * gdf['first']
            df['RepeatedTime'] = gdf['RepeatedTime'].apply(lambda x : math.log(x+1))

            df['prior_KnowledgeTag_frequency'] = df.groupby(['userID','KnowledgeTag']).cumcount()

            df['problem_position'] = df['problem_number'] / df["problem_count"]
            df['solve_order'] = df.groupby(['userID','testId']).cumcount()
            df['solve_order'] = df['solve_order'] - df['problem_count']*(df['solve_order'] > df['problem_count']).apply(int) + 1
            df['retest'] = (df['solve_order'] > df['problem_count']).apply(int)
            T = df['solve_order'] != df['problem_number']
            TT = T.shift(1)
            TT[0] = False
            df['solved_disorder'] = (TT.apply(lambda x : not x) & T).apply(int)
            df['last_problem'] = (df['testId']!=df['testId'].shift(-1)).apply(int)

            df = df.fillna(0)
            # with open(data_path,'wb') as file:
            #     pickle.dump(df,file)
            df.to_csv(data_path,index=False)
        return df

    def load_data_from_file(self, file_name, is_train=True):
        csv_file_path = os.path.join(self.args.data_dir, file_name)
        df = pd.read_csv(csv_file_path, parse_dates=['Timestamp'])#, nrows=100000)
        df = self.__feature_engineering(df,file_name)
        df = self.__preprocessing(df, is_train)
        # 추후 feature를 embedding할 시에 embedding_layer의 input 크기를 결정할때 사용

        self.args.n_questions = len(np.load(os.path.join(self.args.asset_dir,'assessmentItemID_classes.npy')))
        self.args.n_test = len(np.load(os.path.join(self.args.asset_dir,'testId_classes.npy')))
        self.args.n_tag = len(np.load(os.path.join(self.args.asset_dir,'KnowledgeTag_classes.npy')))
        self.args.n_grade = len(np.load(os.path.join(self.args.asset_dir,'grade_classes.npy')))
        self.args.n_other = len(np.load(os.path.join(self.args.asset_dir,'other_classes.npy')))
        self.args.n_cont = len(self.args.cont_col)

        df = df.sort_values(by=self.args.key, axis=0)
        return df

    def load_train_data(self, file_name):
        self.train_data = self.load_data_from_file(file_name)

    def load_test_data(self, file_name):
        self.test_data = self.load_data_from_file(file_name, is_train= False)

class DKTDataset(torch.utils.data.Dataset):
    def __init__(self, data, args, augment):
        self.data = data
        self.args = args
        self.augment = augment

    def __getitem__(self, index):
        row = self.data[index]

        # 각 data의 sequence length
        seq_len = len(row[0])
        do_augment = random.random()
        try :
            if self.augment and do_augment <self.args.augment_rate:
                columns = self.args.cate_col + self.args.cont_col + self.args.temp_col
                l_idx = columns.index('last_problem')
                nx = torch.Tensor(row[l_idx]).numpy()
                last_problems = np.where(nx==max(nx))[0]
                p = np.array([self.test_dist(x) for x in last_problems])
                p = p/sum(p)
                idx = np.random.choice(len(last_problems),1,p=p)[0]
                seq_len = last_problems[idx] if last_problems[idx] > 0 else seq_len
        except :
            print('augmented error in get_item')
            pass

        cols = [row[i][:seq_len] for i in range(0,len(row))]
        # max seq len을 고려하여서 이보다 길면 자르고 아닐 경우 그대로 냅둔다
        if seq_len > self.args.max_seq_len:
            for i, col in enumerate(cols):
                cols[i] = col[-self.args.max_seq_len:]
            mask = np.ones(self.args.max_seq_len, dtype=np.int16)
        else:
            mask = np.zeros(self.args.max_seq_len, dtype=np.int16)
            mask[-seq_len:] = 1

        # mask도 columns 목록에 포함시킴
        cols.append(mask)

        # np.array -> torch.tensor 형변환
        for i, col in enumerate(cols):
            cols[i] = torch.tensor(col)

        return cols

    def __len__(self):
        return len(self.data)

    def test_dist(self,x):
        if self.args.sep_grade == True :
            pmf = [888,293,171,142,130,82,71,57,38,39,27,19,15,7,1,3,1,3,1]
            x = pmf[abs(x)//50 if x < 950 else 18]
        else :
            pmf = [229,108,76,62,51,42,41,28,24,14,4,9,1,1,1]
            x = pmf[abs(x)//100 if x < 1700 else 16]
        return x

from torch.nn.utils.rnn import pad_sequence

def collate(batch):
    col_n = len(batch[0])
    col_list = [[] for _ in range(col_n)]
    max_seq_len = len(batch[0][-1])
    try :
        # batch의 값들을 각 column끼리 그룹화
        for row in batch:
            for i, col in enumerate(row):
                pre_padded = torch.zeros(max_seq_len)
                pre_padded[-len(col):] = col
                col_list[i].append(pre_padded)
    except :
        print("augment error")

    for i, _ in enumerate(col_list):
        col_list[i] = torch.stack(col_list[i])

    return tuple(col_list)

def get_loaders(args, train, valid):

    pin_memory = True
    train_loader, valid_loader = None, None

    if train is not None:
        trainset = DKTDataset(train, args, args.augment)
        train_loader = torch.utils.data.DataLoader(trainset, num_workers=args.num_workers, shuffle=True,
                            batch_size=args.batch_size, pin_memory=pin_memory, collate_fn=collate)
    if valid is not None:
        valset = DKTDataset(valid, args, False)
        valid_loader = torch.utils.data.DataLoader(valset, num_workers=args.num_workers, shuffle=False,
                            batch_size=args.batch_size, pin_memory=pin_memory, collate_fn=collate)

    return train_loader, valid_loader

def add_features(args):

    cate_dict = {
        'base' : ['assessmentItemID', 'testId', 'KnowledgeTag'],
        'cont' : ['assessmentItemID', 'testId', 'KnowledgeTag','grade','problem_number']
    }

    cont_dict = {
        'base' : [],
        'cont' : ['GradeAcc','GradeMElp',"GradeCount", 'GradeElp']
    }

    args.cate_col = list(set(cate_dict[args.feature_type])) # need to check seperate cate and cont
    args.cont_col = list(set(cont_dict[args.feature_type]))
    args.temp_col = []
    args.key = ['userID','grade','Timestamp'] if args.sep_grade else ['userID', 'Timestamp']

    if args.sep_grade == True and args.cate_col.count('grade') == 0 :
        args.temp_col.append('grade')

    if args.augment == True and args.cate_col.count('last_problem') == 0 :
        args.temp_col.append('last_problem')

    if args.interaction_type == 'problem_number' and args.cate_col.count('problem_number') == 0:
        args.temp_col.append('problem_number')

    print("cate_columns : ", args.cate_col)
    print("cont_columns : ", args.cont_col)
    print("temp_columns : ", args.temp_col)

    return args

def post_process(df, args):
    df = df.sort_values(by=args.key, axis=0)
    columns = args.cate_col + args.cont_col + args.temp_col + ['userID','answerCode']
    group = df[columns].groupby(args.key[:-1]).apply(lambda row :
            tuple(row[c].values for c in row.columns if not c == 'userID')
            )

    return group.values

def sweep_apply(args, config):
    args.augment = config.augment
    args.interaction_type = config.interaction_type
    args.sep_grade = config.sep_grade
    args.cv_strategy = config.cv_strategy
    args.loss_type = config.loss_type

    return args


from sklearn.datasets import load_files
from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, confusion_matrix
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn.pipeline import Pipeline
from sklearn.model_selection import cross_val_score
from sklearn.naive_bayes import MultinomialNB
from sklearn.tree import DecisionTreeClassifier
from sklearn.linear_model import SGDClassifier
from sklearn.svm import LinearSVC
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier

def output(x, y):
    print('Accuracy =', accuracy_score(x, y),
          '\nPrecision =', precision_score(x, y),
          '\nRecall =', recall_score(x, y),
          '\nF1-score =', f1_score(x, y),
          '\nError rate =', (1 - accuracy_score(x, y)))

import seaborn
import matplotlib.pyplot as plt

def plot_confusion_matrix(data, labels, output_filename):
    seaborn.set(color_codes=True)
    plt.figure(1, figsize=(9, 6))

    plt.title("Confusion Matrix")

    seaborn.set(font_scale=1.4)
    ax = seaborn.heatmap(data, annot=True, fmt='g', cbar_kws={'label': 'Scale'})

    ax.set_xticklabels(labels)
    ax.set_yticklabels(labels)

    ax.set(ylabel="Predicted class", xlabel="Actual class")

    plt.savefig(output_filename, bbox_inches='tight', dpi=300)
    plt.close()

doc_data = load_files("./train/", encoding="utf-8", decode_error="replace")
X_train, y_train = doc_data.data, doc_data.target

doc_data = load_files("./test/", encoding="utf-8", decode_error="replace")
X_test, y_test = doc_data.data, doc_data.target

# создание конвейеров обучения для моделей
bayes = Pipeline([
    ("count vectorizer", CountVectorizer(stop_words="english", max_features=5000, decode_error="ignore")),
    ("bayes", MultinomialNB())
])
bayes_tfidf = Pipeline([
    ("count vectorizer", TfidfVectorizer(stop_words="english", max_features=5000, decode_error="ignore")),
    ("bayes", MultinomialNB())
])
destree = Pipeline([
    ("count vectorizer", CountVectorizer(stop_words="english", max_features=5000, decode_error="ignore")),
    ("destree", DecisionTreeClassifier())
])
destree_tfidf = Pipeline([
    ("count vectorizer", TfidfVectorizer(stop_words="english", max_features=5000, decode_error="ignore")),
    ("destree", DecisionTreeClassifier())
])
sgd = Pipeline([
    ("count vectorizer", CountVectorizer(stop_words="english", max_features=5000, decode_error="ignore")),
    ("sgd", SGDClassifier(loss="modified_huber"))
])
sgd_tfidf = Pipeline([
    ("tfidf_vectorizer", TfidfVectorizer(stop_words="english", max_features=5000, decode_error="ignore")),
    ("sgd", SGDClassifier())
])
svc = Pipeline([
    ("count_vectorizer", CountVectorizer(stop_words="english", max_features=5000, decode_error="ignore")),
    ("linear svc", LinearSVC())
])
svc_tfidf = Pipeline([
    ("tfidf_vectorizer", TfidfVectorizer(stop_words="english", max_features=5000, decode_error="ignore")),
    ("linear svc", LinearSVC())
])
ranfor = Pipeline([
    ("count_vectorizer", CountVectorizer(stop_words="english", max_features=5000, decode_error="ignore")),
    ("ranfor", RandomForestClassifier())
])
ranfor_tfidf = Pipeline([
    ("tfidf_vectorizer", TfidfVectorizer(stop_words="english", max_features=5000, decode_error="ignore")),
    ("ranfor", RandomForestClassifier())
])
logreg = Pipeline([
    ("count_vectorizer", CountVectorizer(stop_words="english", max_features=5000, decode_error="ignore")),
    ("logreg", LogisticRegression())
])
logreg_tfidf = Pipeline([
    ("tfidf_vectorizer", TfidfVectorizer(stop_words="english", max_features=5000, decode_error="ignore")),
    ("logreg", LogisticRegression())
])
knn = Pipeline([
    ("count_vectorizer", CountVectorizer(stop_words="english", max_features=5000, decode_error="ignore")),
    ("knn", KNeighborsClassifier())
])
knn_tfidf = Pipeline([
    ("tfidf_vectorizer", TfidfVectorizer(stop_words="english", max_features=5000, decode_error="ignore")),
    ("knn", KNeighborsClassifier())
])

all_models = [
    ("bayes", bayes),
    ("bayes_tfidf", bayes_tfidf),
    ("destree", destree),
    ("destree_tfidf", destree_tfidf),
    ("sgd", sgd),
    ("sgd_tfidf", sgd_tfidf),
    ("svc", svc),
    ("svc_tfidf", svc_tfidf),
    ("ranfor", ranfor),
    ("ranfor_tfidf", ranfor_tfidf),
    ("logreg", logreg),
    ("logreg_tfidf", logreg_tfidf),
    ("knn", knn),
    ("knn_tfidf", knn)
]

# нахождение самой производительной модели
unsorted_scores = [(name, cross_val_score(model, X_train, y_train, cv=None).mean()) for name, model in all_models]
scores = sorted(unsorted_scores, key=lambda x: -x[1])
print(scores)

model = svc_tfidf
model.fit(X_train, y_train)
y_pred = model.predict(X_test)
output(y_test, y_pred)

tn, fp, fn, tp = confusion_matrix(list(y_test), list(y_pred), labels=[0, 1]).ravel()

data = [[tp, fp],
        [fn, tn]]
labels = ["private", "public"]

# создание матрицы несоответствий
plot_confusion_matrix(data, labels, "confusion_matrix3.png")

research stages/model training and selection.py


import time
import pyautogui
import pyperclip
from selenium.webdriver import Remote
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.remote.webelement import WebElement
from selenium.webdriver.common.action_chains import ActionChains

TIME_OUT = 5
INTERVAL = .5

class Driver:
    host = ''

    def __init__(self, driver: Remote):
        self.driver = driver

    def goto(self, url:str):
        if url.startswith(('http://', 'https://')):
            return self.driver.get(url)
        if not url.startswith('/'):
            return ValueError('url must start with slash /.')
        url = self.host + url
        return self.driver.get(url)

    def click(self, locator):
        el = self.wait_clickable(locator)
        el.click()
        return self

    def type(self, locator, words):
        el = self.wait_element(locator)
        el.send_keys(words)
        return self

    def double_click(self, locator):
        ac = ActionChains(self.driver)
        el = self.wait_clickable(locator)
        ac.double_click(el).perform()
        return self

    def right_click(self, locator):
        ac = ActionChains(self.driver)
        el = self.wait_clickable(locator)
        ac.context_click(el).perform()
        return self

    def drag(self, start_locator, end_locator):
        ac = ActionChains(self.driver)
        el_start = self.wait_element(start_locator)
        el_end = self.wait_element(end_locator)
        ac.drag_and_drop(el_start, el_end).perform()
        return self

    def move_to(self, locator):
        ac = ActionChains(self.driver)
        el = self.wait_clickable(locator)
        ac.move_to_element(el).perform()
        return self

    def switch_to_window(self, window_name):
        self.driver.switch_to.window(window_name=window_name)
        return self

    def switch_to_frame(self, locator, timeout=TIME_OUT, interval=INTERVAL):
        wait = WebDriverWait(self.driver, timeout=timeout, poll_frequency=interval)
        wait.until(expected_conditions.frame_to_be_available_and_switch_to_it(locator))
        return self

    def switch_to_alert(self, locator, timeout=TIME_OUT, interval=INTERVAL):
        wait = WebDriverWait(self.driver, timeout=timeout, poll_frequency=interval)
        el = wait.until(expected_conditions.alert_is_present)
        return el

    def scroll_to_bottom(self):
        self.driver.execute_script('window.scrollTo(0, document.body.scrollHeight)')
        return self

    def scroll_to(self, width, height):
        self.driver.execute_script(f'window.scrollTo({width}, {height})')
        return self

    def upload(self, locator, file):
        el = self.wait_element(locator)
        if el.tag_name == 'input':
            el.send_keys(file)
            return self
        el.click()
        pyperclip.copy(file)
        time.sleep(.2)
        pyautogui.hotkey('ctrl', 'v')
        pyautogui.press('enter', presses=2)
        return self

    def find(self, locator):
        return self.driver.find_element(*locator)

    def wait_element(self, locator, timeout=TIME_OUT, interval=INTERVAL) -> WebElement:
        used_time = 0
        while used_time < timeout:
            try:
                el = self.driver.find_element(*locator)
                return el
            except NoSuchElementException:
                time.sleep(interval)
                used_time += interval
        raise NoSuchElementException(f"can not find the element by locator:{locator}")

    def wait_visible(self, locator, timeout=TIME_OUT, interval=INTERVAL):
        wait = WebDriverWait(self.driver, timeout=timeout, poll_frequency=interval)
        el = wait.until(expected_conditions.visibility_of_element_located(locator))
        return el

    def wait_presence(self, locator, timeout=TIME_OUT, interval=INTERVAL):
        wait = WebDriverWait(self.driver, timeout=timeout, poll_frequency=interval)
        el = wait.until(expected_conditions.presence_of_element_located(locator))
        return el

    def wait_clickable(self, locator, timeout=TIME_OUT, interval=INTERVAL):
        wait = WebDriverWait(self.driver, timeout=timeout, poll_frequency=interval)
        el = wait.until(expected_conditions.element_to_be_clickable(locator))
        return el

    def quit(self):
        self.driver.quit()

    def screenshot(self):
        self.driver.save_screenshot('demo.png')

def screenshot_required(f):
    def decorator(*args, **kwargs):
        try:
            return f(*args, **kwargs)
        except Exception as e:
            for i in args:
                if isinstance(i, Driver):
                    i.screenshot()
            raise e
    return decorator


# coding:utf-8
import MySQLdb
from eleme import settings
from sqlalchemy import Column, String, Integer, create_engine
from sqlalchemy.orm import sessionmaker
from sqlalchemy.ext.declarative import declarative_base
from eleme.items import RestaurantItem

# 创建对象的基类
Base = declarative_base()

class Restaurant(Base):
    __tablename__ = 'eleme_restaurants'

    id = Column(Integer, primary_key=True)
    name = Column(String(521))
    recent_order_num = Column(Integer)
    average_cost = Column(String(521))
    address = Column(String(1024))
    distance = Column(Integer)

class ElemePipeline(object):
    def open_spider(self, spider):
        engine = create_engine(
            'mysql+mysqldb://%s:%s@%s:3306/%s?charset=utf8' % (
                settings.MYSQL_USER, settings.MYSQL_PASSWORD, settings.MYSQL_HOST,
                settings.MYSQL_DB))
        self.sessionDB = sessionmaker(bind=engine)
        self.session = self.sessionDB()

    def close_spider(self, spider):
        try:
            self.session.commit()
        except:
            self.session.rollback()
            raise
        finally:
            self.session.close()

    def process_item(self, item, spider):
        if isinstance(item, RestaurantItem):
            r = {
                'id': item['restaurant']['id'],
                'name': item['restaurant']['name'],
                'recent_order_num': item['restaurant']['recent_order_num'],
                'average_cost': item['restaurant']['average_cost'],
                'address': item['restaurant']['address'],
                'distance': item['restaurant']['distance'],
            }
            new_r = Restaurant(**r)
            self.session.merge(new_r)
        return item


import numpy as np
import pandas as pd

from random import randint

import matplotlib.pyplot as plt
plt.style.use('seaborn-white')
import seaborn as sns
sns.set_style("white")

from sklearn.model_selection import train_test_split

from skimage.transform import resize

from keras.preprocessing.image import load_img
from keras import Model
from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau
from keras.models import load_model
from keras.optimizers import Adam
from keras.utils.vis_utils import plot_model
from keras.preprocessing.image import ImageDataGenerator
from keras.layers import Input, Conv2D, Conv2DTranspose, MaxPooling2D, concatenate, Dropout

from tqdm import tqdm_notebook
img_size_ori = 101
img_size_target = 101

# removed upsample and downsample code, since resizing is not used
train_df = pd.read_csv("../input/train.csv", index_col="id", usecols=[0])
depths_df = pd.read_csv("../input/depths.csv", index_col="id")

train_df = train_df.join(depths_df)

test_df = depths_df[~depths_df.index.isin(train_df.index)]
train_df["images"] = [np.array(load_img("../input/train/images/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]
train_df["masks"] = [np.array(load_img("../input/train/masks/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]
# Simple split of images into training and testing sets
ids_train, ids_valid, x_train, x_valid, y_train, y_valid = train_test_split(
    train_df.index.values,
    np.array(train_df.images.tolist()).reshape(-1, img_size_target, img_size_target, 1),
    np.array(train_df.masks.tolist()).reshape(-1, img_size_target, img_size_target, 1),
    test_size=0.2)
def build_model(input_layer, start_neurons):
    # standard size 128 -> 64   custome size 101 -> 50
    conv1 = Conv2D(start_neurons * 1, (3, 3), activation="relu", padding="same")(input_layer)
    conv1 = Conv2D(start_neurons * 1, (3, 3), activation="relu", padding="same")(conv1)
    pool1 = MaxPooling2D((2, 2))(conv1)
    pool1 = Dropout(0.25)(pool1)

    # standard size 64 -> 32       custome size 50 -> 25
    conv2 = Conv2D(start_neurons * 2, (3, 3), activation="relu", padding="same")(pool1)
    conv2 = Conv2D(start_neurons * 2, (3, 3), activation="relu", padding="same")(conv2)
    pool2 = MaxPooling2D((2, 2))(conv2)
    pool2 = Dropout(0.5)(pool2)

    # standard size 32 -> 16       custome size 25 -> 12
    conv3 = Conv2D(start_neurons * 4, (3, 3), activation="relu", padding="same")(pool2)
    conv3 = Conv2D(start_n-neurons * 4, (3, 3), activation="relu", padding="same")(conv3)
    pool3 = MaxPooling2D((2, 2))(conv3)
    pool3 = Dropout(0.5)(pool3)

    # standard size 16 -> 8       custome size 12 -> 6
    conv4 = Conv2D(start_neurons * 8, (3, 3), activation="relu", padding="same")(pool3)
    conv4 = Conv2D(start_neurons * 8, (3, 3), activation="relu", padding="same")(conv4)
    pool4 = MaxPooling2D((2, 2))(conv4)
    pool4 = Dropout(0.5)(pool4)

    # Middle
    convm = Conv2D(start_neurons * 16, (3, 3), activation="relu", padding="same")(pool4)
    convm = Conv2D(start_neurons * 16, (3, 3), activation="relu", padding="same")(convm)

    # standard size 8 -> 16         custome size 6-> 12
    deconv4 = Conv2DTranspose(start_neurons * 8, (3, 3), strides=(2, 2), padding="same")(convm)
    uconv4 = concatenate([deconv4, conv4])
    uconv4 = Dropout(0.5)(uconv4)
    uconv4 = Conv2D(start_neurons * 8, (3, 3), activation="relu", padding="same")(uconv4)
    uconv4 = Conv2D(start_neurons * 8, (3, 3), activation="relu", padding="same")(uconv4)

    # standard size 16 -> 32        custome size 12 -> 25
    # Changed padding from "same" to "valid" to round up image to next size
    #deconv3a = Conv2DTranspose(start_neurons * 4, (3, 3), strides=(2, 2), padding="same")(uconv4)
    deconv3 = Conv2DTranspose(start_neurons * 4, (3, 3), strides=(2, 2), padding="valid")(uconv4)
    uconv3 = concatenate([deconv3, conv3])
    uconv3 = Dropout(0.5)(uconv3)
    uconv3 = Conv2D(start_neurons * 4, (3, 3), activation="relu", padding="same")(uconv3)
    uconv3 = Conv2D(start_neurons * 4, (3, 3), activation="relu", padding="same")(uconv3)

    # standard size 32 -> 64   custome size 25 -> 50
    deconv2 = Conv2DTranspose(start_neurons * 2, (3, 3), strides=(2, 2), padding="same")(uconv3)
    uconv2 = concatenate([deconv2, conv2])
    uconv2 = Dropout(0.5)(uconv2)
    uconv2 = Conv2D(start_neurons * 2, (3, 3), activation="relu", padding="same")(uconv2)
    uconv2 = Conv2D(start_neurons * 2, (3, 3), activation="relu", padding="same")(uconv2)

    # standard size 64 -> 128   custome size 50 -> 101
    # Changed padding from "same" to "valid" to round up image to next size
    #deconv1 = Conv2DTranspose(start_neurons * 1, (3, 3), strides=(2, 2), padding="same")(uconv2)
    deconv1 = Conv2DTranspose(start_neurons * 1, (3, 3), strides=(2, 2), padding="valid")(uconv2)
    uconv1 = concatenate([deconv1, conv1])
    uconv1 = Dropout(0.5)(uconv1)
    uconv1 = Conv2D(start_neurons * 1, (3, 3), activation="relu", padding="same")(uconv1)
    uconv1 = Conv2D(start_neurons * 1, (3, 3), activation="relu", padding="same")(uconv1)

    #uconv1 = Dropout(0.5)(uconv1)
    output_layer = Conv2D(1, (1,1), padding="same", activation="sigmoid")(uconv1)

    return output_layer

input_layer = Input((img_size_target, img_size_target, 1))
output_layer = build_model(input_layer, 16)
model = Model(input_layer, output_layer)
model.compile(loss="binary_crossentropy", optimizer="adam", metrics=["accuracy"])
model.summary()

x_train = np.append(x_train, [np.fliplr(x) for x in x_train], axis=0)
y_train = np.append(y_train, [np.fliplr(x) for x in y_train], axis=0)
fig, axs = plt.subplots(2, 10, figsize=(15,3))
for i in range(10):
    axs[0][i].imshow(x_train[i].squeeze(), cmap="Greys")
    axs[0][i].imshow(y_train[i].squeeze(), cmap="Greens", alpha=0.3)
    axs[1][i].imshow(x_train[int(len(x_train)/2 + i)].squeeze(), cmap="Greys")
    axs[1][i].imshow(y_train[int(len(y_train)/2 + i)].squeeze(), cmap="Greens", alpha=0.3)
fig.suptitle("Top row: original images, bottom row: augmented images")
early_stopping = EarlyStopping(patience=10, verbose=2)
filepath="unet_wo_resizing-{epoch:02d}-{val_acc:.2f}.hdf5"
model_checkpoint = ModelCheckpoint(filepath, save_best_only=True, verbose=1)
reduce_lr = ReduceLROnPlateau(factor=0.5, patience=5, min_lr=0.000001)
epochs = 200
batch_size = 32

history = model.fit(x_train, y_train,
                    validation_data=[x_valid, y_valid],
                    epochs=epochs,
                    batch_size=batch_size,
                    verbose=1,
                    callbacks=[early_stopping, model_checkpoint, reduce_lr])
fig, (ax_loss, ax_acc) = plt.subplots(1, 2, figsize=(15,5))
ax_loss.plot(history.epoch, history.history["loss"], label="Train loss")
ax_loss.plot(history.epoch, history.history["val_loss"], label="Validation loss")
ax_acc.plot(history.epoch, history.history["acc"], label="Train accuracy")
ax_acc.plot(history.epoch, history.history["val_acc"], label="Validation accuracy")
# May want to load in top saved model here
# model = load_model("./keras.model")
# Predict the validation set
preds_valid = model.predict(x_valid).reshape(-1, img_size_target, img_size_target)
preds_valid = np.array([x for x in preds_valid])
y_valid_ori = np.array([train_df.loc[idx].masks for idx in ids_valid])
# src: https://www.kaggle.com/aglotero/another-iou-metric
def iou_metric(y_true_in, y_pred_in, print_table=False):
    labels = y_true_in
    y_pred = y_pred_in

    true_objects = 2
    pred_objects = 2

    intersection = np.histogram2d(labels.flatten(), y_pred.flatten(), bins=(true_objects, pred_objects))[0]

    # Compute areas (needed for finding the union between all objects)
    area_true = np.histogram(labels, bins = true_objects)[0]
    area_pred = np.histogram(y_pred, bins = pred_objects)[0]
    area_true = np.expand_dims(area_true, -1)
    area_pred = np.expand_dims(area_pred, 0)

    # Compute union
    union = area_true + area_pred - intersection

    # Exclude background from the analysis
    intersection = intersection[1:,1:]
    union = union[1:,1:]
    union[union == 0] = 1e-9

    # Compute the intersection over union
    iou = intersection / union

    # Precision helper function
    def precision_at(threshold, iou):
        matches = iou > threshold
        true_positives = np.sum(matches, axis=1) == 1   # Correct objects
        false_positives = np.sum(matches, axis=0) == 0  # Missed objects
        false_negatives = np.sum(matches, axis=1) == 0  # Extra objects
        tp, fp, fn = np.sum(true_positives), np.sum(false_positives), np.sum(false_negatives)
        return tp, fp, fn

    # Loop over IoU thresholds
    prec = []
    if print_table:
        print("Thresh\tTP\tFP\tFN\tPrec.")
    for t in np.arange(0.5, 1.0, 0.05):
        tp, fp, fn = precision_at(t, iou)
        if (tp + fp + fn) > 0:
            p = tp / (tp + fp + fn)
        else:
            p = 0
        if print_table:
            print("{:1.3f}\t{}\t{}\t{}\t{:1.3f}".format(t, tp, fp, fn, p))
        prec.append(p)

    if print_table:
        print("AP\t-\t-\t-\t{:1.3f}".format(np.mean(prec)))
    return np.mean(prec)

def iou_metric_batch(y_true_in, y_pred_in):
    batch_size = y_true_in.shape[0]
    metric = []
    for batch in range(batch_size):
        value = iou_metric(y_true_in[batch], y_pred_in[batch])
        metric.append(value)
    return np.mean(metric)
thresholds = np.linspace(0, 1, 50)
ious = np.array([iou_metric_batch(y_valid_ori, np.int32(preds_valid > threshold)) for threshold in tqdm_notebook(thresholds)])
threshold_best_index = np.argmax(ious[9:-10]) + 9
iou_best = ious[threshold_best_index]
threshold_best = thresholds[threshold_best_index]
plt.plot(thresholds, ious)
plt.plot(threshold_best, iou_best, "xr", label="Best threshold")
plt.xlabel("Threshold")
plt.ylabel("IoU")
plt.title("Threshold vs IoU ({}, {})".format(threshold_best, iou_best))
plt.legend()
# Source https://www.kaggle.com/bguberfain/unet-with-depth
def RLenc(img, order='F', format=True):
    """
    img is binary mask image, shape (r,c)
    order is down-then-right, i.e. Fortran
    format determines if the order needs to be preformatted (according to submission rules) or not

    returns run length as an array or string (if format is True)
    """
    bytes = img.reshape(img.shape[0] * img.shape[1], order=order)
    runs = []  ## list of run lengths
    r = 0  ## the current run length
    pos = 1  ## count starts from 1 per WK
    for c in bytes:
        if (c == 0):
            if r != 0:
                runs.append((pos, r))
                pos += r
                r = 0
            pos += 1
        else:
            r += 1

    # if last run is unsaved (i.e. data ends with 1)
    if r != 0:
        runs.append((pos, r))
        pos += r
        r = 0

    if format:
        z = ''

        for rr in runs:
            z += '{} {} '.format(rr[0], rr[1])
        return z[:-1]
    else:
        return runs
x_test = np.array([np.array(load_img("../input/test/images/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(test_df.index)]).reshape(-1, img_size_target, img_size_target, 1)
preds_test = model.predict(x_test)
pred_dict = {idx: RLenc(np.round(preds_test[i] > threshold_best)) for i, idx in enumerate(tqdm_notebook(test_df.index.values))}
sub = pd.DataFrame.from_dict(pred_dict,orient='index')
sub.index.names = ['id']
sub.columns = ['rle_mask']
sub.to_csv('submission.csv')


import numpy as np
import matplotlib.pyplot as plt

# Función de Rosenbrock
def rosenbrock(x, y, a=1, b=100):
    return (a - x) ** 2 + b * (y - x**2) ** 2

# Función para generar una nueva solución vecina (Simulated Annealing)
def neighbor_solution(point, step_size):
    return point + np.random.uniform(-step_size, step_size, point.shape)

# Criterio de Boltzmann para aceptar una nueva solución
def accept_solution(delta, temperature):
    if delta < 0:
        return True
    else:
        return np.random.rand() < np.exp(-delta / temperature)

# Parámetros iniciales
np.random.seed(0)
initial_point = np.random.rand(2) * 2 - 1  # Punto inicial aleatorio (en [-1, 1])
initial_temperature = 1.0  # Temperatura inicial
cooling_rate = 0.99  # Tasa de enfriamiento
step_size = 0.05  # Tamaño de cada paso para generar vecinos
max_iterations = 10000  # Número máximo de iteraciones
min_temperature = 1e-3  # Temperatura mínima para detener el algoritmo

# Simulated Annealing
current_point = initial_point
current_value = rosenbrock(current_point[0], current_point[1])
temperature = initial_temperature
points_history = [current_point]

for iteration in range(max_iterations):
    new_point = neighbor_solution(current_point, step_size)
    new_value = rosenbrock(new_point[0], new_point[1])

    # Se acepta la nueva solución si es mejor
    if accept_solution(new_value - current_value, temperature):
        current_point, current_value = new_point, new_value
        points_history.append(current_point)

    # Se reduce la temperatura
    temperature *= cooling_rate

    # Condición de finalización por temperatura mínima
    if temperature < min_temperature:
        break

# Grafica función de Rosenbrock y el punto mínimo
x = np.linspace(-2, 2, 400)
y = np.linspace(-1, 3, 400)
X, Y = np.meshgrid(x, y)
Z = rosenbrock(X, Y)

plt.figure(figsize=(8, 6))
plt.contour(X, Y, Z, levels=np.logspace(-0.5, 3.5, 20), cmap="viridis")
plt.plot(
    *zip(*points_history),
    marker="o",
    color="r",
    markersize=3,
    linestyle="-",
    linewidth=1,
    label="Camino al mínimo",
)
plt.plot(
    current_point[0],
    current_point[1],
    "bo",
    label=f"Punto mínimo: ({current_point[0]:.4f}, {current_point[1]:.4f})",
)
plt.title("Camino al mínimo de la función de Rosenbrock")
plt.xlabel("x")
plt.ylabel("y")
plt.legend()
plt.show()

# Resultados
print("Punto final:", current_point)
print("Valor en el punto final:", current_value)


# There is one table in the file https://stepik.org/media/attachments/lesson/209723/5.html.
# Sum all the numbers in it. Now we have not only added different tags to change the display style,
# but also rendered invalid HTML (although browsers render it, but BeautifulSoup may have problems).
# Invalid HTML code is not uncommon on the Internet, you need to learn to work with this too.
#
# You can fix the html code or try using a non-standard html parser like html5lib.

# В файле https://stepik.org/media/attachments/lesson/209723/5.html находится одна таблица.
# Просуммируйте все числа в ней. Теперь мы не только добавили разных тегов для изменения стиля отображения,
# но и сделали невалидный HTML-код (правда, браузеры его отображают, а вот с BeautifulSoup могут быть проблемы).
# Невалидный HTML-код - не редкость в интернете, надо учиться работать и с этим.
#
# Вы можете исправить html-код или попробовать использовать нестандартный парсер html, такой как html5lib.

from urllib.request import urlopen, urlretrieve
from bs4 import BeautifulSoup

resp = urlopen('https://stepik.org/media/attachments/lesson/209723/4.html')  # downloading file / скачиваем файл
html = resp.read().decode('utf8')  # read the input / считываем содержимое
soup = BeautifulSoup(html, 'html.parser')  # making a soup / делаем суп

table = soup.find('table', attrs={'class': 'wikitable sortable'})

sum = 0
for tag in soup.find_all('td'):
    sum += int(tag.string)
print(sum)

PythonForPracticaProblems/Module 1 - Web pages scraping/Module 1 Step 4 Task 2 - Using BeautifulSoup .py


import yaml
import argparse
import os
import sys
from os import path

root_folder = path.dirname(path.abspath(__file__))
sys.path.insert(0, path.join(root_folder, "../../../"))
print(sys.path)

from vision_wl import train_vision_wl, run_inference, collect_class_labels, load_model, run_inference_per_patient

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--config_file", type=str, required=True)
    args = parser.parse_args()

    with open(args.config_file, "r") as f:
        config = yaml.safe_load(f)

    dataset_dir = config['args']['dataset_dir']
    train_dataset_dir = os.path.join(dataset_dir, "train")
    test_dataset_dir = os.path.join(dataset_dir, "test")
    # output_dir = config['args']['output_dir']
    output_dir = config['training_args']['output_dir']
    batch_size = config['training_args']['batch_size']
    epochs = config['training_args']['epochs']
    bf16 = config['training_args']['bf16']
    model_name = config['args']['model']

    # do_predict = config['training_args']['do_predict']
    do_predict = config['args']['inference']
    do_predict_per_patient = config['args']['inference_per_patient']
    # do_train = config['training_args']['do_train']
    do_train = config['args']['finetune']

    saved_model_dir = config['args']['saved_model_dir']

    # output_file_test = config['args']['output_file_test_dir']
    output_file_test = config['args']['inference_output']

    # output_file_train = config['args']['output_file_train_dir']
    output_file_train = config['args']['finetune_output']

    # this is one is used for place holder
    vision_int8_inference = 'vision_int_8.yaml' # config['inference_args']['int8_inference']
    class_labels = collect_class_labels(train_dataset_dir)
    if (do_train):
        model, history, dict_metrics = train_vision_wl(train_dataset_dir,
                                                       output_dir, model_name,
                                                       batch_size, epochs, bf16=bf16)
        run_inference(train_dataset_dir, saved_model_dir, class_labels,
                      model_name, vision_int8_inference, output_file_train)

    if (do_predict):
        run_inference(test_dataset_dir, saved_model_dir, class_labels,
                      model_name, vision_int8_inference, output_file_test)
    if (do_predict_per_patient):
        model = load_model(model_name,saved_model_dir)
        # Sample dict
        patient_dict = {'106L':[os.path.join(train_dataset_dir,"Malignant/P106_L_CM_MLO1.jpg")],\
                        '106R':[os.path.join(train_dataset_dir,"Benign/P106_R_CM_CC1.jpg"),\
                                os.path.join(train_dataset_dir,"Benign/P106_R_CM_CC2.jpg")]}
        results = run_inference_per_patient(model, patient_dict,class_labels)

if __name__ == "__main__":
    main()


import arcade

# Constants
SCREEN_WIDTH = 1000
SCREEN_HEIGHT = 550
SCREEN_TITLE = "Platformer"

# Constants used to scale our sprites from their original size
CHARACTER_SCALING = 1
TILE_SCALING = 0.5
COIN_SCALING = 0.5
SPRITE_PIXEL_SIZE = 128
GRID_PIXEL_SIZE = (SPRITE_PIXEL_SIZE * TILE_SCALING)

# How many pixels to keep as a minimum margin between the character
# and the edge of the screen.
LEFT_VIEWPORT_MARGIN = 200
RIGHT_VIEWPORT_MARGIN = 200
BOTTOM_VIEWPORT_MARGIN = 150
TOP_VIEWPORT_MARGIN = 100

# Movement speed of player, in pixels per frame
PLAYER_MOVEMENT_SPEED = 5
GRAVITY = 1
PLAYER_JUMP_SPEED = 20

PLAYER_START_X = 128
PLAYER_START_Y = 300

# number of levels
LEVELS = 3

class MyGame(arcade.Window):
    """
    Main application class.
    """

    def __init__(self):

        # Call the parent class and set up the window
        super().__init__(SCREEN_WIDTH, SCREEN_HEIGHT, SCREEN_TITLE)

        # These are 'lists' that keep track of our sprites. Each sprite should
        # go into a list.
        self.coin_list = None
        self.wall_list = None
        self.foreground_list = None
        self.background_list = None
        self.dont_touch_list = None
        self.player_list = None
        self.enemy_list = None

        # Separate variable that holds the player sprite
        self.player_sprite = None

        # Our physics engine
        self.physics_engine = None

        # Used to keep track of our scrolling
        self.view_bottom = 0
        self.view_left = 0

        # Keep track of the score
        self.score = 0

        # Where is the right edge of the map?
        self.end_of_map = 0

        # Level
        self.level = 1

        # Game state
        self.game_over = False

        # Load sounds
        self.collect_coin_sound = arcade.load_sound("sounds/coin1.wav")
        self.jump_sound = arcade.load_sound("sounds/jump1.wav")
        self.gameover = arcade.load_sound("sounds/gameover4.wav")
        self.enemy_hit = arcade.load_sound("sounds/hurt3.wav")
        self.fall = arcade.load_sound("sounds/fall1.wav")
        self.dont_touch = arcade.load_sound("sounds/hit1.wav")

    def setup(self, level):
        # Used to keep track of our scrolling
        self.view_bottom = 0
        self.view_left = 0

        # Keep track of the score
        self.score = 0

        # Create the Sprite lists
        self.player_list = arcade.SpriteList()
        self.foreground_list = arcade.SpriteList()
        self.background_list = arcade.SpriteList()
        self.wall_list = arcade.SpriteList()
        self.coin_list = arcade.SpriteList()
        self.enemy_list = arcade.SpriteList()

        # Set up the player, specifically placing it at these coordinates.
        self.player_sprite = arcade.Sprite("images/player_1/player_stand.png", CHARACTER_SCALING)
        self.player_sprite.center_x = PLAYER_START_X
        self.player_sprite.center_y = PLAYER_START_Y
        self.player_list.append(self.player_sprite)

        # --- Load in a map from the tiled editor ---

        # Name of the layer in the file that has our platforms/walls
        platforms_layer_name = 'Platforms'
        # Name of the layer that has items for pick-up
        coins_layer_name = 'Coins'
        # Name of the layer that has items for foreground
        foreground_layer_name = 'Foreground'
        # Name of the layer that has items for background
        background_layer_name = 'Background'
        # Name of the layer that has items we shouldn't touch
        dont_touch_layer_name = "Don't Touch"

        # Map name
        map_name = f"Level{level}.tmx"

        # Read in the tiled map
        my_map = arcade.read_tiled_map(map_name, TILE_SCALING)

        # Grab the layer of items we can't move through
        map_array = my_map.layers_int_data[platforms_layer_name]

        # Calculate the right edge of the my_map in pixels
        self.end_of_map = len(map_array[0]) * GRID_PIXEL_SIZE

        # -- Background
        self.background_list = arcade.generate_sprites(my_map, background_layer_name, TILE_SCALING)

        # -- Foreground
        self.foreground_list = arcade.generate_sprites(my_map, foreground_layer_name, TILE_SCALING)

        # -- Platforms
        self.wall_list = arcade.generate_sprites(my_map, platforms_layer_name, TILE_SCALING)

        # -- Coins
        self.coin_list = arcade.generate_sprites(my_map, coins_layer_name, TILE_SCALING)

        # -- Don't Touch Layer
        self.dont_touch_list = arcade.generate_sprites(my_map, dont_touch_layer_name, TILE_SCALING)

        # this line needed to check when player gets to the end of map
        self.end_of_map = (len(map_array[0]) - 1) * GRID_PIXEL_SIZE

        # Set the background color from the map
        if my_map.backgroundcolor:
            arcade.set_background_color(my_map.backgroundcolor)

        # create enemies
        if self.level == 1:
            enemy = arcade.Sprite("images/enemies/slimePurple.png", TILE_SCALING)
            enemy.bottom = GRID_PIXEL_SIZE * 4
            enemy.left = GRID_PIXEL_SIZE * 5
            # right and left boundary limits for the enemy
            enemy.boundary_right = GRID_PIXEL_SIZE * 11
            enemy.boundary_left = GRID_PIXEL_SIZE * 1
            # set enemy speed
            enemy.change_x = 2
            self.enemy_list.append(enemy)

            # create enemies
            enemy = arcade.Sprite("images/enemies/wormGreen.png", TILE_SCALING)
            enemy.bottom = GRID_PIXEL_SIZE * 4
            enemy.left = GRID_PIXEL_SIZE * 25
            # right and left boundary limits for the enemy
            enemy.boundary_right = GRID_PIXEL_SIZE * 31
            enemy.boundary_left = GRID_PIXEL_SIZE * 21
            # set enemy speed
            enemy.change_x = 2
            self.enemy_list.append(enemy)

            # create enemies
            enemy = arcade.Sprite("images/enemies/bee.png", TILE_SCALING)
            enemy.bottom = GRID_PIXEL_SIZE * 7
            enemy.left = GRID_PIXEL_SIZE * 16
            # set enemy speed
            enemy.change_x = -4
            self.enemy_list.append(enemy)

        if self.level == 2:
            enemy = arcade.Sprite("images/enemies/slimeGreen.png", TILE_SCALING)
            enemy.bottom = GRID_PIXEL_SIZE * 3
            enemy.left = GRID_PIXEL_SIZE * 5
            # right and left boundary limits for the enemy
            enemy.boundary_right = GRID_PIXEL_SIZE * 6
            enemy.boundary_left = GRID_PIXEL_SIZE
            # set enemy speed
            enemy.change_x = 2
            self.enemy_list.append(enemy)

            # create enemies
            enemy = arcade.Sprite("images/enemies/saw.png", TILE_SCALING)
            enemy.bottom = GRID_PIXEL_SIZE * 6
            enemy.left = GRID_PIXEL_SIZE * 19
            # set enemy speed
            enemy.change_x = -3
            self.enemy_list.append(enemy)

        if self.level == 3:
            enemy = arcade.Sprite("images/enemies/fishGreen.png", TILE_SCALING)
            enemy.bottom = GRID_PIXEL_SIZE * 4
            enemy.left = GRID_PIXEL_SIZE * 5
            # set enemy speed
            enemy.change_x = -2
            self.enemy_list.append(enemy)

            # create enemies
            enemy = arcade.Sprite("images/enemies/fishPink.png", TILE_SCALING)
            enemy.bottom = GRID_PIXEL_SIZE * 4
            enemy.left = GRID_PIXEL_SIZE * 25
            # right and left boundary limits for the enemy
            enemy.boundary_right = GRID_PIXEL_SIZE * 31
            enemy.boundary_left = GRID_PIXEL_SIZE * 21
            # set enemy speeddw
            enemy.change_x = 2
            #enemy.change_y = 2
            self.enemy_list.append(enemy)

        # Create the 'physics engine'
        self.physics_engine = arcade.PhysicsEnginePlatformer(self.player_sprite,
                                                             self.wall_list,
                                                             GRAVITY)
        # less gravity map
        if self.level == 3:
            self.physics_engine = arcade.PhysicsEnginePlatformer(self.player_sprite,
                                                                 self.wall_list,
                                                                 0.1)

    def draw_game_over(self):
        output1 = "Game Over"
        output2 = "Click to restart game"
        output3 = "Congratulations!"
        output4 = "Click to start again"

        if self.level == LEVELS and self.player_sprite.right >= self.end_of_map:
            arcade.draw_text(output3, 300 + self.view_left, 300 + self.view_bottom, arcade.color.GREEN, 54)
            arcade.draw_text(output4, 320 + self.view_left, 250 + self.view_bottom, arcade.color.RED, 25)
        else:
            arcade.draw_text(output1, 300 + self.view_left, 300 + self.view_bottom, arcade.color.WHITE, 54)
            arcade.draw_text(output2, 320 + self.view_left, 250 + self.view_bottom, arcade.color.RED, 25)

    def on_mouse_press(self, x, y, button, modifiers):
        # move to the first level after successful completion of all levels
        if self.game_over is True and self.level == 3 and self.player_sprite.right >= self.end_of_map:
            self.level = 1
            self.setup(self.level)
            self.game_over = False
        # restart the existing level
        if self.game_over is True:
            self.setup(self.level)
            self.game_over = False

    def draw_game(self):
        """ Render the screen. """

        arcade.start_render()

        # Draw our sprites
        self.wall_list.draw()
        self.background_list.draw()
        self.coin_list.draw()
        self.dont_touch_list.draw()
        self.player_list.draw()
        self.enemy_list.draw()
        self.foreground_list.draw()

        # Draw our score on the screen, scrolling it with the viewport
        score_text = f"Score: {self.score}"
        arcade.draw_text(score_text, 10 + self.view_left, 10 + self.view_bottom,
                         arcade.csscolor.BLACK, 18)

    def on_draw(self):
        # render the screen
        arcade.start_render()

        if not self.game_over:
            self.draw_game()

        else:
            self.draw_game()
            self.draw_game_over()

    def on_key_press(self, key, modifiers):
        """Called whenever a key is pressed. """

        if key == arcade.key.UP or key == arcade.key.W:
            if self.physics_engine.can_jump() and self.level != 3:
                self.player_sprite.change_y = PLAYER_JUMP_SPEED
                arcade.play_sound(self.jump_sound)
            else:
                self.player_sprite.change_y = PLAYER_JUMP_SPEED / 5
                arcade.play_sound(self.jump_sound)
        elif key == arcade.key.LEFT or key == arcade.key.A:
            self.player_sprite.change_x = -PLAYER_MOVEMENT_SPEED
        elif key == arcade.key.RIGHT or key == arcade.key.D:
            self.player_sprite.change_x = PLAYER_MOVEMENT_SPEED

    def on_key_release(self, key, modifiers):
        """Called when the user releases a key. """

        if key == arcade.key.LEFT or key == arcade.key.A:
            self.player_sprite.change_x = 0
        elif key == arcade.key.RIGHT or key == arcade.key.D:
            self.player_sprite.change_x = 0

    def update(self, delta_time):

        # game logic
        if not self.game_over:

            self.physics_engine.update()

            # move enemies
            self.enemy_list.update()

            # Track if we need to change the viewport
            changed_viewport = False

            # See if we hit any coins
            coin_hit_list = arcade.check_for_collision_with_list(self.player_sprite,
                                                                 self.coin_list)
            # Loop through each coin we hit (if any) and remove it
            for coin in coin_hit_list:
                # Remove the coin
                coin.remove_from_sprite_lists()
                # Play a sound
                arcade.play_sound(self.collect_coin_sound)
                # Add one to the score
                self.score += 1

            if self.level != 3:
                for enemy in self.enemy_list:
                    # check if enemy hit a wall, if yes then reverse
                    if len(arcade.check_for_collision_with_list(enemy, self.wall_list)) > 0:
                        enemy.change_x *= -1
                    # check if enemy hit left boundary, reverse
                    elif enemy.boundary_left is not None and enemy.left < enemy.boundary_left:
                        enemy.change_x *= -1
                    # check for right boundary
                    elif enemy.boundary_right is not None and enemy.right > enemy.boundary_right:
                        enemy.change_x *= -1
            else:
                for enemy in self.enemy_list:
                    # check if enemy hit a wall, if yes then reverse
                    if len(arcade.check_for_collision_with_list(enemy, self.dont_touch_list)) > 0:
                        enemy.change_x *= -1
                    # check if enemy hit left boundary, reverse
                    elif enemy.boundary_left is not None and enemy.left < enemy.boundary_left:
                        enemy.change_x *= -1
                    # check for right boundary
                    elif enemy.boundary_right is not None and enemy.right > enemy.boundary_right:
                        enemy.change_x *= -1

            # check if player touched a enemy
            if len(arcade.check_for_collision_with_list(self.player_sprite, self.enemy_list)) > 0:
                # Set the camera to the start
                self.game_over = True
                self.view_left = 0
                self.view_bottom = 0
                arcade.play_sound(self.enemy_hit)

            # Did the player fall off the map?
            if self.player_sprite.center_y < -100 and self.level != 3:
                self.player_sprite.center_x = PLAYER_START_X
                self.player_sprite.center_y = PLAYER_START_Y

                # Set the camera to the start
                self.view_left = 0
                self.view_bottom = 0
                changed_viewport = True
                arcade.play_sound(self.fall)

            # Did the player touch something they should not?
            if arcade.check_for_collision_with_list(self.player_sprite, self.dont_touch_list):
                self.player_sprite.center_x = PLAYER_START_X
                self.player_sprite.center_y = PLAYER_START_Y

                # Set the camera to the start
                self.view_left = 0
                self.view_bottom = 0
                changed_viewport = True
                arcade.play_sound(self.dont_touch)

            # # See if the user got to the end of the level and end of game
            if self.player_sprite.right >= self.end_of_map and self.level < LEVELS:
                # if end of current map, change map
                self.level += 1
                self.setup(self.level)

                # Set the camera to the start
                self.view_left = 0
                self.view_bottom = 0
                changed_viewport = True
            elif self.player_sprite.right >= self.end_of_map and self.level == LEVELS:
                self.game_over = True
                changed_viewport = False
                arcade.play_sound(self.gameover)

            # --- Manage Scrolling ---

            # Scroll left

            left_boundary = self.view_left + LEFT_VIEWPORT_MARGIN
            if self.player_sprite.left < left_boundary:
                self.view_left -= left_boundary - self.player_sprite.left
                changed_viewport = True

            right_boundary = self.view_left + SCREEN_WIDTH - RIGHT_VIEWPORT_MARGIN
            if self.player_sprite.right > right_boundary:
                self.view_left += self.player_sprite.right - right_boundary
                changed_viewport = True

            # Scroll up
            top_boundary = self.view_bottom + SCREEN_HEIGHT - TOP_VIEWPORT_MARGIN
            if self.player_sprite.top > top_boundary:
                self.view_bottom += self.player_sprite.top - top_boundary
                changed_viewport = True

            # Scroll down
            bottom_boundary = self.view_bottom + BOTTOM_VIEWPORT_MARGIN
            if self.player_sprite.bottom < bottom_boundary:
                self.view_bottom -= bottom_boundary - self.player_sprite.bottom
                changed_viewport = True

            if changed_viewport:
                # Only scroll to integers. Otherwise we end up with pixels that
                # don't line up on the screen
                self.view_bottom = int(self.view_bottom)
                self.view_left = int(self.view_left)

            # Do the scrolling
                arcade.set_viewport(self.view_left, SCREEN_WIDTH + self.view_left,
                                self.view_bottom, SCREEN_HEIGHT + self.view_bottom)

def main():
    """ Main method """

    window = MyGame()
    window.setup(window.level)
    arcade.run()

if __name__ == "__main__":
    main()


import git, os, hashlib, logging, io, tempfile, shutil
from django.conf import settings
from django.http import Http404, HttpResponse
from corere.main import models as m
from django.db.models import Max

logger = logging.getLogger(__name__)

# What actually makes something a "helper" here?

def create_manuscript_repo(manuscript):
    repo_path = get_manuscript_repo_path(manuscript)
    repo = git.Repo.init(repo_path, mkdir=True)
    files_path = get_manuscript_files_path(manuscript)
    os.makedirs(files_path, exist_ok=True)
    repo.index.commit("initial commit")

def create_submission_repo(manuscript):
    repo_path = get_submission_repo_path(manuscript)
    repo = git.Repo.init(repo_path, mkdir=True)
    files_path = get_submission_files_path(manuscript)
    os.makedirs(files_path, exist_ok=True)
    repo.index.commit("initial commit")
    # repo.create_head("master")

# #Maybe unused
# def get_manuscript_files_list(manuscript):
#     return _get_files_list(manuscript, get_manuscript_files_path(manuscript))

# #Only used for submission deleteallfiles
# def get_submission_files_list(manuscript):
#     print(get_submission_files_path(manuscript))
#     return _get_files_list(manuscript, get_submission_files_path(manuscript))

# #Only used for get_submission_files_list which is only used for deleteallfiles
# def _get_files_list(manuscript, path):
#     try:
#         print("huh")
#         repo = git.Repo(path+"..")
#         print(repo.__dict__)
#         if 'head' in repo.__dict__: #TODO: This doesn't work currently on the 1st submission because there is no head because we did not make a branch
#             return helper_list_paths(repo.head.commit.tree[0], path, path)
#         else:
#             print("NO COMMITS")
#             yield from () #If no head (e.g. no commits) return empty list
#         #TODO: We may want to handle having a head but no files
#     except git.exc.NoSuchPathError:
#         raise Http404()

# returns md5 of file
def store_manuscript_file(manuscript, file, subdir):
    repo_path = get_manuscript_repo_path(manuscript)
    files_folder = get_manuscript_files_path(manuscript, relative=True)
    return _store_file(repo_path, files_folder + subdir, file)

# returns md5 of file
def store_submission_file(manuscript, file, subdir):
    repo_path = get_submission_repo_path(manuscript)
    files_folder = get_submission_files_path(manuscript, relative=True)
    return _store_file(repo_path, files_folder + subdir, file)

# TODO-REPO: repo_path variable name here is incorrect. Probably other places too

# store file to filesystem, create commit for file
# returns md5 of file
def _store_file(repo_path, subdir, file):
    repo = git.Repo(repo_path)
    full_path = repo_path + subdir

    os.makedirs(full_path, exist_ok=True)
    hash_md5 = hashlib.md5()
    with open(full_path + file.name, "wb+") as destination:
        for chunk in file.chunks():
            destination.write(chunk)
            hash_md5.update(chunk)
    repo.index.add(full_path + file.name)
    repo.index.commit("store file: " + full_path + file.name)
    return hash_md5.hexdigest()

def rename_manuscript_files(manuscript, files_dict_list):
    repo_path = get_manuscript_repo_path(manuscript)
    files_folder = get_manuscript_files_path(manuscript, relative=True)
    return _rename_files(repo_path, files_folder, files_dict_list)

def rename_submission_files(manuscript, files_dict_list):
    repo_path = get_submission_repo_path(manuscript)
    files_folder = get_submission_files_path(manuscript, relative=True)
    return _rename_files(repo_path, files_folder, files_dict_list)

def _rename_files(repo_path, files_folder, files_dict_list):
    repo = git.Repo(repo_path)

    for d in files_dict_list:
        old_path = files_folder + d.get("old")
        new_path = files_folder + d.get("new")

        if os.path.exists(repo_path + new_path):
            logger.warning(
                "Error renaming files, new path already exists. Repo path: "
                + repo_path
                + " . Current file old path: "
                + old_path
                + " . New path: "
                + new_path
                + " ."
            )
            return False
        try:
            os.renames(repo_path + old_path, repo_path + new_path)
            repo.index.add(repo_path + new_path)
            repo.index.remove(repo_path + old_path)
            repo.index.commit("File " + repo_path + old_path + " renamed to " + repo_path + new_path)
        except Exception as e:
            logger.error(
                "Error renaming files. Cause uncertain. Repo path: "
                + repo_path
                + " . Current file old path: "
                + old_path
                + " . New path: "
                + new_path
                + " . Error "
                + str(e)
            )
            return False

        return True

def delete_manuscript_file(manuscript, file_path):
    repo_path = get_manuscript_repo_path(manuscript)
    files_folder = get_manuscript_files_path(manuscript, relative=True)
    # if(not file_path == '/.git'):
    _delete_file(repo_path, repo_path + files_folder + file_path)

    # We recreate our "inside the repo" directory incase our delete recursively deleted it
    files_path = get_manuscript_files_path(manuscript)
    os.makedirs(files_path, exist_ok=True)

def delete_submission_file(manuscript, file_path):
    repo_path = get_submission_repo_path(manuscript)
    files_folder = get_submission_files_path(manuscript, relative=True)
    # if(not file_path == '/.git'):
    _delete_file(repo_path, repo_path + files_folder + file_path)

    # We recreate our "inside the repo" directory incase our delete recursively deleted it
    # TODO: This is likely slow when called many times (e.g. old delete all). If we have performance issues we should only call it once at the end of many deletes
    files_path = get_submission_files_path(manuscript)
    os.makedirs(files_path, exist_ok=True)

def delete_all_submission_files(manuscript):
    submission_files_path = get_submission_files_path(manuscript)
    shutil.rmtree(submission_files_path)
    os.makedirs(submission_files_path)
    repo = git.Repo(get_submission_repo_path(manuscript))
    try:
        repo.index.remove(submission_files_path, r=True)
    except Exception as e:
        if not str(e).endswith("did not match any files'"):
            # We catch the error of there being no files to delete
            raise e
    repo.index.commit("delete all submission files")

# delete file from filesystem, create commit for file
# TODO: This currently will delete the
def _delete_file(repo_path, file_full_path):
    repo = git.Repo(repo_path)
    if repo_path in os.path.realpath(file_full_path):
        if os.path.exists(file_full_path):
            os.remove(file_full_path)
            file_full_folder = file_full_path.rsplit("/", 1)[0]
            try:
                os.removedirs(file_full_folder)  # deletes empty folders recursively. Will never delete repo_path as there is a git folder
            except OSError:
                pass  # If file_full_folder has other files, removedirs will error and fail, as expected
        else:
            logger.error("Attempted to delete file where path did not exist. Path provided: " + file_full_path)
            raise Http404()
    else:
        logger.error("Attempted to delete file above the repo path. Possibly a hack attempt. Path: " + file_full_path)
        raise Http404()

    repo.index.remove(file_full_path)
    repo.index.commit("delete file: " + file_full_path)

def get_manuscript_file(manuscript, file_path, response=False):
    repo_path = get_manuscript_repo_path(manuscript)
    files_folder = get_manuscript_files_path(manuscript, relative=True)
    return _get_file(repo_path, files_folder, file_path, "master", response)

def get_submission_file(submission, file_path, response=False):
    repo_path = get_submission_repo_path(submission.manuscript)
    files_folder = get_submission_files_path(submission.manuscript, relative=True)

    # We have to check whether our submission is the latest submission. If its latest, use master, otherwise use branch name
    max_version_id = submission.manuscript.get_max_submission_version_id()
    if submission.version_id == max_version_id:
        branch_name = "master"
    else:
        branch_name = helper_get_submission_branch_name(submission)
    return _get_file(repo_path, files_folder, file_path, branch_name, response)

def _get_file(repo_path, files_folder, file_path, branch_name, response):
    repo = git.Repo(repo_path)
    branch_commit = repo.commit(branch_name)
    if file_path[0] == "/":
        file_path = file_path[1:]  # [1:] removes leading slash. Could be made more robust
    file_path = files_folder + file_path
    file = branch_commit.tree / file_path

    if response:
        with io.BytesIO(file.data_stream.read()) as f:
            response = HttpResponse(f.read(), content_type=file.mime_type)
            response["Content-Disposition"] = 'attachment; filename="' + file.name + '"'
            return response
    else:
        return file

def download_all_submission_files(submission):
    max_version_id = submission.manuscript.get_max_submission_version_id()
    if submission.version_id == max_version_id:
        branch_name = "master"
    else:
        branch_name = helper_get_submission_branch_name(submission)

    repo_path = get_submission_repo_path(submission.manuscript)
    repo = git.Repo(repo_path)

    tempf = tempfile.TemporaryFile()
    repo.archive(tempf, treeish=branch_name)
    tempf.seek(0)  # you have to return to the start of the temporaryfile after writing to it

    response = HttpResponse(tempf.read(), content_type="application/zip")  # temp.mime_type)
    response["Content-Disposition"] = 'attachment; filename="' + submission.manuscript.slug + "_-_submission_" + str(submission.version_id) + '.zip"'
    return response

def download_all_manuscript_files(manuscript):
    branch_name = "master"
    repo_path = get_manuscript_repo_path(manuscript)
    repo = git.Repo(repo_path)

    tempf = tempfile.TemporaryFile()
    repo.archive(tempf, treeish=branch_name)
    tempf.seek(0)  # you have to return to the start of the temporaryfile after writing to it

    response = HttpResponse(tempf.read(), content_type="application/zip")  # temp.mime_type)
    response["Content-Disposition"] = 'attachment; filename="' + manuscript.slug + '_-_manuscript.zip"'
    return response

# def get_manuscript_repo_name(manuscript):
#     return str(manuscript.id) + "_-_manuscript_-_" + manuscript.slug

# def get_submission_repo_name(manuscript):
#     return str(manuscript.id) + "_-_submission_-_" + manuscript.slug

### The repo contains a sub-folder containing all the files. This is half to support downloading zips with a root folder

#TODO: Maybe rename these path endpoints to communicate they are getting the system path not the relative path in the code folder
def get_manuscript_repo_path(manuscript):
    return settings.GIT_ROOT + "/" + str(manuscript.id) + "_-_manuscript_-_" + manuscript.slug + "/"

def get_manuscript_files_path(manuscript, relative=False):
    rel_path = manuscript.slug + "_-_manuscript/"
    if relative:
        return rel_path
    else:
        return get_manuscript_repo_path(manuscript) + rel_path

def get_submission_repo_path(manuscript):
    return settings.GIT_ROOT + "/" + str(manuscript.id) + "_-_submission_-_" + manuscript.slug + "/"

def get_submission_files_path(manuscript, relative=False):
    rel_path = manuscript.slug + "_-_submission/"
    if relative:
        return rel_path
    else:
        return get_submission_repo_path(manuscript) + rel_path

def create_submission_branch(submission):
    repo = git.Repo(get_submission_repo_path(submission.manuscript))
    repo.create_head(helper_get_submission_branch_name(submission))

# Lists the paths of all the files. Used only to delete_all currently
# When initially called, repo_path and rel_path should be the same.
# TODO: This actually returns a generator, we should probably name it differently or switch it to a list
# def helper_list_paths(root_tree, repo_path, rel_path):
#     for blob in root_tree.blobs:
#         #Split off the system path from the return.
#         yield (rel_path.split(repo_path, 1)[1] + '/' + blob.name)#[1:] #commented code would remove leading slash
#     for tree in root_tree.trees:
#         yield from (helper_list_paths(tree, repo_path, rel_path + '/' + tree.name)) #recursive

# Note: Submission branches are only created when the submission is completed. This will not tell you whether your submission has been completed.
def helper_get_submission_branch_name(submission):
    return "Submission_" + str(submission.version_id)


import os,json,sys,re,string

class nblearn(object):
    def __init__(self, path):
        self.path=path
        self.vocab={}
        self.stop_words=[]
        self.spamtokencount=0
        self.hamtokencount=0
        self.spamfilecount=0
        self.hamfilecount=0

    def generatemodal(self):
        for root, dirs, files in os.walk(self.path):
            if root.endswith('/spam'):
                self.updatedict(root,files,0)
            elif root.endswith('/ham'):
                self.updatedict(root,files,1)

    def updatedict(self,root,files,sh):
        for curfile in files:
            if curfile.endswith(".txt"):
                if sh:
                    self.hamfilecount+=1
                else:
                    self.spamfilecount+=1
                rfile=open(os.path.join(root, curfile), "r", encoding='latin1')
                for data in rfile:
                    contentlist=data.strip().split()
                    for token in contentlist:
                        token=re.sub('[%s]' % re.escape(string.punctuation), '', token)
                        if token not in self.stop_words:
                            if sh:
                                self.hamtokencount+=1
                            else:
                                self.spamtokencount+=1
                            if token in self.vocab:
                                self.vocab[token][sh]+=1
                            else:
                                if sh:
                                    self.vocab[token]=[0,1]
                                else:
                                    self.vocab[token]=[1,0]

    def writemodal(self):
        with open('nbmodal.txt', 'w') as wfile:
            json.dump({'spamfilecount':self.spamfilecount,'spamcount':self.spamtokencount,'hamfilecount':self.hamfilecount,'hamcount':self.hamtokencount,'data':self.vocab}, wfile)

    def load_stop_words(self):
        with open('stop_words.txt') as data_file:
            data = json.load(data_file)
        self.stop_words=data['stop_words']

def main():
    mydir = sys.argv[1]
    learnmodal=nblearn(mydir)
    learnmodal.load_stop_words()
    learnmodal.generatemodal()
    learnmodal.writemodal()

if __name__ == '__main__':
    main()