from typing import Tuple, Any, Union
import functools
import os
import tqdm

import numpy as np

import environments.environment

class Tabular:
    def __init__(self, transition_matrix: np.ndarray, reward_matrix: np.ndarray, terminal_states: np.ndarray,
                 initial_states: np.ndarray):
        self.num_states = transition_matrix.shape[1]
        self.num_actions = transition_matrix.shape[0] // self.num_states
        self.transition_matrix = transition_matrix
        self._reward_matrix = reward_matrix
        self.terminal_states = terminal_states.astype(bool)
        self.initial_states = initial_states

    def sample_initial(self) -> int:
        return np.random.choice(self.num_states, p=self.initial_states)

    def transition(self, state: int, action: int) -> int:
        return np.random.choice(self.num_states, p=self.transition_matrix[self.sa_index(state, action)])

    def reward(self, state: int, action: int, next_state: int) -> float:
        return self._reward_matrix[self.sa_index(state, action), next_state]

    def is_terminal(self, state: int) -> bool:
        return self.terminal_states[state]

    def _wrap_around_transition_matrix(self) -> np.ndarray:
        wrap_around_matrix = self.transition_matrix.copy()
        for state in range(self.num_states):
            if self.is_terminal(state):
                for action in range(self.num_actions):
                    sa = self.sa_index(state, action)
                    wrap_around_matrix[sa, :] = self.initial_states
        return wrap_around_matrix

    def deterministic_policy_transition_matrix(self, deterministic_policy: np.ndarray,
                                               transition_matrix: np.ndarray) -> np.ndarray:
        policy = np.zeros((self.num_states, self.num_states * self.num_actions))
        for state in range(self.num_states):
            policy[state, self.sa_index(state, deterministic_policy[state])] = 1
        return self.policy_transition_matrix(policy, transition_matrix)

    @functools.lru_cache()
    def reward_vector(self) -> np.ndarray:
        rewards = np.zeros((self.num_states * self.num_actions,))
        for state in range(self.num_states):
            for action in range(self.num_actions):
                rewards[state * self.num_actions + action] = \
                    self.transition_matrix[state * self.num_actions + action, :] @ \
                    self._reward_matrix[state * self.num_actions + action, :]
        return rewards

    @staticmethod
    def policy_transition_matrix(policy: np.ndarray, transition_matrix: np.ndarray) -> np.ndarray:
        return policy @ transition_matrix

    def stationary_state_distribution(self, policy: np.ndarray) -> np.ndarray:
        transition_matrix = self.policy_transition_matrix(policy, self._wrap_around_transition_matrix())
        values, vectors = np.linalg.eig(transition_matrix.T)
        for value, vector in zip(values, vectors.T):
            # noinspection PyTypeChecker
            if np.isclose(value, 1):
                if np.max(vector) <= 0:
                    vector *= -1
                if np.min(vector) < 0:
                    vector -= np.min(vector)
                return np.real(vector / np.sum(vector))
        assert False

    def reverse_transition_matrix(self, policy: np.ndarray, stationary_distribution: np.ndarray) -> np.ndarray:
        transition_matrix = self._wrap_around_transition_matrix()

        return transition_matrix.T * \
               (stationary_distribution @ policy)[np.newaxis, :] / stationary_distribution[:, np.newaxis]

    def reverse_state_transition_matrix(self, policy: np.ndarray, stationary_distribution: np.ndarray) -> np.ndarray:
        transition_matrix = policy @ self._wrap_around_transition_matrix()
        return transition_matrix.T * stationary_distribution[np.newaxis, :] / stationary_distribution[:, np.newaxis]

    def log_stationary_derivative(self, policy: np.ndarray, log_policy_derivative: np.ndarray,
                                  stationary_distribution: np.ndarray) -> np.ndarray:
        """
        :param policy: S x SA
        :param log_policy_derivative: SA X parameter_dims
        :param stationary_distribution: S
        :return: S X parameter_dims
        """
        parameter_dims = log_policy_derivative.shape[1:]
        reverse_transitions = self.reverse_transition_matrix(policy, stationary_distribution)
        reverse_state_transitions = sum(
            [reverse_transitions[:, np.arange(0, self.num_actions * self.num_states, self.num_actions) + i]
             for i in range(self.num_actions)]
        )
        # should be solve but is sometimes singular. Probably for numerical reasons?
        lsd = np.linalg.lstsq(np.eye(self.num_states) - reverse_state_transitions,
                              reverse_transitions @ np.reshape(log_policy_derivative,
                                                               newshape=[log_policy_derivative.shape[0], -1]))[0]
        lsd -= np.sum(lsd * stationary_distribution[:, np.newaxis], axis=0, keepdims=True)
        return np.reshape(lsd, newshape=(lsd.shape[0],) + parameter_dims)

    def sa_index(self, state: int, action: int) -> int:
        return self.num_actions * state + action  # don't change

    def sa_from_index(self, index: int) -> Tuple[int, int]:
        return index // self.num_actions, index % self.num_actions

def probabilistic_policy_selector(policy: np.ndarray) -> np.ndarray:
    """
    :param policy: |S|x|A|
    :return: |S|x|S||A|
    """
    selector = np.zeros((policy.shape[0], policy.shape[0] * policy.shape[1]))
    for state in range(policy.shape[0]):
        selector[state, state*policy.shape[1]:(state+1)*policy.shape[1]] = policy[state, :]
    return selector

def evaluate_average_reward_irreducible(policy_selector: np.ndarray, tab: Tabular) -> Tuple[float, np.ndarray]:
    """
    :param policy_selector: |S|x|S||A|
    :return: |S|
    """
    transition_matrix = policy_selector @ tab.transition_matrix
    policy_rewards = policy_selector @ tab.reward_vector()
    long_term_probabilities: np.ndarray = transition_matrix - np.eye(transition_matrix.shape[0])
    long_term_probabilities[:, 0] = -1
    variables = np.linalg.solve(long_term_probabilities, policy_rewards)
    average_reward = variables[0]
    relative_value = np.concatenate((0, variables[1:]))
    return relative_value

def evaluate_probabilistic(policy_selector: np.ndarray, tab: Tabular, discount_factor: float) -> np.ndarray:
    """
    :param policy_selector: |S|x|S||A|
    :return: |S|
    """
    transition_matrix = policy_selector @ tab.transition_matrix
    policy_rewards = policy_selector @ tab.reward_vector()
    long_term_probabilities: np.ndarray = np.eye(tab.num_states) - discount_factor * transition_matrix
    return np.linalg.solve(long_term_probabilities, policy_rewards)

def evaluate_deterministic(policy: np.ndarray, tab: Tabular, discount_factor: float) -> np.ndarray:
    """
    :param policy: |S|
    :return: |S|
    """
    transition_matrix = tab.transition_matrix.reshape(tab.num_states, tab.num_actions,
                                                      tab.num_states)[np.arange(tab.num_states), policy, :]
    policy_rewards = tab.reward_vector().reshape(-1, tab.num_actions)[np.arange(tab.num_states), policy]
    long_term_probabilities: np.ndarray = np.eye(tab.num_states) - discount_factor * transition_matrix
    return np.linalg.solve(long_term_probabilities, policy_rewards)

def policy_iteration(tab: Tabular, discount_factor: float) -> np.ndarray:
    policy = np.zeros((tab.num_states,), dtype=np.int32)
    new_policy = np.argmax(tab.reward_vector().reshape((-1, tab.num_actions)), axis=1)
    progress = tqdm.tqdm()
    while np.any(policy != new_policy):
        policy = new_policy.copy()
        evaluate_deterministic(policy, tab, discount_factor)
        # noinspection PyUnresolvedReferences
        q_values = rewards + discount_factor * tab.transition_matrix @ values
        new_policy = np.argmax(q_values.reshape((-1, tab.num_actions)), axis=1)
        progress.update()
    return policy

def value_iteration(tab: Tabular, discount_factor: float, eps: float = 1e-8) -> np.ndarray:
    rewards = tab.reward_vector()
    q_values = rewards.copy()
    values = np.max(q_values.reshape((-1, tab.num_actions)), axis=1)
    new_q_values = np.zeros((tab.num_states * tab.num_actions,))
    diff = eps + 1
    trange = tqdm.tqdm()
    while diff > eps:
        q_values = new_q_values
        np.argmax(q_values.reshape((-1, tab.num_actions)), axis=1)
        new_q_values = rewards + discount_factor * (1 - tab.terminal_states) * tab.transition_matrix @ values
        values = np.max(new_q_values.reshape((-1, tab.num_actions)), axis=1)
        diff = np.max((q_values - new_q_values) ** 2)
        trange.update()
        trange.set_description(f"diff: {diff}")
    return np.argmax(q_values.reshape((-1, tab.num_actions)), axis=1)

def state_reward_to_matrix(reward_vector: np.ndarray, num_actions: int) -> np.ndarray:
    return np.repeat(np.atleast_2d(reward_vector), len(reward_vector) * num_actions, axis=0)

class TabularEnv:
    def __init__(self, tabular: Tabular):
        self._tabular = tabular

        self._state = self.reset()

    def reset(self) -> int:
        self._state = self._tabular.sample_initial()
        return self._state

    def step(self, action: int) -> Tuple[int, float, bool, Any]:
        next_state = self._tabular.transition(self._state, action)
        reward = self._tabular.reward(self._state, action, next_state)
        self._state = next_state
        return self._state, reward, self._tabular.is_terminal(self._state), None

    @property
    def num_states(self) -> int:
        return self._tabular.num_states

    @property
    def num_actions(self) -> int:
        return self._tabular.num_actions

class OneHotEnv(environments.environment.Environment[int]):
    def __init__(self, env: TabularEnv):
        self._env = env
        self._eye = np.eye(self._env.num_states)

    @classmethod
    def from_tabular(cls, tabular: Tabular):
        return cls(TabularEnv(tabular))

    def reset(self) -> np.ndarray:
        return self._eye[self._env.reset()]

    def step(self, action) -> Tuple[np.ndarray, float, bool, Any]:
        state, reward, is_terminal, info = self._env.step(action)
        return self._eye[state], reward, is_terminal, info

G = 1
S = 2
W = 3

class Gridworld:
    LEFT = 0
    RIGHT = 1
    UP = 2
    DOWN = 3

    def state_index(self, x: int, y: int) -> int:
        return self.width * y + x

    def _pos(self, state_index: int) -> Tuple[int, int]:
        return state_index % self.width, state_index // self.width

    def _consolidate_pos(self, transition_grid, new, original):
        if self.width > new[0] >= 0 and self.height > new[1] >= 0 and transition_grid[new] != W:
            return new
        else:
            return original

    def __init__(self, transition_grid: np.ndarray, reward_grid: np.ndarray, transition_noise: float = 0):
        self.height = transition_grid.shape[0]
        self.width = transition_grid.shape[1]
        self.num_states = self.width * self.height
        self.num_actions = 4

        # Reward matrix
        reward_vector = np.zeros((self.num_states,))
        terminal_vector = np.zeros((self.num_states,))
        initial_vector = np.zeros((self.num_states,))
        for state in range(self.num_states):
            reward_vector[state] = reward_grid[self._pos(state)]
            terminal_vector[state] = transition_grid[self._pos(state)] == G
            initial_vector[state] = transition_grid[self._pos(state)] == S
        reward_matrix = state_reward_to_matrix(reward_vector, self.num_actions)
        initial_vector /= np.sum(initial_vector)

        # Transition matrix
        transition_matrix = np.zeros((self.num_states * self.num_actions, self.num_states))
        for state in range(self.num_states):
            if terminal_vector[state]:
                transition_matrix[state * self.num_actions: (state+1) * self.num_actions, :] = 0
            else:
                pos = self._pos(state)
                left = self.state_index(*self._consolidate_pos(transition_grid, (pos[0] - 1, pos[1]), pos))
                right = self.state_index(*self._consolidate_pos(transition_grid, (pos[0] + 1, pos[1]), pos))
                up = self.state_index(*self._consolidate_pos(transition_grid, (pos[0], pos[1] - 1), pos))
                down = self.state_index(*self._consolidate_pos(transition_grid, (pos[0], pos[1] + 1), pos))

                transition_matrix[state * self.num_actions + self.LEFT, left] += 1 - transition_noise
                transition_matrix[state * self.num_actions + self.LEFT, up] += transition_noise / 2
                transition_matrix[state * self.num_actions + self.LEFT, down] += transition_noise / 2

                transition_matrix[state * self.num_actions + self.RIGHT, right] += 1 - transition_noise
                transition_matrix[state * self.num_actions + self.RIGHT, up] += transition_noise / 2
                transition_matrix[state * self.num_actions + self.RIGHT, down] += transition_noise / 2

                transition_matrix[state * self.num_actions + self.UP, up] += 1 - transition_noise
                transition_matrix[state * self.num_actions + self.UP, left] += transition_noise / 2
                transition_matrix[state * self.num_actions + self.UP, right] += transition_noise / 2

                transition_matrix[state * self.num_actions + self.DOWN, down] += 1 - transition_noise
                transition_matrix[state * self.num_actions + self.DOWN, left] += transition_noise / 2
                transition_matrix[state * self.num_actions + self.DOWN, right] += transition_noise / 2

        self.tabular = Tabular(transition_matrix, reward_matrix, terminal_vector, initial_vector)

    def tabular_env(self):
        return TabularEnv(self.tabular)

    def one_hot_env(self):
        return OneHotEnv.from_tabular(self.tabular)

    def state_repr(self, state: Union[int, np.ndarray]) -> Tuple[int, int]:
        if type(state) is int:
            return self._pos(state)
        else:
            return self._pos(np.asscalar(np.argmax(state)))

class Racetrack:
    width = 33
    height = 9
    num_actions = 5

    @property
    def num_states(self):
        return self.width * self.height * 25 + 1

    def _transition_probabilities(self, state, action):
        assert 0 <= action <= 4

        next_x = state[0] + state[2]
        next_y = state[1] + state[3]
        next_x = min(self.width - 1, max(0, next_x))
        next_y = min(self.height - 1, max(0, next_y))

        if action == 4:
            success_probability = 1
        elif max(abs(state[2]), abs(state[3])) == 2:
            success_probability = 0.2
        else:
            success_probability = 0.9

        next_dx = state[2]
        next_dy = state[3]

        if action == 0:
            next_dx = state[2] + 1
        elif action == 1:
            next_dx = state[2] - 1
        elif action == 2:
            next_dy = state[3] + 1
        elif action == 3:
            next_dy = state[3] - 1

        next_dx = min(2, max(-2, next_dx))
        next_dy = min(2, max(-2, next_dy))

        possible_transitions = [(1 - success_probability, [next_x, next_y, state[2], state[3]]),
                                (success_probability, [next_x, next_y, next_dx, next_dy])]
        return possible_transitions

    def _state_from_index(self, state: int) -> Tuple[int, int, int, int]:
        dy = state % 5 - 2
        state //= 5
        dx = state % 5 - 2
        state //= 5
        x = state % self.width
        y = state // self.width
        return x, y, dx, dy

    def _state_index(self, state: Tuple[int, int, int, int]) -> int:
        x, y, dx, dy = state
        return dy + 2 + 5 * (dx + 2 + 5 * (x + self.width * y))

    @functools.lru_cache(None)
    def transition_probabilities(self, state, action):
        possible_transitions = self._transition_probabilities(self._state_from_index(state), action)
        probability_vector = np.zeros((self.num_states,))
        for p, next_state in possible_transitions:
            probability_vector[self._state_index(next_state)] += p
        return probability_vector

    def transition_matrix(self, reward_matrix: np.ndarray) -> np.ndarray:
        matrix = np.zeros((self.num_states * self.num_actions, self.num_states))
        for state in range(self.num_states):
            for action in range(self.num_actions):
                matrix[state * self.num_actions + action, :] = self.transition_probabilities(state, action)
        indices: np.ndarray = reward_matrix[:-self.num_actions, :] == 5
        indices = np.argwhere(indices)
        for sa, next_state in indices:
            if next_state < self.num_states - 1:
                matrix[sa, -1] += matrix[sa, next_state]
                matrix[sa, next_state] = 0
        for action in range(self.num_actions):
            matrix[(self.num_states - 1) * self.num_actions + action, -1] = 1
        return matrix

    def _reward(self, state: Tuple[int, int, int, int], _: int, next_state: Tuple[int, int, int, int]):
        y = state[1]

        # Finish line
        reward = 0.
        if next_state[0] < 3:
            if y <= 5 and next_state[1] >= 6:
                reward = 5
            elif y >= 6 and next_state[1] <= 5:
                reward = -5

        if not (0 < next_state[0] < self.width - 2 and 0 < next_state[1] < self.height - 2):
            reward = min(reward, -0.1)

        if 3 <= next_state[0] < self.width - 3 and 3 <= next_state[1] < self.height - 3:
            reward = min(reward, -0.1)
        return reward

    def reward_matrix(self):
        matrix = np.zeros((self.num_states * self.num_actions, self.num_states))
        for state in range(self.num_states):
            for next_state in range(self.num_states):
                r = self._reward(self._state_from_index(state), 0, self._state_from_index(next_state))
                for action in range(self.num_actions):
                    matrix[state * self.num_actions + action, next_state] = r
        matrix[:-self.num_actions, -1] = 5
        matrix[-self.num_actions:, :] = 0

        return matrix

    def __init__(self):
        transition_matrix_file = 'transition_matrix_racetrack.npy'
        reward_matrix_file = 'reward_matrix_racetrack.npy'
        if not os.path.isfile(reward_matrix_file):
            print("Creating reward matrix")
            reward_matrix = self.reward_matrix()
            np.save(reward_matrix_file, reward_matrix)
        else:
            reward_matrix = np.load(reward_matrix_file)
        if not os.path.isfile(transition_matrix_file):
            print("Creating transition matrix")
            transition_matrix = self.transition_matrix(reward_matrix)
            np.save(transition_matrix_file, transition_matrix)
        else:
            transition_matrix = np.load(transition_matrix_file)

        terminal_states = np.zeros((self.num_states,))
        terminal_states[-1] = True

        initial_states = np.zeros((self.num_states,))
        initial_states[self._state_index((1, 6, 0, 0))] = 1

        self.tabular = Tabular(transition_matrix, reward_matrix, terminal_states, initial_states)

    def tabular_env(self):
        return TabularEnv(self.tabular)

    def one_hot_env(self):
        return OneHotEnv.from_tabular(self.tabular)

    def optimal_policy(self, discount_factor: float) -> np.ndarray:
        policy_file = f'racetrack_policy_{discount_factor}.npy'
        if os.path.isfile(policy_file):
            return np.load(policy_file)
        else:
            policy = value_iteration(self.tabular, 0.99)
            np.save(policy_file, policy)
            return policy

def noisy_simple_grid1(transition_noise: float) -> Gridworld:
    return Gridworld(
        np.array(
            [[0, 0, 0, 0, G],
             [0, 0, 0, 0, 0],
             [0, 0, 0, 0, 0],
             [0, 0, 0, 0, 0],
             [S, 0, 0, 0, 0]]
        ).T,
        np.array(
            [[0, 0, 0, 0, 1],
             [0, 0, 0, 0, 0],
             [0, 0, 0, 0, 0],
             [0, 0, 0, 0, 0],
             [0, 0, 0, 0, 0]]
        ).T,
        transition_noise
    )

simple_grid1 = noisy_simple_grid1(0)

simple_grid2 = Gridworld(
    np.array(
        [[0, 0, 0, 0, 0, 0, 0, 0, 0, G],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         [S, 0, 0, 0, 0, 0, 0, 0, 0, 0]]
    ).T,
    np.array(
        [[0, 0, 0, 0, 0, 0, 0, 0, 0, 1],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]
    ).T
)

def _run():
    env = simple_grid1.tabular_env()
    policy = value_iteration(simple_grid1.tabular, 0.99)
    print(policy)
    state = env.reset()
    is_terminal = False
    while not is_terminal:
        print(type(state))
        print(simple_grid1.state_repr(state))
        state, reward, is_terminal, _ = env.step(policy[state])
    print(simple_grid1.state_repr(state))

def _solve_racetrack():
    racetrack = Racetrack()
    policy = value_iteration(racetrack.tabular, 0.99)
    np.save('racetrack_policy_0.99.npy', policy)

def _test_racetrack():
    policy = np.load('racetrack_policy_0.99.npy')
    racetrack = Racetrack()
    env = racetrack.tabular_env()
    state = env.reset()
    is_terminal = False
    score = 0
    while not is_terminal:
        state, reward, is_terminal, _ = env.step(policy[state])
        print(reward)
        score += reward
    print("===")
    print(score)

if __name__ == '__main__':
    _test_racetrack()


from django import  forms
from apps.Administrador.models import Usuario,Cuenta,Pais,Artista,Genero,Album,Cancion,AlbCan,Archivo
from django.forms import formset_factory
class RegistrarUsuarioForm(forms.ModelForm):
    class Meta:
        model=Usuario
        fields=[
            'nombre',
            'apellido',
            'contrasena',
            'correo',
            'telefono',
            'fotografia',
            'genero',
            'fecha_nacimiento',
            'direccion',
            'pais',

        ]
        labels={
            'nombre':'Ingrese su nombre',
            'apellido':'Ingreses sus apellidos',
            'contrasena':'Ingrese su contrasena',
            'correo':'Ingrese su direccion de correo electronico',
            'telefono':'Ingrese su numero de telefono',
            'genero':'Ingrese su genero 1-F / 2-M',
            'fecha_nacimiento':'Ingrese su fecha de nacimiento',
            'direccion':'Ingrese su direccion de vivienda',
            'pais':'Ingrese su nacionalidad',
        }

        widgets={

            'nombre':forms.TextInput(attrs={'class':'form-control'}),
            'apellido': forms.TextInput(attrs={'class':'form-control'}),
            'contrasena': forms.TextInput(attrs={'class':'form-control'}),
            'correo': forms.EmailInput(attrs={'class':'form-control'}),
            'telefono': forms.NumberInput(attrs={'class':'form-control'}),
            #'fotografia': forms.FileInput(attrs={'class':'form-control'}),
#            'genero': forms.NumberInput(attrs={'class':'form-control'}),
            'fecha_nacimiento': forms.TextInput(attrs={'class':'form-control'}),
            #'fecha_registro': forms.TextInput(attrs={'class':'form-control'}),
            'direccion': forms.TextInput(attrs={'class':'form-control'}),
            'pais':forms.TextInput(attrs={'class':'form-control'}),

        }

class Congelar(forms.ModelForm):

    nombre=forms.ModelChoiceField(Cuenta.objects.filter(activa='A'))

    Comentario = forms.CharField(widget=forms.Textarea())
    class Meta:
        model=Usuario
        fields=[
            'nombre',

        ]
        labels={
            'nombre':'Seleccione el usuario para dar congelacion',
            'comentario':'Ingrese motivos',
        }
        widgets={
            'nombre':forms.Select(attrs={'class':'form-control'}),
            'comentario':forms.Textarea(attrs={'class':'form-control'}),

        }

class Alta(forms.ModelForm):

    nombre=forms.ModelChoiceField(Cuenta.objects.filter(activa='C'))

    Comentario = forms.CharField(widget=forms.Textarea())
    class Meta:
        model=Usuario
        fields=[
            'nombre',

        ]
        labels={
            'nombre':'Seleccione el usuario para dar de alta',
            'comentario':'Ingrese motivos',
        }
        widgets={
            'nombre':forms.Select(attrs={'class':'form-control'}),
            'comentario':forms.Textarea(attrs={'class':'form-control'}),

        }

class Eliminar(forms.ModelForm):
    nombre = forms.ModelChoiceField(Cuenta.objects.filter())

    Comentario = forms.CharField(widget=forms.Textarea())

    class Meta:
        model = Usuario
        fields = [
            'nombre',

        ]
        labels = {
            'nombre': 'Seleccione el usuario para dar de alta',
            'comentario': 'Ingrese motivos',
        }
        widgets = {
            'nombre': forms.Select(attrs={'class': 'form-control'}),
            'comentario': forms.Textarea(attrs={'class': 'form-control'}),

        }
class pais(forms.ModelForm):
    class Meta:
        model=Pais
        fields=[
            'nombre'
        ]
        labels={
            'nombre':'Ingrese el nombre del pais'

        }
        widgets={
            'nombre':forms.TextInput(attrs={'class':'form-control'}),
        }

class artista(forms.ModelForm):
    nacionalidad=forms.ModelChoiceField(Pais.objects.all())
    class Meta:
        model=Artista
        fields=[
            'nombre',
            'nacionalidad',
            'fecha_nacimiento',
            'foto_artista',

        ]
        labels={
            'nombre':'Ingrese el nombre del artista',
            'nacionalidad':'Seleccione su nacionalidad',
            'fecha_nacimiento':'Ingrese su fecha de nacimiento',
            'foto' :'Seleccione fotografia del artista'

        }
        widgets={
            'nombre': forms.TextInput(attrs={'class': 'form-control'}),
            'fecha_nacimiento':forms.DateInput(format=['%m/%d/%y']),

        }
class genero(forms.ModelForm):
    class Meta:
        model=Genero
        fields=[
            'genero'
        ]
        labels={
            'genero':'Ingrese el genero '
        }
        widgets={
            'genero':forms.TextInput(attrs={'class':'form-control'}),
        }
class album(forms.ModelForm):
    id_artista=forms.ModelChoiceField(Artista.objects.all())
    id_gener=forms.ModelChoiceField(Genero.objects.all())
    class Meta:
        model=Album
        fields=[
            'nombre',
            'fecha',
            'id_artista',
            'id_gener',
            'foto_album',
        ]
        labels={
            'nombre':'Ingrese nombre del album',
            'fecha':'Ingrese la fecha de lanzamiento',
            'id_artista':'Seleccione artista',
            'id_gener':'Seleccione Genero del album',
            'foto_album':'Seleccione foto de portada del album'
        }
        widgets={
            'nombre': forms.TextInput(attrs={'class': 'form-control'}),
            'fecha': forms.DateInput(format=['%m/%d/%y$']),

        }
class cancion(forms.ModelForm):
    id_artista = forms.ModelChoiceField(Artista.objects.all())
    id_genero = forms.ModelChoiceField(Genero.objects.all())
    class Meta:
        model=Cancion
        fields={
            'nombre',
            'fecha_lanzamiento',
            'id_genero',
            'id_artista',
            'path_cancion',
        }
        labels={
            'nombre':'Ingrese el nombre de la cancion',
            'fecha_lanzamiento':'Ingrese fecha de lanzamiento',
            'id_genero':'Ingrese genero de la cancion',
            'id_artista':'Seleccione artista ',
            'path_cancion':'Seleccione archivo de la cancion'
        }
        widgets={
            'nombre': forms.TextInput(attrs={'class': 'form-control'}),
            'fecha_lanzamiento': forms.DateInput(format=['%m/%d/%y$']),
        }

class alb_can(forms.ModelForm):
    id_album=forms.ModelChoiceField(Album.objects.all())

    class Meta:
        model=AlbCan
        fields=[
            'id_album',
            'id_cancion',
        ]
        labels={
            'id_album':'Seleccione que album se le ingresara las canciones ',
            'id_cancion':'Selecciones las canciones',

        }
        widgets={
            'id_cancion':forms.CheckboxSelectMultiple()
        }

class artistaver(forms.ModelForm):
    id_artista=forms.ModelChoiceField(Artista.objects.all())
    class Meta:
        model=Artista
        fields=[
            'id_artista'
        ]

class formCargarArchivo(forms.ModelForm):
    class Meta:
        model = Archivo
        fields=[
            'nombre'
        ]
        labels={
            'nombre':'Seleccione archivo a subir'
        }


from datetime import date, datetime
from django.utils import timezone
from django.db.models import Sum

def kicukiro_ll_fees():
	return Fee.objects.filter(category__code='land_lease', status__code='active', prop__village__cell__sector__district__name__iexact='Kicukiro')

def adjust_payments(payments, date_from=None):
	fees = {}
	receipts = {}

	payments = payments.filter(status__code='active').order_by('paid_date').order_by('id')

	balance = 0
	for payment in payments:

		if not payment.receipt or not payment.paid_date:
			continue

		if date_from and date_from < payment.paid_date:
			balance = payment.bf
			continue

		payment.bf = balance

		fee = fees.get(payment.fee.pk)
		if not fee: # new fee payment
			fee = payment.fee
			fee.penalty = 0
			fee.penalty_paid = 0
			fee.interest_paid = 0
			fee.principle_paid = 0
			fee.residual_interest = 0
			fee.remaining_amount = fee.amount
			fees[fee.pk] = fee

		receipt = receipts.get(payment.pk)
		if not receipt:
			receipt = payment.receipt
			receipts[payment.pk] = receipt
			balance = balance + int(receipt.amount)

		payment.interest = 0
		payment.penalty = 0
		payment.principle = 0

		#calculate amounts owed before payment
		calc_penalty, calc_interest = fee.calc_penalty(payment.paid_date, fee.remaining_amount)
		if not fee.penalty:
			fee.penalty = calc_penalty

		payment.interest_due = calc_interest + fee.residual_interest
		fee.penalty = fee.penalty - fee.penalty_paid
		payment.penalty_due = fee.penalty
		payment.principle_due = fee.remaining_amount

		if balance <= fee.remaining_amount:
			calc_penalty, calc_interest = fee.calc_penalty(payment.paid_date, balance)
			fee.residual_interest = fee.residual_interest + calc_interest
			fee.remaining_amount = fee.remaining_amount - balance
			payment.interest = 0
			payment.principle = balance
			balance = 0

		else: # balance > remaining amount
			balance = balance - fee.remaining_amount
			payment.principle = fee.remaining_amount

			# pay off residual interest
			if fee.residual_interest > 0:
				if balance > fee.residual_interest:
					balance = balance - fee.residual_interest
					payment.interest = fee.residual_interest
					fee.residual_interest = 0
				else:
					fee.residual_interest = fee.residual_interest - balance
					payment.interest = balance
					balance = 0

			# pay off remaining interest
			calc_penalty, calc_interest = fee.calc_penalty(payment.paid_date, fee.remaining_amount)
			if balance > calc_interest:
				balance = balance - calc_interest
				payment.interest = payment.interest + calc_interest
			else:
				fee.residual_interest = fee.residual_interest + calc_interest
				balance = 0

			fee.remaining_amount = 0

		#pay off penalty
		if fee.penalty > 0 and balance > 0:
			if balance >= fee.penalty:
				balance = balance - fee.penalty
				payment.penalty = fee.penalty

			elif balance > 0:
				payment.penalty = balance
				balance = 0

		payment.credit = balance
		payment.amount = payment.penalty + payment.interest + payment.principle
		payment.save()

		fee.penalty_paid += payment.penalty
		fee.interest_paid += payment.interest
		fee.principle_paid += payment.principle

	outstanding_fees = []
	#bring interest  penalty up to date
	for pk, fee in fees.iteritems():
		calc_penalty, calc_interest = fee.calc_penalty(date.today(), fee.remaining_amount)
		if not fee.penalty:
			fee.penalty = calc_penalty
		fee.penalty = fee.penalty - fee.penalty_paid
		fee.interest = calc_interest + fee.residual_interest

		if fee.total_due <=0 and fee.amount > 0:
			fee.is_paid = True
		else:
			fee.is_paid = False
			outstanding_fees.append(fee)

		fee.submit_date =  timezone.make_aware(datetime.today(), timezone.get_default_timezone())
		fee.save()

	#update balance
	if fee.business_id:
		business = fee.business
		business.credit = balance
		business.save(update_fields=('credit',))

	elif fee.prop:
		prop = fee.prop
		prop.credit = balance
		prop.save(update_fields=('credit',))

	return balance


# Loading Data
from pyspark.ml import SparkSession

spark = SparkSession.builder.appName('lr').getOrCreate()
data = spark.read.csv("Ecommerce_Customers.csv",inferSchema=True,header=True)
data.printSchema()
data.show()
data.head()
for item in data.head():
    print(item)

# Data preparation
from pyspark.ml.linalg import Vectors
from pyspark.ml.feature import VectorAssembler

data.columns
assembler = VectorAssembler(
    inputCols=["Avg Session Length", "Time on App",
               "Time on Website",'Length of Membership'],
    outputCol="features")
output = assembler.transform(data)
output.select('features').show()
output.show()

final_data = output.select(['features', 'Yearly Amount Spent'])
train_data, test_data = final_data.randomSplit([0.7, 0.3])
train_data.describe().show()
test_data.describe().show()

# Model Building
from pyspark.ml.regression import LinearRegression
lr = LinearRegression(labelCol='Yearly Amount Spent')
lrModel = lr.fit(train_data)
print("Coefficients: {} Intercept: {}".format(lrModel.coefficients,lrModel.intercept))

#Training Summary
trainingSummary = lrModel.summary
trainingSummary.residuals.show()
print("RMSE: {}".format(trainingSummary.rootMeanSquaredError))
print("r2: {}".format(trainingSummary.r2))

# Prediction
unlabeled_data = test_data.select('features')
predictions = lrModel.transform(unlabeled_data)
predictions.show()

# Model Evaluation
test_results = lrModel.evaluate(test_data)
test_results.residuals.show()
print("RMSE: {}".format(test_results.rootMeanSquaredError))
print("MSE: {}".format(test_results.meanSquaredError))


from selenium import webdriver
import time

class TextValue():
    def test_text_value(self):
        url = 'https://letskodeit.teachable.com/p/practice'
        driver = webdriver.Firefox()
        driver.maximize_window()
        driver.get(url)
        driver.implicitly_wait(5)
        ele_text = driver.find_element_by_xpath("//a[@id='opentab']")
        ele_name_text = ele_text.text
        print('The text in the element is ' + ele_name_text)
        ele_value  = driver.find_element_by_xpath("//input[@id='name']")
        print('The value of class attribute of ele_value is ' + ele_value.get_attribute("class"))
        driver.quit()

tv = TextValue()
tv.test_text_value()


import utils
import predict
import time as tm
import numpy as np

# This file is intended to demonstrate how we would evaluate your code
# The data loader needs to know how many feature dimensions are there
dictSize = 225
(X, y) = utils.loadData( "train", dictSize = dictSize )

# Get error class predictions from predict.py and time the thing
tic = tm.perf_counter()
yPred = predict.findErrorClass( X, 5 )
toc = tm.perf_counter()

print( "Total time taken is %.6f seconds " % (toc - tic) )

preck = utils.getPrecAtK( y, yPred, 5 )
# The macro precision code takes a bit longer to execute due to the for loop over labels
mpreck = utils.getMPrecAtK( y, yPred, 5 )

# According to our definitions, both prec@k and mprec@k should go up as k goes up i.e. for your
# method, prec@i > prec@j if i > j and mprec@i > mprec@j if i > j. See the assignment description
# to convince yourself why this must be the case.

print( "prec@1: %0.3f" % preck[0], "prec@3: %0.3f" % preck[2], "prec@5: %0.3f" % preck[4] )
# Dont be surprised if mprec is small -- it is hard to do well on rare error classes
print( "mprec@1: %0.3e" % mpreck[0], "mprec@3: %0.3e" % mpreck[2], "mprec@5: %0.3e" % mpreck[4] )


import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
from sklearn.linear_model import LinearRegression
from sklearn.preprocessing import PolynomialFeatures

#Obtenha um novo modelo de regressão polinomial utilizando degree=4. E exiba o gráfico aseguir.

#importação da base de dados
dataset = pd.read_csv('HeightVsWeight.csv')

#selecionando as variáveis dependentes e independentes
x = dataset.iloc[:, 0].values.reshape(-1, 1) #x é independente
y = dataset.iloc[:, 1].values.reshape(-1, 1) #y é dependente
print(dataset.shape)

#Comparando os resultados
lr = LinearRegression()
lr.fit(x, y)

#AQUI ENTRA O MODELO DE REGRESSÃO LINEAR POLINOMIAL
poly_features = PolynomialFeatures (degree= 4)
x_poly = poly_features.fit_transform(x)
polyLinearRegression = LinearRegression()
polyLinearRegression.fit(x_poly, y)

#Vizualizar os resultados com regressão linear simples
plt.scatter(x, y, color="red")
plt.plot(x, lr.predict(x), color="blue")
plt.title("Dados por Regressão Linear ")
plt.xlabel("Age")
plt.ylabel("Height")
plt.show()

#Vizualizar os resultados com regressão polinomial
plt.scatter(x, y, color="red")
plt.plot(x, polyLinearRegression.predict(x_poly),color="blue")
plt.title("Dados Regressão Linear Polinomial")
plt.xlabel("Age")
plt.ylabel("Height")
plt.show()


import urllib.request, urllib.parse, urllib.error
import xml.etree.ElementTree as ET
import ssl

# Ignore SSL certificate errors
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE

url = input('Enter url: ')
print('Retrieving', url)
uh = urllib.request.urlopen(url, context=ctx)

data = uh.read()
print('Retrieved', len(data), 'characters')
tree = ET.fromstring(data)

comments = tree.findall('comments/comment/count')
total = 0
for count in comments:
    total = total+int(count.text)
print(total)

Coursera/Python for Everybody/Using Python to Access Web Data/w5-assignment.py


#!/usr/bin/env python3

import numpy as np
from py4j.java_gateway import JavaGateway

# connect to the JVM
gateway = JavaGateway()

# get the HeadlessModel instance
headlessModel = gateway.entry_point

# call the static method exec(), which runs the simulation
result = gateway.jvm.housing.Model.exec(1.5)

# result is a py4j.java_collections.JavaArray, whose elements are floats
print(type(result))

for x in result:
    print(x)


import torch
import torch.nn as nn
from config import args

class BiLSTM(nn.Module):
    def __init__(self, input_size, hidden_size, num_layers, num_dirs):
        super().__init__()
        self.input_size = input_size
        self.hidden_size = hidden_size
        self.num_layers = num_layers
        self.num_dirs = num_dirs
        self.rnn = nn.LSTM(
            input_size=input_size,
            hidden_size=hidden_size // num_dirs,
            num_layers=num_layers,
            bias=True,
            batch_first=True,
            bidirectional=(num_dirs == 2)
        )

    def init_states(self, x, device):
        h0 = torch.zeros(self.num_layers*self.num_dirs, x.size(0),
                         self.hidden_size//self.num_dirs).to(device)
        c0 = torch.zeros(self.num_layers*self.num_dirs, x.size(0),
                         self.hidden_size//self.num_dirs).to(device)
        return (h0, c0)

    def forward(self, x, mask, pack=True):
        s = self.init_states(x, x.device)

        if pack:
            lens = mask.sum(1)
            sorted_lens, sorted_idx = torch.sort(lens, descending=True)
            _, unsorted_idx = torch.sort(sorted_idx)

            x = x.index_select(0, sorted_idx)
            packed_x = nn.utils.rnn.pack_padded_sequence(
                x, sorted_lens.cpu().numpy(), batch_first=True)

            self.rnn.flatten_parameters()
            out, _ = self.rnn(packed_x, s)
            padded_out, _ = nn.utils.rnn.pad_packed_sequence(
                out, batch_first=True)
            out = padded_out.index_select(0, unsorted_idx)
        else:
            self.rnn.flatten_parameters()
            out, _ = self.rnn(x, s)
        return out


import sys
sys.path.append('.')

import argparse
import paddle

from DAIN.predict import VideoFrameInterp
from DeOldify.predict import DeOldifyPredictor
from EDVR.predict import EDVRPredictor

parser = argparse.ArgumentParser(description='Fix video')
parser.add_argument('--input',   type=str,   default=None, help='Input video')
parser.add_argument('--output',   type=str,   default='output', help='output dir')
parser.add_argument('--DAIN_weight',  type=str, default=None, help='Path to the reference image directory')
parser.add_argument('--DeOldify_weight',  type=str, default=None, help='Path to the reference image directory')
parser.add_argument('--EDVR_weight',  type=str, default=None, help='Path to the reference image directory')
# DAIN args
parser.add_argument('--time_step', type=float, default=0.5, help='choose the time steps')
parser.add_argument('--proccess_order',  type=str, default='none', nargs='+', help='Process order')

if __name__ == "__main__":
    args = parser.parse_args()
    print('args...', args)
    orders = args.proccess_order
    temp_video_path = None

    for order in orders:
        if order == 'DAIN':
            predictor = VideoFrameInterp(args.time_step, args.DAIN_weight,
                                        args.input, output_path=args.output)
            frames_path, temp_video_path = predictor.run()
        elif order == 'DeOldify':
            print('frames:', frames_path)
            print('video_path:', temp_video_path)

            paddle.disable_static()
            predictor = DeOldifyPredictor(temp_video_path, args.output, weight_path=args.DeOldify_weight)
            frames_path, temp_video_path = predictor.run()
            print('frames:', frames_path)
            print('video_path:', temp_video_path)
            paddle.enable_static()
        elif order == 'EDVR':
            predictor = EDVRPredictor(temp_video_path, args.output, weight_path=args.EDVR_weight)
            frames_path, temp_video_path = predictor.run()
            print('frames:', frames_path)
            print('video_path:', temp_video_path)


# -*- coding: utf-8 -*-

import time
from datetime import datetime
#from lxml import etree
#import netsvc
from osv import osv, fields, orm
from tools.translate import _
import pooler

import logging
logger = logging.getLogger('DOTCOM_LOGGER')

def validar_referencia_intervencao(cr,uid,context=None):
    pool= pooler.get_pool(cr.dbname)
    intervencao_pool = pool.get('dotcom.internos.intervencao')
    results = intervencao_pool.search(cr,uid,[])
    results=sorted(results,reverse=True)
    new_id=0
    if results !=[]:
        intervencao=intervencao_pool.browse(cr,uid,results[0])
        new_id=intervencao.id+1
    else:
        new_id=new_id+1

    data=datetime.now()
    ano=data.year

    ref=''
    if len(str(new_id))==1:
        ref='IN.0000'+str(new_id)+'-'+str(ano)
    elif len(str(new_id))==2:
        ref='IN.000'+str(new_id)+'-'+str(ano)
    elif len(str(new_id))==3:
        ref='In.00'+str(new_id)+'-'+str(ano)
    elif len(str(new_id))==4:
        ref='IN.0'+str(new_id)+'-'+str(ano)
    return ref

def validar_referencia_contrato(cr,uid,context=None):
    pool= pooler.get_pool(cr.dbname)
    contratos_pool = pool.get('dotcom.internos.cont')
    results = contratos_pool.search(cr,uid,[])
    results=sorted(results,reverse=True)
    new_id=0
    if results !=[]:
        intervencao=contratos_pool.browse(cr,uid,results[0])
        new_id=intervencao.id+1
    else:
        new_id=new_id+1

    data=datetime.now()
    ano=data.year

    ref=''
    if len(str(new_id))==1:
        ref='CNT.000'+str(new_id)+'-'+str(ano)
    elif len(str(new_id))==2:
        ref='CNT.00'+str(new_id)+'-'+str(ano)
    elif len(str(new_id))==3:
        ref='CNT.0'+str(new_id)+'-'+str(ano)
    elif len(str(new_id))==4:
        ref='CNT.'+str(new_id)+'-'+str(ano)
    return ref

def validar_referencia_pedido(cr,uid,context=None):
    pool= pooler.get_pool(cr.dbname)
    contratos_pool = pool.get('dotcom.internos.pedido.intervencao')
    results = contratos_pool.search(cr,uid,[])
    results=sorted(results,reverse=True)
    new_id=0
    if results !=[]:
        intervencao=contratos_pool.browse(cr,uid,results[0])
        new_id=intervencao.id+1
    else:
        new_id=new_id+1

    data=datetime.now()
    ano=data.year

    ref=''
    if len(str(new_id))==1:
        ref='PA.000'+str(new_id)+'-'+str(ano)
    elif len(str(new_id))==2:
        ref='PA.00'+str(new_id)+'-'+str(ano)
    elif len(str(new_id))==3:
        ref='PA.0'+str(new_id)+'-'+str(ano)
    elif len(str(new_id))==4:
        ref='PA.'+str(new_id)+'-'+str(ano)
    return ref

def atribuir_horas_contratos_ilimitados(self,cr,uid,context=None):
    pool= pooler.get_pool(cr.dbname)
    data_hoje=datetime.now()
    dia=data_hoje.day
    pool= pooler.get_pool(cr.dbname)
    contratos_pool = pool.get('dotcom.internos.cont')
    results = contratos_pool.search(cr,uid,[])
    for contrato in results:
        contrato=contratos_pool.browse(cr,uid,contrato)
        if contrato.tipo_conta=='sem_horas' and contrato.state is not 'fechado':
            if dia==1:
                contratos_pool.write(cr,uid,contrato.id,{'horas_diponiveis':contrato.limite_maximo_horas, 'horas_usadas':0, 'state':'em_uso'})
    return True

def validar_disponibilidade_horas_contrato(self,cr,uid,contrato,horas_usadas,context=None):
    if context is None:
        context={}

    pool= pooler.get_pool(cr.dbname)
    contratos_pool = pool.get('dotcom.internos.cont')
    if contrato.tipo_conta=='sem_horas' and (contrato.state=='novo' or contrato.state=='em_uso'):
        logger.info('LINHA %s' % str('STATE'))
        if contrato.horas_diponiveis==0:
            contratos_pool.write(cr,uid,contrato.id,{'state':'cheio'})
        elif contrato.horas_diponiveis<horas_usadas:
            raise osv.except_osv(_('Invalid action !'), _('O Tempo Gasto excedeu as horas disponíveis no Contrato!'))

    elif (contrato.tipo_conta=='sem_horas') and (contrato.state is not 'fechado'):
        if contrato.limite_maximo_horas==0:
            contratos_pool.write(cr,uid,contrato.id,{'state':'cheio'})
        elif contrato.limite_maximo_horas<horas_usadas:
            raise osv.except_osv(_('Invalid action !'), _('O Tempo Gasto excedeu as horas disponíveis no Contrato!'))
    return True

def validar_contrato_cliente(self,cr,uid,cliente,contrato,horas_usadas,context=None):
    if context is None:
        context={}

    if cliente!=contrato.cliente_id:
        raise osv.except_osv(_('Acção Invalida !'), _('O contrato seleccionado não pertence ao cliente em causa!'))
    return True

def validar_intervencoe_facturadas(self,cr,uid,intervencao,context=None):
    if context is None:
        context={}

    pool= pooler.get_pool(cr.dbname)
    venda_pool = pool.get('dotcom.venda')
    results = venda_pool.search(cr,uid,[])
    for venda in results:

        venda=venda_pool.browse(cr,uid,venda)
        #logger.info('VENDA %s' % str(venda.state))
        if venda.state=='cancel':

            continue
        else:
            logger.info('VENDA %s' % str('EM DIA'))
            if venda.intervencao_ids:
                for inter in venda.intervencao_ids:
                    #logger.info('LINHA %s' % str(inter))
                    #logger.info('LINHA %s' % str(intervencao))
                    logger.info('PARAM %s' % str(intervencao))
                    logger.info('VENDA %s' % str(inter.id))
                    if intervencao==inter.id:
                        raise osv.except_osv(_('Accao Invalida !'), _('Intervencao '+ inter.nome +' com documento ja emitido!'))
                        break
    return True


#!/usr/bin/env python
# vim: set sts=4 sw=4 fdm=indent fdl=0 fdn=1 et:

import os
import sys
import re
import math
import shelve
import types
import functools
import itertools
from datetime import datetime
from copy import copy, deepcopy

from SingleBuToKstarMuMuFitter.anaSetup import q2bins
import SingleBuToKstarMuMuFitter.StdFitter as StdFitter
import v2Fitter.Batch.AbsBatchTaskWrapper as AbsBatchTaskWrapper

from SingleBuToKstarMuMuFitter.StdProcess import p
import ROOT
import SingleBuToKstarMuMuFitter.cpp
import SingleBuToKstarMuMuFitter.dataCollection as dataCollection
import SingleBuToKstarMuMuFitter.toyCollection as toyCollection
import SingleBuToKstarMuMuFitter.pdfCollection as pdfCollection
import SingleBuToKstarMuMuFitter.fitCollection as fitCollection
import SingleBuToKstarMuMuFitter.plotCollection as plotCollection
import v2Fitter.Fitter.AbsToyStudier as AbsToyStudier

# Define toyStudier and profilers
# Ref:
#   https://root.cern.ch/how/how-write-ttree-python
#   https://github.com/root-project/root/blob/master/tutorials/pyroot/staff.py
ROOT.gROOT.ProcessLine(
"""struct MyTreeContent {
   Double_t     fl;
   Double_t     afb;
   Double_t     fs;
   Double_t     transAs;
   Double_t     nSig;
   Double_t     nBkgComb;
   Double_t     nll;
};""")
from ROOT import AddressOf

class ProfiledFCToyStudier(AbsToyStudier.AbsToyStudier):
    """"""
    def getSubDataEntries(self, setIdx):
        expectedYield = 0
        try:
            db = shelve.open(self.process.dbplayer.odbfile)
            expectedYield += db['nSig']['getVal']
            expectedYield += db['nBkgComb']['getVal']
        finally:
            db.close()
        yields = ROOT.gRandom.Poisson(expectedYield)
        self.logger.logINFO("SubDataSet has expected yields {0} in set {1}".format(yields, setIdx))
        return yields

    def _preSetsLoop(self):
        """ Just build a tree to keep information of each set of toy """
        self.otree = ROOT.TTree("tree", "")
        self.treeContent = ROOT.MyTreeContent()
        self.otree.Branch("fl", AddressOf(self.treeContent, 'fl'), 'fl/D')
        self.otree.Branch("afb", AddressOf(self.treeContent, 'afb'), 'afb/D')
        self.otree.Branch("fs", AddressOf(self.treeContent, 'fs'), 'fs/D')
        self.otree.Branch("transAs", AddressOf(self.treeContent, 'transAs'), 'as/D')
        self.otree.Branch("nSig", AddressOf(self.treeContent, 'nSig'), 'nSig/D')
        self.otree.Branch("nBkgComb", AddressOf(self.treeContent, 'nBkgComb'), 'nBkgComb/D')
        self.otree.Branch("nll", AddressOf(self.treeContent, 'nll'), 'nll/D')

    def _preRunFitSteps(self, setIdx):
        # Do nothing, there's no need to reset input db file.
        # It should be fine enough to start from previous fit result.
        pass

    def _postRunFitSteps(self, setIdx):
        """ Fill information to otree """
        if math.fabs(self.fitter._nll.getVal()) < 1e20:
            unboundAfb = self.fitter.args.find('unboundAfb').getVal()
            unboundFl = self.fitter.args.find('unboundFl').getVal()
            self.treeContent.fl = StdFitter.unboundFlToFl(unboundFl)
            self.treeContent.afb = StdFitter.unboundAfbToAfb(unboundAfb, self.treeContent.fl)
            self.treeContent.fs = self.fitter.args.find('fs').getVal()
            self.treeContent.transAs = self.fitter.args.find('transAs').getVal()
            self.treeContent.nSig = self.fitter.args.find('nSig').getVal()
            self.treeContent.nBkgComb = self.fitter.args.find('nBkgComb').getVal()
            self.treeContent.nll = self.fitter._nll.getVal()
            self.otree.Fill()

    def _runSetsLoop(self):
        """Customization: No force reset initial db file"""
        # Check if profiling gives valid result
        self.process.dbplayer.resetDB(False)
        for iSet in range(self.cfg['nSetOfToys']):
            self.fitter.hookProcess(self.process)
            self.fitter.customize()
            self.currentSubDataEntries = self.getSubDataEntries(iSet)
            self.fitter.pdf = self.process.sourcemanager.get(self.fitter.cfg['pdf'])
            self.fitter.data = self.getSubData().next()
            self.fitter._bookMinimizer()
            self.fitter._preFitSteps()
            self.fitter._runFitSteps()  # Don't run fitter._postRunFitSteps, db will be overwritten
            self._postRunFitSteps(iSet)
            self.fitter.reset()

    def _postSetsLoop(self):
        """ Write otree to a file """
        ofile = ROOT.TFile("setSummary_{0}.root".format(q2bins[self.process.cfg['binKey']]['label']), 'RECREATE')
        ofile.cd()
        self.otree.Write()
        ofile.Close()

    getSubData = AbsToyStudier.getSubData_random

setupProfiledFCToyStudier = deepcopy(AbsToyStudier.AbsToyStudier.templateConfig())
setupProfiledFCToyStudier.update({
    'name': "profiledFCToyStudier",
    'data': "ToyGenerator.mixedToy",
    'fitter': fitCollection.finalFitter,
    'nSetOfToys': 100,  # Typically 100 Toys * 5 submissions for acceptable precision, in proportion to generating time.
})
profiledFCToyStudier = ProfiledFCToyStudier(setupProfiledFCToyStudier)

setupProfiler = deepcopy(fitCollection.setupFinalFitter)
profiler = StdFitter.StdFitter(setupProfiler)
profiler.name = "profiler"

# Customize batch task

class BatchTaskWrapper(AbsBatchTaskWrapper.AbsBatchTaskWrapper):
    def createJdl(self, parser_args):
        jdl = self.createJdlBase()
        jdl += """
JobBatchName = "{JobBatchName}_{binKey}"
arguments = --binKey {binKey} run $(Process)
queue {nJobs}
""".format(
        JobBatchName="FCBatch",
        binKey=parser_args.binKey,
        nJobs=self.cfg['nJobs'])
        return jdl

setupBatchTask = deepcopy(BatchTaskWrapper.templateCfg())
setupBatchTask.update({
    'nJobs': 1,  # Fix to 1 for profiledFCToyStudier.cfg['nSetOfToys'] sets of toys
    'queue': "workday",
})

class BatchTaskWrapperProfile(AbsBatchTaskWrapper.AbsBatchTaskWrapper):
    def createJdl(self, parser_args):
        jdl = self.createJdlBase()
        jdl += """
JobBatchName = "{JobBatchName}_{binKey}"
arguments = --binKey {binKey} run_profile $(Process)
queue {nJobs}
""".format(
        JobBatchName="FCProfile",
        binKey=parser_args.binKey,
        nJobs=self.cfg['nJobs'])
        return jdl

setupBatchTaskProfile = deepcopy(BatchTaskWrapper.templateCfg())
setupBatchTaskProfile.update({
    'nJobs': 1,
    'queue': "espresso",
})

class BatchTaskWrapperBestFit(AbsBatchTaskWrapper.AbsBatchTaskWrapper):
    def createJdl(self, parser_args):
        jdl = self.createJdlBase()
        jdl += """
JobBatchName = "{JobBatchName}_{binKey}"
arguments = --binKey {binKey} run_bestFit $(Process)
queue {nJobs}
""".format(
        JobBatchName="FCBest",
        binKey=parser_args.binKey,
        nJobs=self.cfg['nJobs'])
        return jdl

setupBatchTaskBestFit = deepcopy(BatchTaskWrapper.templateCfg())
setupBatchTaskBestFit.update({
    'nJobs': 1,  # Fix to 1 for profiledFCToyStudier.cfg['nSetOfToys'] sets of toys
    'queue': "workday",
})

if __name__ == '__main__':
    # First create all profiling point...
    task_dir = "/afs/cern.ch/work/p/pchen/public/BuToKstarMuMu/v2Fitter/SingleBuToKstarMuMuFitter/batchTask_profiledFeldmanCousins"

    for iAfbSet in range(150):
        try:
            os.makedirs("{0}/afb{1:+.3f}".format(task_dir, 0.01 * iAfbSet - 0.745))
        except OSError:
            pass
    for iFlSet in range(100):
        try:
            os.makedirs("{0}/fl{1:+.3f}".format(task_dir, 0.01 * iFlSet + 0.005))
        except OSError:
            pass
    profilePoints = [d for d in os.listdir(task_dir) if re.match(r"^(afb|fl)[+-]0\.\d{3}$", d)]

    parser = AbsBatchTaskWrapper.BatchTaskParser
    parser.add_argument(
        '--binKey',
        dest="binKey",
        default="summary",
        help="Select q2 bin with binKey"
    )

    BatchTaskSubparserSubmitProfile = AbsBatchTaskWrapper.copyAndRegSubparser(AbsBatchTaskWrapper.BatchTaskSubparsers, 'submit', 'submit_profile')
    BatchTaskSubparserSubmitProfile.set_defaults(func=AbsBatchTaskWrapper.submitTask)

    BatchTaskSubparserRunProfile = AbsBatchTaskWrapper.copyAndRegSubparser(AbsBatchTaskWrapper.BatchTaskSubparsers, 'run', 'run_profile')
    BatchTaskSubparserRunProfile.set_defaults(func=AbsBatchTaskWrapper.runJob)

    BatchTaskSubparserSubmitBestFit = AbsBatchTaskWrapper.copyAndRegSubparser(AbsBatchTaskWrapper.BatchTaskSubparsers, 'submit', 'submit_bestFit')
    BatchTaskSubparserSubmitBestFit.set_defaults(func=AbsBatchTaskWrapper.submitTask)

    BatchTaskSubparserRunBestFit = AbsBatchTaskWrapper.copyAndRegSubparser(AbsBatchTaskWrapper.BatchTaskSubparsers, 'run', 'run_bestFit')
    BatchTaskSubparserRunBestFit.set_defaults(func=AbsBatchTaskWrapper.runJob)

    args = parser.parse_args()
    p.cfg['binKey'] = args.binKey

    if args.Function_name in ["submit_profile", "run_profile"]:
        if args.Function_name == "run_profile":
            print("INFO\t: Profiling job {1} with {0}".format(profilePoints[args.jobId], args.jobId))
            parseProfilePoint = re.match(r'(afb|fl)([+-]0\.\d{3})', profilePoints[args.jobId])
            constrained_var, constrained_val = parseProfilePoint.group(1), float(parseProfilePoint.group(2))
            print(constrained_var, constrained_val)

            @StdFitter.decorator_bookMinimizer_addGausConstraints([constrained_var], [constrained_val])
            def customized_bookMinimizer(self):
                StdFitter.StdFitter._bookMinimizer(self)
            profiler._bookMinimizer = types.MethodType(customized_bookMinimizer, profiler)

            def customized_preFitSteps_preFit(self, var, val):
                """ Set proper initial value to reasonable region."""
                unboundAfb = self.args.find("unboundAfb")
                unboundFl = self.args.find("unboundFl")
                if var == "afb":
                    for iFl in range(1, 20):
                        unboundAfb.setVal(StdFitter.afbToUnboundAfb(val, (1. - 1.33 * math.fabs(val)) / 20. * (20. - iFl)))
                        unboundFl.setVal(StdFitter.flToUnboundFl((1. - 1.33 * math.fabs(val)) / 20. * (20. - iFl)))
                        self.FitMigrad()
                        if math.fabs(self._nll.getVal()) < 1e20:
                            break
                elif var == "fl":
                    unboundFl.setVal(StdFitter.flToUnboundFl(val))
                    unboundFl.setConstant(True)
                    for iAfb in list(itertools.chain(*[(i, -i) for i in range(-9, 0)])):
                        unboundAfb.setVal(StdFitter.afbToUnboundAfb(0.75 * (1. - val) / 10. * iAfb, val))
                        self.FitMigrad()
                        if math.fabs(self._nll.getVal()) < 1e20:
                            break
            profiler._preFitSteps_preFit = types.MethodType(functools.partial(customized_preFitSteps_preFit, var=constrained_var, val=constrained_val), profiler)

            def customized_postFitSteps(self):
                """ Leave a log file for skipping failed profiling """
                StdFitter.StdFitter._postFitSteps(self)
                self.process.sourcemanager.get('afb').Print()
                self.process.sourcemanager.get('fl').Print()
                if self.fitResult['profiler.migrad']['status'] != 0 or math.fabs(self.fitResult['profiler.minos']['nll']) > 1e20:
                    with open("failed_in_profile_{0}.txt".format(q2bins[self.process.cfg['binKey']]['label']), 'w') as f:
                        try:
                            db = shelve.open(self.process.dbplayer.odbfile)
                            f.write(db.__repr__().replace("}", "\n"))
                        finally:
                            db.close()
            profiler._postFitSteps = types.MethodType(customized_postFitSteps, profiler)

            plotCollection.plotter.cfg['switchPlots'] = ['simpleBLK']
            plotCollection.plotter.cfg['plots']['simpleBLK']['kwargs'].update({
                'dataPlots': [["dataReader.Fit", ()], ],
                'pdfPlots': [["f_final", plotCollection.plotterCfg_styles['allStyle'] + (ROOT.RooFit.ProjectionRange("Fit"), ROOT.RooFit.Range("Fit"))],
                             ["f_final", plotCollection.plotterCfg_styles['sigStyle'] + (ROOT.RooFit.ProjectionRange("Fit"), ROOT.RooFit.Range("Fit"), ROOT.RooFit.Components('f_sig3D'))],
                             ["f_final", plotCollection.plotterCfg_styles['bkgStyle'] + (ROOT.RooFit.ProjectionRange("Fit"), ROOT.RooFit.Range("Fit"), ROOT.RooFit.Components('f_bkgComb'))],
                             ],
                'marks': [],
            })
        p.setSequence([
            dataCollection.dataReader,
            pdfCollection.stdWspaceReader,
            profiler,
            plotCollection.plotter
        ])
        setupBatchTaskProfile['work_dir'] = profilePoints
        setupBatchTaskProfile['nJobs'] = len(profilePoints)
        wrappedTask = BatchTaskWrapperProfile(
            "myBatchTask",
            "{0}".format(task_dir),
            cfg=setupBatchTaskProfile)
    elif args.Function_name in ["submit", "run"]:
        p.setSequence([
            pdfCollection.stdWspaceReader,
            toyCollection.sigToyGenerator,
            toyCollection.bkgCombToyGenerator,
            profiledFCToyStudier,
        ])
        setupBatchTask['nJobs'] = len(profilePoints)
        wrappedTask = BatchTaskWrapper(
            "myBatchTask",
            "{0}".format(task_dir),
            cfg=setupBatchTask)
        if args.Function_name == "run":
            wrappedTask.task_dir = "{0}/{1}".format(task_dir, profilePoints[args.jobId])
            p.dbplayer.absInputDir = wrappedTask.task_dir
            wrappedTask.cfg['work_dir'] = ['toys_{0}'.format(datetime.utcnow().strftime("UTC-%Y%m%d-%H%M%S"))] * len(profilePoints)

            toyCollection.sigToyGenerator.cfg.update({
                'scale': profiledFCToyStudier.cfg['nSetOfToys'] * 5,
                'db': "{0}/fitResults_{{binLabel}}.db".format(wrappedTask.task_dir),
                'saveAs': "sigToyGenerator_{0}.root".format(q2bins[args.binKey]['label']),
                'preloadFiles': ["{0}/sigToyGenerator_{1}.root".format(wrappedTask.task_dir, q2bins[args.binKey]['label'])],
            })
            toyCollection.bkgCombToyGenerator.cfg.update({
                'scale': profiledFCToyStudier.cfg['nSetOfToys'] * 5,
                'db': "{0}/fitResults_{{binLabel}}.db".format(wrappedTask.task_dir),
                'saveAs': "bkgCombToyGenerator_{0}.root".format(q2bins[args.binKey]['label']),
                'preloadFiles': ["{0}/bkgCombToyGenerator_{1}.root".format(wrappedTask.task_dir, q2bins[args.binKey]['label'])],
            })

            # No run if failed in profiling
            if os.path.exists(wrappedTask.task_dir + "/failed_in_profile_{0}.txt".format(q2bins[args.binKey]['label'])):
                print("INFO\t: Failed in profile step. Abort.\n")
                sys.exit()
        else:
            print("Each submit creates {0} toys. Please do submit many times to ensure sufficient accuracy.".format(profiledFCToyStudier.cfg['nSetOfToys']))
    elif args.Function_name in ['submit_bestFit', 'run_bestFit']:
        nBestFitJobs = 20
        p.setSequence([
            pdfCollection.stdWspaceReader,
            toyCollection.sigToyGenerator,
            toyCollection.bkgCombToyGenerator,
            profiledFCToyStudier,
        ])
        wrappedTask = BatchTaskWrapperBestFit(
            "myBatchTask",
            "{0}/bestFit".format(task_dir),
            cfg=setupBatchTaskBestFit)
        if args.Function_name == "run_bestFit":
            wrappedTask.cfg['work_dir'] = ['toys_{0}'.format(datetime.utcnow().strftime("UTC-%Y%m%d-%H%M%S"))] * nBestFitJobs
            toyCollection.sigToyGenerator.cfg.update({
                'scale': profiledFCToyStudier.cfg['nSetOfToys'] * 5,
                'saveAs': "sigToyGenerator_{0}.root".format(q2bins[args.binKey]['label']),
                'preloadFiles': ["{0}/sigToyGenerator_{1}.root".format(wrappedTask.task_dir, q2bins[args.binKey]['label'])],
            })
            toyCollection.bkgCombToyGenerator.cfg.update({
                'scale': profiledFCToyStudier.cfg['nSetOfToys'] * 5,
                'saveAs': "bkgCombToyGenerator_{0}.root".format(q2bins[args.binKey]['label']),
                'preloadFiles': ["{0}/bkgCombToyGenerator_{1}.root".format(wrappedTask.task_dir, q2bins[args.binKey]['label'])],
            })
        else:
            if args.nJobs:
                print("WARNING\t: Force nJobs={0} in coverage test.".format(nBestFitJobs))
            wrappedTask.cfg['nJobs'] = nBestFitJobs
    else:
        raise ValueError("Unknown function '{0}'".format(args.Function_name))

    parser.set_defaults(
        wrapper=wrappedTask,
        process=p
    )

    args = parser.parse_args()
    args.func(args)

    sys.exit()


from keras.models import load_model
import we_helpers as wh
import numpy as np

m_model = load_model('./models/model1490017738-54-71')
test_case = [
    "I don't like it, i hate it",
    "Amazing! It's a masterpiece",
    "What a boring film, i'll never watch it again",
    "illuminating if overly talky documentary",
    "interesting , but not compelling",
    "offers a breath of the fresh air of true sophistication"
]

x_test = []
for tc in test_case:
    x_test.append(wh.sentence2matrix(tc))

x_test = np.array(x_test)
x_test = x_test.reshape((x_test.shape[0], x_test.shape[1], x_test.shape[2], 1))

result = m_model.predict(x_test)
print(result)


from trains import Task
import csv

import numpy as np
from datetime import datetime
from sklearn import preprocessing

task = Task.init(project_name="diabetes prediction", task_name="ingesting dataset", task_type=Task.TaskTypes.data_processing)

args = {
    'dataset_task_id': 'c9943dc9aba047cea250364277061dbc',
    'test_size': 20
}

task.connect(args)

dataset_upload_task = Task.get_task(task_id=args['dataset_task_id'])
print('Input task id={} artifacts {}'.format(args['dataset_task_id'], list(dataset_upload_task.artifacts.keys())))
# download the csv file
source_ds = dataset_upload_task.artifacts['dataset'].get_local_copy()

x_vals = []
y_vals = []

# Converting CSV file to lists
with open(source_ds) as input_csv:
    csv_reader = csv.reader(input_csv, delimiter=',')
    next(csv_reader)
    for row in csv_reader:
        x_vals.append(float(row[2]))
        y_vals.append(int(row[-1]))

scaler = preprocessing.StandardScaler()
x_vals = np.array(x_vals).reshape(-1, 1)
x_scaled = scaler.fit_transform(x_vals)
y_vals = np.array(y_vals)

x_train = x_scaled[:-args['test_size']]
x_test = x_scaled[-args['test_size']:]

y_train = y_vals[:-args['test_size']]
y_test = y_vals[-args['test_size']:]

date= {'date': str(datetime.date(datetime.now()))}

task.upload_artifact('x_train', artifact_object=x_train, metadata=date)
task.upload_artifact('x_test', artifact_object=x_test, metadata=date)
task.upload_artifact('y_train', artifact_object=y_train, metadata=date)
task.upload_artifact('y_test', artifact_object=y_test, metadata=date)

print('all done!')


import base64
from io import BytesIO
from PIL import Image
import time
import numpy as np

def print_elapsed_time(_time):
    elapsed_time = time.time() - _time
    if elapsed_time < 60:
        print("{:2.1f} sec.".format(elapsed_time), flush=True)
    elif 60 < elapsed_time < 3600:
        print("{:2.1f} min.".format(elapsed_time / 60), flush=True)
    else:
        print("{:2.1f} hr.".format(elapsed_time / 3600), flush=True)

def base64_str_to_numpy(base64_str):
    base64_str = str(base64_str)
    if "base64" in base64_str:
        _, base64_str = base64_str.split(',')
    buf = BytesIO()
    buf.write(base64.b64decode(base64_str))
    buf.seek(0)
    pimg = Image.open(buf)
    img = np.array(pimg)
    return img[:, :, 3]

def resize_img(img_ndarray):
    img = Image.fromarray(img_ndarray)
    img.thumbnail((28, 28), Image.ANTIALIAS)
    return np.array(img)

def reshape_array(img_ndarray):
    img_std = img_ndarray / 255
    digit = np.reshape(img_std, (784, 1))
    return digit

def reshape_array_to_tflearn(img_std):
    letter = np.reshape(img_std, (1, 28, 28, 1))
    return letter


from mtgo_daily import get_alldecks, daterange
import datetime
import numpy as np
import pandas
import pylab as pl
import scipy.cluster

from deck_guess_AKH import deck_guess

bad_cards = ('Collected Company', "Nissa's Pilgrimage",
             "Knight of the White Orchid", "Smuggler's Copter",
             "Emrakul, the Promised End",
             "Reflector Mage",
             "Felidar Guardian",
             "Ojutai's Command", "Tragic Arrogance")

start_date = datetime.date(year=2017,month=1,day=25)

alldecks = get_alldecks(start_date=start_date)

alldecks = {date: results for date, results in alldecks.items()
            if not any(bad in deck['mainboard'] for deck in results.values() for bad in bad_cards)}

card_namespace = set(card
                     for dailies in alldecks.values()
                     for deck in dailies.values() if not any(x in {**deck['mainboard'], **deck['sideboard']} for x in bad_cards)
                     for card in {**deck['mainboard'], **deck['sideboard']})

deckcount = sum(len(x) for x in alldecks.values())

array = np.recarray(deckcount, dtype=[('ID','S20'),('Date','S10'),("Archetype","S25")]+[(name, np.int16) for name in card_namespace])

ii = 0
for date,daily in alldecks.items():
    for deckname,deck in daily.items():
        array['ID'][ii] = deckname.encode('ascii')
        array['Date'][ii] = date

        for card in card_namespace:
            if card in deck['mainboard']:
                array[card][ii] = int(deck['mainboard'][card])+10
            else:
                array[card][ii] = 0

        ii = ii+1

pd = pandas.DataFrame(array)

# get rid of the id column and date column
justdata = pd.T[3:]

distortions = []

# +1 for the "others"
guess_array = np.zeros([len(deck_guess)+1, justdata.shape[0]])
for ii,(deckname, deck) in enumerate(deck_guess.items()):
    for card in deck:
        if card not in pd.columns:
            print("Deck {0} is not represented in the meta".format(deckname))
            #raise ValueError("Card {0} is not real".format(card))
        guess_array[ii,justdata.T.keys() == card] = deck[card] + 10

codebook, distortion = scipy.cluster.vq.kmeans(np.array(justdata.T, 'float'), guess_array)
code, dist = scipy.cluster.vq.vq(np.array(justdata.T, 'int'), codebook)
#cntr, u, u0, d, jm, p, fpc = fuzz.cluster.cmeans(
#    data=justdata, c=ncenters, m=2, error=0.005, maxiter=1000, init=None)

pd['Distortion'] = dist

# Store fpc values for later
distortions.append(distortion)

cluster_membership = code

deck_50_pct = {}
deck_top20s = {}
deck_ids = {}

def top20_match(x):
    x = list(x)
    if len(x) == 0:
        return 0
    return np.sum(x)/len(x)

for ii in range(len(deck_guess)+1):
    mask = cluster_membership==ii
    deck = (justdata.T[mask] > 0).sum(axis=0)
    deck.sort_values(inplace=True)

    deck_top20s[ii] = deck

    name = None
    card_match_fraction = {dk:top20_match(k in deck.keys()[-20:] for k in ks)
                           for dk,ks in deck_guess.items()}
    bestfrac = 0
    for dk, frac in card_match_fraction.items():
        if frac > bestfrac:
            bestfrac = frac
            name = dk
    if bestfrac < 0.8:
        print("Deck {0}  has bad matches {1}".format(ii, bestfrac))
        name = None
    if name is None:
        print("Deck {0}: {1} matches, {2}%".format(ii, mask.sum(), mask.sum()/len(mask)))
        deck_ids[ii]=ii
        deck_50_pct[ii] = mask.sum()/len(mask)
        pd.ix[mask, 'Archetype'] = "Other "+str(ii)
    else:
        if name in deck_50_pct:
            print("**********DUPLICATE**********")
            pd.ix[mask, 'Archetype'] = name + str(ii)
        else:
            pd.ix[mask, 'Archetype'] = name
        deck_50_pct[name] = mask.sum()/len(mask)
        print("Deck {0}={2}: {1} matches, {3}%".format(ii, mask.sum(), name, mask.sum()/len(mask)))
        deck_ids[name]=ii
        #pd['Archetype'][mask] = name
    #print(deck[-20:])

print(len(deck_50_pct), deck_50_pct)

final = pandas.DataFrame.from_dict(list(deck_50_pct.items()))
final.sort_values(by=1, inplace=True)
print(final.sort_values(by=1))

timestep=3
week_starts = [day for day in daterange(start_date, datetime.date.today(), step=timestep)]

weekly_summary = pandas.DataFrame(index=week_starts, columns=deck_guess.keys())

dates = pandas.to_datetime(pd.Date.str.decode('ascii'))

for week_start in week_starts:
    week_end = week_start + datetime.timedelta(timestep)
    date_matches = (dates>=week_start) & (dates < week_end)
    for deck in deck_guess:
        deck_matches = (pd.Archetype == deck) & date_matches
        weekly_summary[deck][week_start] = deck_matches.sum() / date_matches.sum()

weekly_summary.plot(style=[x+'o'+y for x,y in zip('rgbcmykrgbcmykrgbcmyk', ['-']*7 + ['--']*7 + [':']*7)],
                    figsize=[24,20])
pl.legend(loc='center left', bbox_to_anchor=(1.0, 0.5))
pl.xlabel("First date in week")
pl.ylabel("Fraction of 5-0 decks in that week")
pl.savefig("MTGO_5-0_meta.png", bbox_inches='tight')

def get_deck(num):
    deck = pd.loc[num]
    return deck[deck!=0]


import numpy as np
import sympy as sp
import matplotlib.pyplot as plt
from scipy import integrate

KSI = 1/14
A = 1/14
B = 1/15
X_0 = 0
Y_0 = np.array([B * np.pi, A * np.pi])
X_k = np.pi
EPS = 10 ** -4

def f(x, y, symbol=False):
    if symbol:
        return [A * sp.Symbol('y1'), -B * sp.Symbol('y0')]
    return np.array([A * y[1], -B * y[0]])

def two_stage_schema(k, x_0=X_0, y_0=Y_0, x_k=X_k, c_2=KSI):
    x, y = [x_0], [y_0]
    b_2 = 1 / (2 * c_2)
    b_1 = 1 - b_2
    h = (x_k - x_0) / k
    for i in range(k):
        k_1 = h * f(x[i], y[i])
        k_2 = h * f(x[i] + c_2 * h, y[i] + c_2 * k_1)
        x.append(x[i] + h)
        y.append(y[i] + b_1 * k_1 + b_2 * k_2)
    return y

def opponent(k):
    return two_stage_schema(k, c_2=1)

def runge_error(y_dash, y_wave, s=2):
    error = np.linalg.norm((y_wave - y_dash) / (1 - 2 ** -s))
    return error

def solving(method=two_stage_schema, epsilon=EPS):
    k = 2
    y_dash = method(k)[-1]
    k *= 2
    y_wave = method(k)[-1]
    error = runge_error(y_dash, y_wave)
    while error > epsilon:
        k *= 2
        y_dash = y_wave
        y_wave = method(k)[-1]
        error = runge_error(y_dash, y_wave)
    return k, y_dash, error

def step_opt(s=2, epsilon=EPS, method=two_stage_schema):
    k = 2
    y_dash = method(k)[-1]
    y_wave = method(2 * k)[-1]
    h = (X_k - X_0) / k
    h_opt = (h / 2) * ((2 ** s - 1) * epsilon / np.linalg.norm(y_dash - y_wave)) ** (1 / s)
    print(h_opt)
    k_opt = int(np.ceil((X_k - X_0) / h_opt)) - 1
    h_opt = (X_k - X_0) / k_opt
    return k_opt, h_opt

def integrate_auto_step(method=two_stage_schema, delta=10**-5):
    x_list = [X_0]
    y_list = [Y_0]
    k = 2
    len_segmenth = X_k - X_0
    steps_list = [k]
    while X_k > x_list[-1]:
        pass
    return 0

def main():
    true_result = integrate.solve_ivp(
        f, (X_0, X_k), Y_0, dense_output=True, atol=1e-13, rtol=1e-13).sol.__call__(X_k)
    my_result, error = solving()[1:3]
    k_opt, h_opt = step_opt()
    y_nodes = two_stage_schema(k_opt)
    y_nodes_er = two_stage_schema(k_opt*2)[::2]
    my_result_opt = y_nodes[-1]
    x_nodes = [X_0 + h_opt * i for i in range(k_opt + 1)]
    errors = [runge_error(y_nodes[i], y_nodes_er[i]) for i in range(len(x_nodes))]
    plt.plot(x_nodes, errors)
    plt.title('runge error by x')
    plt.grid(True)
    plt.show()
    print('True result:', true_result, '\nMy result:', my_result, '\nAbsolute error:', errors[-1],
          '\nEstimated error:', error, '\nOptimal step:', h_opt, '\nMy result with optimal step:', my_result_opt)
    return 0

main()


import argparse
import enum
import glob
import ninja_syntax
import os
import platform as platform_info
import subprocess
import sys

Platform = enum.Enum('Platform', 'msvc gcc')
BuildType = enum.Enum('BuildType', 'debug release master')

def detect_platform():
	return { 'Windows': Platform.msvc, 'Linux': Platform.gcc }[platform_info.system()]

def get_platform_writer(platform):
	return { Platform.msvc: PlatformMSVC(), Platform.gcc: PlatformGCC() }[platform]

def get_build_type_name(build_type):
	return { BuildType.debug: 'Debug', BuildType.release: 'Release', BuildType.master: 'Master' }[build_type]

def get_code_dir(project_name):
	return os.path.join('Code', project_name, '')

def get_build_dir(project_name, platform_name, build_type):
	return os.path.join('Build', project_name, platform_name, get_build_type_name(build_type), '')

def get_deploy_dir(project_name, platform_name, build_type):
	return os.path.join('Deploy', project_name, platform_name, get_build_type_name(build_type), '')

def get_cpp_files(dir):
	return glob.glob(os.path.join(dir, '**/*.cpp'), recursive = True)

def get_c_files(dir):
	return glob.glob(os.path.join(dir, '**/*.c'), recursive = True)

def get_test_files(dir):
	return [os.path.relpath(f, dir) for f in glob.glob(os.path.join(dir, '**/*.test.cpp'), recursive = True)]

def get_file_stem(filename):
	return os.path.splitext(os.path.basename(filename))[0]

def join_file(*args):
	return os.path.join(*args)

def join_dir(*args):
	return os.path.join(*args, '')

class ProjectStaticLib:
	def __init__(self):
		self.project_name = ''
		self.code_dir = ''
		self.build_dir = ''
		self.deploy_dir = ''
		self.src_files = []
		self.defines = []
		self.inc_dirs = []

	@classmethod
	def from_name(cls, project_name, platform, build_type):
		lib = cls()
		lib.project_name = project_name
		lib.code_dir = get_code_dir(project_name)
		lib.build_dir = get_build_dir(project_name, platform.get_platform_name(), build_type)
		lib.deploy_dir = get_deploy_dir(project_name, platform.get_platform_name(), build_type)
		lib.src_files = get_cpp_files(lib.code_dir) + get_c_files(lib.code_dir)
		return lib

class ProjectExe:
	def __init__(self):
		self.project_name = ''
		self.code_dir = ''
		self.build_dir = ''
		self.deploy_dir = ''
		self.src_files = []
		self.defines = []
		self.inc_dirs = []
		self.standard_libs = []
		self.libs = []
		self.lib_dirs = []

	@classmethod
	def from_name(cls, project_name, platform, build_type):
		exe = cls()
		exe.project_name = project_name
		exe.code_dir = get_code_dir(project_name)
		exe.build_dir = get_build_dir(project_name, platform.get_platform_name(), build_type)
		exe.deploy_dir = get_deploy_dir(project_name, platform.get_platform_name(), build_type)
		exe.src_files = get_cpp_files(exe.code_dir) + get_c_files(exe.code_dir)
		return exe

class PlatformMSVC:

	def get_platform_name(self):
		return 'MSVC'

	def get_lib_name(self, name):
		return name + '.lib'

	def write_rules(self, n):
		n.rule('cc', 'cl.exe -c $cc_flags $cc_build_flags $cc_warning_level $cc_defines $cc_out_obj $cc_out_pdb $cc_in_includes $cc_in_files')
		n.rule('ar', 'lib.exe $ar_flags $ar_out_file $ar_in_files')
		n.rule('rc', 'rc.exe $out_file $in_file')
		n.rule('link', 'link.exe $link_flags $link_build_flags $link_out_exe $link_out_pdb $link_in_lib_dirs $link_in_files $res_file')

	def get_cc_flags(self):
		return '/nologo /Gy /EHsc /WX /std:c++17 /Zc:__cplusplus /utf-8'

	def get_cc_build_flags(self, build_type):
		return { BuildType.debug: '/MTd /Od /Zi', BuildType.release: '/MT /Ox /Zi', BuildType.master: '/MT /Ox' }[build_type]

	def get_cc_defines(self, defines):
		return ' '.join('/D' + d for d in defines)

	def get_cc_warning_level(self, warning_level):
		return '/W' + str(warning_level)

	def get_cc_out_obj(self, path):
		return '/Fo' + path

	def get_cc_out_pdb(self, path):
		return '/Fd' + path

	def get_cc_in_includes(self, paths):
		return ' '.join('/I' + p for p in paths)

	def write_cc_build(self, n, build_type, in_src, in_includes, out_obj, out_pdb, defines, warning_level):
		in_deps = [in_src]
		out_deps = [out_obj, out_pdb]
		n.build(out_deps, 'cc', in_deps,
			variables = [
				('cc_flags', self.get_cc_flags()),
				('cc_build_flags', self.get_cc_build_flags(build_type)),
				('cc_warning_level', self.get_cc_warning_level(warning_level)),
				('cc_defines', self.get_cc_defines(defines)),
				('cc_out_obj', self.get_cc_out_obj(out_obj)),
				('cc_out_pdb', self.get_cc_out_pdb(out_pdb)),
				('cc_in_includes', self.get_cc_in_includes(in_includes)),
				('cc_in_files', in_src)
			])

	def get_ar_flags(self):
		return '/NOLOGO'

	def get_ar_out_file(self, path):
		return '/OUT:' + path

	def write_ar_build(self, n, in_objs, out_lib):
		in_deps = in_objs
		out_deps = [out_lib]
		n.build(out_deps, 'ar', in_deps,
			variables = [
				('ar_flags', self.get_ar_flags()),
				('ar_out_file', self.get_ar_out_file(out_lib)),
				('ar_in_files', ' '.join(in_objs))
			])

	def get_rc_out_file(self, file):
		return '/fo ' + file

	def get_rc_in_file(self, file):
		return file

	def write_rc_call(self, n, build_type, in_file, out_file):
		in_deps = [in_file]
		out_deps = [out_file]
		n.build(out_deps, 'rc', in_deps,
			variables = [
				('out_file', self.get_rc_out_file(out_file)),
				('in_file', self.get_rc_in_file(in_file)),
			])

	def get_link_flags(self):
		return '/NOLOGO /OPT:REF /SUBSYSTEM:WINDOWS'

	def get_link_build_flags(self, build_type):
		return { BuildType.debug: '/DEBUG:FULL', BuildType.release: '/DEBUG:FULL', BuildType.master: '/DEBUG:NONE' }[build_type]

	def get_link_out_exe(self, path):
		return '/OUT:' + path

	def get_link_out_pdb(self, path):
		return '/PDB:' + path

	def get_link_in_lib_dirs(self, paths):
		return ' '.join('/LIBPATH:' + p for p in paths)

	def write_link_build(self, n, build_type, in_objs, in_standard_libs, in_libs, in_lib_dirs, out_exe, out_pdb, res_file):
		in_deps = in_objs + in_libs + [res_file]
		out_deps = [out_exe, out_pdb]
		n.build(out_deps, 'link', in_deps,
			variables = [
				('link_flags', self.get_link_flags()),
				('link_build_flags', self.get_link_build_flags(build_type)),
				('link_out_exe', self.get_link_out_exe(out_exe)),
				('link_out_pdb', self.get_link_out_pdb(out_pdb)),
				('link_in_lib_dirs', self.get_link_in_lib_dirs(in_lib_dirs)),
				('link_in_files', ' '.join(in_objs + in_libs + in_standard_libs)),
				('res_file', res_file),
			])

	def write_static_lib(self, n, build_type, lib, warning_level = '4'):
		src_paths = lib.src_files
		inc_paths = lib.inc_dirs
		obj_paths = [join_file(lib.build_dir, get_file_stem(f)) + '.obj' for f in lib.src_files]
		pdb_paths = [join_file(lib.deploy_dir, get_file_stem(f)) + '.pdb' for f in lib.src_files]

		for src, obj, pdb in zip(src_paths, obj_paths, pdb_paths):
			self.write_cc_build(n, build_type, src, inc_paths, obj, pdb, lib.defines, warning_level)

		lib_path = join_file(lib.deploy_dir, lib.project_name + '.lib')
		self.write_ar_build(n, obj_paths, lib_path)

	def write_exe(self, n, build_type, exe, warning_level = '4', res_file = ''):
		src_paths = exe.src_files
		inc_paths = exe.inc_dirs
		obj_paths = [join_file(exe.build_dir, get_file_stem(f)) + '.obj' for f in exe.src_files]
		pdb_paths = [join_file(exe.build_dir, get_file_stem(f)) + '.pdb' for f in exe.src_files]

		for src, obj, pdb in zip(src_paths, obj_paths, pdb_paths):
			self.write_cc_build(n, build_type, src, inc_paths, obj, pdb, exe.defines, warning_level)

		exe_path = join_file(exe.deploy_dir, exe.project_name + '.exe')
		pdb_final_path = join_file(exe.deploy_dir, exe.project_name + '.pdb')
		self.write_link_build(n, build_type, obj_paths, exe.standard_libs, exe.libs, exe.lib_dirs, exe_path, pdb_final_path, res_file)

	def write(self, n, build_type):

		self.write_rules(n)

		stb = ProjectStaticLib.from_name('stbimage', self, build_type)
		stb.defines = [ 'STBI_WINDOWS_UTF8' ]
		self.write_static_lib(n, build_type, stb, '0')

		glm = ProjectStaticLib.from_name('glm', self, build_type)
		glm.defines = [ 'GLM_FORCE_EXPLICIT_CTOR', 'GLM_FORCE_XYZW_ONLY', 'GLM_FORCE_SIZE_T_LENGTH', 'GLM_FORCE_SILENT_WARNINGS' ]
		glm.inc_dirs = [ glm.code_dir ]
		self.write_static_lib(n, build_type, glm, '4')

		sdl = ProjectStaticLib.from_name('sdl', self, build_type)
		sdl.defines = [ 'HAVE_LIBC', 'SDL_DYNAMIC_API=0', 'DECLSPEC=' ]
		sdl.src_files =  [
			"src/atomic/SDL_atomic.c",
			"src/atomic/SDL_spinlock.c",
			"src/audio/directsound/SDL_directsound.c",
			"src/audio/disk/SDL_diskaudio.c",
			"src/audio/dummy/SDL_dummyaudio.c",
			"src/audio/SDL_audio.c",
			"src/audio/SDL_audiocvt.c",
			"src/audio/SDL_audiodev.c",
			"src/audio/SDL_audiotypecvt.c",
			"src/audio/SDL_mixer.c",
			"src/audio/SDL_wave.c",
			"src/audio/winmm/SDL_winmm.c",
			"src/audio/wasapi/SDL_wasapi.c",
			"src/audio/wasapi/SDL_wasapi_win32.c",
			"src/core/windows/SDL_windows.c",
			"src/core/windows/SDL_xinput.c",
			"src/cpuinfo/SDL_cpuinfo.c",
			"src/dynapi/SDL_dynapi.c",
			"src/events/SDL_clipboardevents.c",
			"src/events/SDL_displayevents.c",
			"src/events/SDL_dropevents.c",
			"src/events/SDL_events.c",
			"src/events/SDL_gesture.c",
			"src/events/SDL_keyboard.c",
			"src/events/SDL_mouse.c",
			"src/events/SDL_quit.c",
			"src/events/SDL_touch.c",
			"src/events/SDL_windowevents.c",
			"src/file/SDL_rwops.c",
			"src/filesystem/windows/SDL_sysfilesystem.c",
			"src/haptic/SDL_haptic.c",
			"src/haptic/windows/SDL_dinputhaptic.c",
			"src/haptic/windows/SDL_windowshaptic.c",
			"src/haptic/windows/SDL_xinputhaptic.c",
			"src/hidapi/windows/hid.c",
			"src/joystick/hidapi/SDL_hidapijoystick.c",
			"src/joystick/hidapi/SDL_hidapi_ps4.c",
			"src/joystick/hidapi/SDL_hidapi_switch.c",
			"src/joystick/hidapi/SDL_hidapi_xbox360.c",
			"src/joystick/hidapi/SDL_hidapi_xboxone.c",
			"src/joystick/SDL_gamecontroller.c",
			"src/joystick/SDL_joystick.c",
			"src/joystick/windows/SDL_dinputjoystick.c",
			"src/joystick/windows/SDL_mmjoystick.c",
			"src/joystick/windows/SDL_windowsjoystick.c",
			"src/joystick/windows/SDL_xinputjoystick.c",
			"src/libm/e_atan2.c",
			"src/libm/e_exp.c",
			"src/libm/e_fmod.c",
			"src/libm/e_log.c",
			"src/libm/e_log10.c",
			"src/libm/e_pow.c",
			"src/libm/e_rem_pio2.c",
			"src/libm/e_sqrt.c",
			"src/libm/k_cos.c",
			"src/libm/k_rem_pio2.c",
			"src/libm/k_sin.c",
			"src/libm/k_tan.c",
			"src/libm/s_atan.c",
			"src/libm/s_copysign.c",
			"src/libm/s_cos.c",
			"src/libm/s_fabs.c",
			"src/libm/s_floor.c",
			"src/libm/s_scalbn.c",
			"src/libm/s_sin.c",
			"src/libm/s_tan.c",
			"src/loadso/windows/SDL_sysloadso.c",
			"src/power/SDL_power.c",
			"src/power/windows/SDL_syspower.c",
			"src/render/direct3d11/SDL_shaders_d3d11.c",
			"src/render/direct3d/SDL_render_d3d.c",
			"src/render/direct3d11/SDL_render_d3d11.c",
			"src/render/direct3d/SDL_shaders_d3d.c",
			"src/render/opengl/SDL_render_gl.c",
			"src/render/opengl/SDL_shaders_gl.c",
			"src/render/opengles2/SDL_render_gles2.c",
			"src/render/opengles2/SDL_shaders_gles2.c",
			"src/render/SDL_d3dmath.c",
			"src/render/SDL_render.c",
			"src/render/SDL_yuv_sw.c",
			"src/render/software/SDL_blendfillrect.c",
			"src/render/software/SDL_blendline.c",
			"src/render/software/SDL_blendpoint.c",
			"src/render/software/SDL_drawline.c",
			"src/render/software/SDL_drawpoint.c",
			"src/render/software/SDL_render_sw.c",
			"src/render/software/SDL_rotate.c",
			"src/SDL.c",
			"src/SDL_assert.c",
			"src/SDL_dataqueue.c",
			"src/SDL_error.c",
			"src/SDL_hints.c",
			"src/SDL_log.c",
			"src/sensor/dummy/SDL_dummysensor.c",
			"src/sensor/SDL_sensor.c",
			"src/stdlib/SDL_getenv.c",
			"src/stdlib/SDL_iconv.c",
			"src/stdlib/SDL_malloc.c",
			"src/stdlib/SDL_qsort.c",
			"src/stdlib/SDL_stdlib.c",
			"src/stdlib/SDL_string.c",
			"src/thread/generic/SDL_syscond.c",
			"src/thread/SDL_thread.c",
			"src/thread/windows/SDL_sysmutex.c",
			"src/thread/windows/SDL_syssem.c",
			"src/thread/windows/SDL_systhread.c",
			"src/thread/windows/SDL_systls.c",
			"src/timer/SDL_timer.c",
			"src/timer/windows/SDL_systimer.c",
			"src/video/dummy/SDL_nullevents.c",
			"src/video/dummy/SDL_nullframebuffer.c",
			"src/video/dummy/SDL_nullvideo.c",
			"src/video/SDL_blit.c",
			"src/video/SDL_blit_0.c",
			"src/video/SDL_blit_1.c",
			"src/video/SDL_blit_A.c",
			"src/video/SDL_blit_auto.c",
			"src/video/SDL_blit_copy.c",
			"src/video/SDL_blit_N.c",
			"src/video/SDL_blit_slow.c",
			"src/video/SDL_bmp.c",
			"src/video/SDL_clipboard.c",
			"src/video/SDL_egl.c",
			"src/video/SDL_fillrect.c",
			"src/video/SDL_pixels.c",
			"src/video/SDL_rect.c",
			"src/video/SDL_RLEaccel.c",
			"src/video/SDL_shape.c",
			"src/video/SDL_stretch.c",
			"src/video/SDL_surface.c",
			"src/video/SDL_video.c",
			"src/video/SDL_vulkan_utils.c",
			"src/video/SDL_yuv.c",
			"src/video/windows/SDL_windowsclipboard.c",
			"src/video/windows/SDL_windowsevents.c",
			"src/video/windows/SDL_windowsframebuffer.c",
			"src/video/windows/SDL_windowskeyboard.c",
			"src/video/windows/SDL_windowsmessagebox.c",
			"src/video/windows/SDL_windowsmodes.c",
			"src/video/windows/SDL_windowsmouse.c",
			"src/video/windows/SDL_windowsopengl.c",
			"src/video/windows/SDL_windowsopengles.c",
			"src/video/windows/SDL_windowsshape.c",
			"src/video/windows/SDL_windowsvideo.c",
			"src/video/windows/SDL_windowsvulkan.c",
			"src/video/windows/SDL_windowswindow.c",
			"src/video/yuv2rgb/yuv_rgb.c",
		]
		sdl.src_files = [ join_file(sdl.code_dir, f) for f in sdl.src_files ]
		sdl.inc_dirs = [
			join_dir(sdl.code_dir, 'include')
		]
		self.write_static_lib(n, build_type, sdl, '3')

		sdlmain = ProjectStaticLib.from_name('sdlmain', self, build_type)
		sdlmain.defines = sdl.defines
		sdlmain.src_files = [
			join_file(sdl.code_dir, 'src/main/windows/SDL_windows_main.c')
		]
		sdlmain.inc_dirs = [
			join_dir(sdl.code_dir, 'include')
		]
		self.write_static_lib(n, build_type, sdlmain, '3')

		sdlmixer = ProjectStaticLib.from_name('sdlmixer', self, build_type)
		sdlmixer.defines = sdl.defines + [ 'MUSIC_WAV' ]
		sdlmixer.inc_dirs = [
			join_dir(sdl.code_dir, 'include')
		]
		self.write_static_lib(n, build_type, sdlmixer, '3')

		rc_file = join_file(get_code_dir('snek'), 'snek.rc')
		res_deploy_dir = get_deploy_dir('res', self.get_platform_name(), build_type)
		res_file = join_file(res_deploy_dir, 'snek.res')
		self.write_rc_call(n, build_type, rc_file, res_file)

		snek = ProjectExe.from_name('snek', self, build_type)
		snek.defines = stb.defines + glm.defines + sdl.defines + [ 'MUSIC_WAV' ]
		snek.inc_dirs = [
			stb.code_dir,
			glm.code_dir,
			join_dir(sdl.code_dir, 'include'),
			sdlmixer.code_dir,
		]
		snek.libs = [
			join_file(stb.deploy_dir, self.get_lib_name(stb.project_name)),
			join_file(sdlmain.deploy_dir, self.get_lib_name(sdlmain.project_name)),
			join_file(sdl.deploy_dir, self.get_lib_name(sdl.project_name)),
			join_file(sdlmixer.deploy_dir, self.get_lib_name(sdlmixer.project_name)),
		]
		snek.standard_libs = [ 'User32.lib', 'Shell32.lib', 'Ole32.lib', 'OpenGL32.lib', 'gdi32.lib', 'Winmm.lib', 'Advapi32.lib', 'Version.lib', 'Imm32.lib', 'Setupapi.lib', 'OleAut32.lib' ]
		self.write_exe(n, build_type, snek, '4', res_file)

class PlatformGCC:

	def get_platform_name(self):
		return 'GCC'

	def get_lib_name(self, name):
		return 'lib' + name + '.a'

	def write_rules(self, n):
		n.rule('cc', 'g++ -c $cc_flags $cc_build_flags $cc_defines $cc_out_obj $cc_in_includes $cc_in_files')
		n.rule('ar', 'ar $ar_flags $ar_out_file $ar_in_files')
		n.rule('ld', 'g++ $ld_flags $ld_build_flags $ld_out_exe $ld_in_lib_dirs $ld_in_files $ld_in_libs')

	def get_cc_flags(self):
		return '-Wall -Werror -std=c++17'

	def get_cc_build_flags(self, build_type):
		return { BuildType.debug: '-g -Og', BuildType.release: '-g -O3', BuildType.master: '-O3' }[build_type]

	def get_cc_defines(self, defines):
		return ' '.join('-D' + d for d in defines)

	def get_cc_out_obj(self, path):
		return '-o' + path

	def get_cc_in_includes(self, paths):
		return ' '.join('-I' + p for p in paths)

	def write_cc_build(self, n, build_type, in_src, in_includes, out_obj, defines):
		in_deps = [in_src]
		out_deps = [out_obj]
		n.build(out_deps, 'cc', in_deps,
			variables = [
				('cc_flags', self.get_cc_flags()),
				('cc_build_flags', self.get_cc_build_flags(build_type)),
				('cc_defines', self.get_cc_defines(defines)),
				('cc_out_obj', self.get_cc_out_obj(out_obj)),
				('cc_in_includes', self.get_cc_in_includes(in_includes)),
				('cc_in_files', in_src)
			])

	def get_ar_flags(self):
		return 'ru'

	def get_ar_out_file(self, path):
		return '-o ' + path

	def write_ar_build(self, n, in_objs, out_lib):
		in_deps = in_objs
		out_deps = [out_lib]
		n.build(out_deps, 'ar', in_deps,
			variables = [
				('ar_flags', self.get_ar_flags()),
				('ar_out_file', self.get_ar_out_file(out_lib)),
				('ar_in_files', ' '.join(in_objs))
			])

	def get_link_flags(self):
		return '-Wall -Werror -std=c++17'

	def get_link_build_flags(self, build_type):
		return { BuildType.debug: '-g -Og', BuildType.release: '-g -O3', BuildType.master: '-O3' }[build_type]

	def get_link_out_exe(self, path):
		return '-o ' + path

	def get_link_in_lib_dirs(self, paths):
		return ' '.join('-L' + p for p in paths)

	def get_link_in_libs(self, paths):
		return ' '.join('-l:' + p for p in paths)

	def get_link_in_standard_libs(self, paths):
		return ' '.join('-l' + p for p in paths)

	def write_link_build(self, n, build_type, in_objs, in_standard_libs, in_libs, in_lib_dirs, out_exe):
		in_deps = in_objs + in_libs
		out_deps = [out_exe]
		n.build(out_deps, 'ld', in_deps,
			variables = [
				('ld_flags', self.get_link_flags()),
				('ld_build_flags', self.get_link_build_flags(build_type)),
				('ld_out_exe', self.get_link_out_exe(out_exe)),
				('ld_in_lib_dirs', self.get_link_in_lib_dirs(in_lib_dirs)),
				('ld_in_files', ' '.join(in_objs)),
				('ld_in_libs', self.get_link_in_libs(in_libs) + ' ' + self.get_link_in_standard_libs(in_standard_libs)),
			])

	def write_static_lib(self, n, build_type, lib):
		src_paths = lib.src_files
		inc_paths = lib.inc_dirs
		obj_paths = [join_file(lib.build_dir, get_file_stem(f) + '.o') for f in lib.src_files]

		for src, obj in zip(src_paths, obj_paths):
			self.write_cc_build(n, build_type, src, inc_paths, obj, lib.defines)

		lib_path = join_file(lib.deploy_dir, self.get_lib_name(lib.project_name))
		self.write_ar_build(n, obj_paths, lib_path)

	def write_exe(self, n, build_type, exe):
		src_paths = exe.src_files
		inc_paths = exe.inc_dirs
		obj_paths = [join_file(exe.build_dir, get_file_stem(f) + '.o') for f in exe.src_files]

		for src, obj in zip(src_paths, obj_paths):
			self.write_cc_build(n, build_type, src, inc_paths, obj, exe.defines)

		exe_path = join_file(exe.deploy_dir, exe.project_name)
		self.write_link_build(n, build_type, obj_paths, exe.standard_libs, exe.libs, exe.lib_dirs, exe_path)

	def write(self, n, build_type):
		self.write_rules(n)

		# ...

def write(platform, build_type, build_file_name):
	try:
		with open(build_file_name, 'w') as build_file:
			n = ninja_syntax.Writer(build_file)
			platform.write(n, build_type)

	except IOError:
		print('failed to write build file: ', build_file_name)
		raise

def main():

	platform = detect_platform()
	platform_writer = get_platform_writer(platform)

	parser = argparse.ArgumentParser()

	parser.add_argument('--build_type',
		required = True,
		choices = [i.name for i in BuildType],
		action = 'store',
		dest = 'build_type',
		help = 'build for \'debug\' or \'release\' mode (affects optimization, debug symbols, etc.)')

	parser.add_argument('--no_build',
		action = 'store_true',
		dest = 'no_build',
		default = False,
		help = 'generate the ninja build file, but don\'t run the build')

	args = parser.parse_args()
	build_type = BuildType[args.build_type]

	try:
		build_file_name = 'build-' + platform.name + '-' + args.build_type + '.ninja'
		write(platform_writer, build_type, build_file_name)

		print('created build file: ', build_file_name)

	except:
		print('failed to generate build file!')
		raise

	if not args.no_build:
		build_result = subprocess.call(['ninja', '-f', build_file_name])

		if build_result != 0:
			print('build failed!')
			return

	print('build succeeded')

	if platform.name == 'msvc':
		copy_result = subprocess.call(['robocopy',
			'/v', '/w:1', '/mir',
			'Data/snek',
			get_deploy_dir('snek', platform_writer.get_platform_name(), build_type) + '/data'])

main()


#!/usr/bin/python3

import logging
import os
import sys
import multiprocessing
import configparser

from gensim.models import Word2Vec
from gensim.models.word2vec import LineSentence

def train():
    program = os.path.basename(sys.argv[0])
    logger = logging.getLogger(program)

    logging.basicConfig(format="%(asctime)s: %(levelname)s: %(message)s")
    logging.root.setLevel(level=logging.INFO)

    logger.info("Running %s" % " ".join(sys.argv))

    inp = "./output/wiki.data.utf8.txt"
    out1 = "./models/data.model"
    out2 = "./models/data.vector"

    config = configparser.ConfigParser()
    config.read("./config/w2v.conf", encoding="utf-8")

    train_size = int(config.get("train", "size"))
    train_window = int(config.get("train", "window"))
    train_min_count = int(config.get("train", "min_count"))

    model = Word2Vec(LineSentence(inp), size=train_size, window=train_window,
                       min_count=train_min_count,workers=multiprocessing.cpu_count())

    model.save(out1)
    model.wv.save_word2vec_format(out2, binary=False)

if __name__ == "__main__":
    train()