import numpy as np

from ...utils.misc import sigma

class LossFunction:
    def negative_gradient(self, y, y_pred):
        pass

    def update_terminal_regions(self, tree, X, y, y_pred, residual, learning_rate=0.1):
        terminal_regions = tree.apply(X)

        for leaf in np.unique(terminal_regions):
            mask = np.where(terminal_regions == leaf)[0]
            self.update_terminal_region(tree.nodes[leaf], X[mask], y[mask], y_pred[mask], residual[mask])

        y_pred += learning_rate * tree.predict(X)

    def update_terminal_region(self, leaf, X, y, y_pred, residual):
        pass

class BinomialDeviance(LossFunction):
    def negative_gradient(self, y, y_pred):
        '''
        y_pred here is logit. Corresponding probability is given by \sigma(y_pred)
        '''
        return y - sigma(y_pred)

    def update_terminal_region(self, leaf, X, y, y_pred, residual):
        '''
        Step given by Newton-Raphson: -grad(L(y, y_pred), y_pred)/grad^2(L(y, y_pred), y_pred)
        -grad(.) is residual
        grad^2(.) is \nabla\sigma = \sigma(1 - \sigma), \sigma = y - residual
        '''
        numerator = np.sum(residual)
        _sigma = y - residual
        denominator = np.sum(_sigma * (1 - _sigma))

        if abs(denominator) < 1e-150:
            leaf.rule.mean = 0.0
        else:
            leaf.rule.mean = numerator / denominator

class LeastSquaresError(LossFunction):
    '''
    L = 1 / 2 * ||y - y_||^2
    '''

    def negative_gradient(self, y, y_pred):
        return y - y_pred

    def update_terminal_regions(self, tree, X, y, y_pred, residual, learning_rate=0.1):
        y_pred += learning_rate * tree.predict(X)

class ObliviousBinomialDeviance:
    def __init__(self, leaf_weight, k):
        leaf_weighters = {
            'root': lambda n: np.sqrt(n / (n + k)),
            'log': lambda n: np.log(1 + n / (n + k)),
            'frac': lambda n: n / (1 + n + np.sqrt(n)),
        }
        self.leaf_weight = leaf_weighters[leaf_weight]

    def negative_gradient(self, y, y_pred):
        return y - sigma(y_pred)

    def update_terminal_regions(self, tree, X, y, y_pred, residual, learning_rate=0.1):
        terminal_regions = tree.apply(X)

        for leaf in np.unique(terminal_regions):
            # mask = np.where(terminal_regions == leaf)[0]
            mask = terminal_regions == leaf
            l_res = residual[mask]
            l_y = y[mask]
            numerator = np.sum(l_res)
            _sigma = l_y - l_res
            denominator = np.sum(_sigma * (1 - _sigma))

            if abs(denominator) < 1e-150:
                tree.leaf_values[leaf] = 0.0
            else:
                tree.leaf_values[leaf] = numerator / denominator * self.leaf_weight(np.sum(mask))

        y_pred += learning_rate * tree.predict(X)


import numpy as np

import matplotlib.pyplot as plt

import confBands

import matplotlib
matplotlib.rcParams.update({'font.size': 15})

orange = '#E69F00'
blue = '#0072B2'
green = '#009E73'
red = '#D55E00'

green2 = '#004D40'
blue2 = '#1E88E5'
orange2 = '#FFC107'
red2 = '#D81B60'

colSimp=orange
colGalExt=green
colGal = 'k'

simpModRunLoc = '../../runs/simpleModel/logNormLikeMstar6to10/'
simpModData = np.genfromtxt('{}combined/mLines.dat'.format(simpModRunLoc))
simpModMs   = np.genfromtxt('{}combined/ms.dat'.format(simpModRunLoc))

_,p05,p25,p50,p75,p95,_ = confBands.getPercentiles(simpModMs,simpModData)

plt.fill_between(simpModMs,p05,p95,alpha=0.3,color=colSimp)
plt.fill_between(simpModMs,p25,p75,alpha=0.3,color=colSimp)
plt.plot(simpModMs,p50,color=colSimp,ls='--')

galExtModRunLoc = '../../runs/galaxyModel_ext/'
galExtModData = np.genfromtxt('{}/m_lines.dat'.format(galExtModRunLoc))
galExtModLogMs = np.genfromtxt('{}/mc.txt'.format(galExtModRunLoc))
galExtModMs = [ 10.0**logM for logM in galExtModLogMs ]

_,p05,p25,p50,p75,p95,_ = confBands.getPercentiles(galExtModMs,galExtModData)

plt.fill_between(galExtModMs,p05,p95,alpha=0.3,color=colGalExt)
plt.fill_between(galExtModMs,p25,p75,alpha=0.3,color=colGalExt)
plt.plot(galExtModMs,p50,color=colGalExt,ls='--')

galExtModPrior = np.genfromtxt('{}/m_linesprior.dat'.format(galExtModRunLoc))
p00p5,_,_,_,_,_,p99p5 = confBands.getPercentiles(galExtModMs,galExtModPrior)
plt.plot(galExtModMs, p00p5, color='k', alpha=0.7, ls=':')
plt.plot(galExtModMs, p99p5, color='k', alpha=0.7, ls=':')

"""
galModRunLoc = '../../runs/galaxyModel/'
galModData = np.genfromtxt('{}/m_lines.dat'.format(galModRunLoc))
galModLogMs = np.genfromtxt('{}/mc.txt'.format(galModRunLoc))
galModMs = [ 10.0**logM for logM in galModLogMs ]

p05,p25,p50,p75,p95 = confBands.getPercentiles(galModMs,galModData)

plt.fill_between(galModMs,p05,p95,alpha=0.3,color=colGal)
plt.fill_between(galModMs,p25,p75,alpha=0.3,color=colGal)
plt.plot(galModMs,p50,color=colGal,ls='--')
"""

plt.ylim(1E-9,1E6)
plt.xlim(1E6,1E11)
plt.yscale('log')
plt.xscale('log')
plt.xlabel(r'$\mathcal{M}~~({\rm M_{\odot}})$')
plt.ylabel(r'${\rm d}n / {\rm d} \log_{10} \mathcal{M}/{\rm M_{\odot}}~~({\rm Mpc}^{-3})$')
plt.tight_layout()
plt.savefig('combinedAnalysisPlots/dndlogM.pdf'.format(simpModRunLoc))
plt.savefig('combinedAnalysisPlots/dndlogM.png'.format(simpModRunLoc))
plt.show()


import torch.nn as nn
import math

class NeuralNet(nn.Module):
    def __init__(self, input_shape: int, layer_size: list, post_layers: list):
        super(NeuralNet, self).__init__()
        layers = []
        first = False
        last_layer = 0
        for x, y in zip(layer_size, post_layers):
            if x == layer_size[0] and first is False:
                layers.append(nn.Linear(input_shape, x))
                last_layer = x
                first = True
            else:
                layers.append(nn.Linear(last_layer, x))
                last_layer = x
            if type(y) == list:
                for item in y:
                    layers.append(item)
            else:
                layers.append(y)
        self.fc = nn.Sequential(*layers)
        self.fc = self.fc.apply(self.init_weights)

    @staticmethod
    def init_weights(m):
        if type(m) == nn.Linear:
            n = m.in_features
            y = 1 / math.sqrt(n)
            m.weight.data.uniform_(-y, y)
            m.bias.data.fill_(0)

    def forward(self, x):
        return self.fc(x)


from __future__ import print_function
from omok_env import OmokEnv
import time
import sys
from collections import deque, defaultdict
import numpy as np
from numpy import random, sqrt, log, argwhere, zeros

N, Q = 0, 1
CURRENT = 0
OPPONENT = 1
COLOR = 2
BLACK = 1
WHITE = 0
BOARD_SIZE = 9
HISTORY = 2
N_SIMUL = 100000
GAME = 1

class MCTS:
    def __init__(self, board_size, n_history, n_simul):
        self.env_simul = OmokEnv(board_size, n_history, display=False)
        self.board_size = board_size
        self.n_simul = n_simul
        self.tree = None
        self.root = None
        self.state = None
        self.board = None
        self.legal_move = None
        self.no_legal_move = None
        self.ucb = None

        # used for backup
        self.key_memory = None
        self.action_memory = None

        # init
        self._reset()
        self.reset_tree()

    def _reset(self):
        self.key_memory = deque(maxlen=self.board_size**2)
        self.action_memory = deque(maxlen=self.board_size**2)

    def reset_tree(self):
        self.tree = defaultdict(
            lambda: zeros((self.board_size**2, 2), 'float'))

    def get_action(self, state, board):
        self.root = state.copy()
        self._simulation(state)
        # init root board after simulatons
        self.board = board
        board_fill = self.board[CURRENT] + self.board[OPPONENT]
        self.legal_move = argwhere(board_fill == 0).flatten()
        self.no_legal_move = argwhere(board_fill != 0).flatten()
        # root state's key
        root_key = hash(self.root.tostring())
        # argmax Q
        action = self._selection(root_key, c_ucb=0)
        print('')
        print(self.ucb.reshape(
            self.board_size, self.board_size).round(decimals=4))
        return action

    def _simulation(self, state):
        start = time.time()
        finish = 0
        for sim in range(self.n_simul):
            print('\rsimulation: {}'.format(sim + 1), end='')
            sys.stdout.flush()
            # reset state
            self.state, self.board = self.env_simul.reset(state)
            done = False
            n_selection = 0
            n_expansion = 0
            while not done:
                board_fill = self.board[CURRENT] + self.board[OPPONENT]
                self.legal_move = argwhere(board_fill == 0).flatten()
                self.no_legal_move = argwhere(board_fill != 0).flatten()
                key = hash(self.state.tostring())
                # search my tree
                if key in self.tree:
                    # selection
                    action = self._selection(key, c_ucb=1)
                    self.action_memory.appendleft(action)
                    self.key_memory.appendleft(key)
                    n_selection += 1
                elif n_expansion == 0:
                    # expansion
                    action = self._expansion(key)
                    self.action_memory.appendleft(action)
                    self.key_memory.appendleft(key)
                    n_expansion += 1
                else:
                    # rollout
                    action = random.choice(self.legal_move)
                self.state, self.board, reward, done = \
                    self.env_simul.step(action)
            if done:
                # backup & reset memory
                self._backup(reward, n_selection + n_expansion)
                self._reset()
                finish = time.time() - start
                # if finish >= self.think_time:
                #     break
        print('\r{} simulations end ({:0.0f}s)'.format(sim + 1, finish))

    def _selection(self, key, c_ucb):
        edges = self.tree[key]
        ucb = self._get_ucb(edges, c_ucb)
        self.ucb = ucb
        if self.board[COLOR][0] == WHITE:
            # black's choice
            action = argwhere(ucb == ucb.max()).flatten()
        else:
            # white's choice
            action = argwhere(ucb == ucb.min()).flatten()
        action = action[random.choice(len(action))]
        return action

    def _expansion(self, key):
        # only select once for rollout
        action = self._selection(key, c_ucb=1)
        return action

    def _get_ucb(self, edges, c_ucb):
        total_N = 0
        ucb = zeros((self.board_size**2), 'float')
        for i in range(self.board_size**2):
            total_N += edges[i][N]
        # black's ucb
        if self.board[COLOR][0] == WHITE:
            for move in self.legal_move:
                if edges[move][N] != 0:
                    ucb[move] = edges[move][Q] + c_ucb * \
                        sqrt(2 * log(total_N) / edges[move][N])
                else:
                    ucb[move] = np.inf
            for move in self.no_legal_move:
                ucb[move] = -np.inf
        # white's ucb
        else:
            for move in self.legal_move:
                if edges[move][N] != 0:
                    ucb[move] = edges[move][Q] - c_ucb * \
                        sqrt(2 * log(total_N) / edges[move][N])
                else:
                    ucb[move] = -np.inf
            for move in self.no_legal_move:
                ucb[move] = np.inf
        return ucb

    def _backup(self, reward, steps):
        # steps is n_selection + n_expansion
        # update edges in my tree
        for i in range(steps):
            edges = self.tree[self.key_memory[i]]
            action = self.action_memory[i]
            edges[action][N] += 1
            edges[action][Q] += (reward - edges[action][Q]) / edges[action][N]

def play():
    env = OmokEnv(BOARD_SIZE, HISTORY)
    mcts = MCTS(BOARD_SIZE, HISTORY, N_SIMUL)
    result = {'Black': 0, 'White': 0, 'Draw': 0}
    for g in range(GAME):
        print('#' * (BOARD_SIZE - 4),
              ' GAME: {} '.format(g + 1),
              '#' * (BOARD_SIZE - 4))
        # reset state
        state, board = env.reset()
        done = False
        while not done:
            env.render()
            # start simulations
            action = mcts.get_action(state, board)
            state, board, z, done = env.step(action)
        if done:
            if z == 1:
                result['Black'] += 1
            elif z == -1:
                result['White'] += 1
            else:
                result['Draw'] += 1
            # render & reset tree
            env.render()
            mcts.reset_tree()
        # result
        print('')
        print('=' * 20, " {}  Game End  ".format(g + 1), '=' * 20)
        blw, whw, drw = result['Black'], result['White'], result['Draw']
        stat = (
            'Black Win: {}  White Win: {}  Draw: {}  Winrate: {:0.1f}%'.format(
                blw, whw, drw,
                1 / (1 + np.exp(whw / (g + 1)) / np.exp(blw / (g + 1))) * 100))
        print(stat, '\n')

if __name__ == '__main__':
    np.set_printoptions(suppress=True)
    np.random.seed(42)
    play()


from os import path, mkdir
import glob
import DatasetStructure as ds
import datetime
import Settings as st
import shutil

def remove_dots_in_filenames(old_filename):
    filename_without_ext = path.splitext(old_filename)[0]
    filename_without_ext = filename_without_ext.replace('.', '_')
    return filename_without_ext + st.file_format

def copyfiles(src_path, dest_path):
    now = datetime.datetime.now()
    print('[{0}] starting to copy {1}...'.format(now, dest_path))
    src_full_path = path.join(st.src_root, src_path)
    if not path.exists(src_full_path):
        print('Folder {0} was not found. Number of classes may need reconsideration.'.format(src_full_path))
    dest_full_path = path.join(st.imbalanced_dataset_root, dest_path)
    if not path.exists(dest_full_path):
        mkdir(dest_full_path)

    scr_samples_path = glob.glob(path.join(src_full_path, st.mask))
    for src_sample_path in scr_samples_path:
        if path.isfile(src_sample_path):
            # exclude dots in file names
            old_filename = path.basename(src_sample_path)
            new_filename = remove_dots_in_filenames(old_filename)
            shutil.copy(src_sample_path, path.join(dest_full_path, new_filename))
    now = datetime.datetime.now()
    print('[{2}] copied {0} samples in {1}'.format(len(scr_samples_path), dest_path, now))

if not path.exists(st.imbalanced_dataset_root):
    mkdir(st.imbalanced_dataset_root)

# region DO2
do2_src_path = 'scr\\1_outer\\0\\'  # ДО2 без дефектов
do2_src_path_cracked_pad = 'scr\\1_outer\\4\\'  # ДО2 с трещиной в подкладке
do2_src_path_turned_pad = 'scr\\1_outer\\5\\'  # ДО2 с разворотом подкладки
do2_src_path_missing_crutch = 'scr\\1_outer\\6\\'  # ДО2 без костыля
do2_src_path_pulled_out_crutch = 'scr\\1_outer\\7\\'  # ДО2 с наддернутым костылем

copyfiles(do2_src_path, ds.do2_path)
if ds.include_defected_and_missing:
    copyfiles(do2_src_path_cracked_pad, ds.do2_path_cracked_pad)
    copyfiles(do2_src_path_turned_pad, ds.do2_path_turned_pad)
    copyfiles(do2_src_path_missing_crutch, ds.do2_path_missing_crutch)
    copyfiles(do2_src_path_pulled_out_crutch, ds.do2_path_pulled_out_crutch)
# endregion

# region DO3
do3_src_path = 'scr\\1_inner\\0\\'  # ДО3 без дефектов
do3_src_path_cracked_pad = 'scr\\1_inner\\4\\'  # ДО3 с трещиной в подкладке
do3_src_path_turned_pad = 'scr\\1_inner\\5\\'  # ДО3 с разворотом подкладки
do3_src_path_missing_crutch = 'scr\\1_inner\\6\\'  # ДО3 без костыля
do3_src_path_pulled_out_crutch = 'scr\\1_inner\\7\\'  # ДО3 с наддернутым костылем

copyfiles(do3_src_path, ds.do3_path)
if ds.include_defected_and_missing:
    copyfiles(do3_src_path_cracked_pad, ds.do3_path_cracked_pad)
    copyfiles(do3_src_path_turned_pad, ds.do3_path_turned_pad)
    copyfiles(do3_src_path_missing_crutch, ds.do3_path_missing_crutch)
    copyfiles(do3_src_path_pulled_out_crutch, ds.do3_path_pulled_out_crutch)
# endregion

# region KB
kb_src_path = 'scr\\5\\0'  # КБ без дефектов
kb_src_path_cracked_pad = 'scr\\5\\4\\'  # КБ с трещиной в подкладке
kb_src_path_turned_pad = 'scr\\5\\5\\'  # КБ с разворотом подкладки
kb_src_path_missing_bolt = 'scr\\5\\8\\'  # КБ без болта
kb_src_path_missing_nut = 'scr\\5\\9\\'  # КБ без гайки
kb_src_path_missing_cleat = 'scr\\5\\11\\'  # КБ без клеммы
kb_src_path_broken_cleat = 'scr\\5\\12\\'  # КБ с изломом клеммы

copyfiles(kb_src_path, ds.kb_path)
if ds.include_defected_and_missing:
    copyfiles(kb_src_path_cracked_pad, ds.kb_path_cracked_pad)
    copyfiles(kb_src_path_turned_pad, ds.kb_path_turned_pad)
    copyfiles(kb_src_path_missing_bolt, ds.kb_path_missing_bolt)
    copyfiles(kb_src_path_missing_nut, ds.kb_path_missing_nut)
    copyfiles(kb_src_path_missing_cleat, ds.kb_path_missing_cleat)
    copyfiles(kb_src_path_broken_cleat, ds.kb_path_broken_cleat)
# endregion

# region KB OP105
kbop105_src_path = 'scr\\6\\0'  # КБ ОП105 без дефектов
kbop105_src_path_cracked_pad = 'scr\\6\\4\\'  # КБ ОП105 с трещиной в подкладке
kbop105_src_path_turned_pad = 'scr\\6\\5\\'  # КБ ОП105 с разворотом подкладки
kbop105_src_path_missing_bolt = 'scr\\6\\8\\'  # КБ ОП105 без болта
kbop105_src_path_missing_nut = 'scr\\6\\9\\'  # КБ ОП105 без гайки
kbop105_src_path_missing_cleat = 'scr\\6\\11\\'  # КБ ОП105 без клеммы
kbop105_src_path_broken_cleat = 'scr\\6\\12\\'  # КБ ОП105 с изломом клеммы

copyfiles(kbop105_src_path, ds.kbop105_path)
if ds.include_defected_and_missing:
    copyfiles(kbop105_src_path_cracked_pad, ds.kbop105_path_cracked_pad)
    copyfiles(kbop105_src_path_turned_pad, ds.kbop105_path_turned_pad)
    copyfiles(kbop105_src_path_missing_bolt, ds.kbop105_path_missing_bolt)
    copyfiles(kbop105_src_path_missing_nut, ds.kbop105_path_missing_nut)
    copyfiles(kbop105_src_path_missing_cleat, ds.kbop105_path_missing_cleat)
    copyfiles(kbop105_src_path_broken_cleat, ds.kbop105_path_broken_cleat)
# endregion

# region JBR
jbr_src_path = 'scr\\10\\0'  # ЖБР без дефектов
jbr_src_path_cracked_pad = 'scr\\10\\4\\'  # ЖБР с трещиной в подкладке
jbr_src_path_turned_pad = 'scr\\10\\5\\'  # ЖБР с разворотом подкладки
jbr_src_path_missing_bolt = 'scr\\10\\8\\'  # ЖБР без болта
jbr_src_path_missing_nut = 'scr\\10\\9\\'  # ЖБР без гайки
jbr_src_path_missing_cleat = 'scr\\10\\11\\'  # ЖБР без клеммы
jbr_src_path_broken_cleat = 'scr\\10\\12\\'  # ЖБР с изломом клеммы

copyfiles(jbr_src_path, ds.jbr_path)
if ds.include_defected_and_missing:
    copyfiles(jbr_src_path_cracked_pad, ds.jbr_path_cracked_pad)
    copyfiles(jbr_src_path_turned_pad, ds.jbr_path_turned_pad)
    copyfiles(jbr_src_path_missing_bolt, ds.jbr_path_missing_bolt)
    copyfiles(jbr_src_path_missing_nut, ds.jbr_path_missing_nut)
    copyfiles(jbr_src_path_missing_cleat, ds.jbr_path_missing_cleat)
    copyfiles(jbr_src_path_broken_cleat, ds.jbr_path_broken_cleat)
# endregion

# region ARS
ars_src_path = 'scr\\18\\0'  # АРС без дефектов
ars_src_path_cracked_pad = 'scr\\18\\4\\'  # АРС с трещиной в подкладке
ars_src_path_turned_pad = 'scr\\18\\5\\'  # АРС с разворотом подкладки
ars_src_path_missing_bolt = 'scr\\18\\8\\'  # АРС без болта
ars_src_path_missing_nut = 'scr\\18\\9\\'  # АРС без гайки
ars_src_path_missing_cleat = 'scr\\18\\11\\'  # АРС без клеммы
ars_src_path_broken_cleat = 'scr\\18\\12\\'  # АРС с изломом клеммы

copyfiles(ars_src_path, ds.ars_path)
if ds.include_defected_and_missing:
    copyfiles(ars_src_path_cracked_pad, ds.ars_path_cracked_pad)
    copyfiles(ars_src_path_turned_pad, ds.ars_path_turned_pad)
    copyfiles(ars_src_path_missing_bolt, ds.ars_path_missing_bolt)
    copyfiles(ars_src_path_missing_nut, ds.ars_path_missing_nut)
    copyfiles(ars_src_path_missing_cleat, ds.ars_path_missing_cleat)
    copyfiles(ars_src_path_broken_cleat, ds.ars_path_broken_cleat)
# endregion

# region fakes and missing
do2_missing_src_path = 'scr\\1_outer\\2\\2\\'  # ДО2 отсутствующий
do3_missing_src_path = 'scr\\1_inner\\3\\2\\'  # ДО3 отсутствующий
kb_missing_src_path = 'scr\\5\\2\\'  # КБ отсутствующий
kbop105_missing_src_path = 'scr\\6\\2\\'  # КБ ОП105 отсутствующий
jbr_missing_src_path = 'scr\\10\\2\\'  # ЖБР
ars_missing_src_path = 'scr\\18\\2\\'  # АРС отсутствующий

fake_src_path = 'scr\\fake\\'

if ds.include_defected_and_missing:
    copyfiles(do2_missing_src_path, ds.missing_path)
    copyfiles(do3_missing_src_path, ds.missing_path)
    copyfiles(kb_missing_src_path, ds.missing_path)
    copyfiles(kbop105_missing_src_path, ds.missing_path)
    copyfiles(jbr_missing_src_path, ds.missing_path)
    copyfiles(ars_missing_src_path, ds.missing_path)

if ds.include_fakes:
    copyfiles(fake_src_path, ds.fake_path)
# endregion


#! /usr/bin/env python
# -*- coding: utf-8 -*-

"""Base class for all models (chainer)."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
from os.path import join, isfile, basename
from glob import glob
import pickle

import logging
logger = logging.getLogger('training')

import numpy as np
import chainer
from chainer import optimizers
from chainer import serializers
from chainer import cuda

from utils.directory import mkdir

OPTIMIZER_CLS_NAMES = {
    "sgd": optimizers.SGD,
    "momentum": optimizers.MomentumSGD,
    "nesterov": optimizers.NesterovAG,
    "adam": optimizers.Adam,
    "adadelta": optimizers.AdaDelta,
    "adagrad": optimizers.AdaGrad,
    "rmsprop": optimizers.RMSprop,
    "rmsprop_graves": optimizers.RMSpropGraves
}

class ModelBase(chainer.Chain):
    """A base class for all models. All models have to inherit this class."""

    def __init__(self):
        raise NotImplementedError

    def __call__(self):
        raise NotImplementedError

    def init_weights(self, parameter_init, distribution,
                     keys=[None], ignore_keys=[None]):
        """Initialize parameters.
        Args:
            parameter_init (float):
            distribution (string): uniform or normal or orthogonal or constant
            keys (list, optional):
            ignore_keys (list, optional):
        """
        for name, param in self.namedparams():
            if keys != [None] and len(list(filter(lambda k: k in name, keys))) == 0:
                continue

            if ignore_keys != [None] and len(list(filter(lambda k: k in name, ignore_keys))) > 0:
                continue

            xp = cuda.get_array_module(param.data)
            if distribution == 'uniform':
                param.data[...] = xp.random.uniform(
                    low=-parameter_init, high=parameter_init, size=param.data.shape)
            elif distribution == 'normal':
                assert parameter_init > 0
                param.data[...] = xp.random.normal(
                    loc=0, scale=parameter_init, size=param.data.shape)
            elif distribution == 'orthogonal':
                flat_shape = (len(param.data), int(
                    np.prod(param.data.shape[1:])))
                a = np.random.normal(size=flat_shape)
                # we do not have cupy.linalg.svd for now
                u, _, v = np.linalg.svd(a, full_matrices=False)
                # TODO: fix bugs
                # pick the one with the correct shape
                q = u if u.shape == flat_shape else v
                param.data[...] = xp.asarray(q.reshape(param.data.shape))
                param.data *= 1.1
            elif distribution == 'constant':
                param.data[...] = xp.asarray(parameter_init)
            else:
                raise NotImplementedError

    def init_forget_gate_bias_with_one(self):
        """Initialize bias in forget gate with 1."""
        for name, param in self.namedparams():
            if 'lstm' in name and ('b1' in name or 'b5' in name):
                param.data[:] = 1.0

    def _inject_weight_noise(self, mean, std):
        raise NotImplementedError

    @property
    def num_params_dict(self):
        if not hasattr(self, '_num_params_dict'):
            self._num_params_dict = {}
            for name, param in self.namedparams():
                self._num_params_dict[name] = param.reshape(-1).shape[0]
        return self._num_params_dict

    @property
    def total_parameters(self):
        if not hasattr(self, '_num_params'):
            self._num_params = 0
            for name, param in self.namedparams():
                self._num_params += param.reshape(-1).shape[0]
        return self._num_params

    @property
    def use_cuda(self):
        return cuda.available and cuda.cudnn_enabled

    def set_cuda(self, gpu_id=0, deterministic=False, benchmark=True):
        """Set model to the GPU version.
        Args:
            gpu_id (int)
            deterministic (bool, optional):
            benchmark (bool, optional): not used
        """
        if self.use_cuda:
            chainer.config.type_check = False

            if deterministic:
                chainer.config.cudnn_deterministic = True
                logger.info('GPU deterministic mode (no cudnn)')
            else:
                chainer.config.cudnn_deterministic = False
                logger.info('GPU mode')
            cuda.get_device_from_id(gpu_id).use()
            self.to_gpu()

            if not chainer.cuda.available:
                raise ValueError('cuda is not available')
            if not chainer.cuda.cudnn_enabled:
                raise ValueError('cudnn is not available')
        else:
            logger.info('CPU mode')

    def set_optimizer(self, optimizer, learning_rate_init,
                      weight_decay=0, clip_grad_norm=5,
                      lr_schedule=None, factor=None, patience_epoch=None):
        """Set the optimizer and add hooks
        Args:
            optimizer (string): sgd or adam or adadelta or adagrad or rmsprop
            learning_rate_init (float): An initial learning rate
            weight_decay (float, optional): L2 penalty
            clip_grad_norm (float):
            lr_schedule: not used here
            factor: not used here
            patience_epoch: not used here
        Returns:
            scheduler ():
        """
        optimizer = optimizer.lower()
        if optimizer not in OPTIMIZER_CLS_NAMES:
            raise ValueError(
                "Optimizer name should be one of [%s], you provided %s." %
                (", ".join(OPTIMIZER_CLS_NAMES), optimizer))

        if optimizer == 'adadelta':
            self.optimizer = optimizers.AdaDelta(rho=0.95, eps=1e-6)
            # TODO: check learning rate
        elif optimizer == 'adagrad':
            self.optimizer = optimizers.AdaGrad(
                lr=learning_rate_init, eps=1e-8)
        elif optimizer == 'adam':
            self.optimizer = optimizers.Adam(
                alpha=0.001, beta1=0.9, beta2=0.999, eps=1e-8)
            # TODO: check learning rate
        elif optimizer == 'sgd':
            self.optimizer = optimizers.MomentumSGD(
                lr=learning_rate_init, momentum=0.9)
        elif optimizer == 'nesterov':
            self.optimizer = optimizers.NesterovAG(
                lr=learning_rate_init, momentum=0.9)
        elif optimizer == 'rmsprop':
            self.optimizer = optimizers.RMSprop(
                lr=learning_rate_init, alpha=0.99, eps=1e-8)
        elif optimizer == 'rmspropgraves':
            self.optimizer = optimizers.RMSpropGraves(
                lr=learning_rate_init, alpha=0.95, momentum=0.9, eps=0.0001)
        else:
            raise NotImplementedError

        self.optimizer.setup(self)

        # Add hook
        self.optimizer.add_hook(
            chainer.optimizer.GradientClipping(clip_grad_norm))
        self.optimizer.add_hook(chainer.optimizer.WeightDecay(weight_decay))
        # self.optimizer.add_hook(chainer.optimizer.GradientNoise(eta=0.01))

        return None

    def set_save_path(self, save_path):
        # Reset model directory
        model_index = 0
        save_path_tmp = save_path
        while True:
            if isfile(join(save_path_tmp, 'complete.txt')):
                # Training of the first model have been finished
                model_index += 1
                save_path_tmp = save_path + '_' + str(model_index)
            elif isfile(join(save_path_tmp, 'config.yml')):
                # Training of the first model have not been finished yet
                model_index += 1
                save_path_tmp = save_path + '_' + str(model_index)
            else:
                break
        self.save_path = mkdir(save_path_tmp)

    def save_checkpoint(self, save_path, epoch, step, lr, metric_dev_best,
                        remove_old_checkpoints=False):
        """Save checkpoint.
        Args:
            save_path (string): path to save a model (directory)
            epoch (int): the currnet epoch
            step (int): the current step
            lr (float):
            metric_dev_best (float):
            remove_old_checkpoints (bool, optional): if True, all checkpoints
                other than the best one will be deleted
        Returns:
            model (string): path to the saved model (file)
        """
        model_path = join(save_path, 'model.epoch-' + str(epoch))

        # Remove old checkpoints
        if remove_old_checkpoints:
            for path in glob(join(save_path, 'model.epoch-*')):
                os.remove(path)

        checkpoint = {
            "epoch": epoch,
            "step": step,
            "lr": lr,
            "metric_dev_best": metric_dev_best
        }

        # Save parameters, optimizer, step index etc.
        # serializers.save_npz(model_path, self)
        # serializers.save_npz(
        #     join(save_path, 'optimizer.epoch-' + str(epoch)), self.optimizer)

        serializer = serializers.DictionarySerializer()

        pickled_params = np.frombuffer(
            pickle.dumps(checkpoint), dtype=np.uint8)
        serializer("checkpoint", pickled_params)

        serializer["model"].save(self)
        serializer["optimizer"].save(self.optimizer)

        np.savez_compressed(model_path, **serializer.target)

        logger.info("=> Saved checkpoint (epoch:%d): %s" % (epoch, model_path))

    def load_checkpoint(self, save_path, epoch=-1, restart=False,
                        load_pretrained_model=False):
        """Load checkpoint.
        Args:
            save_path (string): path to the saved models
            epoch (int, optional): if -1 means the last saved model
            restart (bool, optional): if True, restore the save optimizer
            load_pretrained_model (bool, optional): if True, load all parameters
                which match those of the new model's parameters
        Returns:
            epoch (int): the currnet epoch
            step (int): the current step
            lr (float):
            metric_dev_best (float):
        """
        if int(epoch) == -1:
            # Restore the last saved model
            epochs = [(int(basename(x).split('-')[-1].split('.')[0]), x)
                      for x in glob(join(save_path, 'model.*'))]

            if len(epochs) == 0:
                raise ValueError

            epoch = sorted(epochs, key=lambda x: x[0])[-1][0]

        model_path = join(save_path, 'model.epoch-' + str(epoch) + '.npz')

        if isfile(join(model_path)):
            with np.load(model_path) as f:
                deserializer = serializers.NpzDeserializer(f)

                pickled_params = deserializer("checkpoint", None)
                checkpoint = pickle.loads(pickled_params.tobytes())
                # type: HyperParameters

                # Restore parameters
                if load_pretrained_model:
                    logger.info(
                        "=> Loading pre-trained checkpoint (epoch:%d): %s" % (epoch, model_path))

                    # TODO:
                    # pretrained_dict = checkpoint['state_dict']
                    # model_dict = self.state_dict()
                    #
                    # # 1. filter out unnecessary keys and params which do not match size
                    # pretrained_dict = {
                    #     k: v for k, v in pretrained_dict.items() if k in model_dict.keys() and v.size() == model_dict[k].size()}
                    # # 2. overwrite entries in the existing state dict
                    # model_dict.update(pretrained_dict)
                    # # 3. load the new state dict
                    # self.load_state_dict(model_dict)

                    # for k in pretrained_dict.keys():
                    #     logger.info(k)

                deserializer["model"].load(self)

                # Restore optimizer
                if restart:
                    if hasattr(self, 'optimizer'):
                        deserializer["optimizer"].load(self.optimizer)
                    else:
                        raise ValueError('Set optimizer.')
                else:
                    print("=> Loading checkpoint (epoch:%d): %s" %
                          (epoch, model_path))

        else:
            raise ValueError("No checkpoint found at %s" % model_path)

        return (checkpoint['epoch'] + 1, checkpoint['step'] + 1,
                checkpoint['lr'], checkpoint['metric_dev_best'])

    def _create_var(self, size, fill_value=0, dtype=np.float32):
        """Initialize a variable with zero.
        Args:
            size (tuple):
            fill_value (int or float, optional):
            dtype ():
        Returns:
            var (chainer.Variable, float):
        """
        var = chainer.Variable(self.xp.full(size, fill_value, dtype=dtype))
        return var

    def np2var(self, array, use_cuda=False, volatile=False, dtype=None):
        """Convert form np.ndarray to Variable.
        Args:
            array (np.ndarray): A tensor of any sizes
            use_cuda (bool, optional): if True, use CUDA
            volatile (bool, optional): if True, the history will not be saved.
                This should be used in inference model for memory efficiency.
            type (string, optional): float or long or int
            backend (string, optional): pytorch or chainer
        Returns:
            array (chainer.Variable or list of chainer.Variable):
                pytorch => A tensor of size `[B, T, input_size]`
                chainer => list of `[T_i, input_size]`
        """
        if self.use_cuda:
            if isinstance(array, list):
                array = [chainer.Variable(chainer.cuda.to_gpu(a), requires_grad=False)
                         for a in array]
            else:
                array = chainer.Variable(
                    chainer.cuda.to_gpu(array), requires_grad=False)
        else:
            if isinstance(array, list):
                array = [chainer.Variable(a, requires_grad=False)
                         for a in array]
            else:
                array = chainer.Variable(array, requires_grad=False)

        return array

    def var2np(slef, var):
        """Convert form Variable to np.ndarray.
        Args:
            var (chainer.Variable):
        Returns:
            np.ndarray
        """
        return chainer.cuda.to_cpu(var.data)


import os

import librosa
import numpy as np

from audio.conversion import ms_to_samples, magnitude_to_decibel, normalize_decibel
from audio.features import linear_scale_spectrogram, mel_scale_spectrogram
from audio.io import load_wav
from datasets.dataset_helper import DatasetHelper
from datasets.statistics import collect_decibel_statistics, collect_duration_statistics
from tacotron.params.model import model_params

class CMUDatasetHelper(DatasetHelper):
    """
    Dataset loading helper for the CMU_ARCTIC dataset.
    """
    # Mel. scale spectrogram reference dB over the entire dataset.
    mel_mag_ref_db = 9.33

    # Mel. scale spectrogram maximum dB over the entire dataset.
    mel_mag_max_db = 100.0

    # Linear scale spectrogram reference dB over the entire dataset.
    linear_ref_db = 36.50

    # Linear scale spectrogram maximum dB over the entire dataset.
    linear_mag_max_db = 100.0

    # Raw waveform silence reference signal dB.
    raw_silence_db = None

    def __init__(self, dataset_folder, char_dict, fill_dict):
        super().__init__(dataset_folder, char_dict, fill_dict)

        self._abbreviations = {
            '.': ''
        }

    def load(self, max_samples=None, min_len=None, max_len=None, listing_file_name='train.txt'):
        data_file = os.path.join(self._dataset_folder, listing_file_name)
        wav_folder = os.path.join(self._dataset_folder, 'wav')

        file_paths = []
        sentences = []
        with open(data_file, 'r') as listing_file:
            # Iterate the metadata file.
            for line in listing_file:
                file_id, normalized_sentence = line.split(' ', maxsplit=1)

                # Remove new line characters.
                normalized_sentence = normalized_sentence.strip()

                # Extract the transcription.
                # We do not want the sentence to contain any non ascii characters.
                sentence = self.utf8_to_ascii(normalized_sentence)

                # Skip sentences in case they do not meet the length requirements.
                sentence_len = len(sentence)
                if min_len is not None:
                    if sentence_len < min_len:
                        continue

                # Skip sentences in case they do not meet the length requirements.
                if max_len is not None:
                    if sentence_len > max_len:
                        continue

                sentences.append(sentence)

                # Get the audio file path.
                file_path = '{}.wav'.format(os.path.join(wav_folder, file_id))
                file_paths.append(file_path)

                if max_samples is not None:
                    if len(sentences) == max_samples:
                        break

        # Normalize sentences, convert the characters to dictionary ids and determine their lengths.
        id_sentences, sentence_lengths = self.process_sentences(sentences)

        # for k, v in self._char2idx_dict.items():
        #     print("'{}': {},".format(k, v))

        return id_sentences, sentence_lengths, file_paths

    @staticmethod
    def load_audio(file_path):
        # Window length in audio samples.
        win_len = ms_to_samples(model_params.win_len, model_params.sampling_rate)
        # Window hop in audio samples.
        hop_len = ms_to_samples(model_params.win_hop, model_params.sampling_rate)

        # Load the actual audio file.
        wav, sr = load_wav(file_path.decode())

        # TODO: Determine a better silence reference level for the CMU_ARCTIC dataset (See: #9).
        # Remove silence at the beginning and end of the wav so the network does not have to learn
        # some random initial silence delay after which it is allowed to speak.
        wav, _ = librosa.effects.trim(wav)

        # Calculate the linear scale spectrogram.
        # Note the spectrogram shape is transposed to be (T_spec, 1 + n_fft // 2) so dense layers
        # for example are applied to each frame automatically.
        linear_spec = linear_scale_spectrogram(wav, model_params.n_fft, hop_len, win_len).T

        # Calculate the Mel. scale spectrogram.
        # Note the spectrogram shape is transposed to be (T_spec, n_mels) so dense layers for
        # example are applied to each frame automatically.
        mel_spec = mel_scale_spectrogram(wav, model_params.n_fft, sr, model_params.n_mels,
                                         model_params.mel_fmin, model_params.mel_fmax, hop_len,
                                         win_len, 1).T

        # Convert the linear spectrogram into decibel representation.
        linear_mag = np.abs(linear_spec)
        linear_mag_db = magnitude_to_decibel(linear_mag)
        linear_mag_db = normalize_decibel(linear_mag_db,
                                          CMUDatasetHelper.linear_ref_db,
                                          CMUDatasetHelper.linear_mag_max_db)
        # => linear_mag_db.shape = (T_spec, 1 + n_fft // 2)

        # Convert the mel spectrogram into decibel representation.
        mel_mag = np.abs(mel_spec)
        mel_mag_db = magnitude_to_decibel(mel_mag)
        mel_mag_db = normalize_decibel(mel_mag_db,
                                       CMUDatasetHelper.mel_mag_ref_db,
                                       CMUDatasetHelper.mel_mag_max_db)
        # => mel_mag_db.shape = (T_spec, n_mels)

        # Tacotron reduction factor.
        if model_params.reduction > 1:
            mel_mag_db, linear_mag_db = DatasetHelper.apply_reduction_padding(mel_mag_db,
                                                                              linear_mag_db,
                                                                              model_params.reduction)

        return np.array(mel_mag_db).astype(np.float32), \
               np.array(linear_mag_db).astype(np.float32)

if __name__ == '__main__':
    init_char_dict = {
        'pad': 0,  # padding
        'eos': 1,  # end of sequence
        'a': 2, 'u': 3, 't': 4, 'h': 5, 'o': 6, 'r': 7, ' ': 8, 'f': 9, 'e': 10, 'd': 11, 'n': 12,
        'g': 13, 'i': 14, 'l': 15, ',': 16, 'p': 17, 's': 18, 'c': 19, 'm': 20, 'z': 21, 'w': 22,
        'v': 23, 'k': 24, 'b': 25, "'": 26, 'y': 27, 'j': 28, 'q': 29, 'x': 30, '-': 31, ';': 32
    }

    dataset = CMUDatasetHelper(dataset_folder='/thesis/datasets/cmu_us_slt_arctic',
                               char_dict=init_char_dict,
                               fill_dict=False)

    ids, lens, paths = dataset.load()

    # dataset.pre_compute_features(paths)

    # Print a small sample from the dataset.
    # for p, s, l in zip(paths[:10], ids[:10], lens[:10]):
    #     print(p, np.fromstring(s, dtype=np.int32)[:10], l)

    # Collect and print the duration statistics for all the files.
    # collect_duration_statistics("CMU US SLT Arctic", paths)


import csv
import yaml

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

from pubg_python import PUBG, Shard
from datetime import datetime

config = yaml.safe_load(open('config.yml'))

API_KEY = config['API_KEY']

def convert_timestamp(s):
  """"""
  # return datetime.strptime(s, "%Y-%m-%dT%H:%M:%SZ")
  return datetime.strptime(s, "%Y-%m-%dT%H:%M:%S.%fZ")

def get_player_ids(names, verbose=False):
  """return player ids from usernames"""

  players = api.players().filter(player_names=names)

  if verbose:
    for player in players:
      print('{}: {}'.format(player.name, player.id))

  return players

def participant_names(roster):
  return [participant.name for participant in roster.participants]

def get_winners(rosters):
  for i, roster in enumerate(rosters):
    if roster.won == 'true':
      return i, participant_names(roster)

def print_match_history():
  player_names = ['eponymoose']
  # player_names = ['Stalkyard']

  players = api.players().filter(player_names=player_names)
  player = players[0]

  print("name: {}\nid: {}\n".format(player.name, player.id))

  matches = player.matches

  match_ids = [match.id for match in matches]

  matches = api.matches().filter(match_ids=match_ids)

  for i, match_id in enumerate(match_ids):
    match = api.matches().get(match_id)
    print("{}: {}; {}; {}; {}".format(
      i, match.id, datetime.strptime(match.created_at, "%Y-%m-%dT%H:%M:%SZ"), match.game_mode, match.map))

    rosters = match.rosters

    for roster in rosters:
      if any([participant.name in player_names for participant in roster.participants]):
        print([participant.name for participant in roster.participants])

        for participant in roster.participants:
          print("\t{}: kills={}; death_type={}; kill_place={}".format(
            participant.name, participant.kills, participant.death_type, participant.kill_place))
  #             print("winners(s): {}".format(roster.won))
    try:
      roster_index, winning_participants = get_winners(rosters)
      print("winners(s) [{}]: {}\n".format(roster_index, winning_participants))

    except TypeError:
      print('\nEXCEPTION! None type returned get_winners()\n')
      continue

  print('end')

def write_match_kill_details():

  non_player_kills = [
    "Damage_BlueZone",
    "Damage_Drown",
    "Damage_Explosion_RedZone",
    "Damage_Instant_Fall",
    "Damage_VehicleCrashHit"]

  player_names = ['eponymoose']
  players = api.players().filter(player_names=player_names)
  player = players[0]

  print("name: {}\nid: {}\n".format(player.name, player.id))

  matches = player.matches
  match_ids = [match.id for match in matches]

  match_id = match_ids[0]
  # match_id = "2a0346f6-2493-4deb-beb3-151af50ecf19"  # squad/erangel
  # match_id = "1deb2118-557e-4945-a8e1-81ae70bf62e3"
  # match_id = "42f94823-1e69-423e-983a-02f0973c9534"  # duo/sanhok

  match = api.matches().get(match_id)
  asset = match.assets[0]
  telemetry = api.telemetry(asset.url)

  player_kill_events = telemetry.events_from_type('LogPlayerKill')

  # print(player_kill_events)

  with open('c:/workspace/pubg-analytics/output/pubg_kill_events_{}.csv'.format(match.id), 'w', newline='') as f:
    writer = csv.writer(f, delimiter=',')
    writer.writerow(["killer", "killer_team_id", "victim", "victim_team_id"])
    for i, event in enumerate(player_kill_events):

      # TODO: add in logic for suicides
      killer_team = event.killer.team_id

      if event.damage_type_category in non_player_kills:
        killer_name = event.damage_type_category
        killer_team = 0
      else:
        if event.killer.name == "":
          killer_name = event.damage_type_category + "_MISSING_KILLER"
        else:
          killer_name = event.killer.name

      writer.writerow([killer_name, event.killer.team_id, event.victim.name, event.victim.team_id])

if __name__ in '__main__':

  api = PUBG(API_KEY, Shard.PC_OC)
  names = ['eponymoose', 'Stalkyard', 'Ritalin', 'yakabox']

  # get_player_ids(names, verbose=True)

  write_match_kill_details()


#!/usr/bin/env python2
# -*- coding: utf-8 -*-
"""
Created on Mon Apr 17 22:27:34 2017

@author: bladestone
"""
from flask_apscheduler import APScheduler
from flask import Flask
from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore
import time,json
import urllib2
from flask import request
import traceback
import logging
logging.basicConfig()
scheduler=None

class Config(object):
    # JOBS = [
    #         {
    #            'id':'job1',
    #            'func':'flask-ap:test_data',
    #            'args': 'x',
    #            'trigger': {
    #                 'type': 'cron',
    #                 'day_of_week':"mon-fri",
    #                 'hour':'0-23',
    #                 'minute':'0-59',
    #                 'second': '*/5'
    #             }

    #          }
    #     ]
    SCHEDULER_API_ENABLED = True
    SCHEDULER_JOBSTORES = {
        'default': SQLAlchemyJobStore(url='sqlite:////var/www/jobs.sqlite')
    }

app = Flask(__name__, static_url_path='')

def get(URL):
    response = urllib2.urlopen(URL) #调用urllib2向服务器发送get请求
    return json.loads(response.read()) #获取服务器返回的页面信息

def post(URL,values):
    headers = {'Content-Type': 'application/json'}
    data = json.dumps(values)    #适用urllib对数据进行格式化编码
    req = urllib2.Request(url=URL, headers=headers, data=data)    #生成页面请求的完整数据
    response = urllib2.urlopen(req)     #发送页面请求
    return json.loads(response.read())    #获取服务器返回的页面信息
def task(d):
    print("work param:",d)
    r = get(d)
    print("work res:",r)

def jobfromparm(**jobargs):
    id = str(time.time())
    print('add job: ',id)
    job = scheduler.add_job(func=task,id=id, **jobargs)
    return 'sucess'

@app.route('/pause')
def pausejob():
    scheduler.pause_job('job1')
    return "Success!"

@app.route('/')
def index():
    return "杜萌你好!"

@app.route('/resume')
def resumejob():
    scheduler.resume_job('job1')
    return "Success!"

@app.route('/job', methods=['GET', 'POST'])
def job():
    try:
        if request.method=='POST':
            data = request.get_json(force=True)
            job = jobfromparm(**data)
            return json.dumps({'res':'sucess'})
        elif request.method=='GET':
            res = {'res':[]}
            for job in scheduler.get_jobs():
                t = {}
                t['id'] = job.id
                t['args'] = job.args
                t['trigger'] = str(job.trigger)
                res['res'].append(t)
            return json.dumps(res)
    except:
        traceback.print_exc()

@app.route('/job/<id>', methods=['GET', 'PUT', 'DELETE'])
def job_id(id):
    if request.method=='PUT':
        pass
        return 'sucess'
    elif request.method=='DELETE':
        scheduler.delete_job(id)
        return json.dumps({'res':'sucess'})
    elif request.method=='GET':
        job = scheduler.get_job(id)
        res = {'id':job.id,'args':job.args}
        return json.dumps(res)

if __name__ == '__main__':
    scheduler = APScheduler()
    print("Let us run out of the loop")
    app.config.from_object(Config())

    # it is also possible to enable the API directly
    # scheduler.api_enabled = True
    scheduler.init_app(app)
    scheduler.start()
    # scheduler.add_job('job3', test_data,trigger='cron',second='*/5',args=['z'])
    app.run(host='0.0.0.0',debug=True,threaded=False)


'''
Copyright (C) 2015-2016 Skylable Ltd. <info-copyright@skylable.com>
License: Apache 2.0, see LICENSE for more details.
'''

import hashlib
import hmac
import time

import bottle

from sxswift.config import get_settings
from sxswift.utils.security import const_time_is_equal
from sxswift.lib.info import get_info_data
from sxswift.http_helpers.decorators import sxswift_route

ALLOWED_HMAC_METHODS = {
    'HEAD': ['HEAD', 'GET'],
    'GET': ['GET'],
}

def _get_hmac(method, path, expires, key):
    msg = '\n'.join([method, str(expires), path])
    digestmod = hashlib.sha1
    return hmac.new(key, msg=msg, digestmod=digestmod).hexdigest()

def _validate_signature(path, methods, sig, expires):
    try:
        expires = int(expires)
    except (ValueError, TypeError):
        raise bottle.HTTPError(400, 'expires not an integer')

    if expires < time.time():
        raise bottle.HTTPError(400, 'expires points at the time in past')

    has_valid_signature = False
    admin_key = get_settings().get('this.admin_key')
    for method in methods:
        digested_msg = _get_hmac(method, path, expires, admin_key)
        sigs_are_equal = const_time_is_equal(digested_msg, sig)
        has_valid_signature = has_valid_signature or sigs_are_equal
        # Even though we could break here we leave it as it is to
        # prevent timing attack even further.

    if not has_valid_signature:
        raise bottle.HTTPError(401, 'Incorrect sig for given expires')

@sxswift_route
def get_info():
    sig = bottle.request.params.get('swiftinfo_sig')
    expires = bottle.request.params.get('swiftinfo_expires')
    is_admin = sig or expires
    if is_admin:
        methods = ALLOWED_HMAC_METHODS[bottle.request.method]
        _validate_signature(bottle.request.path, methods, sig, expires)
    return get_info_data(is_admin=is_admin)


# https://github.com/musikalkemist/Deep-Learning-Audio-Application-From-Design-to-Deployment/blob/master/4-%20Making%20Predictions%20with%20the%20Speech%20Recognition%20System/keyword_spotting_service.py

import librosa
import tensorflow as tf
import numpy as np

SAVED_MODEL_PATH = "C:/Users/Brandon/Desktop/Speech/C_model.h5"
SAMPLES_TO_CONSIDER = 22050 * 2

"""Singleton class for keyword spotting inference with trained models.
:param model: Trained model
"""

class _Phrase_Identifier:

    """model = None"""
    _mapping = [
        "KaFeiDaiZou",
        "NiHao",
        "XieXie",
        "ZaiJian",
        "ZaoShangHao"
    ]
    _instance = None

    def preprocess(self, file_path, num_mfcc=13, n_fft=2048, hop_length=512):

        # load audio file
        signal, sample_rate = librosa.load(file_path)

        if len(signal) >= SAMPLES_TO_CONSIDER:
            # ensure consistency of the length of the signal
            signal = signal[:SAMPLES_TO_CONSIDER]

            # extract MFCCs
            MFCCs = librosa.feature.mfcc(signal, sample_rate, n_mfcc=num_mfcc, n_fft=n_fft,
                                         hop_length=hop_length)
        return MFCCs.T

    """Extract MFCCs from audio file.
    :param file_path (str): Path of audio file
    :param num_mfcc (int): # of coefficients to extract
    :param n_fft (int): Interval we consider to apply STFT. Measured in # of samples
    :param hop_length (int): Sliding window for STFT. Measured in # of samples
    :return MFCCs (ndarray): 2-dim array with MFCC data of shape (# time steps, # coefficients)
    """

    def predict(self, file_path):

        # extract MFCC
        MFCCs = self.preprocess(file_path)

        # 4 dimensional array to feed the model for prediction: (# samples, # time steps, # coefficients, 1)
        MFCCs = MFCCs[np.newaxis, ..., np.newaxis]

        # making the prediction and returning the phrase
        predictions = self.model.predict(MFCCs)
        print(predictions)
        predicted_index = np.argmax(predictions)
        predicted_phrase = self._mapping[predicted_index]
        return predicted_phrase

    """
    :param file_path (str): Path to audio file to predict
    :return predicted_keyword (str): Keyword predicted by the model
    """

def Phrase_Identifier():

    # ensure an instance is created only the first time the factory function is called
    if _Phrase_Identifier._instance is None:
        _Phrase_Identifier._instance = _Phrase_Identifier()
        _Phrase_Identifier.model = tf.keras.models.load_model(SAVED_MODEL_PATH)
    return _Phrase_Identifier._instance

    """Factory function for Keyword_Spotting_Service class.
    :return _Keyword_Spotting_Service._instance (_Keyword_Spotting_Service):
    """

if __name__ == "__main__":

    # create 2 instances of the keyword spotting service
    kss = Phrase_Identifier()
    kss1 = Phrase_Identifier()

    # check that different instances of the keyword spotting service point back to the same object (singleton)
    assert kss is kss1

    keyword2 = kss.predict("C:/Users/Brandon/Desktop/Speech/Chinese/ZaoShangHao/zaoshanghao010.wav")
    print(keyword2)

    # make a prediction
    # keyword = kss.predict(TEST_SAMPLE)
    # print(keyword)

    keyword2 = kss.predict("C:/Users/Brandon/Desktop/Speech/Chinese/NiHao/NiHao038.wav")
    print(keyword2)
    keyword2 = kss.predict("C:/Users/Brandon/Desktop/Speech/Chinese/KaFeiDaiZou/KaFeiDaiZou023.wav")
    print(keyword2)
    keyword2 = kss.predict("C:/Users/Brandon/Desktop/Speech/Chinese/XieXie/XieXie021.wav")
    print(keyword2)
    keyword2 = kss.predict("C:/Users/Brandon/Desktop/Speech/Chinese/ZaiJian/ZaiJian025.wav")
    print(keyword2)


import numpy as np
import gym
import pickle

from argparse import ArgumentParser

import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim

from collector import dataCollector
from train import trainer
from test import test

parser = ArgumentParser(description="Parameters for the code - gated trace")
parser.add_argument('--trace_type', type=str, default="gated", help="gated or accumulating" )
parser.add_argument('--gamma', type=float, default=0.99, help="discount factor")
parser.add_argument('--n', type=int, default=8, help="chain length")
parser.add_argument('--t_seeds', type=int, default=25, help="total seeds")
parser.add_argument('--env', type=str, default="gridWorld", help="Environment")
parser.add_argument('--intrst', type=float, default=0.01, help="interest of states while using etd")
parser.add_argument('--hidden', type=int, default=1, help="number of hidden units")
parser.add_argument('--test_every', type=int, default=1, help="calculate MSE after # episode")
parser.add_argument('--episodes', type=int, default=30, help="number of episodes")
parser.add_argument('--lr', type=float, default=0.01, help="learning rate")
parser.add_argument('--log', type=int, default=1, help="compute the results")
parser.add_argument('--save', type=int, default=1, help="save the results")
parser.add_argument('--fo', action='store_true', help="type of env")
args = parser.parse_args()

total_seeds = args.t_seeds

seed_errors = []
for seed in range(total_seeds):
	args.seed = seed
	np.random.seed(args.seed)
	torch.manual_seed(args.seed)
	torch.backends.cudnn.deterministic = True
	torch.backends.cudnn.benchmark = False

	device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

	if args.env == "gridWorld":
		from grid_world import gridWorld
		from networks import gridNet

		env = gridWorld(n=args.n, po = not(args.fo))
		val_net = gridNet(n=args.n, hidden=args.hidden)
		val_net.to(device)

	elif args.env == "gridWorld2":
		from grid_world2 import gridWorld2
		from networks import gridNet

		env = gridWorld2(n=args.n, po = not(args.fo))
		val_net = gridNet(n=args.n, hidden=args.hidden)
		val_net.to(device)

	elif args.env == "lightWorld":
		from light_world import lightWorld
		from networks import lightNet

		env = lightWorld(n=args.n)
		val_net = lightNet(n=args.n)
		val_net.to(device)

	env.seed(args.seed)

	fname_data = "drl_env_"+str(args.env)+"_size_"+str(args.n)+"_seed_"+str(seed)+"_epi_250"
	with open("data/"+fname_data+"_data.pkl", "rb") as f:
		data_list = pickle.load(f)
	optimizer = optim.SGD(val_net.parameters(), lr=args.lr)
	train_cls = trainer(args, data_list, val_net, optimizer, device)
	test_cls = test(args, env, np.ones((env.n**2,env.action_space.n))/env.action_space.n, device)
	val_net, error_list = train_cls.train(test_cls)

	seed_errors.append(error_list)

seed_error_mean = np.array(seed_errors).mean(axis=0)
seed_error_std = np.array(seed_errors).std(axis=0)

if args.log == 1 and args.save == 1:
	if args.trace_type in ["etd", "etd_adaptive"]:
		filename = "drl_"+args.trace_type+"_int_"+str(args.intrst)+"_env_"+str(args.env)+"_size_"+str(args.n)+"_hidden_"+str(args.hidden)+"_lr_"+str(args.lr)+"_seeds_"+str(args.t_seeds)+"_epi_"+str(args.episodes)
	else:
		filename = "drl_"+args.trace_type+"_env_"+str(args.env)+"_size_"+str(args.n)+"_hidden_"+str(args.hidden)+"_lr_"+str(args.lr)+"_seeds_"+str(args.t_seeds)+"_epi_"+str(args.episodes)

	with open("results_"+str(args.env)+"/"+filename+"_errors_mean.pkl", "wb") as f:
		pickle.dump(seed_error_mean, f)

	with open("results_"+str(args.env)+"/"+filename+"_errors_std.pkl", "wb") as f:
		pickle.dump(seed_error_std, f)


"""
Einstein's Riddle

Einstein's Riddle is a logic puzzle. It is often claimed that only 2% of the population can solve the puzzle.

It has several variants, one of them is this:
1. There are five houses.
2. The Mathematician lives in the red house.
3. The Hacker writes in Python.
4. Brackets (code editor) is used in the green house.
5. The Analyst uses Atom (code editor).
6. The green house is immediately to the right of the ivory house.
7. The man who uses Redis (Database System) writes in Java.
8. Cassandra (Database System) is used in the yellow house.
9. Notepad++ (code editor) is used in the middle house.
10. The Developer lives in the first house.
11. The man who uses Hadoop (Database System) lives in the house next to the man writing in JavaScript.
12. Cassandra is used in the house next to the house where the primary language is C#.
13. The man who uses ArangoDB uses Sublime Text (code editor).
14. The Engineer uses MongoDB.
15. The Developer lives next to the blue house.

Now, who uses Vim? Who writes in C++?

Write a program that solves this riddle and outputs the answer.
"""
from itertools import permutations

color = ["ivory", "red", "blue", "green", "yellow"]
profession = ["Developer", "Analyst", "Engineer", "Hacker", "Mathematician"]
language = ["C#", "C++", "Java", "JavaScript", "Python"]
database = ["ArangoDB", "Cassandra", "Hadoop", "MongoDB", "Redis"]
editor = ["Notepad++", "Atom", "Brackets", "Sublime Text", "Vim"]

count = 0
for p in permutations(profession[1:]):
 p = ("Developer", *p)                    #10
 for c in permutations(color):

  if (abs(c.index("green") -
  c.index("ivory"))==1 and                 #6
  c[p.index("Mathematician")]=="red"       #2
  and abs(p.index("Developer") -
  c.index("blue"))==1):                   #15

   for l in permutations(language):
    if l[p.index("Hacker")]=="Python":     #3
     for d in permutations(database):

      if (l[d.index("Redis")]=="Java" and  #7
      c[d.index("Cassandra")]=="yellow"    #8
      and abs(d.index("Hadoop") -
      l.index("JavaScript"))==1           #11
      and abs(d.index("Cassandra") -
      l.index("C#"))==1 and               #12
      d[p.index("Engineer")]=="MongoDB"): #14

       for e in permutations(editor[1:]):
        e = (*e[:2], "Notepad++", *e[2:])  #9

        if (c[e.index("Brackets")]=="green"#4
        and e[p.index("Analyst")]=="Atom"  #5
        and e[d.index("ArangoDB")]==
        "Sublime Text"):                  #13

         count += 1
         print("Solution {}:".format(count))
         print("The {} uses Vim".format(
         p[e.index("Vim")]))
         print("The {} writes in C++".format(
         p[l.index("C++")]))
         print("\nFull solution:")
         for i in range(5):
          print(p[i], c[i], l[i], d[i], e[i])
         print()

"""
Notes: This is a somewhat brute-force algorithm. I have five lists for five attributes: color, profession, language, database, and editor. The nested for loops let me go over all possible rearrangements of each of the 5 lists (which are produced by the permutations() function) and check if all 15 conditions are satisfied. Now I could do all the nested for loops together and then check the conditions, like:

for p in permutations(profession):
  for c in permutations(color):
    for l in permutations(language):
      for d in permutations(database):
        for e in permutations(editor):
          #if(all the 15 conditions):
            #solution found!

But that would take a LOT of time: Each list has 5 elements, so there are factorial(5) = 120 rearragements. With 5 nested for loops, it'll be 120^5 = 24 883 200 000 iterations!

A better idea is to check the conditions as early as possible. So immediately inside the first for loop, I'm making sure Developer is the first profession. Inside the second one, I'm throwing in all the conditions that involve profession and color. Only when those conditions are satisfied, am I allowing my code to progress to the next for loop, and so on.
"""

#CODE FOR SOLVING THIS RIDDLE USING THE constraint MODULE
#https://pypi.org/project/python-constraint/
#(Not available on Sololearn)
#To install: pip install python-constraint

"""
from constraint import Problem, AllDifferentConstraint

problem = Problem()

#5x5 variables
profession = {"Developer", "Analyst", "Engineer", "Hacker", "Mathematician"}
color = {"ivory", "red", "blue", "green", "yellow"}
language = {"C#", "C++", "Java", "JavaScript", "Python"}
database = {"ArangoDB", "Cassandra", "Hadoop", "MongoDB", "Redis"}
editor = {"Notepad++", "Atom", "Brackets", "Sublime Text", "Vim"}

attributes = (profession, color, language, database, editor)

eq = lambda x, y: x==y
nbr = lambda x, y: abs(x-y)==1
val = lambda n: lambda x: x==n

for atr in attributes:
    #range(1, 6) is the domain
    #that is, all possible values of each of
    #the variables in atr
    #corresponds to house number
    problem.addVariables(atr, range(1, 6))
    problem.addConstraint(AllDifferentConstraint(), atr)

problem.addConstraint(eq, ["Mathematician", "red"]) #2
problem.addConstraint(eq, ["Hacker", "Python"]) #3
problem.addConstraint(eq, ["Brackets", "green"]) #4
problem.addConstraint(eq, ["Analyst", "Atom"]) #5
problem.addConstraint(nbr, ["green", "ivory"]) #6
problem.addConstraint(eq, ["Redis", "Java"]) #7
problem.addConstraint(eq, ["Cassandra", "yellow"]) #8
problem.addConstraint(val(3), ["Notepad++"]) #9
problem.addConstraint(val(1), ["Developer"]) #10
problem.addConstraint(nbr, ["Hadoop", "JavaScript"]) #11
problem.addConstraint(nbr, ["Cassandra", "C#"]) #12
problem.addConstraint(eq, ["ArangoDB", "Sublime Text"]) #13
problem.addConstraint(eq, ["Engineer", "MongoDB"]) #14
problem.addConstraint(nbr, ["Developer", "blue"]) #15

#That's it!

for sol in problem.getSolutions():
    #Just print(sol) works, but it's a bit of a mess for this problem
    #Each solution is a dictionary mapping variables to values
    #I tried to make it more readable
    for i in range(1, 6):
        print (i, end = ": ")
        for atr in attributes:
            for var in atr:
                if sol[var] == i:
                    print(var, end = " ")
        print()
    print()

"""


import re
import nltk
from nltk.tokenize import word_tokenize
from nltk import pos_tag

class BaseRules(object):
    punctuation = '[/{}@#$%^&*()_=+–‘’「」－\-—•・●<>\n``]'

    def __init__(self):
        pass

    @staticmethod
    def filter(x, cue_type):
        if cue_type == 'conj':
            for i in x:
                if i.startswith('VB'):
                    return True
        else:
            for i in x:
                if i.startswith('VB') or i.startswith('NN'):
                    return True
        return False

    @staticmethod
    def restore_cue(sent, ex):
        g = re.match('.+:(?P<cue>.+?)\)', ex)
        if g:
            res = g.group('cue')
            words = str(res).split('|')
            for w in words:
                if w in sent:
                    return w
        return ex

    @staticmethod
    def extract(pre_sent, current_sent):
        pass

    @staticmethod
    def segment(sent):
        words = word_tokenize(sent)
        return words

    @staticmethod
    def pos_tagger(sent):
        words = word_tokenize(sent)
        words, postags = zip(*pos_tag(words))
        return words, postags

    @staticmethod
    def split_cue(words, cue):
        i = 0
        while i < len(words):
            if words[i] == cue[0]:
                j, tag = 0, True
                while j < len(cue):
                    if words[i+j] == cue[j]:
                        j += 1
                    else:
                        tag = False
                        break
                if tag:
                    return i, i+len(cue)
            i += 1
        return -1, -1

    @staticmethod
    def get_response(left, right, cue, sent, relation, min_len, max_len):
        response = {}
        left = re.sub(BaseRules.punctuation, '', left)
        right = re.sub(BaseRules.punctuation, '', right)
        left_words, left_postags = BaseRules.pos_tagger(left)
        right_words, right_postags = BaseRules.pos_tagger(right)
        if min_len < len(left_words) < max_len and min_len < len(right_words) < max_len:
            response = {
                'left_words': left_words, 'right_words': right_words, 'left_pos': left_postags,
                'right_pos': right_postags, 'cue': cue, 'sent': sent, 'relation': relation
            }
            return response
        return response


import cv2
import numpy as np
from matplotlib import pyplot as plt
from tkinter import *
from tkinter import messagebox
from PIL import ImageTk, Image

root = Tk()
path="input.jpg"

w = Label(root, text="Ancient Tamil Script Recognition and Translation", bg="black", fg="white", font="20")
w.pack(fill=X)
text1 = Text(root, height=20, width=80)
photo = ImageTk.PhotoImage(Image.open(path))
text1.insert(END,'\n')
text1.image_create(END, image=photo)

text1.pack(side=LEFT)

def Display():
	path1="medianblur.jpg"
	photo1 = ImageTk.PhotoImage(Image.open(path1))
	panel=Label(root,image=photo1,height=500,width=500)
	panel.configure(image=photo1)
	panel.image=photo1
	panel.pack(side=LEFT)

def segment():
	image = cv2.imread('input.jpg')
	blur = cv2.medianBlur(image,5)
	#grayscale
	gray = cv2.cvtColor(blur,cv2.COLOR_BGR2GRAY)
	#binary
	ret,thresh = cv2.threshold(gray,127,255,cv2.THRESH_BINARY_INV)

#find contours
	im2,ctrs, hier = cv2.findContours(thresh.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
#sort contours
	sorted_ctrs = sorted(ctrs, key=lambda ctr: cv2.boundingRect(ctr)[0])
	for i, ctr in enumerate(sorted_ctrs):
    # Get bounding box
    		x, y, w, h = cv2.boundingRect(ctr)
    		rect=cv2.boundingRect(ctr)
    		if rect[2] < 10 and rect[3] < 10: continue
# Getting ROI
    		roi = image[y:y+h, x:x+w]
# show ROI

    		cv2.rectangle(image,(x,y),( x + w, y + h ),(90,0,255),2)

	cv2.imwrite('segoutput.jpg',image)
	messagebox.showinfo("Segmentation","segmentation is done")
	cv2.waitKey(0)

root.bind("<Return>", Display)
root.bind("<Return>", segment)
B1=Button(root,text="After noise removed",command=Display)
B1.pack(side=LEFT)
B2=Button(root,text="Segmentation",command=segment)
B2.pack(side=LEFT)

root.mainloop()


"""Ghost post enrichment of data."""
from datetime import datetime, timedelta
from time import sleep

from fastapi import APIRouter
from fastapi.responses import JSONResponse

from app.moment import get_current_datetime, get_current_time
from app.posts.lynx.parse import batch_lynx_embeds, generate_link_previews
from app.posts.metadata import assign_img_alt, batch_assign_img_alt
from app.posts.update import update_metadata, update_mobiledoc
from clients import gcs, ghost
from clients.log import LOGGER
from config import basedir
from database import rdbms
from database.read_sql import collect_sql_queries, fetch_raw_lynx_posts
from database.schemas import PostUpdate

router = APIRouter(prefix="/posts", tags=["posts"])

@router.post(
    "/",
    summary="Optimize post metadata.",
    description="Performs multiple actions to optimize post SEO. \
                Generates meta tags, ensures SSL hyperlinks, and populates missing <img /> `alt` attributes.",
    response_model=PostUpdate,
)
async def update_post(post_update: PostUpdate):
    """
    Enrich post metadata upon update.

    :param post_update: Request to update Ghost post.
    :type post_update: PostUpdate
    """
    previous_update = post_update.post.previous
    if previous_update:
        current_time = get_current_datetime()
        previous_update_date = datetime.strptime(
            previous_update["updated_at"], "%Y-%m-%dT%H:%M:%S.000Z"
        )
        if previous_update_date and current_time - previous_update_date < timedelta(
            seconds=5
        ):
            LOGGER.warning("Post update ignored as post was just updated.")
            return "Post update ignored as post was just updated."
    post = post_update.post.current
    slug = post.slug
    title = post.title
    feature_image = post.feature_image
    custom_excerpt = post.custom_excerpt
    primary_tag = post.primary_tag
    html = post.html
    mobiledoc = post.mobiledoc
    time = get_current_time()
    body = {
        "posts": [
            {
                "meta_title": title,
                "og_title": title,
                "twitter_title": title,
                "meta_description": custom_excerpt,
                "twitter_description": custom_excerpt,
                "og_description": custom_excerpt,
                "updated_at": time,
            }
        ]
    }
    if primary_tag.slug == "roundup" and feature_image is None:
        feature_image = gcs.fetch_random_lynx_image()
        body["posts"][0].update(
            {
                "feature_image": feature_image,
                "og_image": feature_image,
                "twitter_image": feature_image,
            }
        )
    if html and "http://" in html:
        html = html.replace("http://", "https://")
        body["posts"][0].update({"html": html})
        LOGGER.info(f"Replaced unsecure links in post `{slug}`")
    if feature_image is not None:
        body["posts"][0].update(
            {"og_image": feature_image, "twitter_image": feature_image}
        )
    if mobiledoc:
        mobiledoc = assign_img_alt(body["posts"][0]["mobiledoc"])
        body["posts"][0].update({"mobiledoc": mobiledoc})
    sleep(1)
    time = get_current_time()
    body["posts"][0]["updated_at"] = time
    response, code = ghost.update_post(post.id, body, post.slug)
    LOGGER.success(f"Successfully updated post `{slug}`: {body}")
    return {str(code): response}

@router.get(
    "/",
    summary="Sanitize metadata for all posts.",
    description="Run a sequence of analytics to ensure all posts have proper metadata.",
)
async def batch_update_metadata():
    update_queries = collect_sql_queries("posts/updates")
    update_results, num_updated = rdbms.execute_queries(update_queries, "hackers_prod")
    insert_posts = rdbms.execute_query_from_file(
        f"{basedir}/database/queries/posts/selects/missing_all_metadata.sql",
        "hackers_prod",
    )
    insert_results = update_metadata(insert_posts)
    LOGGER.success(
        f"Inserted metadata for {len(insert_results)} posts, updated {num_updated}."
    )
    return {
        "inserted": {"count": len(insert_results), "posts": insert_results},
        "updated": {"count": num_updated, "posts": update_results},
    }

@router.get(
    "/embed",
    summary="Batch create Lynx embeds.",
    description="Fetch raw Lynx post and generate embedded link previews.",
)
async def batch_lynx_previews():
    posts = fetch_raw_lynx_posts()
    result = batch_lynx_embeds(posts)
    return result

@router.post(
    "/embed",
    summary="Embed Lynx links.",
    description="Generate embedded link previews for a single Lynx post.",
)
async def post_link_previews(post_update: PostUpdate):
    """
    Render anchor tag link previews.

    :param post_update: Request to update Ghost post.
    :type post_update: PostUpdate
    """
    post = post_update.post.current
    post_id = post.id
    slug = post.slug
    html = post.html
    previous = post_update.post.previous
    primary_tag = post.primary_tag
    if primary_tag.slug == "roundup":
        if html is not None and "kg-card" not in html:
            if previous.get("slug", None) is None:
                num_embeds, doc = generate_link_previews(post.__dict__)
                result = rdbms.execute_query(
                    f"UPDATE posts SET mobiledoc = '{doc}' WHERE id = '{post_id}';",
                    "hackers_prod",
                )
                LOGGER.info(f"Generated Previews for Lynx post {slug}: {doc}")
                return result
        LOGGER.warning(f"Lynx post {slug} already contains previews.")
        return JSONResponse(
            {f"Lynx post {slug} already contains previews."},
            status_code=202,
            headers={"content-type": "text/plain"},
        )

@router.get(
    "/alt",
    summary="Populate missing alt text for images.",
    description="Assign missing alt text to embedded images.",
)
async def assign_img_alt_attr():
    """Find <img>s missing alt text and assign `alt`, `title` attributes."""
    return batch_assign_img_alt()

@router.get("/backup")
async def backup_database():
    """Export JSON backup of database."""
    json = ghost.get_json_backup()
    return json


#!/usr/bin/env python
# -*- coding: utf-8 -*-

# #########################################################################
# Copyright (c) 2016, UChicago Argonne, LLC. All rights reserved.         #
#                                                                         #
# Copyright 2016. UChicago Argonne, LLC. This software was produced       #
# under U.S. Government contract DE-AC02-06CH11357 for Argonne National   #
# Laboratory (ANL), which is operated by UChicago Argonne, LLC for the    #
# U.S. Department of Energy. The U.S. Government has rights to use,       #
# reproduce, and distribute this software.  NEITHER THE GOVERNMENT NOR    #
# UChicago Argonne, LLC MAKES ANY WARRANTY, EXPRESS OR IMPLIED, OR        #
# ASSUMES ANY LIABILITY FOR THE USE OF THIS SOFTWARE.  If software is     #
# modified to produce derivative works, such modified software should     #
# be clearly marked, so as not to confuse it with the version available   #
# from ANL.                                                               #
#                                                                         #
# Additionally, redistribution and use in source and binary forms, with   #
# or without modification, are permitted provided that the following      #
# conditions are met:                                                     #
#                                                                         #
#     * Redistributions of source code must retain the above copyright    #
#       notice, this list of conditions and the following disclaimer.     #
#                                                                         #
#     * Redistributions in binary form must reproduce the above copyright #
#       notice, this list of conditions and the following disclaimer in   #
#       the documentation and/or other materials provided with the        #
#       distribution.                                                     #
#                                                                         #
#     * Neither the name of UChicago Argonne, LLC, Argonne National       #
#       Laboratory, ANL, the U.S. Government, nor the names of its        #
#       contributors may be used to endorse or promote products derived   #
#       from this software without specific prior written permission.     #
#                                                                         #
# THIS SOFTWARE IS PROVIDED BY UChicago Argonne, LLC AND CONTRIBUTORS     #
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT       #
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS       #
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL UChicago     #
# Argonne, LLC OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,        #
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,    #
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;        #
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER        #
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT      #
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN       #
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE         #
# POSSIBILITY OF SUCH DAMAGE.                                             #
# #########################################################################

"""
Please make sure the installation :ref:`pre-requisite-reference-label` are met.

This module verifies a configured hd5 file. The data is verified against configured
"limits" file. The limits are applied by processes performing specific quality calculations.

The results is a detail report of calculated values. The indexes of slices that did not pass
quality check are returned back.

"""
import os.path
import json
import sys
import string
from multiprocessing import Queue, Process
import dquality.common.utilities as utils
import dquality.handler as handler
from dquality.common.containers import Data
import dquality.common.report as report

__author__ = "Barbara Frosik"
__copyright__ = "Copyright (c) 2016, UChicago Argonne, LLC."
__docformat__ = 'restructuredtext en'
__all__ = ['init',
           'process_data',
           'verify_file',
           'verify']

def init(config):
    """
    This function initializes global variables. It gets values from the configuration file, evaluates and processes
    the values. If mandatory file or directory is missing, the script logs an error and exits.

    Parameters
    ----------
    config : str
        configuration file name, including path

    Returns
    -------
    logger : Logger
        logger instance

    limits : dictionary
        a dictionary containing limit values read from the configured 'limit' file

    report_file : str
        a report file configured in a given configuration file
    """

    conf = utils.get_config(config)

    logger = utils.get_logger(__name__, conf)

    limitsfile = utils.get_file(conf, 'limits', logger)
    if limitsfile is None:
        sys.exit(-1)

    with open(limitsfile) as limits_file:
        limits = json.loads(limits_file.read())['limits']

    return logger, limits

def process_data(data_type, aggregateq, fp, data_tag, limits):
    """
    This method creates and starts a new handler process. The handler is initialized with data queue,
    the data type, and a result queue. The data type can be 'data_dark', 'data_white' or 'data'.
    After starting the process the function enqueues queue slice by slice into data, until all data is
    queued. The last enqueued element is end of the data marker.

    Parameters
    ----------
    data_type : str
        string indicating what type of data is processed.

    aggregateq : Queue
        result queue; used by handler to enqueue results

    fp : file pointer
        a pointer to the file that is verifier

    data_tag : str
        a data tag to look for

    limits : dict
        a dictionary of limits values

    Returns
    -------
    None
    """
    dt = fp[data_tag]

    dataq = Queue()

    p = Process(target=handler.handle_data, args=(dataq, limits[data_type], aggregateq, ))
    p.start()

    for i in range(0,dt.shape[0]):
        dataq.put(Data(dt[i]))
    dataq.put('all_data')

def verify_file(logger, file, limits, report_file):
    """
    This method creates and starts a new handler process. The handler is initialized with data queue,
    the data type, and a result queue. The data type can be 'data_dark', 'data_white' or 'data'.
    After starting the process the function enqueues queue slice by slice into data, until all data is
    queued. The last enqueued element is end of the data marker.

    Parameters
    ----------
    logger: Logger
        Logger instance.

    file : str
        a filename including path that will be verified

    limits : dict
        a dictionary of limits values

    report_file : str
        a name of a report file

    Returns
    -------
    bad_indexes : dict
        a dictionary of bad indexes per data type

    """
    fp, tags = utils.get_data_hd5(file)

    if report_file is not None:
        try:
            report_file = open(report_file, 'w')
        except:
            logger.warning('Cannot open report file')
            report_file = None

    types = ['data_dark', 'data_white', 'data']
    queues = {}
    bad_indexes = {}

    for type in types:
        data_tag = '/exchange/'+ type
        if data_tag in tags:
            queues[type] = Queue()

    for type in queues.keys():
        queue = queues[type]
        data_tag = '/exchange/'+ type
        process_data(type, queue, fp, data_tag, limits)

    # receive the results
    for type in queues.keys():
        queue = queues[type]
        aggregate = queue.get()
        if report_file is not None:
            report.report_results(aggregate, type, file, report_file)
        report.add_bad_indexes(aggregate, type, bad_indexes)

    return bad_indexes

def verify(conf, file):
    """
    This invokes sequentially data verification processes for data_dark, data_white, and data that is contained
    in configured hd5 file. The calculated values are check against limits, configured in a file.
    Each process gets two queues parameters; one queue to get the data, and second queue to
    pass back the results.
    The function awaits results from the response queues in the matching sequence to how the
    processes started.
    The results are retrieved as json object.
    The indexes of slices that did not pass quality check are reported to the calling function in form of dictionary.

    Parameters
    ----------
    conf : str
        name of the configuration file including path. If contains only directory, 'dqconfig_test.ini' will a default
        file name.

    file : str
        file name to do the quality checks on

    Returns
    -------
    bad_indexes : Dict
        A dictionary containing indexes of slices that did not pass quality check. The key is a type of data.
        (i.e. data_dark, data_white,data)
    """

    logger, limits = init(conf)
    if not os.path.isfile(file):
        logger.error(
            'parameter error: file ' +
            file + ' does not exist')
        sys.exit(-1)

    report_file = file.rsplit(".",)[0] + '.report'

    return verify_file(logger, file, limits, report_file)


from django.contrib import admin
from fix_rest.models import Event

# Register your models here.
class EventAdmin(admin.ModelAdmin):
    fieldsets = [
        (None,               {'fields': ['event']}),
        ('Date information', {'fields': ['created_date']}),
    ]

admin.site.register(Event)


#%% LIBRARIES
#import pandas as pd
from scripts import libraries
from pandas import read_csv, concat
from numpy import sqrt
from IPython.display import display, HTML
from sklearn.ensemble import RandomForestRegressor
from sklearn.tree import DecisionTreeRegressor
from sklearn.linear_model import LinearRegression
from sklearn.ensemble import BaggingRegressor
from sklearn import model_selection
from sklearn.metrics import mean_absolute_error, mean_squared_error
print('Libraries done')

#%% DATA
train_df = read_csv('./data/train_readings.csv')
test_df = read_csv('./data/test_readings.csv')
test_y = read_csv('./data/test_labels.csv')

#%% PREPARE TRAIN DATASET
def prep_train(m_df):
    rul = libraries.calculate_rul(m_df)
    engineered_df = libraries.engineer_features(m_df,15) # Step 2. Calculate SD and MU for sensors only
    whole_df = concat([m_df, engineered_df], axis=1) # Step 3. Adds new features to dataset
    scale = libraries.transform_scale(whole_df) # Step 5. Fit transformation with train dataset.
    norm_df = libraries.normalize_features(whole_df, scale) # Step 6. Normalizes data set
    print('Train Data preparation done')
    return(norm_df, rul, scale)

#%% PREPARE TEST DATASET
# Uses the scaling transformation from the training
def prepare_test(m_df, scale):
    eng_df = libraries.engineer_features(m_df, 15) # Step 2. Engineer Features. mean and sd
    whole_df = concat([m_df, eng_df], axis=1) # Step 3. Add new features to dataset
    whole_df = libraries.select_max(whole_df) # Step 4. Get's only the latest cycle
    norm_df = libraries.normalize_features(whole_df, scale) # Step 5. Normalizes dataset
    print('Test Data preparation done')
    return(norm_df)

########## TRAIN AND TEST MODELS ###########
#%%
train_prep_df, y_train, scale = prep_train(train_df)
test_prep_df, y_test = prepare_test(test_df, scale), test_y

#%% REGRESSION MODELS WITH SOME BASIC PARAMETERS
models = [('Random Forest', RandomForestRegressor(n_estimators=35, min_samples_split=2)),
        ('Decision Tree', DecisionTreeRegressor(max_depth=3, random_state=0)),
        ('Bagging Tree', BaggingRegressor()),
        ('Linear Regression', LinearRegression())]
train_results, models_name, test_results = [], [], []

#%% MODELS
for name, model in models:
    # Step 1. Kfold to get a first idea on performance
    #kfold = model_selection.KFold(n_splits=10, random_state=8)
    #cv_results = model_selection.cross_val_score(model, train_prep_df, y_train, cv=kfold)

    # Step 2. Compare to a full run training and testing
    model.fit(train_prep_df, y_train['RUL'])
    train_pred = model.predict(train_prep_df)
    test_pred = model.predict(test_prep_df)
    train_rmse = sqrt(mean_squared_error(y_train['RUL'], train_pred))
    test_rmse = sqrt(mean_squared_error(y_test['RUL'], test_pred))
    train_results.append(train_rmse)
    test_results.append(test_rmse)
    models_name.append(name)
    print("%s, train:%f, test:%f" %(name, train_rmse, test_rmse))


import pandas as pd
import datetime
import matplotlib.pyplot as plt
plt.rcParams['font.sans-serif']=['SimHei'] #用来正常显示中文标签
today= datetime.date.today()-datetime.timedelta(days=1)

qianxi=pd.read_csv(f'baiduqianxi_country_level_20{today.strftime("%y%m%d")}.csv')
raw_data=pd.read_csv(f'province-day-summary-{today}.csv')

data=raw_data[raw_data['updateTime']==f'{today}'][['provinceName','province_confirmedCount']]
data=data.merge(qianxi, on=('provinceName'))
data['ill_rate']=data['province_confirmedCount']/sum(data['province_confirmedCount'])
data['move_out_ill_rate']=[a*b for a,b in zip(data['move_out'],data['ill_rate'])]
overall=sum(data['move_out_ill_rate'])
data['risk']=[0]*len(data)
for i,row in data.iterrows():
    data.at[i,'risk']=row['move_in']*(overall-row['move_out_ill_rate'])
data.sort_values(by='risk',ascending=False,inplace=True)

data=data[:10]
city_name = list(data['provinceName'])
city_name.reverse()
data = list(data['risk'])
data.reverse()

plt.barh(range(len(data)), data,color='#ff4c00')
plt.yticks(range(len(city_name)),city_name,fontsize='13')
plt.xticks()

plt.title('疫情风险值前十名', loc='center', fontsize='20',
          fontweight='bold')
plt.savefig(f'risk-top10-{today}.jpg', bbox_inches='tight')