from khoaitv.config import Config
from utils.helper import normalize_url, inject_async_session
from custom_request.request import AsyncSession, AsyncRequest
from bs4 import BeautifulSoup
from typing import List, Iterable, Tuple, Dict
import time
import asyncio

def _get_num_pages(content: str, debug=False) -> int:
            n_pages = 1
            try:
                html_parser = BeautifulSoup(content, "html.parser")
                page_last = html_parser.find("li", class_="pag-last").find("a")["href"]
                # anime/page/1
                n_pages = int(page_last.split("/")[-1])
            except Exception as e:
                if debug:
                    print(f"_get_num_pages()\n{repr(e)}")
            return n_pages

def _parse_urls_from_page(content: str, debug=False) -> List[str]:
    """
    get movie urls from a page
    """
    n_pages = 0
    links = []
    try:
        html_parser = BeautifulSoup(content, "html.parser")
        for film_box in html_parser.findAll("a", class_="film-small"):
            links.append(film_box["href"])
        return links;
    except Exception as e:
        if debug:
            print("_parse_urls_from_page()", repr(e))
    return links

class GeneralParser:

    @classmethod
    async def get_categories_page(cls, debug=False) -> List[str]:
        links = []
        try:
            async with AsyncSession() as session:
                res = await session.get(Config.BASE_URL)
                res.raise_for_status()
                html_parser = BeautifulSoup(await res.text(), "html.parser")
                categories = html_parser.find("div", {"id": "bs-example-navbar-collapse-1"}).find("ul").find("li").find("ul")
                for category in categories.findAll("li"):
                    links.append(category.find("a")["href"])
        except Exception as e:
            if debug:
                print(f"get_categories_page() {repr(e)}")
        return links

    @classmethod
    @inject_async_session
    async def get_movie_urls(cls, category_url: str, session=None, debug=False) -> List[str]:

        movie_urls = []
        body, request_info = await AsyncRequest.get(category_url, delay=Config.REQUEST_DELAY, session=session)
        num_pages = _get_num_pages(body)
        pages_content = [body] # first page is already parsed

        # all page links except the first page
        page_links = [Config.CATEGORY_PAGINATION_URL.format(\
                            category_url=normalize_url(category_url), page=page)
                                for page in range(2 ,num_pages+1)]
        parse_routines = await asyncio.gather(*(AsyncRequest.get(url, delay=Config.REQUEST_DELAY, session=session) \
                                                    for url in page_links), return_exceptions=True)

        # filter out failed routines
        for page_url, routine in zip(page_links, parse_routines):
            if isinstance(routine, Exception):
                if debug:
                    print(f"Failed to request for page: {page_url}. Error: \n {repr(routine)}")
                continue

            content, request_info = routine
            pages_content.append(content)

        page_links.insert(0, category_url) # first page
        # populate links
        for page_url, content in zip(page_links, pages_content):
            # routine should be of type list
            parsed_movie_urls = _parse_urls_from_page(content, debug=debug)
            print(f"{page_url} has {len(parsed_movie_urls)} movie links")
            movie_urls += parsed_movie_urls

        return movie_urls

    @classmethod
    @inject_async_session
    async def get_categorized_movie_urls(cls, category_urls: Iterable[str], concurrent=True, session = None, debug=False) -> Tuple[Dict[str, List[str]], int]:
        categorized_movies = {}
        total = 0
        #concurrently scrape movie urls for all categories
        if concurrent:
            parse_routines = await asyncio.gather(*( \
                    cls.get_movie_urls(url, session=session, debug=debug) \
                        for url in category_urls))

            for routine, category in zip(parse_routines, category_urls):
                if isinstance(routine, Exception):
                    if debug:
                        print(f"Failed to grab movie links for category {category}. Error: \n {repr(routine)}")
                    continue
                categorized_movies[category] = routine
                total += len(categorized_movies[category])
        else:
            for category in category_urls:
                try:
                    categorized_movies[category] = await cls.get_movie_urls(category, session=session, debug=debug)
                    total += len(categorized_movies[category])
                except Exception as e:
                    if debug:
                        print(f"Failed to grab movie links for category {category}. Error: \n {repr(e)}")
                    continue

        return categorized_movies, total

if __name__ == "__main__":
    import asyncio
    eloop = asyncio.new_event_loop()
    start = time.time()
    categories = eloop.run_until_complete(GeneralParser.get_categories_page(debug=True))
    categorized_movies_urls, total_links = eloop.run_until_complete(GeneralParser.get_categorized_movie_urls(categories, concurrent=True, debug=True))
    print(f"Parsing completed in {time.time() - start} seconds. "
          f"Total of {total_links} links.")


#! /usr/bin/python3
import numpy as np
from keras.models import load_model
from keras.utils import np_utils
from mnist import MNIST
from random import randint
import matplotlib.pyplot as plt

# construct a dictionary for prediction
letter = {0:'null', 1:'A(a)', 2:'B(b)', 3:'C(c)', 4:'D(d)', 5:'E(e)'
, 6:'F(f)', 7:'G(g)', 8:'H(h)', 9:'I(i)', 10:'J(j)', 11:'K(k)'
, 12:'L(l)', 13:'M(m)', 14:'N(n)', 15:'O(o)', 16:'P(p)', 17:'Q(q)'
, 18:'R(r)', 19:'S(s)', 20:'T(t)', 21:'U(u)', 22:'V(v)', 23:'W(w)'
, 24:'X(x)', 25:'Y(y)', 26:'Z(z)'}

# get the data
emnist_data = MNIST(path='gzip', return_type='numpy')
emnist_data.select_emnist('letters')
x_test, y_test = emnist_data.load_testing()

# data preparing
x_test = x_test.reshape(-1, 1,28, 28)/255.
y_test = np_utils.to_categorical(y_test, num_classes=27)

# load the model been trained
model = load_model('letter_recognize_cnn.h5')

# make a prediction of x_test
y_test = model.predict(x_test)

# randomly generate the data
i = randint(0,20799)
print("\npredict letter: ", letter[np.argmax(y_test[i])])

# plot the x_test image to verify the prediction
plt.imshow(x_test[i].reshape(28,28))
plt.show()

''' more info about recognition

# get model info
loss, accuracy = model.evaluate(x_test, y_test)
print("\nloss", loss)             #loss = 0.20181247522672213
print("accuracy", accuracy)     #accuracy = 0.9401442307692308

# get x_test's shape
print("\n", x_test.shape)

# print the predict answer
print('\nprobability of every portion:\n', y_test[i])
'''


# coding:utf-8
import numpy as np
import matplotlib.pyplot as plt

def calc_y(x, w):
    return np.array([np.sum([w[i] * (x[n] ** i) for i in xrange(w.size)]) for n in xrange(x.size)])

# wを計算
def estimate(x, t, M):
    A = np.array([[(x ** (i + j)).sum() for j in xrange(M + 1)] for i in
                  xrange(M + 1)])
    T = np.array([(x ** (i) * t).sum() for i in xrange(M + 1)])
    return np.linalg.solve(A, T)

# 罰金項ありでwを計算
def estimate_la(x, t, M):
    la = np.exp(-18)
    A = np.array([[(x ** (i + j)).sum() if i != j else la + (x ** (i + j)).sum() for j in xrange(M + 1)] for i in
                  xrange(M + 1)])
    T = np.array([(x ** (i) * t).sum() for i in xrange(M + 1)])
    return np.linalg.solve(A, T)

if __name__ == '__main__':
    # 多項式の次数
    M = 20

    # データ点を10個作る
    x_data = np.linspace(0, 1, 100)
    # 分散0.2のノイズを与える
    t_data = np.sin(2 * np.pi * x_data) + np.random.normal(0, 0.5, x_data.size)

    x_true = np.linspace(0, 1, 500)
    y_true = np.sin(2 * np.pi * x_true)

    w = estimate(x_data, t_data, M)
    w_la = estimate_la(x_data, t_data, M)
    y_data = calc_y(x_true, w)
    y_la_data=calc_y(x_true,w_la)

    plt.plot(x_data, t_data, 'o')
    plt.plot(x_true, y_true, 'g')
    plt.plot(x_true, y_data, 'r')
    plt.plot(x_true,y_la_data,'b')

    plt.show()


import numpy as np
import matplotlib.pyplot as plt
from scipy import stats

N=10000
y=np.zeros(N)
for i in range(N):
    #y[i]=np.mean((np.random.normal(loc=0.0,scale=1.0,size=100)))
    y[i]=np.sum((stats.cauchy.rvs(loc=0.0,scale=1.0,size=1000)))
    #y[i]=np.sum(np.random.standard_cauchy(size=1000))
plt.hist(y,bins=100,normed=True,label='1000')
plt.yscale('log')
#plt.hist(y,normed=True,bins=np.arange(-5,5,0.03),label='50000,finer bins')
#plt.hist(y, bins=np.arange(-1,1,0.003),normed=True)
plt.xlabel('x')
plt.ylabel('pdf')
plt.title('pdf of sum of 10 std. Cauchy rvs, using (10,100,1000,10000) realizations')
plt.show()


# -*- coding: utf-8 -*-
#
# Swiss Open Access Repository
# Copyright (C) 2021 RERO
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""SONAR resources responses."""

from flask import Response, make_response
from flask_resources.responses import Response as ResourceResponse

class StreamResponse(ResourceResponse):
    """Stream response."""

    filename = None

    def __init__(self, serializer=None, **kwargs):
        """Stream response initialization.

        Sets the filename for attachment if present in kwargs.

        :param serializer: Record serializer.
        """
        self.filename = kwargs.pop('filename', 'records')
        super().__init__(serializer)

    def make_list_response(self, content, code=200):
        """Builds a response for a list of objects.

        :param content: Content to send.
        :param code: Status code.
        :returns: Response object.
        """
        response = make_response(
            "" if content is None else Response(
                self.serializer.serialize_object_list(content)),
            code,
            self.make_headers(),
        )
        response.headers[
            'Content-Disposition'] = f'attachment; filename={self.filename}'
        return response


from transformers import DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP
from transformers.tokenization_distilbert import PRETRAINED_VOCAB_FILES_MAP
from transformers.file_utils import get_from_cache
from pathlib import Path
import shutil
import os
import numpy as np
import torch
import subprocess

config_path = DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP["distilbert-base-uncased-finetuned-sst-2-english"]
vocab_path = PRETRAINED_VOCAB_FILES_MAP["vocab_file"]["distilbert-base-uncased"]
weights_path = DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP["distilbert-base-uncased-finetuned-sst-2-english"]

target_path = Path.home() / 'rustbert'

temp_config = get_from_cache(config_path)
temp_vocab = get_from_cache(vocab_path)
temp_weights = get_from_cache(weights_path)

os.makedirs(target_path, exist_ok=True)
shutil.copy(temp_config, target_path / 'config.json')
shutil.copy(temp_vocab, target_path / 'vocab.txt')
shutil.copy(temp_weights, target_path / 'model.bin')

weights = torch.load(temp_weights)
nps = {}
for k, v in weights.items():
    nps[k] = v.cpu().numpy()

np.savez(target_path / 'model.npz', **nps)

source = str(target_path / 'model.npz')
target = str(target_path / 'model.ot')

toml_location = (Path(__file__).resolve() / '..' / '..' / 'Cargo.toml').resolve()

subprocess.call(
    ['cargo', '+nightly', 'run', '--bin=convert-tensor', f'--manifest-path={toml_location}', '--', source,
     target])


"""
Asynchronous Advantage Actor Critic (A3C) with discrete action space, Reinforcement Learning.

The Cartpole example.

View more on my tutorial page: https://morvanzhou.github.io/tutorials/

Using:
tensorflow 1.0
gym 0.8.0
"""

import multiprocessing
import threading
import tensorflow as tf
import numpy as np
import gym
import os
import shutil
import matplotlib.pyplot as plt

from gym.wrappers.time_limit import TimeLimit
import argparse
from baselines import logger
from skimage.color import rgb2grey
from skimage.transform import resize
import copy
import sys
from collections import deque

img_lock = threading.Lock()
log_lock = threading.Lock()

GAME = 'CartPole-v0'
OUTPUT_GRAPH = True
LOG_DIR = './log'
N_WORKERS = multiprocessing.cpu_count()
MAX_GLOBAL_EP = 1000
GLOBAL_NET_SCOPE = 'Global_Net'
UPDATE_GLOBAL_ITER = 100 #10
GAMMA = 0.999 # 0.9
ENTROPY_BETA = 0.01 # 0.001
GLOBAL_RUNNING_R = []
GLOBAL_R = []
GLOBAL_EP = 0

env = gym.make(GAME)
N_S = env.observation_space.shape[0]
N_A = env.action_space.n

class ACNet(object):
    def __init__(self, scope, globalAC=None, hard_share=None, soft_sharing_coeff_actor=0.01, soft_sharing_coeff_critic=0.01, gradient_clip_actor=1.0, gradient_clip_critic=1.0, image_shape=None, stack=1):
        self.hard_share = hard_share
        self.image_shape = image_shape
        self.stack = stack

        def input_placeholders():
            if self.image_shape is not None:
                s = tuple([tf.placeholder(tf.float32, [None, ] + list(self.image_shape), 'S') for _ in range(self.stack)])
            else:
                s = tuple([tf.placeholder(tf.float32, [None, N_S], 'S') for _ in range(self.stack)])
            return s

        if scope == GLOBAL_NET_SCOPE:   # get global network
            with tf.variable_scope(scope):
                self.s = input_placeholders()
                self.a_params, self.c_params = self._build_net(scope)[-2:]
        else:   # local net, calculate losses
            with tf.variable_scope(scope):
                self.s = input_placeholders()
                self.a_his = tf.placeholder(tf.int32, [None, ], 'A')
                self.v_target = tf.placeholder(tf.float32, [None, 1], 'Vtarget')

                self.a_prob, self.v, self.a_params, self.c_params = self._build_net(scope)

                td = tf.subtract(self.v_target, self.v, name='TD_error')
                self.t_td = td
                with tf.name_scope('c_loss'):
                    self.c_loss = tf.reduce_mean(tf.square(td))
                    if soft_sharing_coeff_actor > 0:
                        self.c_loss += soft_sharing_coeff_critic*tf.nn.l2_loss(self.l_a - self.l_c)

                with tf.name_scope('a_loss'):
                    log_prob = tf.reduce_sum(tf.log(self.a_prob) * tf.one_hot(self.a_his, N_A, dtype=tf.float32), axis=1, keep_dims=True)
                    self.t_log_prob = log_prob
                    exp_v = log_prob * tf.stop_gradient(td)
                    self.t_exp_v = exp_v
                    entropy = -tf.reduce_sum(self.a_prob * tf.log(self.a_prob + 1e-5),
                                             axis=1, keep_dims=True)  # encourage exploration
                    self.t_entropy = entropy
                    self.exp_v = ENTROPY_BETA * entropy + exp_v
                    self.t_exp_v2 = self.exp_v
                    self.a_loss = tf.reduce_mean(-self.exp_v)
                    if soft_sharing_coeff_critic > 0:
                        self.a_loss += soft_sharing_coeff_actor*tf.nn.l2_loss(self.l_a - self.l_c)

                with tf.name_scope('local_grad'):
                    self.a_grads, _ = tf.clip_by_global_norm(tf.gradients(self.a_loss, self.a_params), gradient_clip_actor)
                    if gradient_clip_actor > 0:
                        self.a_grads, _ = tf.clip_by_global_norm(self.a_grads, gradient_clip_actor)
                    self.c_grads, _ = tf.clip_by_global_norm(tf.gradients(self.c_loss, self.c_params), gradient_clip_critic)
                    if gradient_clip_critic > 0:
                        self.c_grads, _ = tf.clip_by_global_norm(self.c_grads, gradient_clip_critic)

            with tf.name_scope('sync'):
                with tf.name_scope('pull'):
                    self.pull_a_params_op = [l_p.assign(g_p) for l_p, g_p in zip(self.a_params, globalAC.a_params)]
                    self.pull_c_params_op = [l_p.assign(g_p) for l_p, g_p in zip(self.c_params, globalAC.c_params)]
                with tf.name_scope('push'):
                    self.update_a_op = OPT_A.apply_gradients(zip(self.a_grads, globalAC.a_params))
                    self.update_c_op = OPT_C.apply_gradients(zip(self.c_grads, globalAC.c_params))

    def _build_net(self, scope):
        w_init = tf.random_normal_initializer(0., .1)
        inputs = self.s
        if self.hard_share is not None:
            if self.image_shape is not None:
                with tf.variable_scope('conv_inputs'):
                    if self.stack == 2:
                        diff = inputs[1] - inputs[0]
                        inputs = tf.concat([inputs[1], diff], -1)
                    else:
                        inputs = tf.concat(inputs, -1)
                    conv1 = tf.layers.conv2d(inputs, 64, 5, strides=(3,3), padding='same', activation=tf.nn.relu, kernel_initializer=tf.truncated_normal_initializer(), bias_initializer=tf.constant_initializer(1.0), name='c1')
                    conv2 = tf.layers.conv2d(conv1, 32, 5, strides=(3,3), padding='same', activation=tf.nn.relu, kernel_initializer=tf.truncated_normal_initializer(), bias_initializer=tf.constant_initializer(1.0), name='c2')
                    conv3 = tf.layers.conv2d(conv2, 16, 3, strides=(3,3), padding='same', activation=tf.nn.relu, kernel_initializer=tf.truncated_normal_initializer(), bias_initializer=tf.constant_initializer(1.0), name='c3')
                    conv4 = tf.layers.conv2d(conv3, 16, 3, strides=(3,3), padding='same', activation=tf.nn.relu, kernel_initializer=tf.truncated_normal_initializer(), bias_initializer=tf.constant_initializer(1.0), name='c4')
                    inputs = tf.layers.flatten(conv4, name='p3')
            else:
                inputs = tf.concat(inputs, 1)
            if self.hard_share == 'equal_params':
                with tf.variable_scope('shared'):
                    l_s = tf.layers.dense(inputs, 71, tf.nn.relu6, kernel_initializer=w_init, name='ls')
                with tf.variable_scope('actor'):
                    l_a = tf.layers.dense(l_s, 16, tf.nn.relu6, kernel_initializer=w_init, name='la')
                    a_prob = tf.layers.dense(l_a, N_A, tf.nn.softmax, kernel_initializer=w_init, name='ap')
                with tf.variable_scope('critic'):
                    l_c = tf.layers.dense(l_s, 16, tf.nn.relu6, kernel_initializer=w_init, name='lc')
                    v = tf.layers.dense(l_c, 1, kernel_initializer=w_init, name='v')  # state value
            elif False:
                with tf.variable_scope('shared'):
                    l_s = tf.layers.dense(inputs, 100, tf.nn.relu6, kernel_initializer=w_init, name='ls')
                with tf.variable_scope('actor'):
                    l_a = tf.layers.dense(l_s, 10, tf.nn.relu6, kernel_initializer=w_init, name='la')
                    a_prob = tf.layers.dense(l_a, N_A, tf.nn.softmax, kernel_initializer=w_init, name='ap')
                with tf.variable_scope('critic'):
                    l_c = tf.layers.dense(l_s, 10, tf.nn.relu6, kernel_initializer=w_init, name='lc')
                    v = tf.layers.dense(l_c, 1, kernel_initializer=w_init, name='v')  # state value
            else:
                with tf.variable_scope('shared'):
                    l_s = tf.layers.dense(inputs, 200, tf.nn.relu6, kernel_initializer=w_init, name='ls')
                with tf.variable_scope('actor'):
                    l_a = tf.layers.dense(l_s, 10, tf.nn.relu6, kernel_initializer=w_init, name='la')
                    a_prob = tf.layers.dense(l_a, N_A, tf.nn.softmax, kernel_initializer=w_init, name='ap')
                with tf.variable_scope('critic'):
                    l_c = tf.layers.dense(l_s, 10, tf.nn.relu6, kernel_initializer=w_init, name='lc')
                    v = tf.layers.dense(l_c, 1, kernel_initializer=w_init, name='v')  # state value

            s_params = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=scope + '/shared')
            a_params = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=scope + '/actor')
            c_params = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=scope + '/critic')
            return a_prob, v, a_params+s_params, c_params+s_params
        else:
            if self.image_shape is not None:
                with tf.variable_scope('conv_inputs'):
                    if self.stack == 2:
                        diff = inputs[1] - inputs[0]
                        inputs = tf.concat([inputs[1], diff], -1)
                    else:
                        inputs = tf.concat(inputs, -1)
                    conv1 = tf.layers.conv2d(inputs, 64, 8, strides=(5,5), padding='same', activation=tf.nn.relu, kernel_initializer=tf.truncated_normal_initializer(), bias_initializer=tf.constant_initializer(0.1), name='c1')
                    conv2 = tf.layers.conv2d(conv1, 64, 5, strides=(3,3), padding='same', activation=tf.nn.relu, kernel_initializer=tf.truncated_normal_initializer(), bias_initializer=tf.constant_initializer(0.1), name='c2')
                    conv3 = tf.layers.conv2d(conv2, 32, 3, strides=(2,2), padding='same', activation=tf.nn.relu, kernel_initializer=tf.truncated_normal_initializer(), bias_initializer=tf.constant_initializer(0.1), name='c3')
                    conv4 = tf.layers.conv2d(conv3, 16, 3, strides=(2,2), padding='same', activation=tf.nn.relu, kernel_initializer=tf.truncated_normal_initializer(), bias_initializer=tf.constant_initializer(0.1), name='c4')
                    inputs = tf.layers.flatten(conv4, name='p3')
                    '''
                    # TODO DWEBB add stack support here
                    diff = inputs[1] - inputs[0]
                    inputs = tf.concat([inputs[1], diff], -1)
                    conv1 = tf.layers.conv2d(inputs, 64, 8, name='c1')
                    #pool1 = tf.layers.max_pooling2d(conv1, 5, 1, name='p1')
                    relu1 = tf.nn.relu(conv1, name='relu1')
                    conv2 = tf.layers.conv2d(relu1, 32, 8, name='c2')
                    pool2 = tf.layers.max_pooling2d(conv2, 5, 1, name='p2')
                    relu2 = tf.nn.relu(pool2, name='relu2')
                    conv3 = tf.layers.conv2d(relu2, 32, 5, name='c3')
                    pool3 = tf.layers.max_pooling2d(conv3, 5, 1, name='p3')
                    relu3 = tf.nn.relu(pool3, name='relu3')
                    conv4 = tf.layers.conv2d(relu3, 32, 5, name='c4')
                    pool4 = tf.layers.max_pooling2d(conv4, 5, 1, name='p4')
                    relu4 = tf.nn.relu(pool4, name='relu4')
                    inputs = tf.layers.flatten(relu4, name='p3')
                    '''
            else:
                inputs = tf.concat(inputs, 1)
            with tf.variable_scope('actor'):
                l_a = tf.layers.dense(inputs, 100, tf.nn.relu6, kernel_initializer=w_init, name='la')
                self.l_a = l_a
                a_prob = tf.layers.dense(l_a, N_A, tf.nn.softmax, kernel_initializer=w_init, name='ap')
            with tf.variable_scope('critic'):
                l_c = tf.layers.dense(inputs, 100, tf.nn.relu6, kernel_initializer=w_init, name='lc')
                self.l_c = l_c
                v = tf.layers.dense(l_c, 1, kernel_initializer=w_init, name='v')  # state value
        a_params = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=scope + '/actor')
        c_params = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=scope + '/critic')
        if self.image_shape is not None:
            i_params = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=scope + '/conv_inputs')
            a_params += i_params
            c_params += i_params
        return a_prob, v, a_params, c_params

    def get_stats(self, feed_dict):
        return SESS.run([self.a_loss, self.c_loss, self.t_td, self.c_loss, self.t_log_prob, self.t_exp_v, self.t_entropy, self.t_exp_v2, self.a_loss, self.a_grads, self.c_grads], feed_dict)

    def update_global(self, feed_dict):  # run by a local
        a_loss, c_loss, entropy, _, _ = SESS.run([self.c_loss, self.a_loss, self.t_exp_v, self.update_a_op, self.update_c_op], feed_dict)  # local grads applies to global net
        return a_loss, c_loss, entropy[0, 0]

    def pull_global(self):  # run by a local
        SESS.run([self.pull_a_params_op, self.pull_c_params_op])

    def choose_action(self, s):  # run by a local
        if type(s) is not list:
            import ipdb; ipdb.set_trace()
        temp = [obs[np.newaxis, :] for obs in s ]
        feed_dict = {var: obs for var, obs in zip(self.s, temp)}
        prob_weights = SESS.run(self.a_prob, feed_dict=feed_dict)
        action = np.random.choice(range(prob_weights.shape[1]),
                                  p=prob_weights.ravel())  # select action w.r.t the actions prob
        return action

class Worker(object):
    def __init__(self, name, globalAC, hard_share=None, soft_sharing_coeff_actor=0.0, soft_sharing_coeff_critic=0.0, gradient_clip_actor=0.0, gradient_clip_critic=0.0, debug=False, max_ep_steps=200, image_shape=None, stack=1):
        self.env = gym.make(GAME).unwrapped
        self.env = TimeLimit(self.env, max_episode_steps=max_ep_steps)
        self.name = name
        self.AC = ACNet(name, globalAC, hard_share=hard_share, soft_sharing_coeff_actor=soft_sharing_coeff_actor, soft_sharing_coeff_critic=soft_sharing_coeff_critic, gradient_clip_actor=gradient_clip_actor, gradient_clip_critic=gradient_clip_critic, image_shape=image_shape, stack=stack)
        self.debug = debug
        self.image_shape = image_shape
        self.stack = stack

    def work(self):
        def get_img(fn, *args):
            img_lock.acquire()
            results = fn(*args)
            img = self.env.render(mode='rgb_array')
            img_lock.release()
            img = rgb2grey(img)
            img = resize(img, self.image_shape)
            return img, results

        def env_reset_obs():
            return self.env.reset()

        def env_reset_img():
            img, _ = get_img(env_reset_obs)
            return img

        def env_step_obs(a):
            return self.env.step(a)

        def env_step_img(a):
            img, results = get_img(env_step_obs, a)
            return img, results[1], results[2], results[3]

        if self.image_shape is not None:
            env_reset_fn = env_reset_img
            env_step_fn = env_step_img
        else:
            env_reset_fn = env_reset_obs
            env_step_fn = env_step_obs

        global GLOBAL_RUNNING_R, GLOBAL_R, GLOBAL_EP, MAX_GLOBAL_EP
        total_step = 1
        buffer_s, buffer_a, buffer_r = [], [], []
        while not COORD.should_stop() and GLOBAL_EP < MAX_GLOBAL_EP:
            s = env_reset_fn()
            buffer_s = [s]*self.stack
            ep_r = 0
            while True:
                a = self.AC.choose_action(buffer_s[-self.stack:])
                s_, r, done, info = env_step_fn(a)
                if done: r = -5
                ep_r += r
                buffer_s.append(s)
                buffer_a.append(a)
                buffer_r.append(r)

                if total_step % UPDATE_GLOBAL_ITER == 0 or done:   # update global and assign to local net

                    if done:
                        v_s_ = 0   # terminal
                    else:
                        obs_hist = buffer_s[-(self.stack-1):] + [s_,]
                        feed_dict = {var: obs[np.newaxis, :] for var, obs in zip(self.AC.s, obs_hist)}
                        v_s_ = SESS.run(self.AC.v, feed_dict=feed_dict)[0, 0]

                    buffer_v_target = []
                    for r in buffer_r[::-1]:    # reverse buffer r
                        v_s_ = r + GAMMA * v_s_
                        buffer_v_target.append(v_s_)
                    buffer_v_target.reverse()

                    if self.image_shape is not None:
                        buffer_s_ = [buffer_s_[np.newaxis, :] for buffer_s_ in buffer_s]
                    else:
                        buffer_s_ = copy.deepcopy(buffer_s)
                    obs_columns = [np.vstack(buffer_s_[idx:-(self.stack-idx)]) for idx in range(self.stack)]
                    buffer_a, buffer_v_target = np.array(buffer_a), np.vstack(buffer_v_target)
                    feed_dict = {var: obs for var, obs in zip(self.AC.s, obs_columns)}
                    feed_dict[self.AC.a_his] = buffer_a
                    feed_dict[self.AC.v_target] = buffer_v_target
                    if self.debug and self.name == 'W_0':
                        a_loss, c_loss, t_td, c_loss, t_log_prob, t_exp_v, t_entropy, t_exp_v2, a_loss, a_grads, c_grads = self.AC.get_stats(feed_dict)
                        #print("a_loss: ", a_loss.shape, " ", a_loss, "\tc_loss: ", c_loss.shape, " ", c_loss, "\ttd: ", t_td.shape, " ", t_td, "\tlog_prob: ", t_log_prob.shape, " ", t_log_prob, "\texp_v: ", t_exp_v.shape, " ", t_exp_v, "\tentropy: ", t_entropy.shape, " ", t_entropy, "\texp_v2: ", t_exp_v2.shape, " ", t_exp_v2, "\ta_grads: ", [np.sum(weights) for weights in a_grads], "\tc_grads: ", [np.sum(weights) for weights in c_grads])
                        print("a_loss: ", a_loss.shape, " ", a_loss, "\tc_loss: ", c_loss)
                    c_loss, a_loss, entropy = self.AC.update_global(feed_dict)

                    #import ipdb; ipdb.set_trace()
                    buffer_s, buffer_a, buffer_r = buffer_s[-(self.stack):], [], []
                    self.AC.pull_global()

                s = s_
                total_step += 1
                if done:
                    GLOBAL_R.append(ep_r)
                    if len(GLOBAL_RUNNING_R) == 0:  # record running episode reward
                        GLOBAL_RUNNING_R.append(ep_r)
                    else:
                        GLOBAL_RUNNING_R.append(0.99 * GLOBAL_RUNNING_R[-1] + 0.01 * ep_r)

                    log_lock.acquire()
                    logger.record_tabular("global_ep", GLOBAL_EP)
                    logger.record_tabular("name", self.name)
                    logger.record_tabular("ep_r", ep_r)
                    logger.record_tabular("ep_r_weighted", GLOBAL_RUNNING_R[-1])
                    logger.record_tabular("c_loss", c_loss)
                    logger.record_tabular("a_loss", a_loss)
                    logger.record_tabular("entropy", entropy)
                    logger.dump_tabular()
                    log_lock.release()

                    GLOBAL_EP += 1
                    break

def parse_args():
    global MAX_GLOBAL_EP

    parser = argparse.ArgumentParser(description='Run A3C on discrete cart-pole.')
    parser.add_argument('--hard_share', type=str, default='none', help="Indicates whether the models should have an equal number of parameters ('equal_params'), an equal number of hidden units ('equal_hiddens'), or no sharing ('none' -- default).")
    parser.add_argument('--soft_share', type=float, default=0.0, help='Enables soft sharing of both actor and critic parameters, via L2 loss, with the specificied weight.')
    parser.add_argument('--soft_share_actor', type=float, default=0.0, help='Enables soft sharing of actor parameters, via L2 loss, with the specificied weight.')
    parser.add_argument('--soft_share_critic', type=float, default=0.0, help='Enables soft sharing of critic parameters, via L2 loss, with the specificied weight.')
    parser.add_argument('--gradient_clip', type=float, default=0.0, help='Enables gradient clipping of actor and critic parameters with the specificied maximum gradient.')
    parser.add_argument('--gradient_clip_actor', type=float, default=0.0, help='Enables gradient clipping of actor parameters with the specificied maximum gradient.')
    parser.add_argument('--gradient_clip_critic', type=float, default=0.0, help='Enables gradient clipping of critic parameters with the specificied maximum gradient.')
    parser.add_argument('--debug', default=False, action='store_true', help='Enables debugging output.')
    parser.add_argument('--optimizer', default='adagrad', help='Which optimizer to use: rmsprop, adam, adagrad.')
    parser.add_argument('--lr', type=float, default=0.0, help='Sets the learning rate of the actor and critic.')
    parser.add_argument('--lr_a', type=float, default=0.001, help='Sets the learning rate of the actor.')
    parser.add_argument('--lr_c', type=float, default=0.001, help='Sets the learning rate of the critic.')
    parser.add_argument('--max_global_ep', type=int, default=500, help='Sets the maximum number of episodes to be executed across all threads.')
    parser.add_argument('--log', default=False, action='store_true', help='Enables logging.')
    parser.add_argument('--max_ep_steps', type=int, default=2000, help='The number of time steps per episode before calling the episode done.')
    parser.add_argument('--stack', type=int, default=1, help='Number of observations to use for state.')
    parser.add_argument('--image_shape', nargs='*', default=None, help='Designates that images shoud be used in lieu of observations and what shpae to use for them.')
    parser.add_argument('--debug_worker', default=False, action='store_true')
    args = parser.parse_args()

    if args.hard_share not in ['equal_params', 'equal_hiddens', 'none']:
        raise ValueError("Hard sharing options are 'equal_params' and 'equal_hiddens'.")

    if args.hard_share == 'none':
        args.hard_share = None

    soft_share_actor = args.soft_share_actor
    soft_share_critic = args.soft_share_critic
    if args.soft_share > 0:
        soft_share_actor = args.soft_share
        soft_share_critic = args.soft_share

    gradient_clip_actor = args.gradient_clip_actor
    gradient_clip_critic = args.gradient_clip_critic
    if args.gradient_clip > 0:
        gradient_clip_actor = args.gradient_clip
        gradient_clip_critic = args.gradient_clip

    lr_a = args.lr_a
    lr_c = args.lr_c
    if args.lr > 0:
        lr_a = args.lr
        lr_c = args.lr

    MAX_GLOBAL_EP = args.max_global_ep

    if args.log:
        logger.configure('tmp')
        print("logger dir: ", logger.get_dir())

    if args.debug:
        logger.set_level(logger.DEBUG)

    if args.optimizer == 'rmsprop':
        optimizer_class = tf.train.RMSPropOptimizer
    elif args.optimizer == 'adam':
        optimizer_class = tf.train.AdamOptimizer
    elif args.optimizer == 'adagrad':
        optimizer_class = tf.train.AdagradOptimizer

    if args.stack < 1:
        raise ValueError('Number of frames ({}) to stack must be positive.'.format(args.stack))

    image_shape = None
    if args.image_shape is not None:
        image_shape = tuple(map(int, args.image_shape[0].split(','))) + (1,) # Add the number of channels which will always be 1 for grey scale
        if args.stack < 2:
            args.stack = 2 # Automatically incorporate diffs with previous images

    print("hard_share: ", args.hard_share)
    print("soft_share_actor: ", soft_share_actor)
    print("soft_share_critic: ", soft_share_critic)
    print("gradient_clip_actor: ", gradient_clip_actor)
    print("gradient_clip_critic: ", gradient_clip_critic)
    print("learning rate, actor: ", lr_a)
    print("learning rate, critic: ", lr_c)
    print("max_global_ep: ", MAX_GLOBAL_EP)
    print("optimizer_class: ", optimizer_class)
    print("max_ep_steps: ", args.max_ep_steps)
    print("image_shape: ", image_shape)

    return args, soft_share_actor, soft_share_critic, gradient_clip_actor, gradient_clip_critic, lr_a, lr_c, optimizer_class, image_shape

if __name__ == "__main__":
    args, soft_share_actor, soft_share_critic,  gradient_clip_actor, gradient_clip_critic, lr_a, lr_c, optimizer_class, image_shape = parse_args()
    SESS = tf.Session()

    with tf.device("/cpu:0"):
        OPT_A = optimizer_class(lr_a, name='actor_opt')
        OPT_C = optimizer_class(lr_c, name='critic_opt')
        GLOBAL_AC = ACNet(GLOBAL_NET_SCOPE, hard_share=args.hard_share, image_shape=image_shape, stack=args.stack)  # we only need its params
        workers = []
        # Create worker
        for i in range(N_WORKERS):
            i_name = 'W_%i' % i   # worker name
            workers.append(Worker(i_name, GLOBAL_AC, hard_share=args.hard_share, soft_sharing_coeff_actor=soft_share_actor, soft_sharing_coeff_critic=soft_share_critic, gradient_clip_actor=gradient_clip_actor, gradient_clip_critic=gradient_clip_critic, debug=args.debug, max_ep_steps=args.max_ep_steps, image_shape=image_shape, stack=args.stack))

    COORD = tf.train.Coordinator()
    SESS.run(tf.global_variables_initializer())

    if OUTPUT_GRAPH:
        if os.path.exists(LOG_DIR):
            shutil.rmtree(LOG_DIR)
        tf.summary.FileWriter(LOG_DIR, SESS.graph)

    if args.debug_worker:
        workers[0].work()
        sys.exit()

    worker_threads = []
    for worker in workers:
        job = lambda: worker.work()
        t = threading.Thread(target=job)
        t.start()
        worker_threads.append(t)
    COORD.join(worker_threads)

    plt.subplot(2, 1, 1)
    plt.plot(np.arange(len(GLOBAL_RUNNING_R)), GLOBAL_RUNNING_R)
    plt.subplot(2, 1, 2)
    plt.plot(np.arange(len(GLOBAL_R)), GLOBAL_R)
    plt.xlabel('step')
    plt.ylabel('Total moving reward')
    if args.log:
        name = 'plot_'+str(MAX_GLOBAL_EP)+'_sharing_'
        if args.hard_share is not None:
            name += 'hard'
        elif soft_sharing_coeff_actor > 0. or soft_sharing_coeff_critic > 0.:
            name += 'soft'
        else:
            name += 'none'
        name += '_lra_'+str(lr_a)+'_lrc_'+str(lr_c)+'.png'
        plt.savefig()
    else:
        plt.show()

        env = gym.make(GAME).unwrapped
        env = TimeLimit(env, max_episode_steps=args.max_ep_steps)
        s = env.reset()
        buffer_s = [s]*args.stack
        tidx = 0
        done = False
        while tidx < 1000 and not done:
            a = workers[0].AC.choose_action(buffer_s[-args.stack:])
            env.render()
            s_, r, done, info = env.step(a)
            s = s_
            buffer_s.append(s)
            tidx += 1


from datetime import datetime

from . import db

class Blacklist(db.Model):
    '''
    Model to keep track of blacklisted tokens
    '''

    __tablename__ = 'blacklist'
    id = db.Column(db.Integer, primary_key=True)
    token = db.Column(db.String, unique=True, nullable=False)
    blacklisted_on = db.Column(db.DateTime, nullable=False, default=datetime.now())

    @classmethod
    def check_blacklist(cls, token):
        token = cls.query.filter_by(token=token).first()
        if token:
            return True
        return False


"""
Definition of views.
"""

from django.shortcuts import render
from django.http import HttpRequest
from django.template import RequestContext
from django.utils import timezone
from datetime import timedelta, date
from django.shortcuts import redirect
from datetime import datetime
from .models import apiKey
from .forms import apiKeyForm
from django.contrib.auth.decorators import login_required

import logging

logger = logging.getLogger(__name__)

@login_required
def home(request):
    """Renders the home page."""
    assert isinstance(request, HttpRequest)
    logger.debug('home')
    return render(
        request,
        'app/index.html',
        {
            'title':'Accueil',
            'year':datetime.now().year,
        }
    )

@login_required
def manage_key(request):
    if request.method == "POST":
        form = apiKeyForm(request.POST)
        if form.is_valid():
            apiKey.objects.all().delete()
            key = form.save(commit=False)
            key.date_creation = datetime.now()
            current_year = datetime.now().year
            now = datetime.now()
            key.date_expiration = datetime(current_year + 1, now.month, now.day, now.hour, now.minute, now.second)
            key.save()
            return redirect('manage_key')
    else:
        form = apiKeyForm()
    keys = apiKey.objects.all()
    return render(
        request,
        'app/manage_key.html',
        {
            'keys': keys,
            'form': form,
            'year':datetime.now().year,
        }
    )

def delete_key(request):
    if request.method == "POST":
        key_value = request.POST.get('api_key')
        nb_del, _ = apiKey.objects.get(api_key=key_value).delete()
    return redirect('manage_key')

@login_required
def support(request):
    """Render the Support page."""
    assert isinstance(request, HttpRequest)
    logger.debug('support')
    return render(
        request,
        'app/support.html',
        {
            'title': 'Support',
            'message': '',
        }
    )


from PIL import ImageGrab
from pynput.keyboard import Key, Controller
import time
Keyboard = Controller()

uniy1=148
uniy2=150
presstime=.06

def countdown(sec):

    while sec > 0:
        print(sec)
        time.sleep(1)
        sec-=1

def grabdat():
    countdown(3)

    #leftA = ImageGrab.grab(bbox=(1090,uniy1,1092,uniy2))
    downA = ImageGrab.grab(bbox=(1280,uniy1,1282,uniy2))
    #upA = ImageGrab.grab(bbox=(1455,uniy1,1457,uniy2))
    #rightA = ImageGrab.grab(bbox=(1640,uniy1,1642,uniy2))
    return downA

grab = True

if grab:
    RefD= grabdat()

countdown(1)

while True:
    #leftA = ImageGrab.grab(bbox=(1090,uniy1,1092,uniy2))
    downA = ImageGrab.grab(bbox=(1280,uniy1,1282,uniy2))
    #upA = ImageGrab.grab(bbox=(1455,uniy1,1457,uniy2))
    #rightA = ImageGrab.grab(bbox=(1640,uniy1,1642,uniy2))

    if not(downA==RefD):
        print("down")
        Keyboard.press(Key.down)
        time.sleep(presstime)
        Keyboard.release(Key.down)

'''
while True:
    time.sleep(.01)
    leftA = ImageGrab.grab(bbox=(1090,uniy1,1100,uniy2))
    downA = ImageGrab.grab(bbox=(1280,uniy1,1290,uniy2))
    upA = ImageGrab.grab(bbox=(1455,uniy1,1465,uniy2))
    rightA = ImageGrab.grab(bbox=(1640,uniy1,1650,uniy2))

    leftA.save("leftA.jpg")
    rightA.save("rightA.jpg")
    upA.save("upA.jpg")
    downA.save("downA.jpg")

    if not(filecmp.cmp("leftA.jpg",".Refleft.jpg")):
        #print("left")
        Keyboard.press(Key.left)
        time.sleep(presstime)
        Keyboard.release(Key.left)
    if not(filecmp.cmp("rightA.jpg",".Refright.jpg")):
        #print("right")
        Keyboard.press(Key.right)
        time.sleep(presstime)
        Keyboard.release(Key.right)
    if not(filecmp.cmp("upA.jpg",".Refup.jpg")):
        #print("up")
        Keyboard.press(Key.up)
        time.sleep(presstime)
        Keyboard.release(Key.up)
    if not(filecmp.cmp("downA.jpg",".Refdown.jpg")):
        #print("down")
        Keyboard.press(Key.down)
        time.sleep(presstime)
        Keyboard.release(Key.down)

'''


from odoo import models, fields, api, _
from datetime import datetime, timedelta
from odoo.tools import DEFAULT_SERVER_DATETIME_FORMAT
import pytz

class ImportLeave(models.TransientModel):

    _name = 'import.leave'
    _description = "Import Leave"

    name = fields.Char('File name')
    file = fields.Binary('File')
    state = fields.Selection([('init', 'init'), ('done', 'done')],
                             string='Status', readonly=True, default='init')
    type = fields.Selection([("xlsx", "XLSX")],
                            default="xlsx", string="File type.")
    operation = fields.Selection(
        [('create', 'Create Record'), ('update', 'Update Record')],
        default='create')

    def import_data_through_cron_leave(self):
        self.ensure_one()
        cron_obj = self.env['ir.cron']
        now_time = datetime.now() + timedelta(seconds=1)
        leave_master = self.env['import.leave.master'].create({
            'file': self.file,
            'filename': self.name,
            'type': self.type,
            'file_updated': True,
            'user_id': self._uid,
            'status': 'in_process',
            'operation': self.operation,
        })

        user_tz = self.env.user.tz or str(pytz.utc)
        local = pytz.timezone(user_tz)
        user_time_zone = datetime.strftime(pytz.utc.localize(
            datetime.strptime(datetime.strftime(now_time, '%Y-%m-%d %H:%M:%S'),
                              DEFAULT_SERVER_DATETIME_FORMAT)).astimezone(local),
            DEFAULT_SERVER_DATETIME_FORMAT)
        self.state = 'done'
        self.pool.get("hr.leave").import_data(self,leave_master.id)
        return {
            'name': _('Import Leave'),
            'view_type': 'form',
            'view_mode': 'form',
            'view_id': False,
            'res_model': 'import.leave',
            'domain': [],
            'context': dict(self._context, active_ids=self.ids),
            'type': 'ir.actions.act_window',
            'target': 'new',
            'res_id': self.id,
        }


#!/usr/bin/env python3
import eval_utils
import conlleval_py
import os

import shutil
from collections import OrderedDict
import argparse

def read_sentence(ip_file):
	sentences=[]
	current_sentence = []
	for line in open(ip_file):
		line=line.strip()

		if line=="" and len(current_sentence)>0:
			sentences.append(current_sentence)

			current_sentence=[]
			continue
		line_values = line.split()

		if len(line_values)==2:
			current_sentence.append(line_values)

	return sentences

def print_result(eval_result, perf_file):

	result={}
	result["header"]=["", "Precision", "Recall", "F1"]
	result["rows"]=[]

	over_all_result= eval_result['overall']

	by_category_result = eval_result['by_category']

	for entity in sorted(by_category_result):
		l = [entity, by_category_result[entity]["P"], by_category_result[entity]["R"], by_category_result[entity]["F1"]]
		result["rows"].append(l)

	l=["overall",over_all_result["P"],over_all_result["R"], over_all_result["F1"]]
	result["rows"].append(l)

	eval_utils.tolatex(result)
	eval_utils.save_perf_in_tex(result, perf_file)

def compare_prediction(conll_file_gold, conll_file_pred, perf_file, to_latex):

	gold_sentences = read_sentence(conll_file_gold)
	pred_sentences = read_sentence(conll_file_pred)

	pred_file= "prediction.txt"

	fout=open(pred_file,"w")

	for gold_sent_index in range(len(gold_sentences)):
		gold_sent_info =  gold_sentences[gold_sent_index]
		pred_sent_info = pred_sentences[gold_sent_index]

		for word_index in range(len(gold_sent_info)):
			gold_word_info = gold_sent_info[word_index]
			pred_word_info = pred_sent_info[word_index]

			word=gold_word_info[0]
			gold_tag=gold_word_info[1]
			pred_tag=pred_word_info[1]
			base_tag = gold_tag.replace("I-","").replace("B-","")
			op_line=word+" "+base_tag+" "+gold_tag+" "+pred_tag+"\n"
			fout.write(op_line)
		fout.write("\n\n")
	fout.write("\n\n")
	fout.close()

	eval_result = conlleval_py.evaluate_conll_file(inputFile=pred_file)

	if to_latex:
		print_result(eval_result, perf_file)

	os.remove(pred_file)

def evaluate(input_gold_folder="../../data/test_data/Standoff_Format/", input_pred_folder="../baseline_CRF/Standoff_Outputs/", to_latex=False, pref_file=None):
	conll_folder_gold = "Conll_Format_Data/gold/"
	conll_file_gold = 'Conll_Format_Data/gold.txt'

	conll_folder_pred = "Conll_Format_Data/pred/"
	conll_file_pred = 'Conll_Format_Data/pred.txt'

	eval_utils.preprocess_data_to_merge(input_gold_folder, conll_folder_gold, conll_file_gold,input_pred_folder, conll_folder_pred, conll_file_pred)

	compare_prediction(conll_file_gold, conll_file_pred, pref_file, to_latex)

	shutil.rmtree('Conll_Format_Data/')

if __name__ == '__main__':
	parser = argparse.ArgumentParser()

	parameters_eval = OrderedDict()

	parser.add_argument(
	    "-gold_data", default="../../data/test_data/Standoff_Format/",
	    help="Standoff_Format gold labeled files"
	)

	parser.add_argument(
	    "-pred_data", default="../baseline_CRF/Standoff_Outputs/",
	    help="Standoff_Format prediction files"
	)

	parser.add_argument(
	    "-perf_file", default="performance.tex",
	    help="Output file to store the final latex table"
	)

	parser.add_argument(
	    "-to_latex", default="0",
	    help="if print in latex format (1 to enable, 0 to disable)"
	)

	opts = parser.parse_args()

	parameters_eval["gold_data"]=opts.gold_data
	parameters_eval["pred_data"]=opts.pred_data
	parameters_eval["perf_file"]=opts.perf_file

	if opts.to_latex=="0":
	    parameters_eval["print_latex_format"]=False
	else:
	    parameters_eval["print_latex_format"]=True

	input_gold_folder = parameters_eval["gold_data"]

	input_pred_folder = parameters_eval["pred_data"]

	print_latex_format=parameters_eval["print_latex_format"]

	perf_file = parameters_eval["perf_file"]

	evaluate(input_gold_folder, input_pred_folder, print_latex_format, perf_file)


import re
import urllib.request

from bs4 import BeautifulSoup

def clean_html(raw_source):
    clean = re.compile("<.*?>")
    clean_text = re.sub(clean, ' ' ,raw_source)
    return clean_text

no_right_url = "https://post.naver.com/viewer/postView.nhn?volumeNo=21251585&memberNo=16485321"
data1 = urllib.request.urlopen(no_right_url).read()

yes_right_url = "https://news.naver.com/main/read.nhn?mode=LSD&mid=shm&sid1=100&oid=025&aid=0002916700"
data2 = urllib.request.urlopen(yes_right_url)

soup = BeautifulSoup(data1, "html.parser")
new_soup = BeautifulSoup(data2, "html.parser")

soup_body = soup.html.body

with open("tagRemoved.txt", mode = "w") as f:
    f.write(clean_html(str(soup_body)))
    print("tagRemoved saved")

#textList = soup.find_all("span").text
#parsed = soup.body.find('span', attrs={'class':'container'}).text
with open("wordScrap.txt", mode = "w") as f:
    textList = soup.find_all(re.compile("<([A-Z][A-Z0-9]*)\b[^>]*>(.*?)</\1>"))
    f.write(str(textList))
    #use soup.find_all(re.compile("(a|div)")) to use regular expression function
    print("wordScrap saved")

#will not work properly
with open("useBeautiful.txt", mode = "w") as f:
    filtered_h1 = soup.html.body.h1
    f.write("h1: " + filtered_h1.string)
    filtered_p = soup.html.body.div
    f.write("p: " + filtered_p.string)
    #filtered_p2 = filtered_p.next_sibling
    #f.write("p2: " + filtered_p2.string)
    print("useBeautiful saved")

#find by ID
with open("useBeautiful_withID.txt", mode = "w") as f:
    filtered_wrap = soup.find(id="wrap")
    f.write("wrap: " + str(filtered_wrap))
    filtered_body = soup.find(id="body")
    f.write("body: " + str(filtered_body))
    #filtered_p2 = filtered_p.next_sibling
    #f.write("p2: " + filtered_p2.string)
    print("useBeautiful_withID saved")

#find all tag
with open("useBeautiful_findAll.txt", mode = "w") as f:
    filtered_hyperLink = soup.find_all("a")
    for a in filtered_hyperLink:
        href = a.attrs['href']
        text = a.string
        f.write(str(text) + " > " + str(href) + "\n")
    f.write("\n\n")
    print("useBeautiful_findAll saved")

#test
with open("useBeautiful_findAllImg.txt", mode = "w") as f:
    filtered_src = soup.findAll('img')
    for src in filtered_src:
        try:
            print(a.attrs['src'])
        except:
            print("attribution does not match")
        v = src.get('src', src.get('dfr-src'))
        f.write(str(v)+ "\n")
        if v is None:
            continue
    f.write("\n\n")
    print("useBeautiful_findAllImg saved")

#this will not work - use data frame
with open("scraping b.txt", mode = "w") as f:
    filtered_text = soup.find_all("b")
    f.write("text: " + str(filtered_text))
    """
    values = [(p.find('b').string, p.find('span').string)
             for p in soup.find_all('p', class_='info')]
    value_df = pandas.DataFrame.from_items(values)

    or use

    pandas.DataFrame
    """
    print("scraping b saved")

with open("selectUse.txt", mode = "w") as f:
    start = soup.select("#cont")
    end = start[0].findAll('div', attrs={'class' : 'SEDOC-1560934137262--1172937716'})
    f.write("text: " + str(end))
    print("selectUse saved")

with open("New_soup.txt", mode = "w") as f:
    start = new_soup.select_one("#wrap")
    #<[^>]*>|(.)
    pattern1 = re.compile(']*>(.+)
]*')
    pattern2 = re.compile("<[^>]*>|(.)")
    text_set = pattern1.findall(str(start))
    text_set = ''.join(text_set)
    final_string = pattern2.findall(str(text_set))
    final_string = ''.join(final_string)
    print(type(final_string))
    f.write(final_string)

with open("New_soup.txt", mode = "w") as f:
    start = new_soup.select_one("#wrap")
    #<[^>]*>|(.)
    pattern1 = re.compile(']*>(.+)
]*')
    pattern2 = re.compile("<[^>]*>|(.)")
    text_set = pattern1.findall(str(start))
    text_set = ''.join(text_set)
    final_string = pattern2.findall(str(text_set))
    final_string = ''.join(final_string)
    print(type(final_string))
    f.write(final_string)


# PCA method:-

import pandas as pd
from sklearn.decomposition import PCA

# load data
filename = 'indians-diabetes.data.csv'
names = ['preg', 'plas', 'pres', 'skin', 'test',
         'mass', 'pedi', 'age', 'class']

dataframe = pd.read_csv(filename, names=names)

array = dataframe.values
X = array[:, 0:8]
Y = array[:, 8]

# feature extraction
pca = PCA(n_components=3)

fit = pca.fit(X)

resultX = pca.transform(X)
print("\nResult : \n", resultX)

# summarize components
print("Explained Variance:", fit.explained_variance_ratio_)

5. machine learning/1. Data Preprocessing/4. feature selection/3. Principle Component Analysis.py


import logging
from typing import Optional, Tuple, cast

from confuse.core import ConfigView

from servicex.ConfigSettings import ConfigSettings
from servicex.utils import ServiceXException

class ServiceXConfigAdaptor:
    '''Contains the logic to extract all the configuration information needed, as driven from
    things the user has given us and program parameters. This concentrates all the complex config
    logic in on place, and, hopefully, leaves the logic out of all the other code.
    '''
    def __init__(self, config: Optional[ConfigView] = None):
        '''The config needed for the app.

        Note: The config is held onto and only queired when the information is required.

        Args:
            config (ConfigView): The config information for the app. If null, then we just use the
                                 standard servicex name.
        '''
        self._settings = config if config is not None else ConfigSettings('servicex', 'servicex')

    @property
    def settings(self) -> ConfigView:
        '''Return the config settings.

        Eventually this should not be used other than for testing!

        Returns:
            ConfigSettings: The config settings that gives us a view on everything the user wants
        '''
        return self._settings

    def get_default_returned_datatype(self, backend_type: Optional[str]) -> str:
        '''Return the default return data type, given the backend is a certian type.

        Args:
            backend_type (Optional[str]): The backend type string (`xaod`, `uproot`, etc)

        Returns:
            str: The backend datatype, like `root` or `parquet`.
        '''
        # Check to see if we know about the backend info
        if backend_type is not None:
            info = self.get_backend_info(backend_type, 'return_data')
            if info is not None:
                return info

        if self._settings['default_return_data'].exists():
            return cast(str, self._settings['default_return_data'].as_str_expanded())

        raise ServiceXException('A default default_return_data is missing from config files - is '
                                'servicex installed correctly?')

    def get_backend_info(self, backend_type: str, key: str) -> Optional[str]:
        '''Find an item in the backend info, searching first the backend settings and then
           searching the defaults.

        Args:
            backend_type (str): Backend name
            key (str): The key for the info we are after

        Returns:
            Optional[str]: Return a string for the info we are after, or return None if we can't
                           find it.
        '''

        def find_in_list(c, key) -> Optional[str]:
            if c.exists():
                for ep in c:
                    if ep['type'].exists():
                        if ep['type'].as_str() == backend_type:
                            if key in ep:
                                return ep[key].as_str_expanded()
            return None

        a = find_in_list(self._settings['api_endpoints'], key)
        a = find_in_list(self._settings['backend_types'], key) if a is None else a
        return a

    def get_servicex_adaptor_config(self, backend_type: Optional[str] = None) -> \
            Tuple[str, Optional[str]]:
        '''Return the servicex (endpoint, username, email) from a given backend configuration.

        Args:
            backend_type (str): The backend name (like `xaod`) which we hopefully can find in the
            configuration file.

        Returns:
            Tuple[str, str]: The tuple of info to create a `ServiceXAdaptor`: end point,
            token (optionally).
        '''
        # Find a list of all endpoints.
        # It is an error if this is not specified somewhere.
        endpoints = self._settings['api_endpoints']

        def extract_info(ep) -> Tuple[str, Optional[str]]:
            endpoint = ep['endpoint'].as_str_expanded()
            token = ep['token'].as_str_expanded() if 'token' in ep else None

            # We can default these to "None"
            return (endpoint, token)  # type: ignore

        # If we have a good name, look for exact match
        if backend_type is not None:
            for ep in endpoints:
                if ep['type'].exists() and ep['type'].as_str_expanded() == backend_type:
                    return extract_info(ep)

        # See if one is unlabeled.
        log = logging.getLogger(__name__)
        for ep in endpoints:
            if not ep['type'].exists():
                if backend_type is None:
                    log.warning('No backend type requested, '
                                f'using {ep["endpoint"].as_str_expanded()} - please be explicit '
                                'in the ServiceXDataset constructor')
                else:
                    log.warning(f"No '{backend_type}' backend type found, "
                                f'using {ep["endpoint"].as_str_expanded()} - please add to '
                                'the configuration file (e.g. servicex.yaml)')
                return extract_info(ep)

        if backend_type is not None:
            # They have a labeled backend, and all the end-points are labeled. So that means
            # there really is not match. So throw!
            seen_types = [str(ep['type'].as_str_expanded()) for ep in endpoints
                          if ep['type'].exists()]
            raise ServiceXException(f'Unable to find type {backend_type} '
                                    'in .servicex configuration file. Saw only types'
                                    f': {", ".join(seen_types)}')

        # Nope - now we are going to have to just use the first one there.
        for ep in endpoints:
            log.warning('No backend type requested, '
                        f'using {ep["endpoint"].as_str_expanded()} - please be explicit '
                        'in the ServiceXDataset constructor')
            return extract_info(ep)

        # If we are here - then... it just isn't going to work!
        raise ServiceXException('Not even a default set of configurations are here! Bad install '
                                ' of the servicex package!')


from django.shortcuts import render, redirect, get_object_or_404
from .models import Article, Hashtag
from .forms import ArticleForm, CommentForm
from IPython import embed
from django.http import Http404, HttpResponse
from django.views.decorators.http import require_POST
from django.contrib.auth.decorators import login_required
from itertools import chain

# Create your views here.
@login_required
def index(request):
    visits_num = request.session.get('visits', 0)
    request.session['visits'] = visits_num + 1
    request.session.modified = True
    followings = request.user.followings.all()
    followings_and_me = chain(followings, [request.user])
    articles = Article.objects.filter(user__in=followings_and_me)
    # my_articles = request.user.article_set.all()
    context = {
        'articles': articles,
        'visits': visits_num
    }
    return render(request, 'articles/index.html', context)

# detail
def detail(request, article_pk):
    article = get_object_or_404(Article, pk=article_pk)
    # 만약에 Article.objects.get(pk=article_pk)
    # try:
    #     article = Article.objects.get(pk=article_pk)
    # except Article.DoesNotExist:
    #     raise Http404('해당하는 id의 글이 존재하지 않습니다.')
    comment_form = CommentForm()

    context = {
        'article': article,
        'comment_form': comment_form,
        'comments': article.comment_set.all(),
    }
    return render(request, 'articles/detail.html', context)

@login_required
def create(request):
    if not request.user.is_authenticated:
        return redirect('accounts:login')

    if request.method == 'POST':

        form = ArticleForm(request.POST)
        # 전송된 데이터가 유효한 값인지 검사
        if form.is_valid():
            # title = form.cleaned_data.get('title')
            # content = form.cleaned_data.get('content')
            # article = Article.objects.create(title=title, content=content)
            article = form.save(commit=False)
            article.user = request.user
            article.save()

            # hashtag
            for word in article.content.split():
                if word.startswith('#'):
                    hashtag, created = Hashtag.objects.get_or_create(content=word)
                    article.hashtags.add(hashtag)
            return redirect(article)
        else:
            return redirect('articles:create')
    else:
        form = ArticleForm()
        context = {
            'form': form,
        }
        return render(request, 'articles/create.html', context)

# UPDATE -> articles/:id/update | (PUT) articles/:id
@login_required
def update(request, article_pk):
    article = get_object_or_404(Article, pk=article_pk)

    if article.user == request.user:
        if request.method == 'POST':
            # 실제 DB의 데이터를 수정
            form = ArticleForm(request.POST, instance=article)
            if form.is_valid():
                form.save()
                return redirect(article)

        # 편집 화면
        form = ArticleForm(instance=article)
    else:
        return redirect('articles:index')
    context = {
        'article': article,
        'form': form,
    }
    return render(request, 'articles/update.html', context)

# DELETE -> articles/:id/delete
@require_POST
def delete(request, article_pk):
    if request.user.is_authenticated:
        article = get_object_or_404(Article, pk=article_pk)

        if article.user == request.user:
            if request.method == 'POST':
                article.delete()
                return redirect('articles:index')
        else:
            return redirect(article)
    return HttpResponse('검증되지 않은 유저정보', status=401)

@login_required
def create_comment(request, article_pk):
    article = Article.objects.get(pk=article_pk)

    if request.method == 'POST':
        comment_form = CommentForm(request.POST)
        if comment_form.is_valid():
            comment = comment_form.save(commit=False)
            # comment.article_id = article_pk
            # comment.save()
            comment.user = request.user
            comment.article = article
            comment.save()

    # return redirect('articles:detail', article_pk)
    return redirect(article)

def send_cookie(request):
    # return render() -> html 페이지를 만들어 주는 것.
    # return redirect()
    # return reverse()
    # return HttpResponse()
    res = HttpResponse('과자 받아라')
    res.set_cookie('mycookie', 'oreo')
    return res

def like(request, article_pk):
    # article_pk로 넘어온 글의 like_users에 현재 접속중인 유저를 추가한다.
    article = Article.objects.get(pk=article_pk)
    user = request.user
    # 만약 좋아요 리스트에 현재 접속중인 유저가 있다면,
    # -> 해당 유저는 좋아요를 했다
    # if request.user in article.like_users.all():
    if article.like_users.filter(pk=user.pk).exists():
        article.like_users.remove(user)
    else:
        article.like_users.add(user)

    return redirect(article)
    # request.user.like_articles.add(article)

def explore(request):
    articles = Article.objects.all()
    context = {
        'articles': articles
    }
    return render(request, 'articles/explore.html', context)

def tags(request):
    tags = Hashtag.objects.all()
    context = {
        'tags': tags,
    }
    return render(request, 'articles/tags.html', context)

def hashtag(request, hashtag_pk):
    hashtag = get_object_or_404(Hashtag, pk=hashtag_pk)
    articles = hashtag.article_set.all()
    context = {
        'hashtag': hashtag,
        'articles': articles,
    }
    return render(request, 'articles/hashtag.html', context)


import ssl

from pathlib import Path
from typing import Optional, Dict

from src.util.config import config_path_for_filename

def load_ssl_paths(path: Path, config: Dict):
    try:
        return (
            config_path_for_filename(path, config["ssl"]["crt"]),
            config_path_for_filename(path, config["ssl"]["key"]),
        )
    except Exception:
        pass

    return None

def ssl_context_for_server(
    root_path: Path, config: Dict, require_cert: bool = False
) -> Optional[ssl.SSLContext]:
    paths = load_ssl_paths(root_path, config)
    if paths is None:
        return paths
    private_cert, private_key = paths
    ssl_context = ssl._create_unverified_context(purpose=ssl.Purpose.CLIENT_AUTH)
    ssl_context.load_cert_chain(certfile=private_cert, keyfile=private_key)
    ssl_context.load_verify_locations(private_cert)
    ssl_context.verify_mode = ssl.CERT_REQUIRED if require_cert else ssl.CERT_NONE
    return ssl_context

def ssl_context_for_client(
    root_path: Path, config: Dict, auth: bool
) -> Optional[ssl.SSLContext]:
    paths = load_ssl_paths(root_path, config)
    if paths is None:
        return paths
    private_cert, private_key = paths
    ssl_context = ssl._create_unverified_context(purpose=ssl.Purpose.SERVER_AUTH)
    ssl_context.load_cert_chain(certfile=private_cert, keyfile=private_key)
    if auth:
        ssl_context.verify_mode = ssl.CERT_REQUIRED
        ssl_context.load_verify_locations(private_cert)
    else:
        ssl_context.verify_mode = ssl.CERT_NONE
    return ssl_context


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
    ]

    operations = [
        migrations.CreateModel(
            name='Orden',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('fecha', models.DateField(auto_now=True)),
            ],
        ),
        migrations.CreateModel(
            name='Producto',
            fields=[
                ('codigo', models.CharField(max_length=10, serialize=False, primary_key=True)),
                ('descripcion', models.CharField(max_length=100)),
                ('cantidad', models.IntegerField(default=0)),
                ('orden', models.ForeignKey(to='compras_app.Orden')),
            ],
        ),
        migrations.CreateModel(
            name='Proveedor',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('codigo', models.IntegerField()),
                ('nombre', models.CharField(max_length=50)),
                ('productos', models.ManyToManyField(to='compras_app.Producto')),
            ],
        ),
        migrations.AddField(
            model_name='orden',
            name='proveedor',
            field=models.ForeignKey(to='compras_app.Proveedor'),
        ),
    ]


"""CSC148 Assignment 1: Tests for Customer

=== CSC148 Fall 2019 ===
Department of Computer Science,
University of Toronto

=== Module description ===
This module contains starter code for testing the Customer class.

This code is provided solely for the personal and private use of
students taking the CSC148 course at the University of Toronto.
Copying for purposes other than this use is expressly prohibited.
All forms of distribution of this code, whether as given or with
any changes, are expressly prohibited.

Author: Jacqueline Smith

All of the files in this directory and all subdirectories are:
Copyright (c) 2019 Jacqueline Smith
"""
from store import Customer, Item

# TODO: write your test functions for Customer here

def test_customer_no_item() -> None:
    """test for customer with no item"""
    customer1 = Customer("bruce", [])
    assert customer1.name == "bruce"
    assert customer1.arrival_time == -1
    assert customer1._items == []

def test_customer_one_item() -> None:
    """check for customer with one item"""
    item1 = Item("banana",10)
    customer1 = Customer("bruce", [item1])
    assert customer1.name == "bruce"
    assert customer1.arrival_time == -1
    assert customer1._items == [item1]

def test_customer_multi_item() -> None:
    """check for customer with several items"""
    item1 = Item("banana", 10)
    item2 = Item("apple", 2)
    item3 = Item("orange", 9)
    customer1 = Customer("bruce", [item1, item2, item3])
    assert customer1.name == "bruce"
    assert customer1.arrival_time == -1
    assert customer1._items == [item1, item2, item3]

def test_customer_num_items_with_no_item() -> None:
    """check num item method with no item"""
    customer1 = Customer("bruce", [])
    assert customer1.arrival_time == -1
    assert customer1.num_items() == 0

def test_customer_num_items_with_one_item() -> None:
    """check num item method with one item"""
    item1 = Item("banana", 10)
    customer1 = Customer("bruce", [item1])
    assert customer1.num_items() == 1

def test_customer_num_items_with_multi_item() -> None:
    """check num item method with several items"""
    item1 = Item("banana", 10)
    item2 = Item("apple", 2)
    item3 = Item("orange", 9)
    customer1 = Customer("bruce", [item1, item2, item3])
    assert customer1.num_items() == 3

def test_get_item_time_with_no_item() -> None:
    """check get item time method with no item"""
    customer1 = Customer("bruce", [])
    assert customer1.get_item_time() == 0

def test_get_item_time_with_one_item() -> None:
    """check get item time method with one item"""
    item1 = Item("banana", 10)
    customer1 = Customer("bruce", [item1])
    assert customer1.get_item_time() == 10

def test_get_item_time_with_multi_item() -> None:
    """check get item time method with several items"""
    item1 = Item("banana", 10)
    item2 = Item("apple", 2)
    item3 = Item("orange", 9)
    customer1 = Customer("bruce", [item1, item2, item3])
    assert customer1.get_item_time() == 21

if __name__ == '__main__':
    import pytest
    pytest.main(['test_customer.py'])


# -*- coding: utf-8 -*-

"""
Simplify (AWT) CKAN JSON files to a format more suitable for data.gov.be.
"""

__author__ = "Bart Hanssens <bart.hanssens@fedict.be>"

import argparse, os, logging
import json

logging.basicConfig(filename='awt-simplify.log', level=logging.DEBUG)

def ckanparse(infile, outfile):
	"""Parse CKAN JSON file and store it in a new (simplified) JSON file."""
	logging.info('Parsing ' + str(infile))

	data = {}

	with open(infile, 'r') as f:
		d = json.load(f)
		res = d['result']

		data['title'] = res['title']
		data['body'] = ''

		if 'author' in res and res['author']:
			data['source'] = res['author']

		for k in ('contexts', 'urls', 'fmts', 'tags', 'license'):
			data[k] = set()

		if 'notes' in res and res['notes']:
			data['body'] = '\n'.join((data['body'], res['notes']))

		if 'isopen' in res and res['isopen']:
			data['license'].add('Libre')
		else:
			data['license'].add('Voir le site du fournisseur de données')

		if 'organization' in res:
			if res['organization'] and res['organization']['name']:
				data['org'] = res['organization']['name']

		if 'name' in res:
			data['contexts'].add('http://opendata.awt.be/dataset/' + res['name'])

		for r in res['resources']:
			if 'format' in r:
				data['fmts'].add(r['format'])
			if 'url' in r and r['url']:
				data['urls'].add(r['url'])

		for r in res['tags']:
			if 'name' in r:
				data['tags'].add(r['name'])

	# Copy formats to media, for additional mapping
	data['media'] = list(data['fmts'])

	# Add defaults for required fields
	if not 'org' in data:
		data['org'] = 'andere'
	if not data['body']:
		data['body'] = data['title']
	data['body'] = data['body'].strip()

	if not data['tags']:
		data['tags'] = ['']

	# make entries JSON serializable
	for k in ('contexts', 'urls', 'tags', 'fmts', 'license'):
		data[k] = list(data[k])

	with open(outfile, 'w') as f:
		json.dump(data, f, indent=4, encoding='iso-8859-1')

def main():
	parser = argparse.ArgumentParser(description='Simplify AWT CKAN JSON files')
	parser.add_argument('--indir', help='Input directory', required=True)
	parser.add_argument('--outdir', help='Output directory', required=True)

	args = parser.parse_args()

	for f in os.listdir(args.indir):
		ckanparse(os.path.join(args.indir, f), os.path.join(args.outdir, f))

if __name__ == "__main__":
	main()


import pandas
from keras.models import Sequential
from keras.layers.core import Dense, Activation
from keras.callbacks import TensorBoard
# load dataset
from sklearn.model_selection import train_test_split
import pandas as pd

dataset = pd.read_csv("framingham.csv", header=None).values
import numpy as np

X_train, X_test, Y_train, Y_test = train_test_split(dataset[:,0:15], dataset[:,15],
                                                    test_size=0.33, random_state=87)

np.random.seed(100)
nnokay = Sequential() # create model
nnokay.add(Dense(20, input_dim=15, activation='tanh')) # hidden layer
nnokay.add(Dense(30, activation='tanh')) #add whole layer
nnokay.add(Dense(60, activation='tanh'))
nnokay.add(Dense(20, activation='tanh'))
nnokay.add(Dense(15, activation='tanh'))
nnokay.add(Dense(60, activation='tanh'))
nnokay.add(Dense(1, activation='tanh')) # output layer
nnokay.compile(loss='binary_crossentropy', optimizer='sgd', metrics=['accuracy'])

nnokay.fit(X_train, Y_train, epochs=250, verbose=0,
                                    callbacks=[TensorBoard(log_dir = '/tmp/auto')])
#print(nnokay.summary())
#print(nnokay.evaluate(X_test, Y_test, verbose=0))

score = nnokay.evaluate(X_test, Y_test)
print('test accuracy', score[1])


"""MooseCanvas.py:

Here is my Canvas.

"""

__author__ = "Dilawar Singh"
__copyright__ = "Copyright 2016, Dilawar Singh"
__credits__ = ["NCBS Bangalore"]
__license__ = "GNU GPL"
__version__ = "1.0.0"
__maintainer__ = "Dilawar Singh"
__email__ = "dilawars@ncbs.res.in"
__status__ = "Development"

import sys
import random
import os

try:
    import Tkinter as tk
except ImportError as e:
    import tkinter as tk

import logging

class MooseCanvas( tk.Frame ):

    def __init__(self, root = None):

        self.canvas = tk.Canvas( root , width=800, height=800 )
        self.canvas.create_line( 0, 0, 200, 200 );

        '''
        self.xsb = tk.Scrollbar( self.canvas,
            orient="horizontal", command=self.canvas.xview
            )

        self.ysb = tk.Scrollbar( self.canvas, orient="vertical",
            command=self.canvas.yview
            )
        self.canvas.configure(
            yscrollcommand=self.ysb.set,
            xscrollcommand=self.xsb.set
            )

        self.xsb.grid(row=1, column=0, sticky="ew")
        self.ysb.grid(row=0, column=1, sticky="ns")
        # self.grid_rowconfigure(0, weight=1)
        # self.grid_columnconfigure(0, weight=1)
        '''

        # This is what enables using the mouse:
        self.canvas.bind("<ButtonPress-1>", self.move_start)
        self.canvas.bind("<B1-Motion>", self.move_move)
        # linux scroll
        self.canvas.bind("<Button-4>", self.zoomerP)
        self.canvas.bind("<Button-5>", self.zoomerM)
        # windows scroll
        self.canvas.bind("<MouseWheel>", self.zoomer)
        # self.canvas.bind( "<Key>", self.keyboard )
        self.canvas.bind( "Move", self.keyboard )
        self.canvas.bind( "<Button-1>", self.mouseCallback )
        self.canvas.grid( row = 0, column = 0, sticky = 'news' )
        # self.canvas.pack( )

    # move
    def move_start(self, event):
        self.canvas.scan_mark(event.x, event.y)

    def move_move(self, event):
        self.canvas.scan_dragto(event.x, event.y, gain=1)

    # windows zoom
    def zoomer(self, event):
        if (event.delta > 0):
            self.canvas.scale("all", event.x, event.y, 1.1, 1.1)
        elif (event.delta < 0):
            self.canvas.scale("all", event.x, event.y, 0.9, 0.9)
        self.canvas.configure(scrollregion=self.canvas.bbox("all"))

    # linux zoom
    def zoomerP(self, event):
        self.canvas.scale("all", event.x, event.y, 1.1, 1.1)
        self.canvas.configure(scrollregion=self.canvas.bbox("all"))

    def zoomerM(self, event):
        self.canvas.scale("all", event.x, event.y, 0.9, 0.9)
        self.canvas.configure(scrollregion=self.canvas.bbox("all"))

    def mouseCallback(self, event ):
        logging.info( 'Button pressed %s %s' % ( event.x, event.y ) )

    def keyboard(self, event ):
        logging.info( 'Key pressed %s' % ( event.char ) )

if __name__ == "__main__":
    root = tk.Tk()
    a = MooseCanvas( root )
    # a.plot_random_rects( )
    a.pack(fill="both", expand=True)
    root.mainloop()


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('users', '0002_studentuser_teacheruser'),
    ]

    operations = [
        migrations.CreateModel(
            name='Address',
            fields=[
                ('id', models.AutoField(primary_key=True, auto_created=True, verbose_name='ID', serialize=False)),
                ('num', models.IntegerField()),
                ('street', models.CharField(max_length=100)),
                ('city', models.CharField(max_length=100)),
                ('state', models.CharField(max_length=2)),
                ('zipcode', models.CharField(max_length=5)),
            ],
        ),
        migrations.CreateModel(
            name='Certification',
            fields=[
                ('id', models.AutoField(primary_key=True, auto_created=True, verbose_name='ID', serialize=False)),
                ('name', models.CharField(max_length=255)),
                ('description', models.TextField()),
            ],
        ),
        migrations.CreateModel(
            name='Course',
            fields=[
                ('id', models.AutoField(primary_key=True, auto_created=True, verbose_name='ID', serialize=False)),
                ('catalog_id', models.CharField(max_length=255)),
                ('name', models.CharField(max_length=255)),
                ('description', models.TextField()),
                ('online', models.BooleanField(default=False)),
                ('weekly_work_hours', models.IntegerField()),
                ('weekly_sessions', models.IntegerField()),
                ('start_date', models.DateField()),
                ('end_date', models.DateField()),
                ('prerequisites', models.TextField()),
                ('max_num_students', models.IntegerField()),
            ],
        ),
        migrations.CreateModel(
            name='CourseCertificationRelationship',
            fields=[
                ('id', models.AutoField(primary_key=True, auto_created=True, verbose_name='ID', serialize=False)),
                ('certification', models.ForeignKey(to='courses.Certification')),
                ('course', models.ForeignKey(to='courses.Course')),
            ],
        ),
        migrations.CreateModel(
            name='CourseStudentRelationship',
            fields=[
                ('id', models.AutoField(primary_key=True, auto_created=True, verbose_name='ID', serialize=False)),
                ('course', models.ForeignKey(to='courses.Course')),
                ('student', models.ForeignKey(to='users.StudentUser')),
            ],
        ),
        migrations.CreateModel(
            name='CourseSubjectRelationship',
            fields=[
                ('id', models.AutoField(primary_key=True, auto_created=True, verbose_name='ID', serialize=False)),
                ('course', models.ForeignKey(to='courses.Course')),
            ],
        ),
        migrations.CreateModel(
            name='Price',
            fields=[
                ('id', models.AutoField(primary_key=True, auto_created=True, verbose_name='ID', serialize=False)),
                ('value', models.FloatField()),
                ('currency', models.CharField(max_length=255)),
            ],
        ),
        migrations.CreateModel(
            name='Subject',
            fields=[
                ('id', models.AutoField(primary_key=True, auto_created=True, verbose_name='ID', serialize=False)),
                ('name', models.CharField(max_length=255)),
                ('description', models.TextField()),
            ],
        ),
        migrations.AddField(
            model_name='coursesubjectrelationship',
            name='subject',
            field=models.ForeignKey(to='courses.Subject'),
        ),
        migrations.AddField(
            model_name='course',
            name='cost',
            field=models.ForeignKey(to='courses.Price'),
        ),
        migrations.AddField(
            model_name='course',
            name='location',
            field=models.ForeignKey(to='courses.Address'),
        ),
        migrations.AddField(
            model_name='course',
            name='teacher',
            field=models.ForeignKey(to='users.TeacherUser'),
        ),
    ]


import json
import os
import logging

import boto3

print('Loading function')

dynamo = boto3.client('dynamodb')
logger = logging.getLogger()
logger.setLevel(logging.INFO)

def lambda_handler(event, context):
    '''Event contains the following keys:

      - table: the table the rover made a delivery to
      - order_id: the order id for the delivery
      - delivery_time: the time the customer acknowledged delivery of the item
    '''
    print("Received event: " + json.dumps(event, indent=2))

    if 'table' in event and 'order_id' in event and 'delivery_time' in event:
        dynamo.update_item(
            TableName=os.environ['Dynamo_Fulfilled_Table'],
            Key={'order_id':{'S':event['order_id']}},
            UpdateExpression="SET delivery_time = :t",
            ExpressionAttributeValues={":t":{"S": event['delivery_time']}},
        )
    else:
        logger.error("Something went wrong. Received event: %s", json.dumps(event, indent=2))


# coding: utf-8

# Deep Learning
# =============
#
# Assignment 4
# ------------
#
# Previously in `2_fullyconnected.ipynb` and `3_regularization.ipynb`, we trained fully connected networks to classify [notMNIST](http://yaroslavvb.blogspot.com/2011/09/notmnist-dataset.html) characters.
#
# The goal of this assignment is make the neural network convolutional.

# In[ ]:

# These are all the modules we'll be using later. Make sure you can import them
# before proceeding further.
from __future__ import print_function
import numpy as np
import tensorflow as tf
from six.moves import cPickle as pickle
from six.moves import range
import argparse
import regularization_nn as nn
import os
import vtk
import glob

print("Tensorflow version:", tf.__version__)

parser = argparse.ArgumentParser()
parser.add_argument('--model', help='Model file computed with regularization_train.py', required=True)
parser.add_argument('--sampleMesh', help='Evaluate an image sample in vtk format')
parser.add_argument('--sampleDir', help='Evaluate a directory with vtk files')
parser.add_argument('--out', help='Write output of evaluation', default=None, type=str)

args = parser.parse_args()

sampleMesh = args.sampleMesh
sampleDir = args.sampleDir
outfilename = args.out
model = args.model
num_labels = 8
num_features = 3 + num_labels + 4
batch_size = 1

def load_features(shape):

  try:
      print("Reading:", shape)
      reader_poly = vtk.vtkPolyDataReader()
      reader_poly.SetFileName(shape)
      # print "shape : " + shape

      reader_poly.Update()
      geometry = reader_poly.GetOutput()

      # --------------------------------- #
      # ----- GET ARRAY OF FEATURES ----- #
      # --------------------------------- #

      # *****
      # ***** Get normals (3 useful components) - already normalized *****
      normalArray = geometry.GetPointData().GetNormals()
      nbCompNormal = normalArray.GetElementComponentSize() - 1  # -1 car 4eme comp = 1ere du pt suivant

      # ***** Get distances to each mean group (nbGlicroups components) and normalization *****
      listGroupMean = list()
      for i in range(0, num_labels):
          name = "distanceGroup" + str(i)
          temp = geometry.GetPointData().GetScalars(name)
          temp_range = temp.GetRange()
          temp_min, temp_max = temp_range[0], temp_range[1]
          for j in range(0, geometry.GetNumberOfPoints()):
              if temp_max:
                  temp.SetTuple1(j, 2 * (temp.GetTuple1(j) - temp_min) / (temp_max) - 1)
              else:
                  temp.SetTuple1(j, 2 * (temp.GetTuple1(j) - temp_min) / (1) - 1)     # CHANGEEEER
          listGroupMean.append(temp)

      # ***** Get Curvatures and value for normalization (4 components) *****
      meanCurvName = "Mean_Curvature"
      meanCurvArray = geometry.GetPointData().GetScalars(meanCurvName)
      meanCurveRange = meanCurvArray.GetRange()
      meanCurveMin, meanCurveMax = meanCurveRange[0], meanCurveRange[1]
      meanCurveDepth = meanCurveMax - meanCurveMin

      maxCurvName = "Maximum_Curvature"
      maxCurvArray = geometry.GetPointData().GetScalars(maxCurvName)
      maxCurveRange = maxCurvArray.GetRange()
      maxCurveMin, maxCurveMax = maxCurveRange[0], maxCurveRange[1]
      maxCurveDepth = maxCurveMax - maxCurveMin

      minCurvName = "Minimum_Curvature"
      minCurvArray = geometry.GetPointData().GetScalars(minCurvName)
      minCurveRange = minCurvArray.GetRange()
      minCurveMin, minCurveMax = minCurveRange[0], minCurveRange[1]
      minCurveDepth = minCurveMax - minCurveMin

      gaussCurvName = "Gauss_Curvature"
      gaussCurvArray = geometry.GetPointData().GetScalars(gaussCurvName)
      gaussCurveRange = gaussCurvArray.GetRange()
      gaussCurveMin, gaussCurveMax = gaussCurveRange[0], gaussCurveRange[1]
      gaussCurveDepth = gaussCurveMax - gaussCurveMin

      # For each point of the current shape
      currentData = np.ndarray(shape=(geometry.GetNumberOfPoints(), num_features), dtype=np.float32)
      for i in range(0, geometry.GetNumberOfPoints()):

          # Stock normals in currentData
          for numComponent in range(0, nbCompNormal):
              currentData[i, numComponent] = normalArray.GetComponent(i, numComponent)

          for numComponent in range(0, num_labels):
              currentData[i, numComponent + nbCompNormal] = listGroupMean[numComponent].GetTuple1(i)

          value = 2 * (meanCurvArray.GetTuple1(i) - meanCurveMin) / meanCurveDepth - 1
          currentData[i, num_labels + nbCompNormal] = value

          value = 2 * (maxCurvArray.GetTuple1(i) - maxCurveMin) / maxCurveDepth - 1
          currentData[i, num_labels + nbCompNormal + 1] = value

          value = 2 * (minCurvArray.GetTuple1(i) - minCurveMin) / minCurveDepth - 1
          currentData[i, num_labels + nbCompNormal + 2] = value

          value = 2 * (gaussCurvArray.GetTuple1(i) - gaussCurveMin) / gaussCurveDepth - 1
          currentData[i, num_labels + nbCompNormal + 3] = value

  except IOError as e:
      print('Could not read:', shape, ':', e, '- it\'s ok, skipping.')

  # print('Full dataset tensor:', dataset.shape)
  # print('Mean:', np.mean(dataset))
  # print('Standard deviation:', np.std(dataset))
  return currentData

# Reformat into a TensorFlow-friendly shape:
# - convolutions need the image data formatted as a cube (width by height by #channels)
# - labels as float 1-hot encodings.

# In[ ]:

# in_depth = img_size[3] #zdim
# in_height = img_size[2] #ydim
# in_width = img_size[1] #xdim
# num_channels = img_size[0] #numchannels
# num_channels_labels = 1

# Reformat into a TensorFlow-friendly shape:
# - convolutions need the image data formatted as a cube (depth * width * height * channels)
# - We know that nrrd format
# - labels as float 1-hot encodings.

if sampleMesh != None:
  valid_dataset = load_features(sampleMesh)
  valid_dataset = valid_dataset.reshape(1, -1)
  batch_size = 1
else:
  vtklist = glob.glob(os.path.join(sampleDir, "*.vtk"))
  valid_dataset = []
  for vtkfile in vtklist:
    print("Reading:", vtkfile)
    valid_dataset.append(load_features(vtkfile))
  valid_dataset = np.array(valid_dataset)
  valid_dataset = valid_dataset.reshape(valid_dataset.shape[0], -1)
  batch_size = valid_dataset.shape[0]

size_features = valid_dataset.shape[1]

print('Validation set', valid_dataset.shape)

# Let's build a small network with two convolutional layers, followed by one fully connected layer. Convolutional networks are more expensive computationally, so we'll limit its depth and number of fully connected nodes.

# In[ ]:

#batch_size = 64
# patch_size = 8
# depth = 32
# depth2 = 64
# num_hidden = 256
# stride = [1, 1, 1, 1]

# def evaluate_accuracy(prediction, labels):
#   accuracy = tf.reduce_sum(tf.squared_difference(prediction, labels))
#   return accuracy.eval()

graph = tf.Graph()

with graph.as_default():

# run inference on the input data
  x = tf.placeholder(tf.float32,shape=(batch_size, size_features))

  keep_prob = tf.placeholder(tf.float32)

  #tf_valid_dataset = tf.constant(valid_dataset)
  # tf_test_dataset = tf.constant(test_dataset)

  y_conv = nn.inference(x, size_features, num_labels, keep_prob, batch_size)
  predict = tf.argmax(y_conv, 1)

# calculate the loss from the results of inference and the labels
  #loss = nn.loss(y_conv, y_)

  #accuracy_eval = nn.evaluation(y_conv, y_)

  #tf.summary.scalar(loss.op.name, loss)

  #intersection_sum, label_sum, example_sum, precision = nn.evaluation(y_conv, y_)

  #tf.summary.scalar ("Precision op", precision)

# setup the training operations
  #train_step = nn.training(loss, learning_rate, decay_steps, decay_rate)
  # setup the summary ops to use TensorBoard

  # setup the training operations
  #train_step = nn.training(loss, learning_rate, decay_steps, decay_rate)
  #train_step = nn.training(loss, learning_rate, decay_steps, decay_rate)

  # intersection_sum, label_sum, example_sum = evaluation(y_conv, y_)

  # valid_prediction = model(tf_valid_dataset)
  #cross_entropy = tf.reduce_sum(tf.squared_difference(y_conv, y_))

  #regularizers = tf.nn.l2_loss(W_fc1) + tf.nn.l2_loss(W_fc2)
  #cross_entropy += 0.1 * regularizers

  #cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(y_conv, y_))
  #train_step = tf.train.GradientDescentOptimizer(1e-4).minimize(cross_entropy)

  # accuracy = cross_entropy

  # valid_prediction = model(tf_valid_dataset)
  # evaluation(valid_prediction)
  # test_prediction = model(tf_test_dataset)

  with tf.Session() as sess:
    sess.run(tf.initialize_all_variables())
    saver = tf.train.Saver()
    saver.restore(sess, model)

    pred = sess.run([predict], feed_dict={x: valid_dataset, keep_prob: 1})

    if outfilename != None:
      with open(outfilename, "w") as outfile:
        for pr in pred[0]:
          outfile.write(str(pr))
          outfile.write("\n")
    else:
      for pr in pred[0]:
        print(pr)

    #test_accuracy = evaluate_accuracy(test_prediction.eval(feed_dict={keep_prob: 1.0}), test_labels)
    #print("test accuracy %g"%test_accuracy)


#coding:utf-8

import os
import logging
import FolderClass
#logging.disable(logging.INFO)

class File:

	total_file = 0

	def __init__(self, dirEntry):
		self._path = dirEntry.path
		self._content = ""
		self.dirEntry = dirEntry
		self.name = dirEntry.name
		self.lbx_name = ""

		File.total_file += 1
		logging.info('FILE - __init__ %s'% self.name)

		self.update_lbx_name()
		self.update_content()

	def __repr__(self):
		return 'FILE "{}"'.format(self.name, self.path)

	"""-------------------- File instance method -------------------------"""

	def update_content(self):
		logging.info('FILE - "{}" Update content.'.format(self.name))
		try:
			with open(self.path, "r", encoding='utf-8') as ofi:
				self.content = ofi.read()

		except Exception as exception:
			logging.error('FILE - {} Update content FAILED.\n'.format(exception))

	def update_lbx_name(self):
		logging.info('FILE - update_lbx_name({}).'.format(self.name))

		try:
			parentFolder = os.path.dirname(self.path)
			parentFolder = FolderClass.Folder.get_folder_in_foldList(os.path.split(parentFolder)[1])

			if parentFolder != None:
				while parentFolder != None:
					self.lbx_name += '   '
					parentFolder = FolderClass.Folder.get_folder_in_foldList(os.path.split(\
											os.path.dirname(parentFolder.path))[1])
				self.lbx_name += self.name
				logging.info('FILE - lbx_name -> %s'%(self.lbx_name))
			else:
				self.lbx_name = self.name
				logging.info('FILE - lbx_name -> %s'%(self.lbx_name))

		except Exception as exception:
			logging.error('FOLD - update_lbx_name({}) FAILED:\n{}'.format(self.name, exception))

	"""-------------------- File Attribute property ----------------------"""

	def _get_File_path(self):

		try:
			return self._path
		except:
			logging.error('FILE - Get path FAILED.')

	def _set_File_path(self, newPath):

		try:
			self._path = newPath
		except:
			logging.error('FILE - Set newPath  FAILED.')

	def _get_File_content(self):
		try:
			return self._content
		except:
			logging.error('FILE - Get content FAILED.')

	def _set_File_content(self, newContent):
		try :
			self._content = newContent
		except:
			logging.error('FILE - Set content FAILED.')

	path = property(_get_File_path, _set_File_path)
	content = property(_get_File_content, _set_File_content)


import pandas as pd
import numpy as np
import scanpy as sc
import matplotlib.pylab as plt
from scipy import stats
import skimage.filters as skif
from sklearn.preprocessing import normalize
import glob

def reorder_AnnData(AnnData, descending = True):
    AnnData.obs['total_counts'] = AnnData.X.sum(axis=1)
    if(descending==True):
        AnnData = AnnData[np.argsort(AnnData.obs['total_counts'])[::-1]].copy()
        AnnData.uns['Order'] = 'Descending'
    elif(descending==False):
        AnnData = AnnData[np.argsort(AnnData.obs['total_counts'])[:]].copy()
        AnnData.uns['Order'] = 'Ascending'
    return(AnnData)

def read_dropest(dir_path,reorder=True):
    data_matrix = glob.glob(dir_path+"/*.mtx")[0]
    data_genes = glob.glob(dir_path+"/*features.*")[0]
    data_barcodes = glob.glob(dir_path+"/*barcodes.*")[0]
    adata = sc.read_mtx(data_matrix).T
    adata.var.index = pd.read_csv(data_genes,header=None)[0].values
    adata.obs.index = pd.read_csv(data_barcodes,header=None)[0].values
    adata.obs.index.name = 'Cells'
    adata.var.index.name = 'Genes'
    adata = reorder_AnnData(adata,descending=True)
    adata.raw = adata
    return(adata)

def find_inflection(adata_in,mito_tag = "MT-",run_qc=True):
    if(run_qc):
        print("Calculating QC Metrics")
        adata_in.var['Mitochondrial'] = adata_in.var.index.str.startswith(mito_tag)
        sc.pp.calculate_qc_metrics(adata_in,qc_vars=['Mitochondrial'],use_raw=True,inplace=True)
    tc_ngbc_ratio = adata_in.obs['total_counts']/adata_in.obs['n_genes_by_counts']
    data_cumsum = np.cumsum(adata_in.obs['total_counts'])
    x_vals=np.arange(0,adata_in.n_obs)
    secant_coef=data_cumsum[adata_in.obs.shape[0]-1]/adata_in.obs.shape[0]
    secant_line=secant_coef*x_vals
    secant_dist=abs(data_cumsum-secant_line)
    inflection_points = secant_dist.argsort()[::-1]
    inflection_percentiles = [0,15,30]
    inflection_percentiles_inds = np.percentile(x_vals[x_vals>inflection_points[0]],inflection_percentiles).astype(int)
    color=plt.cm.tab10(np.linspace(0,1,adata_in.n_obs))

    fig = plt.figure( figsize=(20,6))
    ax1 = fig.add_subplot(121)
    ax2 = fig.add_subplot(122)
    ax1.plot(np.array(data_cumsum), label="Cumulative Sum")
    ax1.plot(np.array(secant_dist), label="Secant Distance")
    for percentile in inflection_percentiles_inds:
        ax1.axvline(x=percentile,ymin=0,c=color[percentile],linestyle='--',linewidth=2,label="Inflection point {}".format(percentile))
    ax1.legend()
    ax1.set_xlabel("Cell Rank")
    ax1.set_ylabel("Total Counts per Cell")
    ax1.set_title('Inflection Curve')

    ax2.plot(np.sort(tc_ngbc_ratio.values)[::-1],label='Total Counts/N Genes By Counts')
    #ax2.plot(np.sort(adata_in.obs['pct_counts_Mitochondrial'])[::-1]/100)
    ax2.set_xscale('log')
    ax2.set_yscale('log')
    ax2.set_title('Total Counts/N Genes By Counts')
    ax2.set_xlabel("Cell Rank")
    ax2.set_ylabel("Total Counts/N Genes By Counts")
    ax2.legend()

    print("Inflection point at {} for {} percentiles of greatest secant distances".format(inflection_percentiles_inds,inflection_percentiles))
    return(inflection_percentiles_inds)

def relative_diversity(adata_in,zscore_minimum = 0,log_scale=True):
    adata_in.obs['n_genes_by_counts_zscore'] = stats.zscore(adata_in.obs['n_genes_by_counts'])
    diversity_cutoff = skif.threshold_minimum(adata_in.obs['n_genes_by_counts_zscore'][adata_in.obs['n_genes_by_counts_zscore'] >= zscore_minimum])

    fig = plt.figure(figsize=(4,4))
    ax0 = plt.subplot(111)

    hist_plot_div = ax0.hist(adata_in.obs['n_genes_by_counts_zscore'],bins=np.sqrt(adata_in.n_obs).astype(int),log=log_scale)
    #ax0.set_ylim([0,max(hist_plot_div[0])])
    #ax0.set_xlim([0,max(hist_plot_div[1])])
    ax0.axvspan(diversity_cutoff,max(hist_plot_div[1]), alpha=0.25, color='green')
    ax0.vlines(diversity_cutoff,0,max(hist_plot_div[0]),color='green')
    ax0.set_title("Relative Transcript Diversity")
    adata_in.obs['relative_transcript_diversity_threshold'] = adata_in.obs['n_genes_by_counts_zscore']>=diversity_cutoff
    return(adata_in)

def subset_cleanup(adata_in,selection='Cell_Selection'):
    adata_in.X = adata_in.raw.X.copy()
    adata_in = adata_in[adata_in.obs[selection]].copy()
    adata_in = sc.AnnData(adata_in.to_df())
    return(adata_in)