#!/usr/bin/env python

# python train_dcgan.py --gpu=0 --epoch=500 --dataset=motion2img/walk_images --snapshot_interval=100
import pickle

import argparse
import os, sys

import chainer
from chainer import training, serializers
from chainer.training import extensions

from net_jp import Discriminator as Discriminator
from net_jp import Generator as Generator
from net_jp2 import Discriminator as Discriminator2
from net_jp2 import Generator as Generator2
from net_jp3 import Discriminator as Discriminator3
from net_jp3 import Generator as Generator3
from updater import PRGANUpdater_jp
from visualize import out_generated_image_jp
from dataset import BVHDataset_jp

import numpy as np

def train_prgan():
    #コマンドライン引数のparser
    parser = argparse.ArgumentParser(description='Chainer example: DCGAN')
    parser.add_argument('--batchsize', '-b', type=str, default=32,
                        help='Number of images in each mini-batch')
    parser.add_argument('--epochs', '-e', type=str, default=100,
                        help='Number of sweeps over the dataset to train')
    parser.add_argument('--gpu', '-g', type=int, default=0,
                        help='GPU ID (negative value indicates CPU)')
    parser.add_argument('--depth', '-d', type=str, default=0,
                        help='Depth of networks')
    parser.add_argument('--dataset', '-i', default='',
                        help='Directory of image files.  Default is cifar-10.')
    parser.add_argument('--out', default='result_prgan',
                        help='Directory to output the result')
    parser.add_argument('--net', '-o', default='net_jp3',
                            help='network')
    parser.add_argument('--snapshot_interval', type=int, default=100,
                        help='Interval of snapshot')
    parser.add_argument('--display_interval', type=int, default=10,
                        help='Interval of displaying log to console')
    parser.add_argument('--l_smo', type=float, default=0.01)
    parser.add_argument('--l_rec', type=float, default=0.1)
    parser.add_argument('--l_key', type=float, default=1.0)
    parser.add_argument('--alpha', type=float, default=0)
    parser.add_argument('--delta', type=float, default=0.00005)
    parser.add_argument('--frame_nums', type=int, default=64)
    #parser.add_argument('--gen', type=str, default=None)
    #parser.add_argument('--dis', type=str, default=None)
    #parser.add_argument('--pose_dis', type=str, default=None)
    # parser.add_argument('--optg', type=str, default=None)
    # parser.add_argument('--optd', type=str, default=None)
    # parser.add_argument('--optpd', type=str, default=None)
    args = parser.parse_args()

    print('GPU: {}'.format(args.gpu))
    print('# Minibatch-size: {}'.format(args.batchsize))
    print('# frame_nums: {}'.format(args.frame_nums))
    print('# depth: {}'.format(args.depth))
    print('# epochs: {}'.format(args.epochs))
    print('')

    depthlist = args.depth.split(',')
    epochlist = args.epochs.split(',')
    batchsizelist = args.batchsize.split(',')

    for depthnum in range(len(depthlist)):
        depth = int(depthlist[depthnum])
        epoch = int(epochlist[depthnum])
        batchsize = int(batchsizelist[depthnum])
        # Set up a neural network to train
        if args.net == 'net_jp':
            gen = Generator(depth=depth)
            dis = Discriminator(depth=depth)
        elif args.net == 'net_jp2':
            gen = Generator2(depth=depth)
            dis = Discriminator2(depth=depth)
        elif args.net == 'net_jp3':
            gen = Generator3(depth=depth)
            dis = Discriminator3(depth=depth)
        if depth > 0:
            print('loading generator model fromn ' + f'{depth-1}_gen.npz')
            serializers.load_npz(os.path.join(args.out, f'{depth-1}_gen.npz'), gen)

        if depth > 0:
            print('loading discriminator model from ' + f'{depth-1}_dis.npz')
            serializers.load_npz(os.path.join(args.out, f'{depth-1}_dis.npz'), dis)

        #pose_dis = PosewiseDiscriminator()
        #pose_disはProgressiveじゃないから前の学習のを読むと初めから強すぎる？
        #if args.pose_dis is not None:
        #    print('loading pose discriminator model from ' + args.pose_dis)
        #    serializers.load_npz(args.pose_dis, pose_dis)

        if args.gpu >= 0:
            # Make a specified GPU current
            chainer.backends.cuda.get_device_from_id(args.gpu).use()
            gen.to_gpu()  # Copy the model to the GPU
            dis.to_gpu()
            #pose_dis.to_gpu()

        # Setup an optimizer
        def make_optimizer(model, alpha=0.0002, beta1=0.5):
            optimizer = chainer.optimizers.Adam(alpha=alpha, beta1=beta1)
            optimizer.setup(model)
            return optimizer
        opt_gen = make_optimizer(gen, alpha=0.0002, beta1=0.5)
        #if args.optg is not None:
        #    print('loading generator optimizer from ' + args.optg)
        #    serializers.load_npz(args.optg, opt_gen)
        opt_dis = make_optimizer(dis, alpha=0.0002, beta1=0.5)
        #if args.optd is not None:
        #    print('loading discriminator optimizer from ' + args.optd)
        #    serializers.load_npz(args.optd, opt_dis)
        #opt_pose_dis = make_optimizer(pose_dis, alpha=0.0002, beta1=0.5)
        #if args.optpd is not None:
        #    print('loading pose discriminator optimizer from ' + args.optpd)
        #    serializers.load_npz(args.optpd, opt_pose_dis)

        head, ext = os.path.splitext(args.dataset)
        head, data_name = os.path.split(head)

        if os.path.exists('minmax_{0}.npz'.format(data_name)):
            with np.load('minmax_{0}.npz'.format(data_name)) as data:
                datamin = data['min']
                datamax = data['max']
        else:
            print('minmax_{0}.npz not found.'.format(data_name))
            sys.exit()

        train = BVHDataset_jp(args.dataset, args.frame_nums, depth, datamin, datamax)
            # train = chainer.datasets\
            #     .ImageDataset(paths=image_files[:50000], root=args.dataset)
            # train = np.random.permutation(train)
        print('{} contains {} samples'.format(args.dataset, len(train)))
            # test = chainer.datasets\
            #     .ImageDataset(paths=image_files[50000:], root=args.dataset)

        train_iter = chainer.iterators.SerialIterator(train, batchsize)

        # Set up a trainer
        updater = PRGANUpdater_jp(
            models=(gen, dis),#, pose_dis),
            iterator=train_iter,
            optimizer={
                'gen': opt_gen, 'dis': opt_dis},#, 'pose_dis': opt_pose_dis},
            alpha=args.alpha,
            delta=args.delta,
            depth=depth,
            datamin = datamin,
            datamax = datamax,
            frame_nums=args.frame_nums,
            device=args.gpu,
            mode = 'wgan-gp',
            l_smo = args.l_smo,
            l_rec = args.l_rec,
            l_key = args.l_key)
        trainer = training.Trainer(updater, (epoch, 'epoch'), out=args.out)

        snapshot_interval = (args.snapshot_interval, 'iteration')
        display_interval = (args.display_interval, 'iteration')
        #trainer.extend(
        #    extensions.snapshot(filename='snapshot_iter_{.updater.iteration}.npz'),
        #    trigger=snapshot_interval)
        trainer.extend(extensions.snapshot_object(
            gen, 'gen_iter_{.updater.epoch}.npz'), trigger=(10, 'epoch'))
        trainer.extend(extensions.snapshot_object(
            dis, 'dis_iter_{.updater.epoch}.npz'), trigger=(10, 'epoch'))
        #trainer.extend(extensions.snapshot_object(
        #    pose_dis, 'pose_dis_iter_{.updater.epoch}.npz'), trigger=(10, 'epoch'))
        # trainer.extend(extensions.snapshot_object(
        #     opt_gen, 'opt_gen_iter_{.updater.iteration}.npz'), trigger=(1, 'epoch'))
        # trainer.extend(extensions.snapshot_object(
        #     opt_dis, 'opt_dis_iter_{.updater.iteration}.npz'), trigger=(1, 'epoch'))
        # trainer.extend(extensions.snapshot_object(
        #     opt_pose_dis, 'opt_pose_dis_iter_{.updater.iteration}.npz'), trigger=(1, 'epoch'))
        trainer.extend(extensions.LogReport(trigger=display_interval))
        trainer.extend(extensions.PrintReport([
            'epoch', 'iteration', 'gen_loss', 'rec_loss', 'key_loss', 'smooth_loss', 'ad_loss', 'dis_loss', 'alpha'
        ]), trigger=display_interval)
        trainer.extend(extensions.ProgressBar(update_interval=10))
        trainer.extend(
            out_generated_image_jp(
                gen, train, depth, args.frame_nums, datamin, datamax,
                args.out),
            trigger=snapshot_interval)

        # Run the training
        trainer.run()

        #save_model
        modelname = './{0}/{1}_gen.npz'.format(args.out,depth)
        print('saving generator model to ' + modelname)
        serializers.save_npz(modelname, gen)

        modelname = './{0}/{1}_dis.npz'.format(args.out,depth)
        print('saving discriminator model to ' + modelname)
        serializers.save_npz(modelname, dis)

        #modelname = './{0}/{1}_pose_dis.npz'.format(args.out,depth)
        #print('saving posewise_discriminator model to ' + modelname)
        #serializers.save_npz(modelname, pose_dis)

        # #optname = './{0}/{1}_opt_gen.npz'.format(args.out,depth)
        # print( 'saving generator optimizer to ' + optname )
        # serializers.save_npz(optname, opt_gen)
        #
        # optname = './{0}/{1}_opt_dis.npz'.format(args.out,depth)
        # print( 'saving discriminator optimizer to ' + optname )
        # serializers.save_npz(optname, opt_dis)
        #
        # optname = './{0}/{1}_opt_pose_dis.npz'.format(args.out,depth)
        # print( 'saving posewise_discriminator optimizer to ' + optname )
        # serializers.save_npz(optname, opt_pose_dis)

if __name__ == '__main__':
    train_prgan()


"""Implementation of the VQ-VAE [1] and VQ-VAE-2 [2] models.

The Vector Quantized Variational Autoencoder (VQ-VAE) extends the vanilla
Variational Autoencoder by mapping the input to a discrete (instead of
continuous) latent space. The discretization is performed using vector
quantization where continuous outputs from an encoder are mapped to the nearest
vectors in a learned codebook. The main intuition behind using discretized
latents is that the underlying domain we are trying to model is usually
discrete (e.g. words in a sentence, objects in an image, etc).

VQ-VAE-2 further extends the original VQ-VAE by encoding the input to a
hierarchy of latent spaces. The top latent space is responsible for encoding
high level semantic details with progressively lower level details being encoded
as down the hierarchy.

References (used throughout the code):
  [1]: https://arxiv.org/pdf/1711.00937.pdf
  [2]: https://arxiv.org/pdf/1906.00446.pdf
"""

import torch
from torch import nn

from pytorch_generative import nn as pg_nn

class ResidualBlock(nn.Module):
  """A simple residual block."""

  def __init__(self, n_channels, hidden_channels):
    """Initializes a new ResidualBlock instance.

    Args:
      n_channels: Number of input and output channels.
      hidden_channels: Number of hidden channels.
    """
    super().__init__()
    self._net = nn.Sequential(
        nn.ReLU(),
        nn.Conv2d(in_channels=n_channels, out_channels=hidden_channels,
                  kernel_size=3, padding=1),
        nn.ReLU(),
        nn.Conv2d(in_channels=hidden_channels, out_channels=n_channels,
                  kernel_size=1))

  def forward(self, x):
    return x + self._net(x)

class ResidualStack(nn.Module):
  """A stack of multiple ResidualBlocks."""

  def __init__(self, n_channels, hidden_channels, n_residual_blocks=1):
    """Initializes a new ResidualStack instance.

    Args:
      n_channels: Number of input and output channels.
      hidden_channels: Number of hidden channels.
      n_residual_blocks: Number of residual blocks in the stack.
    """
    super().__init__()
    self._net = nn.Sequential(*[
        ResidualBlock(n_channels, hidden_channels)
        for _ in range(n_residual_blocks)
    ] + [nn.ReLU()])

  def forward(self, x):
    return self._net(x)

class Encoder(nn.Module):
  """A feedforward encoder which downsamples its input."""

  def __init__(self,
               in_channels,
               out_channels,
               hidden_channels,
               n_residual_blocks,
               residual_channels,
               stride):
    """Initializes a new Encoder instance.

    Args:
      in_channels: Number of input channels.
      out_channels: Number of output channels.
      hidden_channels: Number of channels in non residual block hidden layers.
      n_residual_blocks: Number of residual blocks in each residual stack.
      residual_channels: Number of hidden channels in residual blocks.
      stride: Stride to use in the downsampling convolutions. Must be even.
    """
    super().__init__()
    assert stride % 2 == 0, '"stride" must be even.'

    net = []
    for i in range(stride//2):
      first, last = 0, stride // 2 - 1
      in_c = in_channels if i == first else hidden_channels//2
      out_c = hidden_channels //2 if i < last else hidden_channels
      net.append(
          nn.Conv2d(in_channels=in_c, out_channels=out_c, kernel_size=4,
                    stride=2, padding=1))
      net.append(nn.ReLU())
    net.append(
        nn.Conv2d(in_channels=hidden_channels, out_channels=hidden_channels,
                  kernel_size=3, padding=1))
    net.append(
        ResidualStack(n_channels=out_channels,
                      hidden_channels=residual_channels,
                      n_residual_blocks=n_residual_blocks))
    self._net = nn.Sequential(*net)

  def forward(self, x):
    return self._net(x)

class Decoder(nn.Module):
  """A feedforward encoder which upsamples its input."""

  def __init__(self,
               in_channels,
               out_channels,
               hidden_channels,
               n_residual_blocks,
               residual_channels,
               stride):
    """Initializes a new Decoder instance.

    Args:
      in_channels: Number of input channels.
      out_channels: Number of output channels.
      hidden_channels: Number of channels in (non residual block) hidden layers.
      n_residual_blocks: Number of residual blocks in each residual stack.
      residual_channels: Number of hidden channels in residual blocks.
      stride: Stride to use in the upsampling (i.e. transpose) convolutions.
        Must be even.
    """
    super().__init__()

    assert stride % 2 == 0, '"stride" must be even.'

    net = [
      nn.Conv2d(in_channels=in_channels, out_channels=hidden_channels,
                kernel_size=3, padding=1),
      ResidualStack(n_channels=hidden_channels,
                    hidden_channels=residual_channels,
                    n_residual_blocks=n_residual_blocks)
    ]
    for i in range(stride//2):
      first, last = 0, stride//2 - 1
      in_c = hidden_channels if i == first else hidden_channels//2
      out_c = hidden_channels//2 if i < last else out_channels
      net.append(
          nn.ConvTranspose2d(in_channels=in_c, out_channels=out_c,
                             kernel_size=4, stride=2, padding=1))
      if i < last:
        net.append(nn.ReLU())
    self._net = nn.Sequential(*net)

  def forward(self, x):
    return self._net(x)

class Quantizer(nn.Module):
  """Wraps a VectorQuantizer to handle input with arbitrary channels."""

  def __init__(self, in_channels, n_embeddings, embedding_dim):
    """Initializes a new Quantizer instance.

    Args:
      in_channels: Number of input channels.
      n_embeddings: Number of VectorQuantizer embeddings.
      embedding_dim: VectorQuantizer embedding dimension.
    """
    super().__init__()
    self._net = nn.Sequential(
        nn.Conv2d(in_channels=in_channels, out_channels=embedding_dim,
                  kernel_size=1),
        pg_nn.VectorQuantizer(n_embeddings, embedding_dim))

  def forward(self, x):
    return self._net(x)

class VQVAE(nn.Module):
  """The Vector Quantized Variational Autoencoder (VQ-VAE) model."""

  def __init__(self,
               in_channels=1,
               out_channels=1,
               hidden_channels=128,
               n_residual_blocks=2,
               residual_channels=32,
               n_embeddings=128,
               embedding_dim=16):
    """Initializes a new VQVAE instance.

    Args:
      in_channels: Number of input channels.
      out_channels: Number of output channels.
      hidden_channels: Number of channels in (non residual block) hidden layers.
      n_residual_blocks: Number of residual blocks in each residual stack.
      residual_channels: Number of hidden channels in residual blocks.
      n_embeddings: Number of VectorQuantizer embeddings.
      embedding_dim: Dimension of the VectorQuantizer embeddings.
    """
    super().__init__()
    self._encoder = Encoder(in_channels=in_channels,
                            out_channels=hidden_channels,
                            hidden_channels=hidden_channels,
                            n_residual_blocks=n_residual_blocks,
                            residual_channels=residual_channels,
                            stride=4)
    self._quantizer = Quantizer(in_channels=hidden_channels,
                                n_embeddings=n_embeddings,
                                embedding_dim=embedding_dim)
    self._decoder = Decoder(in_channels=embedding_dim,
                            out_channels=out_channels,
                            hidden_channels=hidden_channels,
                            n_residual_blocks=n_residual_blocks,
                            residual_channels=residual_channels,
                            stride=4)

  def forward(self, x):
    x = self._encoder(x)
    quantized, quantization_loss = self._quantizer(x)
    return self._decoder(quantized), quantization_loss

class VQVAE2(nn.Module):
  """The VQ-VAE-2 model with a latent hierarchy of depth 2."""

  def __init__(self,
               in_channels=1,
               out_channels=1,
               hidden_channels=128,
               n_residual_blocks=2,
               residual_channels=32,
               n_embeddings=128,
               embedding_dim=16):
    """Initializes a new VQVAE2 instance.

    Args:
      in_channels: Number of input channels.
      out_channels: Number of output channels.
      hidden_channels: Number of channels in (non residual block) hidden layers.
      n_residual_blocks: Number of residual blocks in each residual stack.
      residual_channels: Number of hidden channels in residual blocks.
      n_embeddings: Number of VectorQuantizer embeddings.
      embedding_dim: Dimension of the VectorQuantizer embeddings.
    """
    super().__init__()

    self._encoder_b = Encoder(in_channels=in_channels,
                              out_channels=hidden_channels,
                              hidden_channels=hidden_channels,
                              n_residual_blocks=n_residual_blocks,
                              residual_channels=residual_channels,
                              stride=2)
    self._encoder_t = Encoder(in_channels=hidden_channels,
                              out_channels=hidden_channels,
                              hidden_channels=hidden_channels,
                              n_residual_blocks=n_residual_blocks,
                              residual_channels=residual_channels,
                              stride=2)
    self._quantizer_t = Quantizer(in_channels=hidden_channels,
                                  n_embeddings=n_embeddings,
                                  embedding_dim=embedding_dim)
    self._quantizer_b = Quantizer(in_channels=hidden_channels,
                                  n_embeddings=n_embeddings,
                                  embedding_dim=embedding_dim)
    self._decoder_t = Decoder(in_channels=embedding_dim,
                              out_channels=hidden_channels,
                              hidden_channels=hidden_channels,
                              n_residual_blocks=n_residual_blocks,
                              residual_channels=residual_channels,
                              stride=2)
    self._conv = nn.Conv2d(in_channels=in_channels, out_channels=embedding_dim,
                          kernel_size=1)
    self._decoder_b = Decoder(in_channels=2*embedding_dim,
                              out_channels=out_channels,
                              hidden_channels=hidden_channels,
                              n_residual_blocks=n_residual_blocks,
                              residual_channels=residual_channels,
                              stride=2)

  def forward(self, x):
    encoded_b = self._encoder_b(x)
    encoded_t = self._encoder_t(encoded_b)

    quantized_t, vq_loss_t = self._quantizer_t(encoded_t)
    quantized_b, vq_loss_b = self._quantizer_b(encoded_b)

    decoded_t = self._decoder_t(quantized_t)
    xhat = self._decoder_b(
        torch.cat((self._conv(decoded_t), quantized_b), dim=1))
    return xhat, .5 * (vq_loss_b + vq_loss_t) + F.mse_loss(decoded_t, encoded_b)


import argparse
import asyncio
import logging.config
import os
import sys
import typing
from collections import namedtuple

import aiohttp
import lxml.html
from yarl import URL

from . import storage
from .book import Book
from .notifier import Notifier

class Watcher:
    def __init__(self, notifier: Notifier, logger: logging.Logger = logging.getLogger('Watcher')):
        self.notifier = notifier
        self.logger = logger

    async def get_shelf_books(self, session: aiohttp.ClientSession, shelf: str) -> typing.Dict[str, 'Book']:
        async def create_book_from_book_bom(book_bom: lxml.html.Element) -> Book:
            title = ''.join(book_bom.xpath('td[2]/a/text()'))
            author = ''.join(book_bom.xpath('td[3]/text()'))
            publisher = ''.join(book_bom.xpath('td[4]/text()'))
            publish_date = ''.join(book_bom.xpath('td[5]/text()'))
            call_no = ''.join(book_bom.xpath('td[6]/text()'))
            marc_no_url = URL(''.join(book_bom.xpath('td[2]/a/@href')))
            marc_no = marc_no_url.query.get('marc_no')
            state = await self.get_book_state(session, marc_no)
            return Book(title, author, publisher, publish_date, call_no, marc_no, state)

        url = URL('http://210.32.33.91:8080/opac/show_user_shelf.php').with_query(classid=shelf)
        async with session.get(url) as response:
            try:
                html = await response.text()
            except Exception as e:
                self.logger.debug('Unable to access shelf {} page'.format(shelf), exc_info=True)
                raise e
            else:
                self.logger.debug('Access {} shelf page success'.format(shelf))

                bom = lxml.html.fromstring(html)
                book_bom_list = bom.xpath('//*[@id="container"]/table/tr')[1:]

                _loop = asyncio.get_event_loop()
                book_tasks = [_loop.create_task(create_book_from_book_bom(book_bom)) for book_bom in book_bom_list]
                finished, unfinished = await asyncio.wait(book_tasks)

                book_dict = {}  # type: typing.Dict[str, 'Book']
                for r in finished:
                    book = r.result()
                    if not isinstance(book, Book):
                        raise TypeError('{} is not a Book Object', )
                    book_dict[book.call_no] = book
                self.logger.debug('shelf book is {}'.format(book_dict))
                return book_dict

    async def get_book_state(self, session: aiohttp.ClientSession, marc_no: str):
        url = URL('http://210.32.33.91:8080/opac/ajax_item.php').with_query(marc_no=marc_no)
        async with session.get(url) as response:
            try:
                html = await response.text()
            except Exception as e:
                self.logger.warning('Unable to access ajax-page of {}'.format(marc_no), exc_info=True)
                raise e
            else:
                self.logger.debug('Access {} ajax-page success'.format(marc_no))
                dom = lxml.html.fromstring(html)
                state_list = dom.xpath('//*[@id="item"]/tr/td[5]/font/text() | //*[@id="item"]/tr/td[5]/text()')
                state = '可借' in state_list
                self.logger.debug('Parse {} result is {} {}'.format(marc_no, state, state_list))
                return state

    async def send_all_status_loop(self, loop_time=7 * 24 * 3600):
        remain_time = loop_time
        while remain_time >= 3600:
            remain_time = remain_time - 3600
            await asyncio.sleep(3600)
        store_books = storage.load()
        self.logger.debug('Send All Status')
        books = [Notifier.Notify(book, None) for book in store_books.values()]
        await self.notifier.send_all_status(books)
        asyncio.get_event_loop().create_task(self.send_all_status_loop())

    async def check_loop(self, shelf: str, loop_time: int = 3600):
        self.logger.info('-----------------------')
        self.logger.info('Watcher start')

        session = aiohttp.ClientSession()
        try:
            while True:
                try:
                    self.logger.debug('Checking...')

                    books = await self.get_shelf_books(session, shelf)  # type: typing.Dict[str, 'Book']
                    self.logger.debug('Get {} shelf books'.format(len(books)))

                    with storage.FLock():
                        store_books = storage.load()  # type: typing.Dict[str, 'Book']
                        self.logger.debug('Load {} stored books'.format(len(store_books)))

                        del_key_list = []
                        for call_no, store_book in store_books.items():
                            if store_book not in books.values():
                                self.logger.info('Stop track {}'.format(store_book))
                                # self.notifier.collect_notify(store_book, '停止追踪')
                                del_key_list.append(call_no)

                        for del_key in del_key_list:
                            del store_books[del_key]

                        for call_no, book in books.items():
                            if book not in store_books.values():
                                self.logger.info('Begin track {}'.format(book))
                                # self.notifier.collect_notify(book, '开始追踪')
                                store_books[call_no] = book
                            else:
                                if book.state != store_books.get(call_no).state:
                                    self.logger.info('Tracked book {} state change'.format(book))
                                    self.notifier.collect_notify(book, None)
                                    store_books[call_no] = book
                                else:
                                    self.logger.debug('Tracked book {} state not change'.format(book))

                        storage.dump(store_books)
                        self.logger.debug('Store {} books to file'.format(len(store_books.values())))

                    await self.notifier.send_notify()

                except TimeoutError:
                    self.logger.error('Watcher occurs error', exc_info=True)

                except Exception as e:
                    self.logger.error('Watcher occurs error', exc_info=True)
                    raise e

                finally:
                    self.logger.debug('Wait for {}s'.format(loop_time))
                    await asyncio.sleep(loop_time)

        finally:
            await session.close()
            self.logger.info('Watcher stop')
            self.logger.info('-----------------------')

def get_args():
    parser = argparse.ArgumentParser(description='Book Watcher')
    parser.add_argument('-s', '--shelf_number', required=True,
                        help='[REQUIRED]The class id of Book Shelf you watched.')
    parser.add_argument('-t', '--loop_time', type=int, default=3600,
                        help='Check loop interval [s]')

    parser.add_argument('-w', '--weixin_notify', action='store_true',
                        help='Notify with weixin')
    _weixin_notify = ('-w' in sys.argv) | ('--weixin_notify' in sys.argv)
    parser.add_argument('-wk', '--weixin_notify_key', required=_weixin_notify,
                        help='Get Weixin Notify Key from ServerChan')

    parser.add_argument('-m', '--mail_notify', action='store_true',
                        help='Notify with mail')
    _mail_notify = ('-m' in sys.argv) | ('--mail_notify' in sys.argv)
    parser.add_argument('-mh', '--mail_notify_host', required=_mail_notify,
                        help='SMTP mail host')
    parser.add_argument('-mu', '--mail_notify_user', required=_mail_notify,
                        help='SMTP mail username')
    parser.add_argument('-mp', '--mail_notify_pass', required=_mail_notify,
                        help='SMTP mail password')
    parser.add_argument('-ms', '--mail_notify_sender', required=_mail_notify,
                        help='SMTP mail sender')
    parser.add_argument('-mr', '--mail_notify_receiver', required=_mail_notify,
                        help='SMTP mail receiver')

    args = parser.parse_args()

    _weixin = Notifier.Weixin(args.weixin_notify_key) \
        if _weixin_notify else None
    _mail = Notifier.Mail(args.mail_notify_host, args.mail_notify_user, args.mail_notify_pass,
                          args.mail_notify_sender,
                          args.mail_notify_receiver) \
        if _mail_notify else None

    return namedtuple('args', ['shelf_number', 'loop_time', 'weixin', 'mail'])(
        args.shelf_number, args.loop_time, _weixin, _mail
    )

def init_logger():
    try:
        logging.config.fileConfig('src/hdu_library_watcher/logging.ini')
    except FileNotFoundError:
        os.mkdir('log')
        logging.config.fileConfig('src/hdu_library_watcher/logging.ini')
    return logging.getLogger('Watcher')

def main():
    args_tuple = get_args()
    logger = init_logger()
    notifier = Notifier(mail=args_tuple.mail, weixin=args_tuple.weixin)
    watcher = Watcher(notifier, logger)
    loop = asyncio.get_event_loop()
    loop.create_task(watcher.send_all_status_loop())
    loop.run_until_complete(watcher.check_loop(shelf=args_tuple.shelf_number, loop_time=args_tuple.loop_time))

if __name__ == '__main__':
    main()


# -*- coding: utf-8 -*-
"""
Created on Thu Feb  6 08:42:57 2020

@author: valramon
"""

import pandas as pd
import bokeh
from bokeh.plotting import figure
import numpy as np
from collections import defaultdict
from bokeh.models import ColumnDataSource, HoverTool, ResetTool, SaveTool, DataTable, TableColumn, BoxZoomTool
from bokeh.models.widgets import CheckboxGroup, Button, Select, Slider, Div, TextInput
from bokeh.models.ranges import Range1d
from bokeh.models import PolyDrawTool
from bokeh_callbacks import *
import os, glob, copy, pickle, platform

def create_map(n):
    """
        Esta función recibe un n predeterminado para crear el mapa de zonas. En caso de existir un archivo que ya
        contenga las zonas, simplemente lo carga y crea el mapa sobre el mismo.
    """
    #path donde se encuentre el csv con los datos de las zonas
    pathData = os.environ['USERPROFILE']+'/Downloads/bokeh_flask/bokeh_flask_vm/static/config/'

    archivosCsv = glob.glob(pathData + '*.csv')
    dic_cuadrados = defaultdict(list)

    archivosDb = glob.glob(pathData + '*.db')
    dictHistoricos = defaultdict(list)
    with open(archivosDb[-1], 'rb') as fileO:
        try:
            dataHist = pickle.load(fileO)
        except:
            dataHist = pickle.load(fileO, encoding = 'bytes')
        for i in range(len(dataHist)):
            dictHistoricos['id'].append(str(dataHist[i][1]))
            dictHistoricos['vehi'].append(str(dataHist[i][10]))
            dictHistoricos['progre'].append(str(round(dataHist[i][2], 2)))
            dictHistoricos['fecha'].append(str(dataHist[i][5].strftime("%Y/%m/%d - %H:%M:%S")))
            dictHistoricos['estado'].append(str(dataHist[i][3]))
    dataframe_historicos = pd.DataFrame(dictHistoricos)

    source_historicos = ColumnDataSource(dataframe_historicos)
    source_historicos_muestra = ColumnDataSource(copy.deepcopy(dataframe_historicos))
    del source_historicos.data['index']
    del source_historicos_muestra.data['index']
    keys = dictHistoricos.keys()
    columns = [
        TableColumn(field='id', title='ID'),
        TableColumn(field='vehi', title='Vehiculo'),
        TableColumn(field='progre', title='Progresiva'),
        TableColumn(field='fecha', title='Fecha'),
        TableColumn(field='estado', title='Estado')
    ]

    data_table = DataTable(source=source_historicos_muestra, columns=columns, width=300, height=400)
    data_table.visible = False
    x, y = [], []
    mayor = 0
    #carga los datos de las zonas en un diccionario, lo carga como string, por eso la separacion por comas
    if len(archivosCsv) != 0:
        if len(archivosCsv) > 1:
            for archivo in archivosCsv:
                try:
                    if int(archivo.split(' ')[-1].split('.')[0][1:-1]) > mayor:
                        mayor = int(archivo.split(' ')[-1].split('.')[0][1:-1])
                        archNuevo = archivo
                except:
                    pass
        else:
            archNuevo = archivosCsv[0]
        archCsv = open(archNuevo, 'r')
        data = archCsv.read()
        archCsv.close()
        keys = data.split('\n')[0].split(',')
        data = data.split('\n')[1:]
        dictData = defaultdict(list)
        j = 0
        for i in range(len(data)):
            for key in keys:
                dictData[key].append(data[i].split(',')[j])
                j += 1
            j = 0
        dic_datos_lineas = dictData
        dic_datos_lineas['RT_intensity'] = []
        largo_ducto = int(float(dic_datos_lineas['RT'][-1].split('pk')[-1]))
        n = len(dic_datos_lineas['RT'])
        lista_zonas = [(1.0,dic_datos_lineas['Intensity_tip'][k],dic_datos_lineas['Eventos_permitidos'][k]) for k in range(n)]
        for i in range(len(dic_datos_lineas['RT'])):
            dic_datos_lineas['RT_intensity'].append(np.arange(0, 50))
            dic_datos_lineas['RT'][i] = (float(dic_datos_lineas['RT'][i])) * np.ones(50)
            dic_datos_lineas['rel_int'][i] = float(dic_datos_lineas['rel_int'][i])
            x.append(dic_datos_lineas['RT'][i])
            y.append(dic_datos_lineas['RT_intensity'][i])
            if i != 0:
                dic_cuadrados['x'].append([[[dic_datos_lineas['RT'][i - 1][0], dic_datos_lineas['RT'][i - 1][0], dic_datos_lineas['RT'][i][0], dic_datos_lineas['RT'][i][0]]]])
            else:
                dic_cuadrados['x'].append([[[0, 0, dic_datos_lineas['RT'][i][0], dic_datos_lineas['RT'][i][0]]]])
            dic_cuadrados['y'].append([[[0, 50, 50, 0]]])
            if ("Ninguno" in dic_datos_lineas['Eventos_permitidos'][i]):
                dic_cuadrados['color'].append('White')
            else:
                dic_cuadrados['color'].append('Gray')
            dic_cuadrados['cont'].append(i)

    else:
        #si no existe un archivo, crea un diccionario con valores random para crear la figura del mapa
        largo_ducto = 30000
        dic_datos_lineas = defaultdict(list)
        int_fict=np.arange(50)
        order_loc=np.ones(50)

        n=int(n)
        choices=['Ninguno','Retroexcavadora/Camion']
        lista_zonas=[(1.0,'Zona '+str(k),np.random.choice(choices)) for k in range(n)]

        i = 0
        for scale, mz, evt_list in lista_zonas:
            dic_datos_lineas["RT"].append(order_loc*largo_ducto/n)
            dic_datos_lineas['ProgresivaIni'].append('pk'+str(round((order_loc[0] - 1)*int(largo_ducto/1000/n) + 338, 2)))
            dic_datos_lineas['ProgresivaFin'].append('pk'+str(round(order_loc[0]*int(largo_ducto/1000/n) + 338, 2)))
            dic_cuadrados['x'].append([[[(order_loc[0] - 1)*largo_ducto/n]]])
            dic_cuadrados['x'][i][0][0].append((order_loc[0] - 1)*largo_ducto/n)
            x.append(order_loc)
            order_loc=order_loc+1
            dic_cuadrados['x'][i][0][0].append((order_loc[0] - 1)*largo_ducto/n)
            dic_cuadrados['x'][i][0][0].append((order_loc[0] - 1)*largo_ducto/n)
            dic_datos_lineas["RT_intensity"].append(int_fict*scale)#norm(loc=120.4).pdf(RT_x) *
            y.append(int_fict*scale)
            dic_cuadrados['y'].append([[[0, 50, 50, 0]]])
            dic_datos_lineas['Eventos_permitidos'].append(evt_list)
            dic_datos_lineas['Intensity_tip'].append(mz)
            dic_datos_lineas['rel_int'].append(round(np.random.normal(loc=0.9,scale=0.01),2))
            if ("Ninguno" in evt_list):
                dic_datos_lineas['color'].append('Gray')
                dic_cuadrados['color'].append('White')
            else:
                dic_cuadrados['color'].append('Gray')
            if (len(evt_list.split('/')) == 1 and "Ninguno" not in evt_list):
                dic_datos_lineas['color'].append('Orange')
            if (len(evt_list.split('/')) == 2 and "Ninguno" not in evt_list):
                dic_datos_lineas['color'].append('Orange')
            if (len(evt_list.split('/')) == 3 and "Ninguno" not in evt_list):
                dic_datos_lineas['color'].append('Orange')
            dic_cuadrados['cont'].append(i)
            i += 1
        dic_datos_lineas['RT'][-1].fill(largo_ducto)
    i = 0
    coor = {'x': x, 'y':y}
    data = pd.DataFrame(dic_datos_lineas)
    data2 = pd.DataFrame(coor)
    zona_elegida = ['Zona 0']
    data_modifica = {'zona': zona_elegida}
    #transforma diccionarios a dataframes
    data_a_modificar = pd.DataFrame(data_modifica)
    data_respaldo = pd.DataFrame(copy.deepcopy(dic_datos_lineas))
    data_cuadrados = pd.DataFrame(dic_cuadrados)
    #Create the data sources for plotting the data
    #luego transforma los dataframes en data sources para poder trabajarlos en bokeh
    source = ColumnDataSource(data)
    source2_respaldo = ColumnDataSource(data2)
    source3 = ColumnDataSource(data_a_modificar)
    source_respaldo = ColumnDataSource(data_respaldo)
    source_cuadrados = ColumnDataSource(data_cuadrados)

    x_limit_left=largo_ducto+0.1

    #aqui setea las herramientas a utilizar en el mapa
    tools = [ResetTool(), SaveTool(), BoxZoomTool()]

    #crea la figura del mapa
    p = figure(plot_width=1200, plot_height=600,x_range=(0,x_limit_left), y_range=Range1d(0,49), tools = tools)

    #le incrusta la imagen del mapa y crea las lineas de las zonas
    p.image_url(url=['static/imgs/mapa_vm3.jpg'], x=0, y=0, w=x_limit_left, h=49,anchor='bottom_left')
    ml=p.multi_line(xs='RT', ys='RT_intensity',  # estas 4 lineas dibujan las lines de las zonas
                 line_width=5, line_color='color', line_alpha=0.7,
                 hover_line_color='pink', hover_line_alpha=1.0,
                 source=source)

    #setea las etiquetas para ver en la herramienta que sirve para cuando se pasa el mouse sobre una linea de zona
    hover_t=HoverTool(show_arrow=False, tooltips=[
        ('Zona', '@Intensity_tip'),
        ('Inicio de zona', '@ProgresivaIni'),
        ('Fin de zona', '@ProgresivaFin'),
        ('Eventos permitidos', '@Eventos_permitidos'),
        ('Intensidad relativa', '@rel_int')
    ], renderers = [ml])

    #crea los sombreados de las zonas
    poli = p.multi_polygons(xs='x', ys = 'y', color = 'color', line_alpha = 0,fill_alpha = 0.3, source = source_cuadrados)
    poli.visible = True

    #agrega la posibilidad de manipular las lineas de las zonas
    draw_tool_l1 = PolyDrawTool(renderers=[ml])

    p.add_tools(draw_tool_l1)

    p.xaxis.bounds=(0,n)
    p.yaxis.visible=False
    p.xaxis.visible=False

    oculta_zonas = ['Mostrar zonas']
    labels = [zona[1] for zona in lista_zonas]

    #crea los callbacks en javascript para cuando el script esta en ejecucion
    callback_checkbox = checkbox(source, source2_respaldo, source_respaldo, oculta_zonas)
    callback_guarda_zona = guarda_zona(source3)
    callback_evento = evento(source, source3, source_respaldo, source_cuadrados)
    callback_int = intensidad(source, source3, source_respaldo)

    eventos_perm = ["Ninguno", "Camioneta", "Camion", "Retroexcavadora"]

    #crea los distintos botones y herramientas a utilizar para el manejo del mapa, se crean en bokeh y funcionan con callbacks en javascript
    select = Select(title="Modificar zona:", value=source3.data['zona'][0], options=labels, callback = callback_guarda_zona)
    checkbox_group = CheckboxGroup(labels=oculta_zonas, active=[i for i in range (1)], callback=callback_checkbox)
    input_eventos = CheckboxGroup(labels=eventos_perm, active=[0], callback=callback_evento)
    slider_intensidad = Slider(start=0, end=1, value=0.5, step=.05, title="Intensidad relativa", callback = callback_int)

    callback_deshace_cambios = deshace_cambios(source, source_respaldo, source_cuadrados, input_eventos, slider_intensidad, eventos_perm, source3, draw_tool_l1)
    callback_download = download(source)
    text_input_prog_ini = TextInput(value='340', title='Progresiva de inicio')
    text_input_prog_fin = TextInput(value='342', title='Progresiva de fin')
    text_input_prog_fin.visible = False
    text_input_prog_ini.visible = False

    callback_confirma_zona = confirma_zona(source, source_respaldo, source_cuadrados, select, text_input_prog_ini, text_input_prog_fin)
    boton_confirma_zona = Button(label="Confirmar zona", button_type="success", callback = callback_confirma_zona)
    boton_confirma_zona.visible = False

    callback_agrega_zona = agrega_zona(text_input_prog_ini, text_input_prog_fin, boton_confirma_zona)

    callback_elimina_zona = elimina_zona(source, select)

    button = Button(label = "Rehacer mapa", button_type="success", callback = callback_deshace_cambios)

    button2 = Button(label="Aplicar cambios", button_type="warning", callback = callback_download)

    boton_agregar_zona = Button(label="Agregar zonas", button_type="warning", callback = callback_agrega_zona)

    boton_elimina_zona = Button(label="Eliminar zona", button_type="danger", callback = callback_elimina_zona)
    boton_elimina_zona.disabled = True

    div2 = Div(text="""
Ultimos 10 eventos de zona:
""", width=300, height=50)
    div2.visible = False

    callback_muestra_eventos_zona = muestra_eventos_zona(source, source_historicos, source_historicos_muestra, data_table, select, div2)

    boton_muestra_eventos_zona = Button(label='Mostrar ultimos eventos de zona', button_type='success', callback=callback_muestra_eventos_zona)
    boton_muestra_eventos_zona.disabled = True

    callback_mouse = mouse_drag(source, source_cuadrados, source3, select, poli, slider_intensidad, input_eventos, eventos_perm, source_respaldo, largo_ducto)
    callback_mouse_end = mouse_end(poli, source_cuadrados, source, largo_ducto, draw_tool_l1, boton_elimina_zona, source_respaldo)
    callback_mouse_move = mouse_move(source, source_respaldo, source_cuadrados)
    callback_tap = tap(source, source_respaldo, source3, source_cuadrados, slider_intensidad, input_eventos, eventos_perm, select, poli, draw_tool_l1, boton_elimina_zona, boton_muestra_eventos_zona, data_table, div2)

    #detecta eventos realizados con mouse en la figura para llamar a callbacks que modificar la figura del mapa
    p.js_on_event(bokeh.events.Tap, callback_tap)
    p.js_on_event(bokeh.events.Pan, callback_mouse)
    p.js_on_event(bokeh.events.MouseMove, callback_mouse_move)
    p.js_on_event(bokeh.events.PanEnd, callback_mouse_end)
    div = Div(text="""Eventos Permitidos:""", width=200, height=10)

    return p, checkbox_group, select, input_eventos, slider_intensidad, button, button2, div, div2, data_table, boton_muestra_eventos_zona, hover_t


import json
from gensim.parsing.preprocessing import remove_stopwords
from gensim.utils import simple_preprocess
from gensim.parsing.porter import PorterStemmer
from gensim import corpora
from gensim.models import Word2Vec
import torch
from torch.utils.data import (TensorDataset, DataLoader, RandomSampler,
                              SequentialSampler)

w2vMyModel = Word2Vec.load('./w2vTrainedModel/w2v.10th.10vec.model')

def splitToTrainTestFun(wordSentenceDbLi):
    splitPoint = int(len(wordSentenceDbLi) * 0.95)
    trainXYLi = wordSentenceDbLi[:splitPoint]
    testXYLi = wordSentenceDbLi[splitPoint:]
    return trainXYLi, testXYLi

def dataToXYListRead(fileName):
    with open(fileName) as file:
        porter_stemmer = PorterStemmer()
        lineCount = 0
        wordSentenceDbLi = []
        while True:
            line = file.readlines(1)
            if not line:
                break
            # if lineCount == 20:
            #     break
            jsonLine = json.loads(line[0])

            # noStopWords = remove_stopwords(jsonLine['text'])
            # stemWords = porter_stemmer.stem(noStopWords)
            stemWords = porter_stemmer.stem(jsonLine['text'])
            tokenWords = simple_preprocess(stemWords, deacc=True)

            # print(tokenWords)
            wordSentenceDbLi.append(tokenWords)
            lineCount += 1
        # yelpDic = corpora.Dictionary(wordSentenceDbLi)
        # yelpDic.save('yelpDictionary.dict')
        # print(yelpDic.token2id)
        # print(yelpDic[8])

        return wordSentenceDbLi

def createWordVecModelFun(fileName):
    print("reading data====>>>")
    wordSentenceDbLi = dataToXYListRead(fileName)

    print("training w2v.....")
    w2v_model = Word2Vec(wordSentenceDbLi, size=100, workers=10, window=3, sg=1)
    w2v_model.save('./w2vTrainedModel/w2v.10th.100vec.model')
    print("end of training word2Vec---->>>")

def wordToXYvecFun(xList, yWord):
    QinWVmodel = 1
    xVec = []
    yVec = []
    for word in xList:
        try:
            x10Vec = w2vMyModel[word]
            xVec.append(x10Vec)
            # print("x converting--->", word, x10Vec)
        except KeyError:
            # print("The word X___", word,  "does not appear in this model")
            QinWVmodel = 0
    #
    try:
        yVec = w2vMyModel[yWord]
        # print("y converting-->", yWord, yVec)
    except KeyError:
        # print("The word Y--->", word, "does not appear in this model")
        QinWVmodel = 0

    return xVec, yVec, QinWVmodel

def processData(xyList):
    xVecDbList = []
    yVecList = []
    xLenth = 4
    for sentence in xyList:
        if len(sentence) < xLenth:
            break
        xSentence = sentence[0:4]
        yWord = sentence[4]
        # print(xSentence, "->", yWord)
        xVecSeqList, yWordVec, QinWVmodel = wordToXYvecFun(xSentence, yWord)
        if QinWVmodel == 1:
            xVecDbList.append(xVecSeqList)
            yVecList.append(yWordVec)

        newSentence = sentence[4:]

        while(len(newSentence) > 5):
            xSentence = newSentence[0:4]
            yWord = newSentence[4]
            xVecSeqList, yWordVec, QinWVmodel = wordToXYvecFun(xSentence, yWord)
            if QinWVmodel == 1:
                xVecDbList.append(xVecSeqList)
                yVecList.append(yWordVec)

            newSentence = newSentence[4:]

    # print("xxxx->", xVecDbList)
    # print('yyyy->', yVecList)
    return xVecDbList, yVecList

def myDataLoader(xVecDbList, yVecList, batchSize):
    xVecDbList = torch.FloatTensor(xVecDbList).to('cuda')
    yVecList = torch.FloatTensor(yVecList).to('cuda')
    xyTensor = TensorDataset(xVecDbList, yVecList)
    trainSampler = RandomSampler(xyTensor)
    xyDataLoader = DataLoader(xyTensor, sampler=trainSampler, batch_size=batchSize)

    return xyDataLoader

class NextWordRNN(torch.nn.Module):
    def __init__(self, batchSize, numSequence, numFeature):
        super(NextWordRNN, self).__init__()

        self.hiddenDim = 3
        self.numLayers = 1
        # self.inputSize = 5 # number of input in a x
        self.inputSize = numFeature # number of input in a x
        self.NumSequence = numSequence
        self.NumFeature = numFeature

        self.rnn = torch.nn.RNN(self.inputSize, self.hiddenDim, batch_first=True)
        # self.fc = torch.nn.Linear(self.hiddenDim, 5)
        self.fcHidden = torch.nn.Linear(self.hiddenDim, 1)
        self.fcFinal = torch.nn.Linear(self.NumSequence, self.NumFeature)

    def forward(self, x):
        batchSize = len(x)
        hidden = self.init_hidden(batchSize)

        out, hidden = self.rnn(x, hidden)
        out = self.fcHidden(out)
        out = out.squeeze()
        out = self.fcFinal(out)

        # print("before view==>", out.shape )
        out = out.view(-1, self.NumFeature)
        # print("after view==>", out.shape )

        return out, hidden

    def init_hidden(self, batchSize):
        hidden = torch.zeros(self.numLayers, batchSize, self.hiddenDim, device='cuda')
        return hidden

def get_accuracy(logit, target, batch_size):
    corrects = (torch.max(logit, 1)[1].view(target.size()).data == target.data).sum()
    accuracy = 100.0 * corrects/batch_size
    return accuracy.item()


# -*- coding: utf-8 -*-
"""
Created on Wed Jan 24 15:07:16 2018

@author: wel12
"""

import sys
from sklearn.neighbors import KNeighborsClassifier
from sklearn.model_selection import train_test_split
from sklearn import datasets
from sklearn.neighbors import KNeighborsClassifier
from pandas import read_csv
import matplotlib
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd

def readData(location):
    h = 'carEvaluation\\attributes.csv'
    a = read_csv(h, header=None)
    #temp = a[a.columns[0]]
    #headers = temp.values
    #headers = np.append(headers, '')
    dataset = read_csv(location, header=None,  na_values=" ?") #, names=headers
    return dataset

def numberfyData(raw):
    keys = []
    out = raw
    for column in raw:
        keys.append(set(raw[column]))
        out[column] = raw[column].replace(set(raw[column]),range(len(set(raw[column]))))
    return out, keys

def testModle(data_train, data_test, targets_train, targets_test, classifier):
    modle = classifier.fit(data_train, targets_train)
    return modle.predict(data_test)

def displayTest(targets_predicted,targets_test):
    simCount = 0
    difCount = 0
    print ("predicted")
    print (targets_predicted)
    print("test")
    test = np.array(targets_test)
    print(test)
    for i in range(len(targets_predicted)):
        print (targets_predicted[i], "==", test[i])
        #simCount +=1
        if targets_predicted[i] == test[i]:
            simCount += 1
        if targets_predicted[i] != test[i]:
            difCount += 1
    print("\tsim =\t", simCount)
    print("\tdif =\t", difCount)
    print("\ttotal =\t", len(targets_predicted))
    print("\t\t%",simCount/len(targets_predicted) * 100)
    return simCount/len(targets_predicted) * 100

class KNNClasifier:
    def __init__(self):
        pass

    def fit(self, data_train, targets_train):
        return KNNModle(data_train, targets_train)

def eDistance(a, b):
    out = 0
    if not(np.isscalar(a)):
        r = len(a)
        for point in range(r):
            out = out + (int(a[point])-int(b[point]))**2
    else:
        out = (a-b)**2
    return out

class KNNModle:
    def __init__(self,data_train, targets_train):
        self.data_train    = data_train
        self.targets_train = targets_train
        self.k = 3
        return None

    def getNebors(self, data_test_row):
        nebors = []
        distances = []
        #find how far everything is from eachother
        test = np.array(self.data_train)
        for data_train_row in test:
            #distances.append(np.asscalar(LA.norm(data_train_row - data_test_row)))  #Euclidean distance: LA.norm(row - data_train_row
            distances.append(eDistance(data_train_row, data_test_row))
        #combine rlivent data
        nebors = list(zip(distances, self.targets_train, self.data_train))

        #sort based on distance
        nebors = sorted(nebors, key=lambda k:k[0])
        # cut so their are only k number # default is 3
        nebors = nebors[:self.k]
        return nebors

    def determinSimilarity(self, nebors):
        # get list of top targets
        possabilities = list(zip(*nebors))[1]
        #find unique values of top targets
        targetRange = set(possabilities)
        count = []
        #loop through values of top targets and count how many there are
        for num in targetRange:
            count.append(list(possabilities).count(num))
        # combine values and counts
        out = list(zip(targetRange, count))
        # sort
        out = sorted(out, key=lambda k:k[1], reverse=True)
        # return
        return out[0][0]

    def predict(self, data_test):
        targets = []
        test = np.array(data_test)
        for data_test_row in test:
            nebors = self.getNebors(data_test_row)
            targets.append(self.determinSimilarity(nebors))
        return targets

def main(argv):

    rawData = readData('carEvaluation\car.data')
    numData, key = numberfyData(rawData)
    target = numData.iloc[:,-1]

    print ("DATA :")
    print (numData)
    print ("TARGETS:")
    print (target)
    data_train, data_test, targets_train, targets_test = train_test_split(numData, target, test_size = .3)
    print ("DATA Train:")
    print (data_train)
    print ("DATA Test:")
    print (data_test)
    print ("targts Train:")
    print (targets_train)
    print ("targets test:")
    print (targets_test)

    print("My KNN")
    classifier = KNNClasifier()
    targets_predicted = testModle(data_train, data_test, targets_train, targets_test, classifier)
    displayTest(targets_predicted,targets_test)
    return None

if __name__ == "__main__":
    main(sys.argv)


# -*- coding: utf-8 -*-
# Copyright (c) Hebes Intelligence Private Company

# This source code is licensed under the Apache License, Version 2.0 found in the
# LICENSE file in the root directory of this source tree.

from kedro.pipeline import Pipeline, node, pipeline

from .nodes import apply_day_typing, get_daily_matrix_profile, prepare_data

daytype = Pipeline(
    [
        node(
            func=prepare_data,
            inputs=["preprocessed_data", "params:of_prepare_data"],
            outputs="data_for_daytyping",
            name="prepare_data",
        ),
        node(
            func=get_daily_matrix_profile,
            inputs="data_for_daytyping",
            outputs="matrix_profile_scores",
            name="get_daily_matrix_profile",
        ),
        node(
            func=apply_day_typing,
            inputs=[
                "data_for_daytyping",
                "params:daytyping_window",
                "params:of_find_prototypes",
                "params:of_distance_learning",
            ],
            outputs=["prototype_days", "distance_metrics"],
            name="apply_day_typing",
        ),
    ]
)

daytype_train = pipeline(daytype, namespace="train")

def create_pipeline(**kwargs):
    return Pipeline([daytype_train], tags="train")


import os
import random
import shutil
import warnings
from glob import glob

import PIL
import cv2
import torch
import torchvision
from PIL import Image
from PIL import ImageFile
from torch.utils.data import Dataset
from utils.transformer import *

warnings.filterwarnings('ignore')
ImageFile.LOAD_TRUNCATED_IMAGES = True

class BinaryClassifierDataset(torch.utils.data.Dataset):
    def __init__(self, img_path, input_size, mode=None, transforms=None, argument_path=False):
        self.mode = mode
        if self.mode == 'test':
            self.img_list = glob(os.path.join(img_path, "*.jpg"))
            self.img_list += glob(os.path.join(img_path, "*.png"))
        else:
            self.img_list = glob(os.path.join(img_path, "*", "*.jpg"))
            self.img_list += glob(os.path.join(img_path, "*", "*.png"))
            if argument_path:
                self.img_list += glob(os.path.join("./data/old_train", "*", "*.jpg"))
        if transforms is None:
            mean, std = [0.485, 0.456, 0.406], [0.229, 0.224, 0.225]
            self.transforms = torchvision.transforms.Compose([
                # torchvision.transforms.Resize(size=input_size),  # 缩放
                Resize(size=input_size),  # 等比填充缩放
                # torchvision.transforms.RandomCrop(size=input_size),
                # torchvision.transforms.RandomResizedCrop(size=input_size, scale=(0.6, 1.0)),
                torchvision.transforms.RandomHorizontalFlip(),
                # RandomGaussianBlur(),
                torchvision.transforms.ToTensor(),
                torchvision.transforms.Normalize(mean=mean, std=std),
            ])
        else:
            self.transforms = transforms

    def __len__(self):
        return len(self.img_list)

    def __getitem__(self, idx):
        try:
            img = PIL.Image.open(self.img_list[idx])
            if img.format == 'PNG' or img.format == 'GIF':
                img = img.convert("RGB")
            elif img.layers == 1:
                img = cv2.imread(self.img_list[idx], 0)
                img = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR)
                img = Image.fromarray(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))
            img = self.transforms(img)
        except OSError:
            print("OSError at combined_mask path ", self.img_list[idx])
            return None

        if self.mode == 'train' or self.mode == 'eval':
            if self.img_list[idx].split('/')[-2] == "phone":
                label = 2
            elif self.img_list[idx].split('/')[-2] == "smoke":
                label = 1
            elif self.img_list[idx].split('/')[-2] == "normal":
                label = 0
            elif self.img_list[idx].split('/')[-2] == "calling_images":
                label = 2
            elif self.img_list[idx].split('/')[-2] == "smoking_images":
                label = 1
            elif self.img_list[idx].split('/')[-2] == "normal_images":
                label = 0
            # elif self.img_list[idx].split('/')[-2] == "5k_cigarettes":
            #     label = 1
            else:
                print(self.img_list[idx].split('/')[-2])
                raise ValueError
            sample = {"image": img, "label": label, "name": self.img_list[idx].split('/')[-1]}
        else:
            sample = {"image": img, "name": self.img_list[idx].split('/')[-1]}

        return sample

def build_dataset_train(data_path, input_size):
    return BinaryClassifierDataset(os.path.join(data_path, "train"), input_size, mode='train')

def build_dataset_eval(data_path, input_size):
    return BinaryClassifierDataset(os.path.join(data_path, "eval"), input_size, mode='eval')

def build_dataset_test(data_path, input_size):
    return BinaryClassifierDataset(os.path.join(data_path, "test"), input_size, mode='test')

def split_old_dataset():
    train_path = "../data/train/"
    eval_path = "../data/eval/"
    if not os.path.exists(eval_path):
        os.mkdir(eval_path)
    if not os.path.exists(eval_path + "calling_images/"):
        os.mkdir(eval_path + "calling_images/")
    if not os.path.exists(eval_path + "normal_images/"):
        os.mkdir(eval_path + "normal_images/")
    if not os.path.exists(eval_path + "smoking_images/"):
        os.mkdir(eval_path + "smoking_images/")

    for p in ["calling_images/", "normal_images/", "smoking_images/"]:
        img_list = glob(os.path.join(train_path + p, "*.jpg"))
        random.shuffle(img_list)
        img_list = img_list[:int(0.2 * len(img_list))]
        for img in img_list:
            new_img = img.replace("train", "eval")
            shutil.move(img, new_img)

def split_dataset():
    train_path = "../data/train/"
    eval_path = "../data/eval/"
    if not os.path.exists(eval_path):
        os.mkdir(eval_path)
    if not os.path.exists(eval_path + "phone/"):
        os.mkdir(eval_path + "phone/")
    if not os.path.exists(eval_path + "normal/"):
        os.mkdir(eval_path + "normal/")
    if not os.path.exists(eval_path + "smoke/"):
        os.mkdir(eval_path + "smoke/")

    for p in ["phone/", "normal/", "smoke/"]:
        img_list = glob(os.path.join(train_path + p, "*.jpg"))
        img_list += glob(os.path.join(train_path + p, "*.png"))
        random.shuffle(img_list)
        img_list = img_list[:int(0.2 * len(img_list))]
        for img in img_list:
            new_img = img.replace("train", "eval")
            shutil.move(img, new_img)

if __name__ == '__main__':
    split_dataset()


# uniform content loss + adaptive threshold + per_class_input + recursive G
# improvement upon cqf37
from __future__ import division
import os, scipy.io
import tensorflow as tf
import tensorflow.contrib.slim as slim
import numpy as np
import glob
from PIL import Image

input_dir = './low-light/'
checkpoint_dir = './checkpoint/Sony/'
result_dir = './enhanced/'

def lrelu(x):
    return tf.maximum(x * 0.2, x)

def upsample_and_concat(x1, x2, output_channels, in_channels):
    pool_size = 2
    deconv_filter = tf.Variable(tf.truncated_normal([pool_size, pool_size, output_channels, in_channels], stddev=0.02))
    deconv = tf.nn.conv2d_transpose(x1, deconv_filter, tf.shape(x2), strides=[1, pool_size, pool_size, 1])

    deconv_output = tf.concat([deconv, x2], 3)
    deconv_output.set_shape([None, None, None, output_channels * 2])

    return deconv_output

def network(input):
    conv1 = slim.conv2d(input, 32, [3, 3], rate=1, activation_fn=lrelu, scope='g_conv1_1')
    conv1 = slim.conv2d(conv1, 32, [3, 3], rate=1, activation_fn=lrelu, scope='g_conv1_2')
    pool1 = slim.max_pool2d(conv1, [2, 2], padding='SAME')

    conv2 = slim.conv2d(pool1, 64, [3, 3], rate=1, activation_fn=lrelu, scope='g_conv2_1')
    conv2 = slim.conv2d(conv2, 64, [3, 3], rate=1, activation_fn=lrelu, scope='g_conv2_2')
    pool2 = slim.max_pool2d(conv2, [2, 2], padding='SAME')

    conv3 = slim.conv2d(pool2, 128, [3, 3], rate=1, activation_fn=lrelu, scope='g_conv3_1')
    conv3 = slim.conv2d(conv3, 128, [3, 3], rate=1, activation_fn=lrelu, scope='g_conv3_2')
    pool3 = slim.max_pool2d(conv3, [2, 2], padding='SAME')

    conv4 = slim.conv2d(pool3, 256, [3, 3], rate=1, activation_fn=lrelu, scope='g_conv4_1')
    conv4 = slim.conv2d(conv4, 256, [3, 3], rate=1, activation_fn=lrelu, scope='g_conv4_2')
    pool4 = slim.max_pool2d(conv4, [2, 2], padding='SAME')

    conv5 = slim.conv2d(pool4, 512, [3, 3], rate=1, activation_fn=lrelu, scope='g_conv5_1')
    conv5 = slim.conv2d(conv5, 512, [3, 3], rate=1, activation_fn=lrelu, scope='g_conv5_2')

    up6 = upsample_and_concat(conv5, conv4, 256, 512)
    conv6 = slim.conv2d(up6, 256, [3, 3], rate=1, activation_fn=lrelu, scope='g_conv6_1')
    conv6 = slim.conv2d(conv6, 256, [3, 3], rate=1, activation_fn=lrelu, scope='g_conv6_2')

    up7 = upsample_and_concat(conv6, conv3, 128, 256)
    conv7 = slim.conv2d(up7, 128, [3, 3], rate=1, activation_fn=lrelu, scope='g_conv7_1')
    conv7 = slim.conv2d(conv7, 128, [3, 3], rate=1, activation_fn=lrelu, scope='g_conv7_2')

    up8 = upsample_and_concat(conv7, conv2, 64, 128)
    conv8 = slim.conv2d(up8, 64, [3, 3], rate=1, activation_fn=lrelu, scope='g_conv8_1')
    conv8 = slim.conv2d(conv8, 64, [3, 3], rate=1, activation_fn=lrelu, scope='g_conv8_2')

    up9 = upsample_and_concat(conv8, conv1, 32, 64)
    conv9 = slim.conv2d(up9, 32, [3, 3], rate=1, activation_fn=lrelu, scope='g_conv9_1')
    conv9 = slim.conv2d(conv9, 32, [3, 3], rate=1, activation_fn=lrelu, scope='g_conv9_2')

    conv10 = slim.conv2d(conv9, 12, [1, 1], rate=1, activation_fn=None, scope='g_conv10')
    out = tf.depth_to_space(conv10, 2)
    return out

def pack_raw(img):
    im = Image.open(input_dir+img)
    im = im.convert(mode="I")
    im = np.asarray(im, dtype=np.float32)
    bll = max(1, (3*np.amax(im) // 100))
    im = np.maximum(im - bll, 0) / (np.amax(im) - bll)

    im = np.expand_dims(im, axis=2)
    img_shape = im.shape
    H = img_shape[0]
    W = img_shape[1]

    out = np.concatenate((im[0:H:2, 0:W:2, :],
                          im[0:H:2, 1:W:2, :],
                          im[1:H:2, 1:W:2, :],
                          im[1:H:2, 0:W:2, :]), axis=2)
    return out

def init():
    sess = tf.Session()
    in_image = tf.placeholder(tf.float32, [None, None, None, 4])
    out_image = network(in_image)

    saver = tf.train.Saver()
    sess.run(tf.global_variables_initializer())
    ckpt = tf.train.get_checkpoint_state(checkpoint_dir)
    if ckpt:
         print('loaded ' + ckpt.model_checkpoint_path)
         saver.restore(sess, ckpt.model_checkpoint_path)

    if not os.path.isdir(result_dir):
        os.makedirs(result_dir)

    tf.contrib.quantize.create_eval_graph()
    return sess, out_image, in_image

def enhance(img, sess, out_image, in_image):
    #ratio = min((10 / 0.04), 300)
    ratio = 255

    input_full = np.expand_dims(pack_raw(img), axis=0) * ratio
    input_full = np.minimum(input_full, 1.0)

    output = sess.run(out_image, feed_dict={in_image: input_full})
    output = np.minimum(np.maximum(output, 0), 1)

    output = output[0, :, :, :]
    Image.fromarray(np.uint8(output*255), mode=None).save(result_dir + img)


# Copyright 2013-2017 The Salish Sea MEOPAR contributors
# and The University of British Columbia

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#    http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Salish Sea NEMO nowcast worker that uploads the forcing files
for a nowcast run to the HPC/cloud facility where the run will be
executed assuming that the previous nowcasts were not run there
"""
import argparse
import glob
import logging
import os
import traceback

import arrow
import zmq

from nowcast import lib
from nemo_nowcast import WorkerError

worker_name = lib.get_module_name()

logger = logging.getLogger(worker_name)

context = zmq.Context()

def main():
    # Prepare the worker
    base_parser = lib.basic_arg_parser(
        worker_name, description=__doc__, add_help=False)
    parser = configure_argparser(
        prog=base_parser.prog,
        description=base_parser.description,
        parents=[base_parser],
    )
    parsed_args = parser.parse_args()
    config = lib.load_config(parsed_args.config_file)
    lib.configure_logging(config, logger, parsed_args.debug)
    logger.info('running in process {}'.format(os.getpid()))
    logger.info('read config from {.config_file}'.format(parsed_args))
    lib.install_signal_handlers(logger, context)
    socket = lib.init_zmq_req_rep_worker(context, config, logger)
    # Do the work
    try:
        checklist = upload_all_files(
            parsed_args.host_name, parsed_args.run_date,
            config)
        logger.info(
            'Nowcast ALL files upload to {0.host_name} completed'
            .format(parsed_args), extra={
                'host_name': parsed_args.host_name,
                'date': parsed_args.run_date,
            })
        # Exchange success messages with the nowcast manager process
        lib.tell_manager(
            worker_name, 'success', config, logger, socket, checklist)
    except WorkerError:
        logger.critical(
            'Nowcast ALL files upload to {0.host_name} failed'
            .format(parsed_args), extra={
                'host_name': parsed_args.host_name,
                'date': parsed_args.run_date,
            })
        # Exchange failure messages with the nowcast manager process
        lib.tell_manager(worker_name, 'failure', config, logger, socket)
    except SystemExit:
        # Normal termination
        pass
    except:
        logger.critical('unhandled exception:')
        for line in traceback.format_exc().splitlines():
            logger.error(line)
        # Exchange crash messages with the nowcast manager process
        lib.tell_manager(worker_name, 'crash', config, logger, socket)
    # Finish up
    context.destroy()
    logger.info('task completed; shutting down')

def configure_argparser(prog, description, parents):
    parser = argparse.ArgumentParser(
        prog=prog, description=description, parents=parents)
    parser.add_argument(
        'host_name', help='Name of the host to upload the files to')
    parser.add_argument(
        '--run-date', type=lib.arrow_date, default=arrow.now(),
        help='''
        Date of the run to upload files from;
        use YYYY-MM-DD format.
        Defaults to %(default)s.
        ''',
    )
    return parser

def upload_all_files(host_name, run_date, config):
    host = config['run'][host_name]
    ssh_client, sftp_client = lib.sftp(
        host_name, host['ssh key name']['nowcast'])
    # Neah Bay sea surface height
    for day in range(-1, 2):
        filename = config['ssh']['file template'].format(
            run_date.replace(days=day).date())
        dest_dir = 'obs' if day == -1 else 'fcst'
        localpath = os.path.join(config['ssh']['ssh_dir'], dest_dir, filename)
        remotepath = os.path.join(host['ssh_dir'], dest_dir, filename)
        try:
            upload_file(sftp_client, host_name, localpath, remotepath)
        except OSError:
            if dest_dir != 'obs':
                raise
            # obs file does not exist, to create symlink to corresponding
            # forecast file
            fcst = os.path.join(config['ssh']['ssh_dir'], 'fcst', filename)
            os.symlink(fcst, localpath)
            logger.warning(
                'ssh obs file not found; created symlink to {}'.format(fcst),
                extra={
                    'host_name': host_name,
                    'date': run_date,
                })
            upload_file(sftp_client, host_name, localpath, remotepath)
    # Rivers runoff
    for tmpl in config['rivers']['file templates'].values():
        filename = tmpl.format(run_date.replace(days=-1).date())
        localpath = os.path.join(config['rivers']['rivers_dir'], filename)
        remotepath = os.path.join(host['rivers_dir'], filename)
        upload_file(sftp_client, host_name, localpath, remotepath)
    # Weather
    for day in range(-1, 2):
        filename = config['weather']['file template'].format(
            run_date.replace(days=day).date())
        dest_dir = '' if day <= 0 else 'fcst'
        localpath = os.path.join(
            config['weather']['ops_dir'], dest_dir, filename)
        remotepath = os.path.join(host['weather_dir'], dest_dir, filename)
        upload_file(sftp_client, host_name, localpath, remotepath)
    # Live Ocean Boundary Conditions
    for day in range(-1, 2):
        filename = config['temperature salinity']['file template'].format(
            run_date.replace(days=day).date())
        dest_dir = '' if day <= 0 else 'fcst'
        localpath = os.path.join(
            config['temperature salinity']['bc dir'], dest_dir, filename)
        remotepath = os.path.join(
            host['forcing']['bc dir'], dest_dir, filename)
        upload_file(sftp_client, host_name, localpath, remotepath)

    # Restart File
    prev_run_id = run_date.replace(days=-1).date()
    prev_run_dir = prev_run_id.strftime('%d%b%y').lower()
    local_dir = os.path.join(
        config['run']['results archive']['nowcast'], prev_run_dir)
    localpath = glob.glob(os.path.join(local_dir, '*restart.nc'))
    filename = os.path.basename(localpath[0])
    remote_dir = os.path.join(host['results']['nowcast'], prev_run_dir)
    remotepath = os.path.join(remote_dir, filename)
    make_remote_directory(sftp_client, host_name, remote_dir)
    upload_file(sftp_client, host_name, localpath[0], remotepath)

    sftp_client.close()
    ssh_client.close()
    return {host_name: True}

def upload_file(sftp_client, host_name, localpath, remotepath):
    sftp_client.put(localpath, remotepath)
    sftp_client.chmod(remotepath, lib.PERMS_RW_RW_R)
    logger.debug(
        '{local} uploaded to {host} at {remote}'
        .format(local=localpath, host=host_name, remote=remotepath))

def make_remote_directory(sftp_client, host_name, remote_dir):
    sftp_client.mkdir(remote_dir, mode=lib.PERMS_RWX_RWX_R_X)
    logger.debug(
        '{remote} directory made on {host}'
        .format(remote=remote_dir, host=host_name))

if __name__ == '__main__':
    main()


from django import template
from shop.models import ProductQuantity

register = template.Library()

@register.filter(name='isIncart')
def isProductInCart(product, cart):
    print(cart)
    productId = str(product.product_id)
    print(productId)
    if productId in cart:
        return True
    return False

@register.filter(name='cartCount')
def cartCountByProduct(product, cart):
    productId = str(product.product_id)
    return cart[productId]

@register.filter(name='totalCartCount')
def totalCartCount(cart):
    sum = 0
    if cart:
        for i in cart:
            sum = sum + cart[i]
    return sum

@register.filter(name='isEmpty')
def isEmpty(cart):
    print(cart)
    sum = totalCartCount(cart)
    print(sum)
    if sum > 0:
        return False
    return True

@register.filter(name='cartPrice')
def cartPrice(products, cart):
    price = 0
    for product in products:
        quantity = cartCountByProduct(product, cart)
        price = price+(product.price*quantity)
        print(price)
    return price

@register.filter(name='productquantity')
def productQuantity(product, order):
    orderid = order.order_id
    prodquantity = ProductQuantity.objects.filter(order=order).filter(
        product=product).first()
    if prodquantity:
        return prodquantity.quantity
    return 0

@register.filter(name='belongsToOrder')
def belongsToOrder(product, order):
    orderid = order.order_id
    prodquantity = ProductQuantity.objects.filter(order=order).filter(
        product=product).first()
    if prodquantity:
        return True
    return False

@register.filter(name='priceByQuantity')
def priceByQuantity(product, order):
    price = product.price
    quantity = productQuantity(product, order)
    return price*quantity