"""
the general training framework
"""

from __future__ import print_function

import os
import argparse
import socket
import time

import tensorboard_logger as tb_logger
import torch
import torch.optim as optim
import torch.nn as nn
import torch.backends.cudnn as cudnn

from models import model_dict
from models.util import FeatTrans
from models.util import Connector, Translator, Paraphraser

from dataset.cifar100 import get_cifar100_dataloaders, get_cifar100_dataloaders_sample

from helper.util import adjust_learning_rate

from distiller_zoo import DistillKL, HintLoss, Attention, Similarity, Correlation, VIDLoss, RKDLoss
from distiller_zoo import PKT, ABLoss, FactorTransfer, KDSVD, FSP, NSTLoss
from crd.criterion import CRDLoss

from helper.loops import train_sftn as train, validate
from helper.pretrain import init

def parse_option():

    hostname = socket.gethostname()

    parser = argparse.ArgumentParser('argument for training')

    parser.add_argument('--print_freq', type=int, default=100, help='print frequency')
    parser.add_argument('--tb_freq', type=int, default=500, help='tb frequency')
    parser.add_argument('--save_freq', type=int, default=40, help='save frequency')
    parser.add_argument('--batch_size', type=int, default=64, help='batch_size')
    parser.add_argument('--num_workers', type=int, default=8, help='num of workers to use')
    parser.add_argument('--epochs', type=int, default=240, help='number of training epochs')
    parser.add_argument('--init_epochs', type=int, default=30, help='init training for two-stage methods')

    # optimization
    parser.add_argument('--learning_rate', type=float, default=0.05, help='learning rate')
    parser.add_argument('--lr_decay_epochs', type=str, default='150,180,210', help='where to decay lr, can be a list')
    parser.add_argument('--lr_decay_rate', type=float, default=0.1, help='decay rate for learning rate')
    parser.add_argument('--weight_decay', type=float, default=5e-4, help='weight decay')
    parser.add_argument('--momentum', type=float, default=0.9, help='momentum')

    # dataset
    parser.add_argument('--dataset', type=str, default='cifar100', choices=['cifar100'], help='dataset')

    # model
    parser.add_argument('--model_s', type=str, default='resnet110',
                        choices=['resnet8', 'resnet14', 'resnet20', 'resnet32', 'resnet44', 'resnet56', 'resnet110',
                                 'resnet8x4', 'resnet32x4', 'wrn_16_1', 'wrn_16_2', 'wrn_40_1', 'wrn_40_2',
                                 'vgg8', 'vgg11', 'vgg13', 'vgg16', 'vgg19',
                                 'MobileNetV2', 'ShuffleV1', 'ShuffleV2', ])
    parser.add_argument('--model_t', type=str, default=None, help='teacher model snapshot')
    parser.add_argument('--trial', type=str, default='1', help='trial id')

    parser.add_argument('-r', '--gamma', type=float, default=1, help='weight for teacher classification')
    parser.add_argument('-a', '--alpha', type=float, default=3, help='weight balance for KD between teacher and student logit')
    parser.add_argument('-b', '--beta', type=float, default=1, help='weight for student classification')

    # KL distillation
    parser.add_argument('--kd_T', type=float, default=1, help='temperature for student aware training')

    # hint layer
    parser.add_argument('--hint_layer', default=2, type=int, choices=[0, 1, 2, 3, 4])

    # gpu configuration
    parser.add_argument('--gpu_idx', default="0", type=str)

    opt = parser.parse_args()

    # set the path according to the environment
    opt.model_path = './save/sftn'
    opt.tb_path = './save/sftn_tensorboards'

    iterations = opt.lr_decay_epochs.split(',')
    opt.lr_decay_epochs = list([])
    for it in iterations:
        opt.lr_decay_epochs.append(int(it))

    opt.model_name = 'S:{}_T:{}_sftn_{}_r:{}_a:{}_b:{}_{}'.format(opt.model_s, opt.model_t, opt.dataset,
                                                                opt.gamma, opt.alpha, opt.beta, opt.trial)

    opt.tb_folder = os.path.join(opt.tb_path, opt.model_name)
    if not os.path.isdir(opt.tb_folder):
        os.makedirs(opt.tb_folder)

    opt.save_folder = os.path.join(opt.model_path, opt.model_name)
    if not os.path.isdir(opt.save_folder):
        os.makedirs(opt.save_folder)

    return opt

def main():
    best_acc = 0

    opt = parse_option()
    os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
    os.environ["CUDA_VISIBLE_DEVICES"] = opt.gpu_idx

    # tensorboard logger
    logger = tb_logger.Logger(logdir=opt.tb_folder, flush_secs=2)

    # dataloader
    if opt.dataset == 'cifar100':
        train_loader, val_loader, n_data = get_cifar100_dataloaders(batch_size=opt.batch_size,
                                                                    num_workers=opt.num_workers,
                                                                    is_instance=True)
        n_cls = 100
    else:
        raise NotImplementedError(opt.dataset)

    # teacher model
    model_t = model_dict[opt.model_t](num_classes=n_cls) # random init t model

    # student branches
    branch1, branch2 = model_dict[opt.model_s+"_branches"](num_classes=n_cls)

    # feature transform layer
    model_s = model_dict[opt.model_s](num_classes=n_cls)

    data = torch.randn(2, 3, 32, 32)
    model_t.eval()
    model_s.eval()
    feat_t, _ = model_t(data, is_feat=True)
    feat_s, _ = model_s(data, is_feat=True)
    transform1 = FeatTrans(feat_s[1].shape, feat_t[1].shape)
    transform2 = FeatTrans(feat_s[2].shape, feat_t[2].shape)

    module_list = nn.ModuleList([])
    module_list.append(model_t)
    module_list.append(branch1)
    module_list.append(branch2)
    module_list.append(transform1)
    module_list.append(transform2)

    trainable_list = nn.ModuleList([])
    trainable_list.append(model_t)
    trainable_list.append(branch1)
    trainable_list.append(branch2)
    trainable_list.append(transform1)
    trainable_list.append(transform2)

    criterion_T_CE = nn.CrossEntropyLoss()
    criterion_S_KL = DistillKL(opt.kd_T)
    criterion_S_CE = nn.CrossEntropyLoss()

    criterion_list = nn.ModuleList([])
    criterion_list.append(criterion_T_CE)    # classification loss
    criterion_list.append(criterion_S_KL)    # KL divergence loss, original knowledge distillation
    criterion_list.append(criterion_S_CE)     # other knowledge distillation loss

    # optimizer
    optimizer = optim.SGD(trainable_list.parameters(),
                          lr=opt.learning_rate,
                          momentum=opt.momentum,
                          weight_decay=opt.weight_decay)

    if torch.cuda.is_available():
        module_list.cuda()
        criterion_list.cuda()
        cudnn.benchmark = True

    # routine
    for epoch in range(1, opt.epochs + 1):

        adjust_learning_rate(epoch, opt, optimizer)
        print("==> training...")

        time1 = time.time()
        train_acc, train_loss = train(epoch, train_loader, module_list, criterion_list, optimizer, opt)
        time2 = time.time()
        print('epoch {}, total time {:.2f}'.format(epoch, time2 - time1))

        logger.log_value('train_acc', train_acc, epoch)
        logger.log_value('train_loss', train_loss, epoch)

        test_acc, tect_acc_top5, test_loss = validate(val_loader, model_t, criterion_T_CE, opt)

        logger.log_value('test_acc', test_acc, epoch)
        logger.log_value('test_loss', test_loss, epoch)
        logger.log_value('test_acc_top5', tect_acc_top5, epoch)

        # save the best model
        if test_acc > best_acc:
            best_acc = test_acc
            state = {
                'epoch': epoch,
                'model': model_t.state_dict(),
                'best_acc': best_acc,
            }
            save_file = os.path.join(opt.save_folder, '{}_best.pth'.format(opt.model_s))
            print('saving the best model!')
            torch.save(state, save_file)

        # regular saving
        if epoch % opt.save_freq == 0:
            print('==> Saving...')
            state = {
                'epoch': epoch,
                'model': model_t.state_dict(),
                'accuracy': test_acc,
            }
            save_file = os.path.join(opt.save_folder, 'ckpt_epoch_{epoch}.pth'.format(epoch=epoch))
            torch.save(state, save_file)

    # This best accuracy is only for printing purpose.
    # The results reported in the paper/README is from the last epoch.
    print('best accuracy:', best_acc)

    # save model
    state = {
        'opt': opt,
        'model': model_t.state_dict(),
    }
    save_file = os.path.join(opt.save_folder, '{}_last.pth'.format(opt.model_s))
    torch.save(state, save_file)

if __name__ == '__main__':
    main()


import numpy as np
import matplotlib.pylab as plt
import os

def nucleoPropuestoPrimo(f0, nHarmonics, fs, N, decayP=0.5, amplitudeLimit=0.5):
    return getPrimeKernel(float(f0), nHarmonics, fs, N, decayP, amplitudeLimit)

def nucleoPropuestoNoPrimo(f0, nHarmonics, fs, N, decayP=0.5, amplitudeLimit=0.5):
    return getNonPrimeKernel(float(f0), nHarmonics, fs, N, decayP, amplitudeLimit)

def isPrime(x):
    xp = x + 0.0
    return 1.0 == xp or 2.0 == xp or 3.0 == xp or 5.0 == xp or 7.0 == xp or 11.0 == xp or 13.0 == xp

def getPrimeKernel(f0, A, fs, N, decayP, amplitudeLimit):
    # nucleo propuesto con armonicas PRIMAS

    k = np.zeros([N, 1])
    fmax = fs / 2.0
    for n in range(0, N):
        f = n * fmax / N
        fNorm = f / f0
        if (0.25 < fNorm and fNorm < 0.75) or (A + 0.25 < fNorm and fNorm < A + 0.75):
            k[n] = amplitudeLimit * np.cos(2 * np.pi * 1 / f0 * f)
        elif (0.75 < fNorm and fNorm < A + 0.25) and (isPrime(np.round(fNorm)) or 0.25 <= np.abs(round(fNorm) - fNorm)):
            k[n] = np.cos(2 * np.pi * 1 / f0 * f)

        # 0.9 para alcanzar un error de -0.013202 con 5 armonicas y
        # conservar el peso 0.5 en la primer subarmonica
        decaimiento = fNorm ** -decayP if 0 < fNorm else 1
        k[n] *= decaimiento

    return k

def getNonPrimeKernel(f0, A, fs, N, decayP, amplitudeLimit):
    # nucleo propuesto con todas las armonicas

    k = np.zeros([N, 1])
    fmax = fs / 2.0
    for n in range(0, N):
        f = n * fmax / N
        fNorm = f / f0
        if (0.25 < fNorm and fNorm < 0.75) or (A + 0.25 < fNorm and fNorm < A + 0.75):
            k[n] = amplitudeLimit * np.cos(2 * np.pi * 1 / f0 * f)
        elif (0.75 < fNorm and fNorm < A + 0.25):
            k[n] = np.cos(2 * np.pi * 1 / f0 * f)

        # 0.9 para alcanzar un error de -0.013202 con 5 armonicas y
        # conservar el peso 0.5 en la primer subarmonica
        decaimiento = fNorm ** -decayP if 0 < fNorm else 1
        k[n] *= decaimiento

    return k

def run(A=7, outImgFolderPath=None):
    # A: number of harmonics
    N = 256
    fs = 22050.0
    f0 = 1000.0
    f = np.linspace(0, fs / 2, N)

    bestDecaP = 0.5
    bestAmpLim = 0.5
    k1 = nucleoPropuestoPrimo(f0, A, fs, N, bestDecaP, bestAmpLim)
    k2 = nucleoPropuestoNoPrimo(f0, A, fs, N, bestDecaP, bestAmpLim)

    # plot kernel
    plt.figure()
    plt.subplot(211)
    plt.title('f0=%.1f kHz, A=%i, dec.=k^-%.1f, ampLim=%.1f, sum=%.2f vs. %.2f' % (
    f0 / 1000.0, A, bestDecaP, bestAmpLim, np.sum(k1), np.sum(k2)))
    plt.plot(f, k1)
    plt.xlim([0, A * f0 * 1.5])
    plt.ylabel('Amplitud')
    plt.subplot(212)
    plt.plot(f, k2)
    plt.ylabel('Amplitud')
    plt.xlabel('Frecuencia (Hz)')
    plt.xlim([0, A * f0 * 1.5])

    # guardar en archivo para graficar en formato vectorial
    hilera_k1 = "f, p"
    hilera_k2 = "f, p"
    for n in range(0, N):
        hilera_k1 = "%s\n%f, %f" % (hilera_k1, f[n], k1[n])
        hilera_k2 = "%s\n%f, %f" % (hilera_k2, f[n], k2[n])

    if outImgFolderPath is None:
        plt.show()
    else:
        outImgPath = '%s/proposedKernels_%iHarm.pdf' % (outImgFolderPath, A)
        text_file_k1 = open('%s/prime_%iHarmKernel.txt' % (outImgFolderPath, A), "w")
        text_file_k2 = open('%s/nonPrime_%iHarmKernel.txt' % (outImgFolderPath, A), "w")
        text_file_k1.write(hilera_k1)
        text_file_k2.write(hilera_k2)
        text_file_k1.close()
        text_file_k2.close()
        plt.savefig(outImgPath)

# testing
if __name__ == "__main__":
    # save current working directory and change to script's directory
    cwd = os.getcwd()
    os.chdir(os.path.dirname(os.path.realpath(__file__)))

    run()

    os.chdir(cwd)  # restore directory


#!/usr/bin/python3
#
# unilaiva-copy-audio.py3
#
# Part of Unilaiva songbook system.
#
# Copies the .MIDI files created by Lilypond to another location and creates
# audio files (mp3) from those files in the new location also.
#
# For audio creation, executables 'ffmpeg' and 'fluidsynth', and a sound font
# file are required.
#
# Run with '-h' argument for usage info.
#

import argparse
from argparse import RawDescriptionHelpFormatter
import sys
import os
import subprocess
import shutil
import pathlib
import re
import unicodedata

defaultsffile = '/usr/share/sounds/sf2/FluidR3_GM.sf2'

def strip_accents(text):
    """
    Strip accents from input string.

    Parameters
    ----------
      text : str
        The input string.

    Returns
    -------
    str
      The processed string.
    """
    try:
        text = unicode(text, 'utf-8')
    except (TypeError, NameError): # unicode is a default on Python 3
        pass
    text = unicodedata.normalize('NFD', text)
    text = text.encode('ascii', 'ignore')
    text = text.decode('utf-8')

    return str(text)

def cleanstr(text):
    """
    Returns the string given as parameter converted to clean ASCII string
    containing only numbers, A-Z, a-z, - and _. Accented characters will
    be returned without their accents and spaces converted to underscores.
    Other unallowed characters will be dropped.

    Parameters
    ----------
    text : str
      The input string.

    Returns
    -------
    str
      The processed string.
    """
    text = strip_accents(text)
    text = re.sub('[ ]+', '_', text)
    text = re.sub('[^0-9a-zA-Z_-]', '', text)

    return text

class Song:
  """Represents one song with a MIDI file associated to it"""
  def __init__(self, chaptertitle, title, nr, midifile):
    """
    Parameters
    ----------
    chapter title : str
      Title of the chapter this song belongs to
    title : str
      Title of the song.
    nr : int
      The number of the song in it's chapter
    midifile : pathlib.Path
      The MIDI file for the song
    """

    self.title = title.split('\\\\',1)[0].strip()
    self.nr = nr
    self.nrstr = str(nr)
    self.originalmidifile = midifile
    self.chaptertitle = chaptertitle
    while len(self.nrstr) < 3:
      self.nrstr = '0' + self.nrstr
    self.newmidifilename = self.nrstr + '_-_' + cleanstr(self.title) + '.midi'
    self.newmp3filename = self.nrstr + '_-_' + cleanstr(self.title) + '_midi' + '.mp3'

  def copymidifile(self, destdir):
    """
    Copies the MIDI file associated with this song to another directory. The
    directory must exist and be writable. If the destination file exists, it
    will be overwritten.

    Parameters
    ----------
    destdir : pathlib.Path
      The destination directory to copy the MIDI file into
    """

    if not self.originalmidifile.is_file():
      print ('ERROR    : source MIDI file is not available')
      exit(4)
    shutil.copy2(self.originalmidifile, destdir.joinpath(self.newmidifilename))
    print(destdir.name + ' : ' + self.newmidifilename)

  def createaudiofile(self, destdir):
    """
    Creates an mp3 audio file from the MIDI file associated with this song to
    another directory using external software (fluidsynth and ffmpeg). The
    destination directory must exist and be writable. If the destination file
    exists, it will be overwritten. The sound font file is picked from global
    variable args.

    Parameters
    ----------
    destdir : pathlib.Path
      The destination directory to create the mp3 file into
    """

    if not self.originalmidifile.is_file():
      print('ERROR    : source MIDI file is not available')
      exit(4)

    try:
      "Check options with: fluidsynth -o help"
      mp3process = subprocess.run(
        'fluidsynth -a file -T raw -O s16 -E little -r 48000 \
        -g 2 -o synth.reverb.level=.9 -o synth.reverb.room-size=0.3 -o synth.reverb.width=1 \
        -F - \"%s\" \"%s\"\
        | ffmpeg -f s32le -ac 1 -ar 48000 -i -\
        -codec:a libmp3lame -f mp3 -q:a 1\
        -id3v2_version 3 -write_id3v2 -write_id3v1\
        -metadata title=\"%s\"\
        -metadata artist=\"Unilaiva audio generator\"\
        -metadata album_artist=\"Unilaiva audio generator\"\
        -metadata track=%d\
        -metadata album=\"%s\"\
        -y \"%s\"'
          %(args.sf, self.originalmidifile,
            self.title + ' [generated from notation]',
            self.nr,
            self.chaptertitle + ' [generated from notation]',
            destdir.joinpath(self.newmp3filename)),
          capture_output=True, shell=True, check=True)
    except subprocess.CalledProcessError as e:
      print('ERROR    : mp3 coversion fails with error %i'%e.returncode)
      print('\n' + e.stderr.decode('utf-8'))
      exit(5)
    print(destdir.name + ' : ' + self.newmp3filename)

class Chapter:
  """Represents a song chapter, a collection of songs"""
  def __init__(self, title, basedir, chapterfile):
    """
    Parameters
    ----------
    title : str
      The title of the chapter
    basedir : pathlib.Path
      Base (root) directory under which midi files can be found; the directory
      where the .dep file resides
    chapterfile : pathlib.Path
      The file containing the songs in this chapter (between beginsong and
      endsong tags)
    """

    self.title = title
    self.cleantitle = cleanstr(self.title)
    self.chapterfile = chapterfile
    self.songs = []

    with open(chapterfile, 'r') as file:
      rawsongs = file.read().split('\endsong')

    songnr = 1
    for rawsong in rawsongs:
      pretext = rawsong.split('\\beginsong',1)[0]
      while '\\setcounter' in pretext:
        pretext = pretext.split('\\setcounter',1)[1]
        countername = pretext.split('{',1)[1].split('}',1)[0].strip()
        if countername == 'songnum':
          #print(countername)
          songnr = int(pretext.split('{',2)[2].split('}',1)[0].strip())
          #print(songnr)
      if 'lilypondbook' not in rawsong:
        continue
      songtitle = rawsong.split('\\beginsong',1)[1].split('{',1)[1].split('}',1)[0].strip()
      inputarg = rawsong.split('lilypondbook',1)[1].split('\\input{',1)[1].split('}',1)[0].strip()
      subdir = inputarg.split('/',1)[0]
      midifilename = inputarg.split('-systems.tex',1)[0] + '.midi'
      midifile = basedir.joinpath(midifilename)
      if midifile.is_file(): # add song only if there is a midi file for it
        self.songs.append(Song(self.title, songtitle, songnr, midifile))
      songnr += 1

  def copymidifiles(self, destdir, createsubdir):
    """
    Copies the midi files associated to this chapter to a new directory,
    optionally inside a subdirectory named after the title of this chapter.

    Parameters
    ----------
    destdir : pathlib.Path
      Destination directory. It must exist.
    createsubdir : boolean
      If true, a subdirectory will be created for this chapter's songs under
      the destination directory. The subdir is named after the title of this
      chapter.
    """

    if len(self.songs) == 0:
      return None
    fullsubdir = destdir
    if createsubdir and len(self.songs) > 0:
      fullsubdir = destdir.joinpath(self.title)
      os.makedirs(fullsubdir, 0o777, True)
    for song in self.songs:
      song.copymidifile(fullsubdir)

  def createaudiofiles(self, destdir, createsubdir):
    """
    Creates audio files based on the midi files associated to this chapter to a
    directory, optionally inside a subdirectory named after the title of
    this chapter.

    Parameters
    ----------
    destdir : pathlib.Path
      Destination directory. It must exist.
    createsubdir : boolean
      If true, a subdirectory will be created for this chapter's songs under
      the destination directory. The subdir is named after the title of this
      chapter.
    """

    if len(self.songs) == 0:
      return None
    fullsubdir = destdir
    if createsubdir:
      fullsubdir = destdir.joinpath(self.title)
      os.makedirs(fullsubdir, 0o777, True)
    for song in self.songs:
      song.createaudiofile(fullsubdir)

def execute():
  """
  Executes everything: creates chapters and initiates midi file copying and/or
  mp3 creation for each of them. Gets arguments from global variable args.
  NOTE: Only songs defined in content/songs_*.tex files are included, and each
        such a file is considered a chapter.
  """

  # TODO: Get chapter titles from the main .tex file

  basedir = args.depfile.parent
  with open(args.depfile, 'r') as depfile:
    deplines = depfile.read().strip().split(' ')

  chapters = []

  for line in deplines:
    if line.startswith('content/songs_'):
      title = line.split('content/songs_',1)[1].split('.tex',1)[0]
      chapters.append(Chapter(title, basedir, basedir.joinpath(line)))

  for chapter in chapters:
    if args.midi:
      chapter.copymidifiles(args.destdir, True)
    if args.audio:
      chapter.createaudiofiles(args.destdir, True)

# Main program start

# Parse arguments and check for their validity.

parser = argparse.ArgumentParser(
  description='This utility is part of Unilaiva songbook system.\n\n' +
              'Copies the MIDI files created by lilypond-book to another location, \n' +
              'and creates MP3 audio from those files in the new location also.\n' +
              'Note that only songs defined in content/songs_*.tex files are included,\n' +
              'and each such a file is considered a chapter by which a subdirectory\n' +
              'is created for the songs contained within.',
  formatter_class=RawDescriptionHelpFormatter)
parser.add_argument('depfile', type=pathlib.Path,
                    help='The .dep file created by lilypond-book')
parser.add_argument('destdir', type=pathlib.Path, help='Destination directory under \
                    where the midi and audio files will be copied as a set of \
                    subdirectories named after chapters. The directory must exist.')
parser.add_argument('--sf', type=pathlib.Path, default=defaultsffile,
                    help='Path to a soundfont file. If this argument is not present, \
                    the default %s is used'%defaultsffile)
parser.add_argument('--midi', action='store_true', help='Copy MIDI files')
parser.add_argument('--audio', action='store_true', help='Create MP3 files')

args = parser.parse_args()

if not args.depfile.is_file():
  print('ERROR    : ' + args.depfile.__str__() + ' does not exist')
  exit(2)
if not args.destdir.is_dir():
  print('ERROR    : destination directory does not exist')
  exit(2)
if not args.midi and not args.audio:
  print('ERROR    : No --midi nor --audio argument present: nothing to do!')
  print('           At least one of them is needed.')
  exit(2)
if args.audio:
  if shutil.which('fluidsynth') == None:
    print("ERROR    : 'fluidsynth' executable not found in path!")
    print('           It is required for audio (MP3) creation. Either install it or')
    print('           skip audio creation using by not specifying --audio argument.')
    exit(3)
  if shutil.which('ffmpeg') == None:
    print("ERROR    : 'ffmpeg' executable not found in path!")
    print('           It is required for audio (MP3) creation. Either install it or')
    print('           skip audio creation using by not specifying --audio argument.')
    exit(3)
  if not args.sf.is_file():
    print('ERROR    : Soundfont file not found: %s'%(args.sf))
    print('           It is required for audio (MP3) creation. Either install it, or')
    print('           specify another soundfont file (location) using --sf argument, or')
    print('           skip audio creation using by not specifying --audio argument.')
    exit(3)

# Do what we came here for!
execute()


import torch
from models import MobileNet
from utils import utils

from torchsummary import summary

import time

if __name__ == "__main__":

    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

    #model = MobileNet.MobileNetV1Conv224(num_classes=30)
    #model = MobileNet.MobileNetV1Dw224(num_classes=30)
    model = torch.hub.load('pytorch/vision', 'MobileNetV2')

    model = model.to(device)
    model.eval()

    summary(model, (3, 224, 224))

    input()

    prediction_time = utils.Average()
    start_time = 0

    with torch.no_grad():
        for i in range(10000):

            image = torch.randn(1, 3, 224, 224).to(device)

            start_time = time.time()

            prediction = model(image)

            prediction_time.add_value(time.time()-start_time)

            print("[{0}/{1}]".format(i,1000))

    print("Average prediction time: {0:.4f}".format(prediction_time.get_average()))
    print("Min prediction time: {0:.4f}".format(prediction_time.min))
    print("Max prediction time: {0:.4f}".format(prediction_time.max))


from gui.util.multiselect import Multiselect
from scripts.util.util import parse_options, write_options
from gui.util.script_progress import ScriptProgress
from gui.util.util import bind_button
from gui.util.select_directory import SelectDirectory
from scripts.upload_files import upload_files
from tkinter import *
from tkinter import ttk

class UploadFilesScreen(ttk.Frame):

    def __init__(self, parent, *args, **kwargs):
        ttk.Frame.__init__(self, parent, *args, **kwargs)

        # Parse upload files options
        self.options = parse_options("upload_options.json")

        # Directory selection
        self.select_directory = SelectDirectory(self, path=self.options["resources-directory"], title='Select Resources Folder')
        self.select_directory.grid(row=0, column=0, columnspan=2, sticky=(N, E, S, W), padx=20, pady=10)

        # Mode selection + title match checkbox
        self.mode = IntVar()
        self.mode.set(self.options["mode"])
        self.title_match = BooleanVar()
        self.title_match.set(self.options["require-titles-match"])
        mode_frame = ttk.Labelframe(self, text='Select Mode')
        update = ttk.Radiobutton(mode_frame, text='Update Files', variable=self.mode, value=0)
        add = ttk.Radiobutton(mode_frame, text='Add Files', variable=self.mode, value=1)
        update_and_add = ttk.Radiobutton(mode_frame, text='Add and Update', variable=self.mode, value=2)
        title_match_checkbox = ttk.Checkbutton(mode_frame, text='Enforce Matching Titles', variable=self.title_match)

        mode_frame.grid(row=1, column=0, columnspan=2, sticky=(N, E, S, W), padx=20, pady=10)
        update.grid(row=0, column=0)
        add.grid(row=0, column=1)
        update_and_add.grid(row=0, column=2)
        title_match_checkbox.grid(row=0, column=3, padx=20)

        # New chartz selection
        self.new_chartz = Multiselect(self, input=self.options['new-chartz'],
                                        title='Create New Chartz',
                                        key1='name',
                                        key2='to',
                                        header='Chart Name',
                                        addText='Add New Chart',
                                        height=3,
                                        archive=True)
        self.new_chartz.grid(row=2, column=0, columnspan=2, sticky=(N, E, S, W), padx=20, pady=10)

        # File extensions selection
        self.file_extensions = Multiselect(self, input=self.options["supported-file-types"],
                                            title='Supported File Types',
                                            header='File Type',
                                            addText='Add File Type',
                                            warn=True,
                                            height=6)
        self.file_extensions.grid(row=3, column=0, columnspan=2, sticky=(N, E, S, W), padx=20, pady=10)

        # Run script/Close buttons
        close_button = ttk.Button(self, text='Close', command=lambda: parent.master.destroy())
        bind_button(close_button)
        run_script_button = ttk.Button(self, text='Upload Files', command=self.run_script)
        bind_button(run_script_button)

        close_button.grid(row=4, column=0, sticky=(S, W), padx=20, pady=10)
        run_script_button.grid(row=4, column=1, sticky=(S, E), padx=20, pady=10)

        # Allow resizing
        for row in range(5):
            self.rowconfigure(row, weight="1")
        self.columnconfigure(0, weight="1")
        for col in range(4):
            mode_frame.columnconfigure(col, weight="1", minsize="80")

    def run_script(self, *args):
        # Update and write to options
        self.options["mode"] = self.mode.get()
        self.options["require-titles-match"] = self.title_match.get()
        self.options["resources-directory"] = self.select_directory.get_path()
        self.options["new-chartz"] = self.new_chartz.get_chosen_values("name", "to")
        self.options["supported-file-types"] = self.file_extensions.get_chosen_values()
        write_options(self.options, "upload_options.json")

        # Re-show main window
        def callback(code):
            self.master.master.deiconify()
            print(f"Thread Finished with code {code}")

        ScriptProgress(self, script=upload_files, callback=callback, title="Uploading Files", name="Upload Files", safe=self.options['mode'] == 0)
        self.master.master.withdraw()


from typing import Optional

import click

import clinica
import clinica.pydra.engine_utils as pydra_utils
import clinica.pydra.statistics_volume.pipeline as pydra_statistics_volume
from clinica.pipelines import cli_param
from clinica.pipelines.engine import clinica_pipeline

pipeline_name = "pydra-statistics-volume"

@clinica_pipeline
@click.command(name=pipeline_name, hidden=True)
@cli_param.argument.caps_directory
@cli_param.argument.group_label
@cli_param.argument.orig_input_data_volume
@cli_param.argument.subject_visits_with_covariates_tsv
@cli_param.argument.contrast
@cli_param.option_group.pipeline_specific_options
@cli_param.option_group.option(
    "-dartel",
    "--group_label_dartel",
    default="*",
    show_default=True,
    help="Name of the DARTEL template that Clinica needs to use to grab the input files.",
)
@cli_param.option_group.option(
    "-fwhm",
    "--full_width_at_half_maximum",
    default=8,
    show_default=True,
    help="Full Width at Half Maximum (FWHM) of the smoothing used in your input files.",
)
@cli_param.option_group.custom_pipeline_options(
    "Pipeline options if you use inputs from pet-volume pipeline"
)
@cli_param.option.acq_label
@cli_param.option.suvr_reference_region
@cli_param.option.use_pvc_data
@cli_param.option_group.custom_pipeline_options(
    "Pipeline options if you selected custom-pipeline"
)
@cli_param.option_group.option(
    "-cf",
    "--custom_file",
    help=(
        "Custom file string. Specify filename using * when the subject or session name "
        "appears e.g. '*_trc-18FFDG_pet_space-Ixi549Space_pet.nii.gz' will grab "
        "the corresponding file in all the subjects/sessions. "
        "This flag must be specified with the --measure_label flag). "
        "See Wiki for an example."
    ),
)
@cli_param.option_group.option(
    "-ml",
    "--measure_label",
    help=(
        "Name of the feature type, it will be saved on the CAPS "
        "_measure-MEASURE_LABEL key-value association. "
        "This flag must be specified with the --custom_file flag). "
        "See Wiki for an example."
    ),
)
@cli_param.option_group.common_pipelines_options
@cli_param.option.working_directory
@cli_param.option.n_procs
@cli_param.option_group.advanced_pipeline_options
@cli_param.option_group.option(
    "-ct",
    "--cluster_threshold",
    default=0.001,
    show_default=True,
    help="Threshold to define a cluster in the process of cluster-wise correction.",
)
def cli(
    caps_directory: str,
    group_label: str,
    orig_input_data_volume: str,
    subject_visits_with_covariates_tsv: str,
    contrast: str,
    group_label_dartel: str = "*",
    full_width_at_half_maximum: int = 8,
    acq_label: Optional[str] = None,
    suvr_reference_region: Optional[str] = None,
    use_pvc_data: bool = False,
    custom_file: Optional[str] = None,
    measure_label: Optional[str] = None,
    cluster_threshold: float = 1e-3,
    working_directory: Optional[str] = None,
    n_procs: Optional[int] = None,
) -> None:
    """Volume-based mass-univariate analysis with SPM.

       GROUP_LABEL is an user-defined identifier to target a specific group of subjects.

       The type of volume-based feature can be defined by using the third argument: t1-volume to use gray matter maps, pet-volume to use PET data or custom-pipeline for you own data in CAPS directory.

       SUBJECT_VISITS_WITH_COVARIATES_TSV is a TSV file containing a list of subjects with their sessions and all the covariates and factors of the model.

       CONTRAST is a string defining the contrast matrix or the variable of interest for the GLM.

    See https://aramislab.paris.inria.fr/clinica/docs/public/latest/Pipelines/Stats_Volume/
    """
    from clinica.utils.exceptions import ClinicaException

    parameters = {
        # Clinica compulsory arguments
        "group_label": group_label,
        "orig_input_data_volume": orig_input_data_volume,
        "contrast": contrast,
        # Optional arguments
        "group_label_dartel": group_label_dartel,
        "full_width_at_half_maximum": full_width_at_half_maximum,
        # Optional arguments for inputs from pet-volume pipeline
        "acq_label": acq_label,
        "use_pvc_data": use_pvc_data,
        "suvr_reference_region": suvr_reference_region,
        # Optional arguments for custom pipeline
        "measure_label": measure_label,
        "custom_file": custom_file,
        # Advanced arguments
        "cluster_threshold": cluster_threshold,
        "tsv_file": subject_visits_with_covariates_tsv,
        "base_dir": working_directory,
    }

    statistics_volume_pipeline = pydra_statistics_volume.build_core_workflow(
        name="pydra-statistics-volume",
        input_dir=None,
        output_dir=caps_directory,
        parameters=parameters,
    )
    pydra_utils.run(statistics_volume_pipeline, n_procs=n_procs)

if __name__ == "__main__":
    cli()


"""Button platform for La Marzocco espresso machines."""

import logging

from homeassistant.components.button import ButtonEntity

from .const import (
    DOMAIN,
    ENTITY_FUNC,
    ENTITY_ICON,
    ENTITY_MAP,
    ENTITY_NAME,
    ENTITY_TYPE,
    TYPE_START_BACKFLUSH,
    MODEL_GS3_AV,
    MODEL_LM,
)
from .entity_base import EntityBase
from .services import async_setup_entity_services, call_service

_LOGGER = logging.getLogger(__name__)

ENTITIES = {
    "start_backflush": {
        ENTITY_NAME: "Start Backflush",
        ENTITY_MAP: {
            MODEL_GS3_AV: None,
            MODEL_LM: None,
        },
        ENTITY_TYPE: TYPE_START_BACKFLUSH,
        ENTITY_ICON: "mdi:coffee-maker",
        ENTITY_FUNC: "set_start_backflush",
    },
}

async def async_setup_entry(hass, config_entry, async_add_entities):
    """Set up button entities and services."""

    lm = hass.data[DOMAIN][config_entry.entry_id]
    async_add_entities(
        LaMarzoccoButton(lm, button_type, hass)
        for button_type in ENTITIES
        if lm.model_name in ENTITIES[button_type][ENTITY_MAP]
    )

    await async_setup_entity_services(lm)

class LaMarzoccoButton(EntityBase, ButtonEntity):
    """Button supporting backflush."""

    def __init__(self, lm, button_type, hass):
        """Initialise buttons."""
        self._object_id = button_type
        self._hass = hass
        self._lm = lm
        self._entities = ENTITIES
        self._entity_type = self._entities[self._object_id][ENTITY_TYPE]

    async def async_press(self, **kwargs) -> None:
        """Press button."""
        await call_service(
            getattr(self._lm, self._entities[self._object_id][ENTITY_FUNC])
        )


from __future__ import print_function, division
from keras.models import Model
from keras.layers import Concatenate, Add, Average, Input, Dense, Flatten, BatchNormalization, Activation, LeakyReLU
from keras.layers.convolutional import Convolution2D, MaxPooling2D, UpSampling2D, Convolution2DTranspose
from keras import backend as K
from keras.utils.np_utils import to_categorical
import keras.callbacks as callbacks
import keras.optimizers as optimizers
from advanced import HistoryCheckpoint, SubPixelUpscaling, non_local_block, TensorBoardBatch
import img_utils
import numpy as np
import os
import time
import warnings

try:
    import cv2
    _cv2_available = True
except:
    warnings.warn('Could not load opencv properly. This may affect the quality of output images.')
    _cv2_available = False

train_path = img_utils.output_path
validation_path = img_utils.validation_output_path
path_X = img_utils.output_path + "X/"
path_Y = img_utils.output_path + "y/"

def PSNRLoss(y_true, y_pred):

    return -10. * K.log(K.mean(K.square(y_pred - y_true))) / K.log(10.)

def psnr(y_true, y_pred):
    assert y_true.shape == y_pred.shape, "Cannot calculate PSNR. Input shapes not same." \
                                         " y_true shape = %s, y_pred shape = %s" % (str(y_true.shape),
                                                                                str(y_pred.shape))

    return -10. * np.log10(np.mean(np.square(y_pred - y_true)))

class BaseSuperResolutionModel(object):

    def __init__(self, model_name, scale_factor):

        self.model = None # type: Model
        self.model_name = model_name
        self.scale_factor = scale_factor
        self.weight_path = None

        self.type_scale_type = "norm" # Default = "norm" = 1. / 255
        self.type_requires_divisible_shape = False
        self.type_true_upscaling = False

        self.evaluation_func = None
        self.uses_learning_phase = False

    def create_model(self, height=32, width=32, channels=3, load_weights=False, batch_size=128) -> Model:

        if self.type_requires_divisible_shape and height is not None and width is not None:
            assert height * img_utils._image_scale_multiplier % 4 == 0, "Height of the image must be divisible by 4"
            assert width * img_utils._image_scale_multiplier % 4 == 0, "Width of the image must be divisible by 4"

        if K.image_dim_ordering() == "th":
            if width is not None and height is not None:
                shape = (channels, width * img_utils._image_scale_multiplier, height * img_utils._image_scale_multiplier)
            else:
                shape = (channels, None, None)
        else:
            if width is not None and height is not None:
                shape = (width * img_utils._image_scale_multiplier, height * img_utils._image_scale_multiplier, channels)
            else:
                shape = (None, None, channels)

        init = Input(shape=shape)

        return init

    def fit(self, batch_size=128, nb_epochs=100, save_history=True, history_fn="Model History.txt") -> Model:

        samples_per_epoch = img_utils.image_count()
        val_count = img_utils.val_image_count()
        if self.model == None: self.create_model(batch_size=batch_size)

        callback_list = [callbacks.ModelCheckpoint(self.weight_path, monitor='val_PSNRLoss', save_best_only=True,
                                                   mode='max', save_weights_only=True, verbose=2)]
        if save_history:
            callback_list.append(HistoryCheckpoint(history_fn))

            if K.backend() == 'tensorflow':
                log_dir = './%s_logs/' % self.model_name
                tensorboard = TensorBoardBatch(log_dir, batch_size=batch_size)
                callback_list.append(tensorboard)

        print("Training model : %s" % (self.__class__.__name__))
        self.model.fit_generator(img_utils.image_generator(train_path, scale_factor=self.scale_factor,
                                                           small_train_images=self.type_true_upscaling,
                                                           batch_size=batch_size),
                                 steps_per_epoch=samples_per_epoch // batch_size + 1,
                                 epochs=nb_epochs, callbacks=callback_list,
                                 validation_data=img_utils.image_generator(validation_path,
                                                                           scale_factor=self.scale_factor,
                                                                           small_train_images=self.type_true_upscaling,
                                                                           batch_size=batch_size),
                                 validation_steps=val_count // batch_size + 1)

        return self.model

    def upscale(self, img_path, save_intermediate=False, return_image=False, suffix="scaled",
                patch_size=8, mode="patch", verbose=True):

        import os
        from scipy.misc import imread, imresize, imsave

        # Destination path
        path = os.path.splitext(img_path)
        #print(path[0])
        filename = path[0] + "_" + suffix + "(%dx)" % (self.scale_factor) + path[1]

        # Read image
        scale_factor = int(self.scale_factor)
        true_img = imread(img_path, mode='RGB')
        init_dim_1, init_dim_2 = true_img.shape[0], true_img.shape[1]
        if verbose: print("Old Size : ", true_img.shape)
        if verbose: print("New Size : (%d, %d, 3)" % (init_dim_1 * scale_factor, init_dim_2 * scale_factor))

        img_dim_1, img_dim_2 = 0, 0

        if mode == "patch" and self.type_true_upscaling:
            # Overriding mode for True Upscaling models
            mode = 'fast'
            print("Patch mode does not work with True Upscaling models yet. Defaulting to mode='fast'")

        if mode == 'patch':
            # Create patches
            if self.type_requires_divisible_shape:
                if patch_size % 4 != 0:
                    print("Deep Denoise requires patch size which is multiple of 4.\nSetting patch_size = 8.")
                    patch_size = 8

            images = img_utils.make_patches(true_img, scale_factor, patch_size, verbose)

            nb_images = images.shape[0]
            img_dim_1, img_dim_2 = images.shape[1], images.shape[2]
            print("Number of patches = %d, Patch Shape = (%d, %d)" % (nb_images, img_dim_2, img_dim_1))
        else:
            # Use full image for super resolution
            img_dim_1, img_dim_2 = self.__match_autoencoder_size(img_dim_1, img_dim_2, init_dim_1, init_dim_2,
                                                                 scale_factor)

            images = imresize(true_img, (img_dim_1, img_dim_2))
            images = np.expand_dims(images, axis=0)
            print("Image is reshaped to : (%d, %d, %d)" % (images.shape[1], images.shape[2], images.shape[3]))

        # Save intermediate bilinear scaled image is needed for comparison.
        # intermediate_img = None
        # if save_intermediate:
        #     if verbose: print("Saving intermediate image.")
        #     fn = path[0] + "_intermediate_" + path[1]
        #     intermediate_img = imresize(true_img, (init_dim_1 * scale_factor, init_dim_2 * scale_factor))
        #     imsave(fn, intermediate_img)

        # Transpose and Process images
        if K.image_dim_ordering() == "th":
            img_conv = images.transpose((0, 3, 1, 2)).astype(np.float32) / 255.
        else:
            img_conv = images.astype(np.float32) / 255.

        model = self.create_model(img_dim_2, img_dim_1, load_weights=True)
        if verbose: print("Model loaded.")

        # Create prediction for image patches
        result = model.predict(img_conv, batch_size=128, verbose=verbose)

        if verbose: print("De-processing images.")

         # Deprocess patches
        if K.image_dim_ordering() == "th":
            result = result.transpose((0, 2, 3, 1)).astype(np.float32) * 255.
        else:
            result = result.astype(np.float32) * 255.

        # Output shape is (original_width * scale, original_height * scale, nb_channels)
        if mode == 'patch':
            out_shape = (init_dim_1 * scale_factor, init_dim_2 * scale_factor, 3)
            result = img_utils.combine_patches(result, out_shape, scale_factor)
        else:
            result = result[0, :, :, :] # Access the 3 Dimensional image vector

        result = np.clip(result, 0, 255).astype('uint8')

        if _cv2_available:
            # used to remove noisy edges
            result = cv2.pyrUp(result)
            result = cv2.medianBlur(result, 3)
            result = cv2.pyrDown(result)

        if verbose: print("\nCompleted De-processing image.")

        if return_image:
            # Return the image without saving. Useful for testing images.
            return result

        if verbose: print("Saving image.")
        imsave(filename, result)

    def __match_autoencoder_size(self, img_dim_1, img_dim_2, init_dim_1, init_dim_2, scale_factor):
        if self.type_requires_divisible_shape:
            if not self.type_true_upscaling:
                # AE model but not true upsampling
                if ((init_dim_2 * scale_factor) % 4 != 0) or ((init_dim_1 * scale_factor) % 4 != 0) or \
                        (init_dim_2 % 2 != 0) or (init_dim_1 % 2 != 0):

                    print("AE models requires image size which is multiple of 4.")
                    img_dim_2 = ((init_dim_2 * scale_factor) // 4) * 4
                    img_dim_1 = ((init_dim_1 * scale_factor) // 4) * 4

                else:
                    # No change required
                    img_dim_2, img_dim_1 = init_dim_2 * scale_factor, init_dim_1 * scale_factor
            else:
                # AE model and true upsampling
                if ((init_dim_2) % 4 != 0) or ((init_dim_1) % 4 != 0) or \
                        (init_dim_2 % 2 != 0) or (init_dim_1 % 2 != 0):

                    print("AE models requires image size which is multiple of 4.")
                    img_dim_2 = ((init_dim_2) // 4) * 4
                    img_dim_1 = ((init_dim_1) // 4) * 4

                else:
                    # No change required
                    img_dim_2, img_dim_1 = init_dim_2, init_dim_1
        else:
            # Not AE but true upsampling
            if self.type_true_upscaling:
                img_dim_2, img_dim_1 = init_dim_2, init_dim_1
            else:
                # Not AE and not true upsampling
                img_dim_2, img_dim_1 = init_dim_2 * scale_factor, init_dim_1 * scale_factor

        return img_dim_1, img_dim_2,

class DistilledResNetSR(BaseSuperResolutionModel):

    def __init__(self, scale_factor):
        super(DistilledResNetSR, self).__init__("DistilledResNetSR", scale_factor)

        # Treat this model as a denoising auto encoder
        # Force the fit, evaluate and upscale methods to take special care about image shape
        self.type_requires_divisible_shape = True
        self.uses_learning_phase = False

        self.n = 32
        self.mode = 2

        self.weight_path = "weights/DistilledResNetSR %dX.h5" % (self.scale_factor)
        self.type_true_upscaling = True

    def create_model(self, height=32, width=32, channels=3, load_weights=False, batch_size=128):
        init =  super(DistilledResNetSR, self).create_model(height, width, channels, load_weights, batch_size)

        x0 = Convolution2D(self.n, (3, 3), activation='relu', padding='same', name='student_sr_res_conv1')(init)

        x = self._residual_block(x0, 1)

        x = Add(name='student_residual')([x, x0])
        x = self._upscale_block(x, 1)

        x = Convolution2D(3, (3, 3), activation="linear", padding='same', name='student_sr_res_conv_final')(x)

        model = Model(init, x)
        # dont compile yet
        if load_weights: model.load_weights(self.weight_path, by_name=True)

        self.model = model
        return model

    def _residual_block(self, ip, id):
        mode = False if self.mode == 2 else None
        channel_axis = 1 if K.image_data_format() == 'channels_first' else -1
        init = ip

        x = Convolution2D(self.n, (3, 3), activation='linear', padding='same',
                          name='student_sr_res_conv_' + str(id) + '_1')(ip)
        x = BatchNormalization(axis=channel_axis, name="student_sr_res_batchnorm_" + str(id) + "_1")(x, training=mode)

        x = Activation('relu', name="student_sr_res_activation_" + str(id) + "_1")(x)

        x = Convolution2D(self.n, (3, 3), activation='linear', padding='same',
                          name='student_sr_res_conv_' + str(id) + '_2')(x)
        x = BatchNormalization(axis=channel_axis, name="student_sr_res_batchnorm_" + str(id) + "_2")(x, training=mode)

        m = Add(name="student_sr_res_merge_" + str(id))([x, init])

        return m

    def _upscale_block(self, ip, id):
        init = ip

        channel_dim = 1 if K.image_data_format() == 'channels_first' else -1
        channels = init._keras_shape[channel_dim]

        x = UpSampling2D(name='student_upsampling_%d' % id)(init)
        x = Convolution2D(self.n * 2, (3, 3), activation="relu", padding='same', name='student_sr_res_filter1_%d' % id)(x)

        return x

    def fit(self, batch_size=128, nb_epochs=100, save_history=True, history_fn="Distilled ResNetSR History.txt"):
        super(DistilledResNetSR, self).fit(batch_size, nb_epochs, save_history, history_fn)


import gym
import numpy as np
from gym.envs.registration import register

def make(
    domain_name,
    task_name,
    noise,
    mult_factor,
    idx,
    seed=1,
    visualize_reward=True,
    from_pixels=False,
    height=84,
    width=84,
    camera_id=0,
    frame_skip=1,
    episode_length=1000,
    environment_kwargs=None
):
    env_id = 'dmc_%s_%s_factor%s-v1' % (domain_name, task_name, idx)

    if from_pixels:
        assert not visualize_reward, 'cannot use visualize reward when learning from pixels'

    # shorten episode length
    max_episode_steps = (episode_length + frame_skip - 1) // frame_skip

    if not env_id in gym.envs.registry.env_specs:
        register(
            id=env_id,
            entry_point='dmc2gym.wrappers:DMCWrapper',
            kwargs={
                'domain_name': domain_name,
                'task_name': task_name,
                'noise': noise,
                'mult_factor': mult_factor,
                'idx': idx,
                'task_kwargs': {
                    'random': seed
                },
                'environment_kwargs': environment_kwargs,
                'visualize_reward': visualize_reward,
                'from_pixels': from_pixels,
                'height': height,
                'width': width,
                'camera_id': camera_id,
                'frame_skip': frame_skip,
            },
            max_episode_steps=max_episode_steps
        )
    return gym.make(env_id)


import wikipedia
from bs4 import BeautifulSoup
import pandas as pd
import time
import urllib.parse
import random

teams = [
    # {
    #     "team": "Czech Republic",
    #     "players": {
    #         "goalkeepers": ["Ales Mandous", "Jiri Pavlenka", "Tomas Vaclik"],
    #         "defenders": ["Jan Boril", "Jakub Brabec", "Ondrej Celustka"," Vladimir Coufal",
    #                       "Pavel Kaderabek", "Tomas Kalas", "Ales Mateju", "David Zima"],
    #         "midfielders": ["Tomas Holes", "Antonin Barak", "Vladimir Darida", "Jakub Jankto", "Alex Kral",
    #                         "Lukas Masopust", "Jakub Pesek", "Michal Sadilek",
    #                         "Petr Sevcik", "Tomas Soucek", "Adam Hlozek"],
    #         "forwards": ["Michael Krmencik", "Tomas Pekhart", "Patrik Schick", "Matej Vydra"]
    #     }
    # },
    # {
    #     'team': 'France',
    #     "players": {
    #         "goalkeepers": ["Hugo Lloris", "Mike Maignan", "Steve Mandanda"],
    #         "defenders": ["Lucas Digne", "Léo Dubois", "Lucas Hernandez", "Presnel Kimpembe", "Jules Koundé",
    #                       "Clément Lenglet", "Benjamin Pavard", "Raphaël Varane", "Kurt Zouma"],
    #         "midfielders": ["Golo Kanté", "Thomas Lemar", "Paul Pogba", "Adrien Rabiot", "Moussa Sissoko",
    #                         "Corentin Tolisso"],
    #         "forwards": ["Kingsley Coman", "Wissam Ben Yedder", "Karim Benzema", "Ousmane Dembélé", "Olivier Giroud",
    #                      "Antoine Griezmann", "Kylian Mbappé", "Marcus Thuram"]
    #     }
    # },
    # {
    #     'team': 'Switzerland',
    #     "players": {
    #         "goalkeepers": ["Yann Sommer", "Yvon Mvogo", "Jonas Omlin"],
    #         "defenders": ["Manuel Akanji", "Loris Benito", "Eray Cömert", "Nico Elvedi", "Jordan Lotomba",
    #                       "Kevin Mbabu", "Becir Omeragic", "Ricardo Rodríguez", "Fabian Schär", "Silvan Widmer"],
    #         "midfielders": ["Christian Fassnacht", "Edimilson Fernandes", "Remo Freuler", "Admir Mehmedi",
    #                         "Xherdan Shaqiri", "Djibril Sow", "Ruben Vargas", "Granit Xhaka", "Denis Zakaria",
    #                         "Steven Zuber"],
    #         "forwards": ["Breel Embolo", "Mario Gavranović", "Haris Seferovic"]
    #     }
    # },
    # {
    #     "team": "Sweden",
    #     "players": {
    #         "goalkeepers": ["Karl-Johan Johnsson", "Kristoffer Nordfeldt", "Robin Olsen"],
    #         "defenders": ["Ludwig Augustinsson", "Marcus Danielson", "Andreas Granqvist", "Filip Helander",
    #                       "Pontus Jansson", "Emil Krafth", "Mikael Lustig", "Victor Lindelöf", "Pierre Bengtsson"],
    #         "midfielders": ["Jens Cajuste", "Albin Ekdal", "Emil Forsberg", "Dejan Kulusevski", "Sebastian Larsson",
    #                         "Kristoffer Olsson", "Mattias Svanberg", "Gustav Svensson"],
    #         "forwards": ["Marcus Berg", "Viktor Claesson", "Alexander Isak",
    #                      "Jordan Larsson", "Robin Quaison", "Ken Sema"]
    #     }
    # },
    # {
    #     "team": "Ukraine",
    #     "players": {
    #         "goalkeepers": ["Georgiy Bushchan", "Andriy Pyatov", "Anatolii Trubin"],
    #         "defenders": ["Oleksandr Karavaev", "Serhiy Kryvtsov", "Mykola Matviyenko", "Vitaliy Mykolenko",
    #                       "Denys Popov", "Eduard Sobol", "Oleksandr Tymchyk", "Illia Zabarnyi"],
    #         "midfielders": ["Oleksandr Zinchenko", "Roman Bezus", "Yevhen Makarenko", "Ruslan Malinovskyi",
    #                         "Marlos", "Mykola Shaparenko", "Taras Stepanenko", "Heorhii Sudakov", "Serhiy Sydorchuk",
    #                         "Viktor Tsygankov", "Andriy Yarmolenko", "Oleksandr Zubkov"],
    #         "forwards": ["Artem Besedin", "Artem Dovbyk", "Roman Yaremchuk"]
    #     }
    # },
    # {
    #     "team": "Spain",
    #     "players": {
    #         "goalkeepers": ["David de Gea", "Unai Simón", "Robert Sánchez"],
    #         "defenders": ["José Gayà", "Jordi Alba", "Pau Torres", "Aymeric Laporte", "Eric García", "Diego Llorente",
    #                       "César Azpilicueta", "Marcos Llorente"],
    #         "midfielders": ["Thiago Alcântara", "Sergio Busquets", "Koke", "Dani Olmo", "Rodri",
    #                         "Fabián Ruiz", "Pedri"],
    #         "forwards": ["Pablo Sarabia", "Ferran Torres", "Adama Traoré", "Álvaro Morata", "Gerard Moreno",
    #                      "Mikel Oyarzabal"]
    #     }
    # },
    # {
    #     "team": "England",
    #     "players": {
    #         "goalkeepers": ["Aaron Ramsdale", "Sam Johnstone", "Jordan Pickford"],
    #         "defenders": ["Ben Chilwell", "Conor Coady", "Reece James", "Harry Maguire", "Tyrone Mings",
    #                       "Luke Shaw", "John Stones", "Kieran Trippier", "Kyle Walker", "Ben White"],
    #         "midfielders": ["Jude Bellingham", "Phil Foden", "Jack Grealish", "Jordan Henderson", "Mason Mount",
    #                         "Kalvin Phillips", "Declan Rice", "Bukayo Saka", "Jadon Sancho"],
    #         "forwards": ["Dominic Calvert-Lewin", "Harry Kane", "Marcus Rashford", "Raheem Sterling"]
    #     }
    # },
    # {
    #     "team": "Denmark",
    #     "players": {
    #         "goalkeepers": ["Kasper Schmeichel", "Jonas Lössl", "Frederik Rønnow"],
    #         "defenders": ["Jens Stryger Larsen", "Simon Kjær", "Andreas Christensen", "Joachim Andersen", "Daniel Wass",
    #                       "Mathias Jørgensen", "Joakim Mæhle", "Jannik Vestergaard", "Nicolai Boilesen"],
    #         "midfielders": ["Mathias Jensen", "Christian Nørgaard", "Pierre-Emile Højbjerg", "Thomas Delaney",
    #                         "Anders Christiansen", "Christian Eriksen", "Mikkel Damsgaard", "Robert Skov"],
    #         "forwards": ["Martin Braithwaite", "Andreas Cornelius", "Andreas Skov Olsen", "Yussuf Poulsen",
    #                      "Kasper Dolberg", "Jonas Wind"]
    #     }
    # },
    # {
    #     "team": "Belgium",
    #     "players": {
    #         "goalkeepers": ["Thibaut Courtois", "Simon Mignolet", "Matz Sels"],
    #         "defenders": ["Toby Alderweireld", "Dedryck Boyata", "Jason Denayer", "Thomas Vermaelen",
    #                       "Jan Vertonghen", "Timothy Castagne", "Thomas Meunier"],
    #         "midfielders": ["Kevin De Bruyne", "Leander Dendoncker", "Dennis Praet",
    #                         "Youri Tielemans", "Hans Vanaken", "Axel Witsel"],
    #         "forwards": ["Nacer Chadli", "Yannick Carrasco", "Thorgan Hazard", "Michy Batshuayi",
    #                      "Christian Benteke", "Jérémy Doku", "Eden Hazard", "Romelu Lukaku",
    #                      "Dries Mertens", "Leandro Trossard"]
    #     }
    #
    # },
    {
        "team": "Italy",
        "players": {
            "goalkeepers": ["Gianluigi Donnarumma", "Salvatore Sirigu", "Alex Meret"],
            "defenders": ["Francesco Acerbi", "Alessandro Bastoni", "Leonardo Bonucci", "Giorgio Chiellini",
                          "Giovanni Di Lorenzo", "Alessandro Florenzi", "Leonardo Spinazzola", "Rafael Tolói",
                          "Emerson Palmieri"],
            "midfielders": ["Nicolò Barella", "Bryan Cristante", "Jorginho", "Manuel Locatelli", "Gaetano Castrovilli",
                            "Matteo Pessina", "Marco Verratti"],
            "forwards": ["Andrea Belotti", "Domenico Berardi", "Federico Bernardeschi", "Federico Chiesa",
                         "Ciro Immobile", "Giacomo Raspadori"]
        }
    }

]
players = []
for team in teams:
    for key, values in team['players'].items():
        for value in values:
            time.sleep(random.randint(3, 7))
            age_started_senior_career = None
            age_started_youth_career = None
            print(value)
            result = wikipedia.search(value, results=1)
            print(result)
            # content = wikipedia.page(result).html()
            # soup = BeautifulSoup(content, 'lxml')
            # table = soup.find('table', {'class': 'infobox vcard'})
            # print(table)

            processed_name = urllib.parse.quote(result[0])
            # processed_name = processed_name.encode('utf-8').decode('ascii')
            url = "https://en.wikipedia.org/wiki/"+processed_name
            print(url)
            try:
                table = pd.read_html(url, header=0)[0]

                print(table.columns)
                print(table)
                table.columns = [0, 1, 2, 3]
            except ValueError:
                continue
            if not table.empty:

                age_row_index = table.index[table[0] == 'Date of birth'].to_list()[0]
                age_row = table.iloc[age_row_index]
                print(age_row)

                if __name__ == '__main__':
                    age_value = age_row[1].replace('(', '').replace(')', '').\
                        replace('[2]', '').replace('[1]', '').replace('[3]', '')

                age = [int(s) for s in age_value.split() if s.isdigit()][0]
                print(age)

                senior_career = None
                try:
                    senior_career_index = table.index[
                        table[0].str.lower().str.contains('senior career', na=False)
                    ].to_list()[0]
                    senior_career = table.iloc[senior_career_index+2][0]

                    print(senior_career)
                    # print(senior_career[0])
                    # senior_career_start = int(senior_career[0].split('–')[0].replace('-', ''))
                    # print(senior_career_start)
                    # if senior_career_start:
                    #     age_started_senior_career = age - (2021 - senior_career_start)

                except Exception:
                    pass

                youth_career = None
                had_youth_career = False
                youth_career_index = table.index[table[0].str.lower().str.contains('youth career', na=False)].to_list()
                print("youth_career_index list", youth_career_index)
                if youth_career_index:
                    had_youth_career = True
                    print("table", table.iloc[youth_career_index[0]:senior_career_index])
                    youth_career = str(table.iloc[youth_career_index[0] + 1][0])
                    # youth_career = youth_career.replace('–', ' ')

                    if not youth_career.isdigit():
                        youth_career = youth_career.split('-')[0]

                    print('youth', youth_career)
                    if 'nan' in youth_career:
                        youth_career_next_row = str(table.iloc[youth_career_index[0] + 2][0])
                        if 'senior' not in youth_career_next_row.lower():
                            youth_career = youth_career_next_row
                        print('youth next row', youth_career)
                player_data = {
                    'team': team['team'],
                    'name': value,
                    'age': age,
                    'senior_career': senior_career,
                    'youth_career': youth_career,
                    'had_youth_career': had_youth_career
                }
                players.append(player_data)

players_df = pd.DataFrame(players)
players_df.to_csv('data/Italy.csv', index=False)


# -*- coding: utf-8 -*-
import scrapy
from ..items import CodeItem

class JpointSpider(scrapy.Spider):
    name = 'jpoint'
    allowed_domains = ['www.javatpoint.com']
    start_urls = ['https://www.javatpoint.com/operators-in-java']

    def parse(self, response):
       # tk=response.css('.codeblock')
       # dess=response.css('p::text').extract()
        #titlelist=dict()
        title=response.css('#h1::text,.h1 ::text')[0].extract()
        language=response.xpath('//*[@id="smoothmenu1"]/ul/li[5]/a/text()').extract_first()
        subtitles=response.xpath('//h2[not(contains(.,"Javatpoint Services") or contains(.,"Training For College Campus") or contains(.,"Please Share") or contains(.,"Learn Latest Tutorials"))]/text()').extract()
        count = 1
        for t in subtitles:
            #code = t.css('.java::text').extract_first()
            description = response.xpath('//p[count(preceding-sibling::h2)=' + str(count) + ']/text()').extract()

            #print(code)
            Item=CodeItem()
            Item['Title']=title + ' - ' + t

            Item['Code']= response.xpath('//div[count(preceding-sibling::h2)=' + str(count) + ']/textarea[@class="java"]/text()').extract()
            Item['Language']=language
            Item['Description']=description
            Item['URL']= response.request.url
            count+=1
            yield Item

        NextLinkSelector= response.xpath('//*[@id="bottomnextup"]/a[1]/@href')
        if NextLinkSelector:
           NextLink=NextLinkSelector.extract_first()
           yield scrapy.Request(url=response.urljoin(NextLink))


import numpy as np
import os
import collections
import string
import re

EMBED_DIM = 128

def normalize_answer(s):
    """Lower text and remove punctuation, articles and extra whitespace."""
    def remove_articles(text):
        regex = re.compile(r'\b(a|an|the)\b', re.UNICODE)
        return re.sub(regex, ' ', text)

    def white_space_fix(text):
        return ' '.join(text.split())

    def remove_punc(text):
        exclude = set(string.punctuation)
        return ''.join(ch for ch in text if ch not in exclude)

    def lower(text):
        return text.lower()

    return white_space_fix(remove_articles(remove_punc(lower(s))))

def get_tokens(s):
    if not s:
        return []

    return normalize_answer(s).split()

def compute_exact(a_gold, a_pred):
    return int(normalize_answer(a_gold) == normalize_answer(a_pred))

def compute_f1(a_gold, a_pred):
    gold_toks = get_tokens(a_gold)
    pred_toks = get_tokens(a_pred)
    common = collections.Counter(gold_toks) & collections.Counter(pred_toks)
    num_same = sum(common.values())
    if len(gold_toks) == 0 or len(pred_toks) == 0:
        # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
        return int(gold_toks == pred_toks)

    if num_same == 0:
        return 0

    precision = 1.0 * num_same / len(pred_toks)
    recall = 1.0 * num_same / len(gold_toks)
    f1 = (2 * precision * recall) / (precision + recall)

    return f1

def calculate_accuracies(answer_array, predicted_answer_array, document_array, word_dictionary):
    """
    This method calculates the F1 score and the Exact Match (EM) accuracy over the input batch.
    Inputs:
    answer_array - The start and end indices of the ground truth answers
    predicted_answer_array - The start and end indices of the predicted answers
    document_array - Document tokens for each example in the batch
    Output:
    f1_score - The harmonic mean of precision and recall over the current batch, where:
    Precision = True Positives / (True Positives + False Positives)
    Recall    = True Positives / (True Positives + False negatives)
    f1_score  = 2 * (Precision * Recall) / (Precision + Recall)

    Exact Match Accuracy, which is:
    1 - if answer == prediction
    0 otherwise

    Both measures of accuracy disregard:
    Capitalization              - "CAT"     == "cat"
    Punctuation                 - ".cat,"   == "cat"
    Articles ("a", "an", "the") - "the cat" == "cat"
    White space                 - "   cat " == "cat"
    """

    # Create the inverse dictionary, which maps numbers to (string) words
    inv_dictionary = dict(zip(word_dictionary.values(), word_dictionary.keys()))

    # Initialize batch f1 score
    f1_score_batch = 0
    exact_match_batch = 0

    # Loop over examples in the batch, calculate f1 score for each and accumulate it
    for i in range(answer_array.shape[0]):
        current_answer_indices = answer_array[i]
        current_prediction_indices = predicted_answer_array[i]
        current_document = document_array[i]

        current_answer = current_document[current_answer_indices[0]:current_answer_indices[1]+1]
        current_answer_string = " ".join([inv_dictionary[words] for words in current_answer])

        current_prediction = \
            current_document[current_prediction_indices[0]:current_prediction_indices[1] + 1]
        current_prediction_string = " ".join([inv_dictionary[words] for words in
                                              current_prediction])

        f1_score_batch += compute_f1(current_answer_string, current_prediction_string)
        exact_match_batch += compute_exact(current_answer_string, current_prediction_string)

        # if i == 0:
        #     inv_document = " ".join([inv_dictionary[words] for words in current_document])
        #     print("\n\n{:-^100}".format(" DEBUG {} ".format(i)))
        #     print("doc, answer and prediction numbers: {}, {}, {}".format(current_document,
        #                                                                   current_answer,
        #                                                                   current_prediction))
        #     # print("Document: {}".format(inv_document))
        #     print("Answer string: {}".format(current_answer_string))
        #     print("Prediction string: {}\n".format(current_prediction_string))
        #     print("F1 and EM accuracy: {}, {}".format(compute_f1(current_answer_string,
        #                                                          current_prediction_string),
        #                                               compute_exact(current_answer_string,
        #                                                             current_prediction_string)))

    # Calculate averages for exact match accuracy and f1 score by dividing with the
    # number of examples in the current batch

    f1_score_batch /= document_array.shape[0]
    exact_match_batch /= document_array.shape[0]

    return f1_score_batch, exact_match_batch

# def calculate_f1_score(answer_array, predicted_answer_array, document_mask):
#     """
#     This method calculates the F1 score over the input batch.
#     Inputs:
#     answer_array - The start and end indices of the ground truth answers
#     predicted_answer_array - The start and end indices of the predicted answers
#     document_mask - The mask of document tokens over the current batch of documents
#     Output:
#     f1_score - The harmonic mean of precision and recall over the current batch, where:
#     Precision = True Positives / (True Positives + False Positives)
#     Recall    = True Positives / (True Positives + False negatives)
#     f1_score  = 2 * (Precision * Recall) / (Precision + Recall)
#     """
#     # Initialize counters for true positives, false positives and false negatives
#     TP = FP = FN = 0
#
#     # Loop over the current batch examples
#     for i in range(answer_array.shape[0]):
#         current_answer_indices = answer_array[i]
#         current_prediction_indices = predicted_answer_array[i]
#
#         # Create answer and prediction masks
#         answer_mask = np.zeros(shape=(document_mask.shape[1]))
#         answer_mask[current_answer_indices[0]:current_answer_indices[1] + 1] = 1
#
#         # TODO: Check if this works as intended in logged accuracies
#         # If prediction end index is lower than start, flip them around
#         if current_prediction_indices[1] < current_prediction_indices[0]:
#             current_prediction_indices[0], current_prediction_indices[1] = \
#                 current_prediction_indices[1], current_prediction_indices[0]
#
#         prediction_mask = np.zeros(shape=(document_mask.shape[1]))
#         prediction_mask[current_prediction_indices[0]:current_prediction_indices[1] + 1] = 1
#
#         # Calculate TP, FP and FN for the current example in the batch
#         # True Positives, the count of overlapping 1's in answer and prediction
#         current_TP = np.count_nonzero(prediction_mask * answer_mask)
#         # False Positives, the count of indices where predictions is 1 but answer is 0
#         current_FP = np.count_nonzero(prediction_mask * (answer_mask - 1))
#         # False Negatives, the count of indices where prediction is 0 but answer is 1
#         current_FN = np.count_nonzero((prediction_mask - 1) * answer_mask)
#
#         # Increment counters for the whole batch
#         TP += current_TP
#         FP += current_FP
#         FN += current_FN
#
#     # Calculate precision and recall over the whole batch
#     # If conditions are for handling zero division cases
#     # TODO: Print answer and prediction mask when one of these conditions is true
#     if (TP + FP) == 0:
#         print("\nTP+FP is 0!")
#         print("Answer and Prediction indices of current batch: {}\n{}\n{:-^100}".format(
#             answer_array, predicted_answer_array, " End of example "))
#         precision = 0
#     else:
#         precision = TP / (TP + FP)
#
#     if (TP + FN) == 0:
#         print("\nTP+FN is 0!")
#         print("Answer and Prediction indices of current batch: {}\n{}\n{:-^100}".format(
#             answer_array, predicted_answer_array, " End of example "))
#         recall = 0
#     else:
#         recall = TP / (TP + FN)
#
#     # Calculate F1 score
#     if (precision + recall) == 0:
#         f1_score = 0
#     else:
#         f1_score = 2 * (precision * recall) / (precision + recall)
#
#     return f1_score

def load_word2vec_embeddings(dictionary, vocab_embed_file):
    if vocab_embed_file is None:
        return None, EMBED_DIM

    fp = open(vocab_embed_file, encoding='utf-8')

    info = fp.readline().split()
    embed_dim = int(info[1])
    # vocab_embed: word --> vector
    vocab_embed = {}
    for line in fp:
        line = line.split()
        vocab_embed[line[0]] = np.array(
            list(map(float, line[1:])), dtype='float32')
    fp.close()

    vocab_size = len(dictionary)
    W = np.random.randn(vocab_size, embed_dim).astype('float32')
    n = 0
    for word, value in dictionary.items():
        if word in vocab_embed:
            W[value, :] = vocab_embed[word]
            n += 1
    print("{}/{} vocabs are initialized with word2vec embeddings."
          .format(n, vocab_size))
    return W, embed_dim

def check_dir(*args, exit_function=False):
    """
    check the existence of directories
    Args:
    - args: (list) paths of directories
    - exit_function: (bool) action to take
    """
    for dir_ in args:
        if not os.path.exists(dir_):
            if not exit_function:
                os.makedirs(dir_)
            else:
                raise ValueError("{} does not exist!".format(dir_))

# def prepare_input(document_array, query_array):
#     f = np.zeros(document_array.shape[:2]).astype('int32')
#     for i in range(document_array.shape[0]):
#         f[i, :] = np.in1d(document_array[i, :, 0], query_array[i, :, 0])
#     return f


#! /usr/bin/env python

import sys
import os
import pygame
from pygame.locals import *

sys.path.append(os.path.join(os.path.dirname(__file__), os.path.pardir))

import game_lib

'''
def load_image(image_path):

    from PIL import Image
    from PIL import ImageDraw
    image = Image.open(image_path)
    draw = ImageDraw.Draw(image)
    mode = image.mode
    size = image.size
    data = image.tobytes()
    return pygame.image.fromstring(data, size, mode)
'''

pygame.init()

FPS = 30 # frames per second setting
fpsClock = pygame.time.Clock()

# set up the window
DISPLAYSURF = pygame.display.set_mode((400, 300), 0, 32)
pygame.display.set_caption('Animation')

WHITE = (255, 255, 255)

#catImg = pygame.image.load('cat.png')
catImg = game_lib.load_image('cat.png')
catx = 10
caty = 10
direction = 'right'

while True: # the main game loop
    DISPLAYSURF.fill(WHITE)

    if direction == 'right':
        catx += 5
        if catx == 280:
            direction = 'down'
    elif direction == 'down':
        caty += 5
        if caty == 220:
            direction = 'left'
    elif direction == 'left':
        catx -= 5
        if catx == 10:
            direction = 'up'
    elif direction == 'up':
        caty -= 5
        if caty == 10:
            direction = 'right'

    DISPLAYSURF.blit(catImg, (catx, caty))

    for event in pygame.event.get():
        if event.type == QUIT:
            pygame.quit()
            sys.exit()

    pygame.display.update()
    fpsClock.tick(FPS)


import numpy as np
import re
import time
import utils
from bpe import BPE

bpe = BPE()
bpe.load('data/words.bpe')
endToken = bpe.str_to_token['\n']
numTokens = len(bpe.str_to_token)

raw_data = utils.get_dataset(low_it=True)

SAVE_NAME = 'data/train_data_bpe_edition.npz'

DATA_TO_PROCESS = int(input('data to process, {} max? '.format(len(raw_data))))
BATCH_SIZE = int(input('batch size? '))

if DATA_TO_PROCESS > len(raw_data):
	DATA_TO_PROCESS = len(raw_data)

vocab = utils.vocab

xi, xp, y = utils.encodeData(raw_data[:DATA_TO_PROCESS], bpe, endToken)

print ('to start processing {} of data, press any key'.format(len(xi)))
input()
print ('{:=^40}'.format(' starting '))

start = time.time()

x1_shape, x2_shape, y_shape = utils.encoded2xy(xi[:1], xp[:1], y[:1], endToken, numTokens)

Xp = np.zeros(x1_shape.shape, dtype=np.float32)
Xu = np.zeros(x2_shape.shape, dtype=np.float32)
Y = np.zeros(y_shape.shape, dtype=np.float32)

while len(xi) > 0:
	X_past_on_batch, X_user_on_batch, Y_on_batch = utils.encoded2xy(xi[:BATCH_SIZE], xp[:BATCH_SIZE], y[:BATCH_SIZE], endToken, numTokens)

	#print (X.shape, X_on_batch.shape)

	Xp = np.concatenate((Xp, X_past_on_batch), axis=0)
	Xu = np.concatenate((Xu, X_user_on_batch), axis=0)
	Y = np.concatenate((Y, Y_on_batch), axis=0)

	#print (X, Y)
	print ('data loaded:', Xp[1:].shape, Xu[1:].shape, Y[1:].shape)
	print ('data available:', len(xi), len(xp), len(y))
	xi = xi[BATCH_SIZE:]
	xp = xp[BATCH_SIZE:]
	y = y[BATCH_SIZE:]

utils.save_train_data2(SAVE_NAME, Xp[1:], Xu[1:], Y[1:])
print ('{:=^40}'.format(' data saved '))

print ('data saved:', Xp[1:].shape, Xu[1:].shape, Y[1:].shape)
print ('data left:', len(xi), len(xp), len(y))

print ('total time: {:.4f}s'.format(time.time()-start))

print ('(っ・ω・）っ≡≡≡≡≡≡☆')


"""Implementation referenced from ESIPy under BSD-3-Clause License"""
import hashlib
import inspect
import logging
import pickle
from email.utils import parsedate
from dataclasses import dataclass
from datetime import datetime, timedelta
from typing import Coroutine, Union, Callable

from eve_tools.data.db import ESIDBManager

logger = logging.getLogger(__name__)

def hash_key(key) -> str:
    """Default hashing function for a key. Using sha256 as hash function."""
    return "esi_cache-" + hashlib.sha256(pickle.dumps(key)).hexdigest()

class _CacheRecordBaseClass:
    """Base class holding all cache instances.
    Provides stats over all instances."""

    instances = set()

    @property
    def record(self):
        ret = [_cache.record for _cache in self.instances]
        return ret

CacheStats = _CacheRecordBaseClass()

@dataclass
class _CacheRecord:
    """Records hits and misses of a cache instance."""

    caller: str = None
    id: int = 0
    hits: int = 0
    miss: int = 0

class BaseCache(_CacheRecordBaseClass):
    """Specifies BaseCache object used by other caching implimentation.

    Requires set and get method for api(s) to access the cache.
    evict is useful for testing. Expired entries should be deleted in get or set,
    so user would not be required to call evict all the time.
    User of BaseCache should call super().__init__() to register instance under cache stats.
    """

    def __init__(self):
        self.__class__.instances.add(self)
        self._record = _CacheRecord()
        module = inspect.getmodule(inspect.stack(0)[2][0])
        if module is not None:
            module = module.__name__
        self._record.caller = module
        self._record.id = id(self)

    def set(self, key, value, expires):
        raise NotImplementedError

    def get(self, key, default):
        raise NotImplementedError

    def evict(self, key):
        raise NotImplementedError

    @property
    def record(self):
        return self._record

    @property
    def hits(self):
        return self._record.hits

    @property
    def miss(self):
        return self._record.miss

    @hits.setter
    def hits(self, val):
        self._record.hits = val

    @miss.setter
    def miss(self, val):
        self._record.miss = val

class SqliteCache(BaseCache):
    """A Sqlite implementation of cache.

    Each api cache entry is stored in a single line in cache.db.
    The value is serialized/deserialized using pickle.
    Expired entries are deleted in get().
    """

    def __init__(self, esidb: ESIDBManager, table: str):
        self.c = esidb
        self.table = table

        self._last_used = None  # used for testing
        super().__init__()

    def set(self, key, value, expires: Union[str, int] = None):
        """Sets k/v pair with expires.

        Args:
            key: An object returned from make_cache_key(), which is pickled and hashed using hash_key().
            value: An object to put into cache. Needs to be pickle(able).
            expires: A str or int.
                If a string is provided, it should be in any standard datetime format.
                ESI returns a datetime string in RFC7231 format.
                If an int is provided, it should be a custom expire threshold in seconds, such as 1200 for 20 minutes.
                If not provided, default expire in 20 minutes.
        """
        if not expires:
            expires = datetime.utcnow().replace(microsecond=0) + timedelta(seconds=1200)
        elif isinstance(expires, int):
            expires = datetime.utcnow().replace(microsecond=0) + timedelta(
                seconds=expires
            )
        else:
            expires = datetime(*parsedate(expires)[:6])

        _h = hash_key(key)
        self.c.cursor.execute(
            f"INSERT OR REPLACE INTO {self.table} VALUES(?,?,?)",
            (_h, pickle.dumps(value), expires),
        )
        self.c.conn.commit()
        logger.info("Cache entry set: %s", _h)

    def get(self, key, default=None):
        """Gets the value from cache.

        Attempts to return value with key from cache. A default value is returned if unsuccessful.
        Deletes all expired keys everytime get() is called.

        Args:
            key: An object returned from make_cache_key(), which is pickled and hashed using hash_key().
            default: If cache returns nothing, returns a default value. Default None.
        """
        self.c.cursor.execute(f"DELETE FROM {self.table} WHERE expires < DATE('now')")
        self.c.conn.commit()

        _h = hash_key(key)
        row = self.c.cursor.execute(
            f"SELECT * FROM {self.table} WHERE key=?", (_h,)
        ).fetchone()
        if not row:
            logger.info("Cache MISS: %s", _h)
            self.miss += 1
            return default

        expires = datetime.strptime(row[2], "%Y-%m-%d %H:%M:%S")
        if datetime.utcnow() > expires:
            logger.info("Cache MISS: %s", _h)
            self.miss += 1
            return default  # expired
        else:
            self._last_used = _h
            logger.info("Cache HIT: %s", _h)
            self.hits += 1
            return pickle.loads(row[1])  # value

    def evict(self, key):
        """Deletes cache entry with key. Useful in testing."""
        _h = hash_key(key)
        self.c.cursor.execute(f"DELETE FROM {self.table} WHERE key=?", (_h,))
        self.c.conn.commit()
        logger.info("Cache entry evicted: %s", _h)

def make_cache_key(func: Union[Callable, Coroutine], *args, **kwd):
    """Hashes a function and its arguments using sha256.

    The function is hashed using function_hash() to a string.
    The args and kwds are pickled to bytes.
    If any of args or kwd is a function, this function encodes them by inspecting source code.
    If the source code is changed, this function encodes a different value.

    Args:
        func: A function or a coroutine.
        args: Argument passed to the api function in argument form.
        kwd: Keyword arguments passed to the api function.

    Returns:
        A set containing (function_hash, str(args), str(kwd))

    Examples:
    >>> # Encode: get_market_history("The Forge", reduces=reduce_volume)
    >>> key_before = make_cache_key(get_market_history, "The Forge", reduce_volume)
    >>> # If reduce_volume is changed, even if a space or an extra line is added
    >>> key_after = make_cache_key(get_market_history, "The Forge", reduce_volume)
    >>> assert key_before != key_after
    """
    func_args = list(args)
    func_kwd = kwd.copy()
    for i in range(len(func_args)):
        if isinstance(func_args[i], Callable):
            func_args[i] = function_hash(func_args[i])
        if isinstance(func_args[i], list):
            func_args[i] = list(set(func_args[i]))
    for k in func_kwd:
        if isinstance(func_kwd[k], Callable):
            func_kwd[k] = function_hash(func_kwd[k])
        if isinstance(func_kwd[k], list):
            func_kwd[k] = list(set(func_kwd[k]))
    _h = function_hash(func)

    ret = (_h, pickle.dumps(func_args), pickle.dumps(func_kwd))
    logger.info("Making cache key for %s", func.__qualname__)
    return ret

def function_hash(func: Union[Callable, Coroutine]):
    """Hashes a function.

    Hashes a function based on its source code. If the source code is modified in any way,
    this function hashes to a different value.
    """
    return "esi_function-{}-{}".format(
        func.__qualname__,
        hashlib.sha256(inspect.getsource(func).encode("utf-8")).hexdigest(),
    )


from django.shortcuts import render
from django.contrib.auth.decorators import login_required
from django.http import HttpResponse, HttpResponseRedirect, Http404
from intranet.decorators import role_required
from .models import *
from django.urls import reverse
from django.contrib import messages
import csv
from .forms import *
from .filters import answers_filter

# Create your views here.
@login_required(login_url="/login/")

def index(request):
    context = {}
    placement = Placements.objects.all()
    context['title'] = 'Companies'
    context['placement'] = placement
    return render(request, 'control/placead.html', context)

@login_required(login_url="/login/")
def details(request, id=None):
    if request.method == "GET":
        try:
            placement = Placements.objects.get(id=id)
        except:
            raise Http404
        form =ans_form()
        context = {}
        context['placement'] = placement
        context['form'] = form
        return render(request, 'student/detail.html', context)
    if request.method == "POST":
        user_id = request.user
        form = ans_form(request.POST)
    if form.is_valid():
        post = form.save()
        post.save()
        messages.success(request, "Successful")

def answer(request, id=None):
    if request.method == "GET":
        placement = Placements.objects.get(id=id)
        answer = Answer.objects.all()
        myFilter = answers_filter(request.GET, queryset=answer)
        answer = myFilter.qs
        context ={}
        context['myfilter'] = myFilter
        context['answer'] = answer
        context['user'] = request.user
        return render(request, 'control/answer.html', context)

def answer_backup(request):
    if request.method == "POST":
        response = HttpResponse(content_type='text/csv')
    writer = csv.writer(response)
    writer.writerow(['Usernumber', 'URN', 'First Name', 'Last Name', 'Company'])
    for ans in Answer.objects.all().values_list('placement','user','username', 'firstname', 'lastname'):
        writer.writerow(ans)
    response['Content-Disposition'] = 'attachment; filename="student_names.csv"'
    return response

@login_required(login_url="/login/")
def single_choice(request, id=None):
    if request.method == 'POST':
        form = ans_form(request.POST)
        if form.is_valid():
            post = form.save()
            post.save()
            return HttpResponseRedirect(reverse('placement'))
    else:
        form = ans_form()
    return render(request, 'student/detail.html', {'form': form})

@login_required(login_url="/login/")
@role_required(allowed_roles = "Admin")
def add_placement(request):
    return render(request, 'control/add.html')

@login_required(login_url="/login/")
@role_required(allowed_roles = "Admin")
def create_placement(request):
    title = request.POST["title"]
    info = request.POST["info"]
    creator = request.user
    start_date = request.POST["start_date"]
    end_date = request.POST["end_date"]
    start_time = request.POST["start_time"]
    end_time = request.POST["end_time"]

    placement_info = Placements(title=title, information=info, created_by=creator,
    start_date=start_date, end_date=end_date, start_time=start_time, end_time=end_time)
    placement_info.save()
    messages.success(request, "Successful")
    return render(request, 'control/add.html')

@login_required(login_url="/login/")
@role_required(allowed_roles = "Admin")
def placement_delete(request, id=None):
    placement = Placements.objects.get(id=id)

    if request.method == 'POST':
        placement.delete()
        messages.success(request, "Successful")
        return HttpResponseRedirect(reverse('placement'))
    else:
        context = {}
        context['placement'] = placement
        return render(request, 'control/delete.html', context)

def add_choice(request):
    if request.method == 'POST':
        form = choice_form(request.POST)
        if form.is_valid():
            post = form.save()
            post.save()
            return HttpResponseRedirect(reverse('add_choice'))
    else:
        form = choice_form()
    return render(request, 'control/add_choice.html', {'form': form})


"""File containing tests specific for the djWasabi/container.py file."""

import sys
import os
import requests
import pytest

currentPath = os.path.dirname(os.path.realpath(__file__))
rootPath = os.path.join(currentPath, "..")
libraryDir = os.path.join(rootPath, "lib")
sys.path.append(libraryDir)
from djWasabi import djWasabi

def test_validateDockerRunning():
    """Test if Docker is running.
    :returns:
    """
    assert djWasabi.container.validateDockerRunning()

def test_container_getValueArg_owner():
    """Test the to execute command to do an ls
    :return:
    """
    value = "owner"
    owner = "dj-wasabi"
    repository = "dj-wasabi-release"
    output = djWasabi.container.getValueArg(
        value=value,
        owner=owner,
        repository=repository
    )
    assert output == "dj-wasabi"

def test_container_getValueArg_repository():
    """Test the to execute command to do an ls
    :return:
    """
    value = "repository"
    owner = "dj-wasabi"
    repository = "dj-wasabi-release"
    output = djWasabi.container.getValueArg(
        value=value,
        owner=owner,
        repository=repository
    )
    assert output == "dj-wasabi-release"

def test_container_getValueArg_none():
    """Test the to execute command to do an ls
    :return:
    """
    value = "notexisting"
    owner = "dj-wasabi"
    repository = "dj-wasabi-release"
    output = djWasabi.container.getValueArg(
        value=value,
        owner=owner,
        repository=repository
    )
    assert output is None

def test_container_getValueArg_version():
    """Test the to execute command to do an ls
    :return:
    """
    value = "version"
    owner = "dj-wasabi"
    repository = "dj-wasabi-release"
    version = "1.2.3"
    output = djWasabi.container.getValueArg(
        value=value,
        owner=owner,
        repository=repository,
        version=version
    )
    assert output == "1.2.3"

def test_container_createContainerCommand():
    """Test the docker run with only Docker image.
    :return:
    """
    configuration = {
        "image": "dj-wasabi/consul"
    }
    owner = "dj-wasabi"
    repository = "dj-wasabi-release"
    container = djWasabi.container.createContainerCommand(
        configuration=configuration,
        owner=owner,
        repository=repository
    )
    output = djWasabi.generic.getString(data=container)
    assert output == "docker run --rm dj-wasabi/consul"

def test_container_createContainerCommand_environment():
    """Test the docker run with only Docker image.
    :return:
    """
    configuration = {
        "image": "dj-wasabi/consul",
        "environment": ["DJWASABI"]

    }
    owner = "dj-wasabi"
    repository = "dj-wasabi-release"
    container = djWasabi.container.createContainerCommand(
        configuration=configuration,
        owner=owner,
        repository=repository
    )
    output = djWasabi.generic.getString(data=container)
    assert output == "docker run --rm -e DJWASABI=test dj-wasabi/consul"

def test_container_createContainerCommand_volumes():
    """Test the docker run with only Docker image.
    :return:
    """

    configuration = {
        "image": "dj-wasabi/consul",
        "volumes": {
            "PWD": "/data",
            "/data": "/data"
        }

    }
    owner = "dj-wasabi"
    repository = "dj-wasabi-release"
    container = djWasabi.container.createContainerCommand(
        configuration=configuration,
        owner=owner,
        repository=repository
    )
    output = djWasabi.generic.getString(data=container)
    value = "docker run --rm -v {v}:/data -v /data:/data dj-wasabi/consul".format(v=os.getcwd())
    assert output == value

def test_container_createContainerCommand_noImage():
    """Test the _delete function without providing url.
    :return:
    """
    configuration = {
        "volumes": {
            "PWD": "/data",
            "/data": "/data"
        }

    }
    owner = "dj-wasabi"
    repository = "dj-wasabi-release"
    with pytest.raises(ValueError, match="Please provide the Docker image."):
        djWasabi.container.createContainerCommand(
            configuration=configuration,
            owner=owner,
            repository=repository
        )

def test_container_createContainerCommand_arguments():
    """Test the docker run with provided arguments.
    :return:
    """

    configuration = {
        "image": "dj-wasabi/consul",
        "arguments": {
            "repository": "-p"
        }

    }
    owner = "dj-wasabi"
    repository = "dj-wasabi-release"
    container = djWasabi.container.createContainerCommand(
        configuration=configuration,
        owner=owner,
        repository=repository
    )
    output = djWasabi.generic.getString(data=container)
    value = "docker run --rm dj-wasabi/consul -p {r}".format(r=repository)
    assert output == value