import tensorflow as tf
import metadata
import preprocess
from tensorflow.python.feature_column import feature_column

def create_feature_columns():
    """Creates tensorflow feature_column definitions based on the metadata of the features.

    the tensorflow feature_column objects are created based on the data types of the features
    defined in the metadata.py module. Extended featured (if any) are created, based on the base features,
    as the preprocess.extend_feature_columns method is called.

    Returns:
      {string: tf.feature_column}: dictionary of name:feature_column .
    """

    numeric_columns = {feature_name: tf.feature_column.numeric_column(feature_name)
                       for feature_name in metadata.NUMERIC_FEATURE_NAMES}

    categorical_column_with_vocabulary = {item[0]: tf.feature_column.categorical_column_with_vocabulary_list(item[0], item[1])
                                          for item in metadata.CATEGORICAL_FEATURE_NAMES_WITH_VOCABULARY.items()}

    categorical_column_with_hash_bucket = {item[0]:tf.feature_column.categorical_column_with_hash_bucket(item[0], item[1])
                                           for item in metadata.CATEGORICAL_FEATURE_NAMES_WITH_HASH_BUCKET.items()}

    feature_columns = {}

    if numeric_columns is not None:
        feature_columns.update(numeric_columns)

    if categorical_column_with_vocabulary is not None:
        feature_columns.update(categorical_column_with_vocabulary)

    if categorical_column_with_hash_bucket is not None:
        feature_columns.update(categorical_column_with_hash_bucket)

    # add extended feature definitions before returning the feature_columns list
    return preprocess.extend_feature_columns(feature_columns)

def get_deep_and_wide_columns(feature_columns, embedding_size=0, use_indicators=True):
    """Creates deep and wide feature column lists.

    given a list of feature_column, each feature_column is categorised as either:
    1) dense, if the column is tf.feature_column._NumericColumn,
    2) categorical, if the column is tf.feature_column._VocabularyListCategoricalColumn or tf.feature_column._BucketizedColumn, or
    3) sparse, if the column is tf.feature_column._HashedCategoricalColumn or tf.feature_column._CrossedColumn.

    if use_indicators=True, then  categorical_columns are converted into indicator_column.
    if embedding_size > 0, then sparse_columns are converted to tf.feature_column.embedding_column (using the embedding_size).

    deep_columns = dense_columns + indicator_columns + embedding_columns
    wide_columns = categorical_columns + sparse_columns

    Args:
        feature_columns: [tf.feature_column] - A list of tf.feature_column objects.
        embedding_size: int - if greater than 0, then sparse_columns are converted to tf.feature_column.embedding_column.
        use_indicators: bool - if True, then categorical_columns are converted into tf.feature_column.indicator_column.
    Returns:
        [tf.feature_column],[tf.feature_column]: deep and wide feature_column lists.
    """
    dense_columns = list(
        filter(lambda column: isinstance(column, feature_column._NumericColumn),
               feature_columns
        )
    )

    categorical_columns = list(
        filter(lambda column: isinstance(column,feature_column._VocabularyListCategoricalColumn) |
                              isinstance(column, feature_column._BucketizedColumn),
                   feature_columns)
    )

    sparse_columns = list(
        filter(lambda column: isinstance(column,feature_column._HashedCategoricalColumn) |
                              isinstance(column, feature_column._CrossedColumn),
               feature_columns)
    )

    indicator_columns = []

    if use_indicators:

        indicator_columns = list(
            map(lambda column: tf.feature_column.indicator_column(column),
                categorical_columns)
        )

    embedding_columns = []

    if embedding_size > 0:

        embedding_columns = list(
            map(lambda sparse_column: tf.feature_column.embedding_column(sparse_column, dimension=embedding_size),
                sparse_columns)
        )

    deep_columns = dense_columns + indicator_columns + embedding_columns
    wide_columns = categorical_columns + sparse_columns

    return deep_columns, wide_columns


# from PIL import ImageGrab
import pyautogui
from time import sleep

class lab4:
	def __init__(self):
		self.coordinates = {
			'time': ((1305, 328), (1437, 358)),
			'mass': ((1395, 391), (1320, 391)),
			'radius': ((1395, 450), (1320, 450)),
			'height': ((1395, 506), (1320, 506)),
			'start': (1420, 569),
			'stop': (1420, 623),
			'reset': (1420, 679),
			'googleedit': (674, 275),
		}

	# def get_screen(self, rect):
	# 	return ImageGrab.grab(rect)

	def click(self, coords):
		pyautogui.moveTo(coords)
		sleep(0.1)
		pyautogui.click()
		sleep(0.1)

	def press(self, *keys):
		pyautogui.hotkey(*keys, interval=0.1)
		sleep(0.1)

	def highlight(self, rect):
		pyautogui.moveTo(rect[0][0], rect[0][1])
		sleep(0.5)
		pyautogui.dragTo(rect[1][0], rect[1][1])
		sleep(0.1)

	def erase(self, rect):
		self.highlight(rect)
		self.press('backspace')

	def copy(self, rect):
		self.highlight(rect)
		self.press('ctrl', 'c')

	def write(self, string):
		for char in string:
			self.press(char)

	def save(self, newcolumn=False, newgroup=False):
		self.press('ctrl', 'shift', 'tab')
		self.press('ctrl', 'v')

		self.press('enter')
		self.press('backspace')
		self.press('backspace')
		self.press('backspace')
		self.press('home')
		self.press('delete')
		self.press('delete')

		self.press('enter')

		if newcolumn:
			self.press('right')
			for _ in range(5):
				self.press('up')

		if newgroup:
			self.press('down')
			for _ in range(5):
				self.press('left')

		self.press('ctrl', 'tab')

	def solve(self):
		self.erase(self.coordinates['height'])
		self.write('0.7')

		masses = ['0.1', '0.3', '0.6', '0.9']
		radiuses = ['0.07', '0.1', '0.13', '0.16', '0.19', '0.22']
		sleeps = [
			[10, 12, 14, 16, 17.5, 21],
			[4.5, 5.5, 6.5, 7.5, 9, 10],
			[3.2, 4.2, 4.5, 5.3, 6, 6.8],
			[2.6, 3.2, 3.7, 4.3, 4.8, 5.5]
		]

		for m, mass in enumerate(masses):
			self.erase(self.coordinates['mass'])
			self.write(mass)

			for h, radius in enumerate(radiuses):
				self.erase(self.coordinates['radius'])
				self.write(radius)

				for i in range(5):
					self.click(self.coordinates['start'])
					sleep(sleeps[m][h])

					self.copy(self.coordinates['time'])
					self.save(newcolumn=(i == 4 and radius != radiuses[-1]),
							  newgroup=(radius == radiuses[-1] and i == 4))

					self.click(self.coordinates['reset'])

if __name__ == '__main__':
	laba = lab4()
	sleep(3)
	laba.solve()


from __future__ import absolute_import, division, generators, nested_scopes, print_function, unicode_literals, with_statement

from django.contrib.auth import login
from django.core.urlresolvers import reverse
from django.http import HttpResponseRedirect, Http404
from django.shortcuts import get_object_or_404
from django.utils.safestring import mark_safe
from django.utils.translation import ugettext_lazy as _

from ..forms.clubs import ClubForm, ClubFilterForm, ClubRegistrationForm, ClubRegistrationPublicForm, ClubJournalEntryForm, ClubJournalLeaderEntryForm
from ..models import Club, ClubPeriod, ClubJournalEntry, ClubJournalLeaderEntry, ClubRegistration, Participant

from .generic import FilteredListView, DetailView, CreateView, UpdateView, ConfirmUpdateView, DeleteView, TemplateView, PdfView

class ClubListView(FilteredListView):
    model               = Club
    form_class          = ClubFilterForm
    preview_template    = 'domecek/club_preview.html'
    template_name       = 'domecek/club_list.html'
    message_empty       = _('No clubs matching given filter.')
    paginate_by         = 10

    def get_title(self):
        return _('Clubs in school year {}').format(self.request.school_year)

    def get_queryset(self):
        qs = super(ClubListView, self).get_queryset()
        qs = qs.filter(school_year=self.request.school_year)
        if not self.request.user.is_staff:
            qs = qs.filter(public=True)
        form = self.get_form()
        if form.is_valid():
            qs = form.filter_queryset(self.request, qs)
        return qs

class ClubListMineView(ClubListView):
    def get_queryset(self):
        return super(ClubListMineView, self).get_queryset().filter(leaders=self.request.leader)

    def get_title(self):
        return _('My clubs in school year {}').format(self.request.school_year)

class ClubAlternatingView(TemplateView):
    template_name       = 'domecek/club_alternating.html'

    def get_title(self):
        return _('Alternating in school year {}').format(self.request.school_year)

    def get_context_data(self, **kwargs):
        context = super(ClubAlternatingView, self).get_context_data(**kwargs)
        context['alternate_leader_entries'] = self.request.leader.get_alternate_leader_entries(self.request.school_year)
        return context

class ClubDetailView(DetailView):
    model = Club

    def get_queryset(self):
        qs = super(ClubDetailView, self).get_queryset()
        if not self.request.user.is_staff:
            qs = qs.filter(public=True)
        return qs

class ClubParticipantsView(DetailView):
    model = Club
    template_name_suffix = '_participants'

    def get_queryset(self):
        qs = super(ClubParticipantsView, self).get_queryset()
        if not self.request.user.is_staff:
            qs = qs.filter(leaders=self.request.leader)
        return qs

class ClubJournalView(DetailView):
    model = Club
    template_name_suffix = '_journal'

    def get_queryset(self):
        qs = super(ClubJournalView, self).get_queryset()
        if not self.request.user.is_staff:
            qs = qs.filter(leaders=self.request.leader)
        return qs

class ClubUpdateView(UpdateView):
    model       = Club
    form_class  = ClubForm
    title = _('Change club')

    def get_queryset(self):
        qs = super(ClubUpdateView, self).get_queryset()
        if not self.request.user.is_staff:
            qs = qs.filter(leaders=self.request.leader)
        return qs

    def get_message(self, form):
        return _('The club {} has been updated.').format(self.object)

class ClubJournalEntryCreateView(CreateView):
    model       = ClubJournalEntry
    form_class  = ClubJournalEntryForm
    template_name = 'domecek/clubjournalentry_form.html'
    title = _('New journal entry')

    def dispatch(self, request, *args, **kwargs):
        if self.request.user.is_staff:
            self.club = get_object_or_404(Club,
                id = int(kwargs.pop('club')),
            )
        else:
            self.club = get_object_or_404(Club,
                id = int(kwargs.pop('club')),
                leaders = self.request.leader,
            )
        return super(ClubJournalEntryCreateView, self).dispatch(request, *args, **kwargs)

    def get_form_kwargs(self):
        kwargs = super(ClubJournalEntryCreateView, self).get_form_kwargs()
        kwargs['club'] = self.club
        return kwargs

    def get_message(self, form):
        return _('The journal entry has been created.')

class ClubJournalEntryUpdateView(UpdateView):
    model       = ClubJournalEntry
    form_class  = ClubJournalEntryForm
    template_name = 'domecek/clubjournalentry_form.html'
    title = _('Change journal entry')

    def get_object(self):
        obj = super(ClubJournalEntryUpdateView, self).get_object()
        if (self.request.user.is_staff
            or self.request.leader in obj.club.all_leaders + obj.all_alternates):
            return obj
        else:
            raise Http404()

    def get_message(self, form):
        return _('The journal entry has been updated.')

class ClubJournalEntryDeleteView(DeleteView):
    model = ClubJournalEntry
    title = _('Delete journal entry')

    def get_queryset(self):
        return super(ClubJournalEntryDeleteView, self).get_queryset().filter(
            club__leaders = self.request.leader,
        )

    def get_object(self):
        obj = super(ClubJournalEntryDeleteView, self).get_object()
        if obj.timesheets.filter(submitted = True).exists():
            raise Http404()
        return obj

    def get_question(self):
        return _('Do You really want to delete club journal entry?')

    def get_message(self):
        return _('The journal entry has been deleted.')

class ClubJournalLeaderEntryUpdateView(UpdateView):
    model       = ClubJournalLeaderEntry
    form_class  = ClubJournalLeaderEntryForm
    template_name = 'domecek/clubjournalleaderentry_form.html'
    title = _('Change timesheet entry')

    def get_object(self):
        obj = super(ClubJournalLeaderEntryUpdateView, self).get_object()
        if self.request.user.is_staff \
        or obj.timesheet.leader == self.request.leader \
        or self.request.leader in obj.club_entry.club.all_leaders:
            return obj
        else:
            raise Http404()

    def get_message(self, form):
        return _('The timesheet entry has been updated.')

class ClubJournalLeaderEntryDeleteView(DeleteView):
    model = ClubJournalLeaderEntry
    title = _('Delete timesheet entry')

    def get_queryset(self):
        return super(ClubJournalLeaderEntryDeleteView, self).get_queryset().filter(
            timesheet__leader = self.request.leader,
            timesheet__submitted = False,
        )

    def get_question(self):
        return _('Do You really want to delete timesheet entry?')

    def get_message(self):
        return _('The timesheet entry has been deleted.')

class ClubRegistrationPublicFormView(CreateView):
    model           = ClubRegistration
    form_class      = ClubRegistrationPublicForm
    template_name   = 'domecek/registration_form.html'

    def get_title(self):
        return _('Registration for club {}').format(self.club.name)

    def dispatch(self, request, *args, **kwargs):
        club_kwargs = {
            'id':           int(kwargs.pop('club')),
            'school_year':  self.request.school_year,
            'reg_active':   True,
        }
        if not self.request.user.is_staff:
            club_kwargs['public'] = True
        self.club = get_object_or_404(Club, **club_kwargs)
        if self.request.user.is_authenticated() and not self.request.toolbar.use_draft:
            return HttpResponseRedirect(reverse('domecek:club_detail', args=(self.club.id, )))
        return super(ClubRegistrationPublicFormView, self).dispatch(request, *args, **kwargs)

    def get_form_kwargs(self):
        kwargs  = super(ClubRegistrationPublicFormView, self).get_form_kwargs()
        kwargs['club'] = self.club
        return kwargs

    def form_valid(self, form):
        response = super(ClubRegistrationPublicFormView, self).form_valid(form)
        user = form.instance.participant.user
        user.backend = 'django.contrib.auth.backends.ModelBackend'
        login(self.request, user)
        return response

    def get_message(self, form):
        return _('The registration has been accepted.')

class ClubRegistrationFormView(CreateView):
    model       = ClubRegistration
    form_class  = ClubRegistrationForm

    def get_title(self):
        return _('Registration for club {}').format(self.club.name)

    def dispatch(self, request, *args, **kwargs):
        club_kwargs = {
            'id':           int(kwargs.pop('club')),
            'school_year':  self.request.school_year,
            'reg_active':   True,
        }
        if not self.request.user.is_staff:
            club_kwargs['public'] = True
        self.club = get_object_or_404(Club, **club_kwargs)
        self.participant = get_object_or_404(Participant,
            user    = self.request.user,
            id      = int(kwargs.pop('participant')),
        )
        # user may get back to this page after successful registration
        if self.club.registrations.filter(participant=self.participant).exists():
            return HttpResponseRedirect(reverse('domecek:summary'))
        return super(ClubRegistrationFormView, self).dispatch(request, *args, **kwargs)

    def get_form_kwargs(self):
        kwargs = super(ClubRegistrationFormView, self).get_form_kwargs()
        kwargs['club']          = self.club
        kwargs['participant']   = self.participant
        return kwargs

    def get_message(self, form):
        return _('The registration has been accepted.')

class ClubRegistrationConfirmView(DetailView):
    model = ClubRegistration
    template_name_suffix = '_confirm'

class ClubRegistrationPdfView(PdfView):
    model = ClubRegistration
    template_name_suffix = '_pdf'

class ClubRegistrationCancelView(ConfirmUpdateView):
    model   = ClubRegistration
    title   = _('Cancellation request')

    def get_queryset(self):
        return super(ClubRegistrationCancelView, self).get_queryset().filter(participant__user=self.request.user)

    def get_question(self):
        return _('Are you sure You want to cancel the registration "{}"?').format(self.object)

    def get_message(self, form):
        return _('The cancellation request for {} has been saved.').format(self.object)

    def confirmed(self):
        self.object.cancel_request = True
        self.object.save()


import os
import re

from citeproc import CitationStylesStyle, CitationStylesBibliography
from citeproc import Citation, CitationItem
from citeproc import formatter
from citeproc.source.json import CiteProcJSON

from website.preprints.model import PreprintService
from website.settings import CITATION_STYLES_PATH, BASE_PATH, CUSTOM_CITATIONS

def clean_up_common_errors(cit):
    cit = re.sub(r"\.+", '.', cit)
    cit = re.sub(r" +", ' ', cit)
    return cit

def display_absolute_url(node):
    url = node.absolute_url
    if url is not None:
        return re.sub(r'https?:', '', url).strip('/')

def preprint_csl(preprint, node):
    csl = node.csl

    csl['id'] = preprint._id
    csl['publisher'] = preprint.provider.name
    csl['URL'] = display_absolute_url(preprint)

    if csl.get('DOI'):
        csl.pop('DOI')

    doi = preprint.article_doi
    if doi:
        csl['DOI'] = doi

    return csl

def render_citation(node, style='apa'):
    """Given a node, return a citation"""
    csl = None
    if isinstance(node, PreprintService):
        csl = preprint_csl(node, node.node)
        data = [csl, ]
    else:
        data = [node.csl, ]

    bib_source = CiteProcJSON(data)

    custom = CUSTOM_CITATIONS.get(style, False)
    path = os.path.join(BASE_PATH, 'static', custom) if custom else os.path.join(CITATION_STYLES_PATH, style)
    bib_style = CitationStylesStyle(path, validate=False)

    bibliography = CitationStylesBibliography(bib_style, bib_source, formatter.plain)

    citation = Citation([CitationItem(node._id)])

    bibliography.register(citation)

    bib = bibliography.bibliography()
    cit = unicode(bib[0] if len(bib) else '')

    title = csl['title'] if csl else node.csl['title']
    if cit.count(title) == 1:
        i = cit.index(title)
        prefix = clean_up_common_errors(cit[0:i])
        suffix = clean_up_common_errors(cit[i + len(title):])
        cit = prefix + title + suffix
    elif cit.count(title) == 0:
        cit = clean_up_common_errors(cit)

    return cit


import flair
from flair.embeddings import WordEmbeddings
import sqlite3
import torch
import re
import os
from tqdm import tqdm

class WordEmbeddingsStore:
    """
    class to simulate a WordEmbeddings class from flair.

    Run this to generate a headless (without word embeddings) model as well a stored word embeddings:

    >>> from flair.inference_utils import WordEmbeddingsStore
    >>> from flair.models import SequenceTagger
    >>> import pickle
    >>> tagger = SequenceTagger.load("multi-ner-fast")
    >>> WordEmbeddingsStore.create_stores(tagger)
    >>> pickle.dump(tagger, open("multi-ner-fast-headless.pickle", "wb"))

    Then this can be used as follows:

    >>> from flair.data import Sentence
    >>> tagger = pickle.load(open("multi-ner-fast-headless.pickle", "rb"))
    >>> WordEmbeddingsStore.load_stores(tagger)
    >>> text = "Schade um den Ameisenbären. Lukas Bärfuss veröffentlicht Erzählungen aus zwanzig Jahren."
    >>> sentence = Sentence(text)
    >>> tagger.predict(sentence)
    >>> print(sentence.get_spans('ner'))
    """

    def __init__(self, embedding, verbose=True):
        # some non-used parameter to allow print
        self._modules = dict()
        self.items = ""

        # get db filename from embedding name
        self.name = embedding.name
        self.store_filename = WordEmbeddingsStore._get_store_filename(embedding)
        if verbose:
            print("store filename:", self.store_filename)

        # if embedding database already exists
        if os.path.isfile(self.store_filename):
            self.db = sqlite3.connect(self.store_filename)
            cursor = self.db.cursor()
            cursor.execute("SELECT * FROM embedding LIMIT 1;")
            result = list(cursor)
            self.k = len(result[0]) - 1
            return

        # otherwise, push embedding to database
        self.db = sqlite3.connect(self.store_filename)
        pwe = embedding.precomputed_word_embeddings
        self.k = pwe.vector_size
        self.db.execute(f"DROP TABLE IF EXISTS embedding;")
        self.db.execute(
            f"CREATE TABLE embedding(word text,{','.join('v'+str(i)+' float' for i in range(self.k))});"
        )
        vectors_it = (
            [word] + pwe.get_vector(word).tolist() for word in pwe.vocab.keys()
        )
        if verbose:
            print("load vectors to store")
        self.db.executemany(
            f"INSERT INTO embedding(word,{','.join('v'+str(i) for i in range(self.k))}) \
        values ({','.join(['?']*(1+self.k))})",
            tqdm(vectors_it),
        )
        self.db.execute(f"DROP INDEX IF EXISTS embedding_index;")
        self.db.execute(f"CREATE INDEX embedding_index ON embedding(word);")
        self.db.commit()
        self.db.close()

    def _get_vector(self, word="house"):
        db = sqlite3.connect(self.store_filename)
        cursor = db.cursor()
        word = word.replace('"','')
        cursor.execute(f'SELECT * FROM embedding WHERE word="{word}";')
        result = list(cursor)
        db.close()
        if not result:
            return [0.0] * self.k
        return result[0][1:]

    def embed(self, sentences):
        for sentence in sentences:
            for token in sentence:
                t = torch.tensor(self._get_vector(word=token.text.lower()))
                token.set_embedding(self.name, t)

    @staticmethod
    def _get_store_filename(embedding):
        """
        get the filename of the store
        """
        embedding_filename = re.findall(".flair(/.*)", embedding.name)[0]
        store_filename = str(flair.cache_root) + embedding_filename + ".sqlite"
        return store_filename

    @staticmethod
    def create_stores(model):
        """
        creates database versions of all word embeddings in the model and
        deletes the original vectors to save memory
        """
        for embedding in model.embeddings.embeddings:
            if type(embedding) == WordEmbeddings:
                WordEmbeddingsStore(embedding)
                del embedding.precomputed_word_embeddings

    @staticmethod
    def load_stores(model):
        """
        loads the db versions of all word embeddings in the model
        """
        for i, embedding in enumerate(model.embeddings.embeddings):
            if type(embedding) == WordEmbeddings:
                model.embeddings.embeddings[i] = WordEmbeddingsStore(embedding)

    @staticmethod
    def delete_stores(model):
        """
        deletes the db versions of all word embeddings
        """
        for embedding in model.embeddings.embeddings:
            store_filename = WordEmbeddingsStore._get_store_filename(embedding)
            if os.path.isfile(store_filename):
                print("delete store:", store_filename)
                os.remove(store_filename)


#!/usr/bin/env python
# -*- coding: utf-8 -*-
import argparse
import logging
import os

from dwi_ml.data.dataset.utils import prepare_multisubjectdataset
from dwi_ml.experiment_utils.timer import Timer
from dwi_ml.models.projects.transforming_tractography import TransformerSrcAndTgtModel
from dwi_ml.training.projects.transformer_trainer import TransformerTrainer
from dwi_ml.training.utils.batch_loaders import prepare_batch_loader
from dwi_ml.training.utils.batch_samplers import prepare_batch_sampler
from dwi_ml.training.utils.trainer import run_experiment

def prepare_arg_parser():
    p = argparse.ArgumentParser(description=__doc__,
                                formatter_class=argparse.RawTextHelpFormatter)
    p.add_argument('experiments_path',
                   help='Path from where to load your experiment, and where to'
                        'save new results.\nComplete path will be '
                        'experiments_path/experiment_name.')
    p.add_argument('experiment_name',
                   help='If given, name for the experiment. Else, model will '
                        'decide the name to \ngive based on time of day.')

    p.add_argument('--new_patience', type=int, metavar='new_p',
                   help='If a checkpoint exists, patience can be increased '
                        'to allow experiment \nto continue if the allowed '
                        'number of bad epochs has been previously reached.')
    p.add_argument('--new_max_epochs', type=int,
                   metavar='new_max',
                   help='If a checkpoint exists, max_epochs can be increased '
                        'to allow experiment \nto continue if the allowed '
                        'number of epochs has been previously reached.')

    p.add_argument('--logging', dest='logging_choice', default='WARNING',
                   choices=['ERROR', 'WARNING', 'INFO', 'DEBUG'],
                   help="Logging level. Note that, for readability, not all "
                        "debug logs are printed in DEBUG mode.")

    return p

def init_from_checkpoint(args, checkpoint_path):

    # Loading checkpoint
    checkpoint_state = TransformerTrainer.load_params_from_checkpoint(
        args.experiments_path, args.experiment_name)

    # Stop now if early stopping was triggered.
    TransformerTrainer.check_stopping_cause(
        checkpoint_state, args.new_patience, args.new_max_epochs)

    # Prepare data
    args_data = argparse.Namespace(**checkpoint_state['dataset_params'])
    dataset = prepare_multisubjectdataset(args_data)

    # Setting log level to INFO maximum for sub-loggers, else it become ugly
    sub_loggers_level = args.logging_choice
    if args.logging_choice == 'DEBUG':
        sub_loggers_level = 'INFO'

    # Prepare model
    model = TransformerSrcAndTgtModel.load_params_and_state(
        os.path.join(checkpoint_path, 'model'), sub_loggers_level)

    # Prepare batch sampler
    _args = argparse.Namespace(**checkpoint_state['batch_sampler_params'])
    batch_sampler = prepare_batch_sampler(dataset, _args, sub_loggers_level)

    # Prepare batch loader
    _args = argparse.Namespace(**checkpoint_state['batch_loader_params'])
    batch_loader = prepare_batch_loader(dataset, model, _args, sub_loggers_level)

    # Instantiate trainer
    with Timer("\n\nPreparing trainer", newline=True, color='red'):
        trainer = TransformerTrainer.init_from_checkpoint(
            model, args.experiments_path, args.experiment_name,
            batch_sampler,  batch_loader,
            checkpoint_state, args.new_patience, args.new_max_epochs,
            args.logging_choice)
    return trainer

def main():
    p = prepare_arg_parser()
    args = p.parse_args()

    # Setting root logger with high level but we will set trainer to
    # user-defined level.
    logging.getLogger().setLevel(level=logging.INFO)

    # Verify if a checkpoint has been saved.
    checkpoint_path = os.path.join(
            args.experiments_path, args.experiment_name, "checkpoint")
    if not os.path.exists(checkpoint_path):
        raise FileNotFoundError("Experiment's checkpoint not found ({})."
                                .format(checkpoint_path))

    trainer = init_from_checkpoint(args, checkpoint_path)

    run_experiment(trainer)

if __name__ == '__main__':
    main()


"""
Deepmind FullyConv Agent Implementation from SC2LE paper.
(https://deepmind.com/documents/110/sc2le.pdf)
"""

import torch
import torch.nn as nn
import numpy as np
from pysc2.lib import features
import torch.nn.functional as F

class EncodeSpatial(nn.Module):
    """
    """

    def __init__(self, n_features):
        super().__init__()
        self.preconv_layers = [
            nn.Conv2d(n_f, 1, kernel_size=1, stride=1, padding=0)
            for n_f in n_features]

        self.conv1 = nn.Conv2d(len(n_features), 16,
                               kernel_size=5, stride=1, padding=2)
        self.conv2 = nn.Conv2d(16, 32, kernel_size=3, stride=1, padding=1)

    def forward(self, input):
        """

        :param input: List of 3d tensors of feature
        :return:
        """
        feature_maps = [F.relu(self.preconv_layers[i](x))
                        for i, x in enumerate(input)]
        x = torch.cat(feature_maps, dim=1)

        x = F.relu(self.conv1(x))
        x = F.relu(self.conv2(x))
        return x

class Encode(nn.Module):
    """
    ...
    Args:
        n_screen_features: List of feature sizes of screen
        n_minimap_features: List of feature sizes of minimap
        n_nonspatial_features: Non-spatial feature size
    """

    def __init__(self, n_screen_features, n_minimap_features,
                 n_nonspatial_features):
        super().__init__()
        self.screen = EncodeSpatial(n_screen_features)
        self.minimap = EncodeSpatial(n_minimap_features)
        self.non_spatial_features = nn.Linear(n_nonspatial_features, 64)

    def forward(self, input):
        screen_map = self.screen(input[0])
        mini_map = self.minimap(input[1])
        assert screen_map.shape[-2:] == mini_map.shape[-2:], (
            "Spatial size of screen must"
            " be the same with mini-map's")

        non_spatial = torch.tanh(self.non_spatial_features(input[2]))
        non_spatial = non_spatial.unsqueeze(-1).unsqueeze(-1)
        non_spatial = non_spatial.repeat(1, 1, *screen_map.shape[-2:])

        return torch.cat([screen_map, mini_map, non_spatial], dim=1)

    def obs_to_torch(self, input, device="cpu"):
        spatials = [[torch.from_numpy(
            np.expand_dims(feature_map, 0)).float().to(device)
            for feature_map in obs] for obs in input[:2]]
        return (*spatials, torch.from_numpy(
            np.expand_dims(input[2], 0)).float().to(device))

class Output(nn.Module):
    def __init__(self, input_channels, spatial_y, spatial_x, n_actions):
        super().__init__()
        self.spatial_conv = nn.Conv2d(input_channels, 1, kernel_size=1)
        self.non_spatial_fc = nn.Linear(
            input_channels * spatial_y * spatial_x, 256)
        self.value = nn.Linear(256, 1)
        self.policy = nn.Linear(256, n_actions)

    def forward(self, input):
        spatial = self.spatial_conv(input)
        spatial = spatial.view(
            *spatial.shape[:2], spatial.shape[-1] * spatial.shape[-2])
        spatial = F.softmax(spatial, dim=-1)

        input = input.view(input.shape[0], -1)
        non_spatial = self.non_spatial_fc(input)
        value = self.value(non_spatial)
        policy = self.policy(non_spatial)

        return spatial, value, policy


from copy import deepcopy

from rest_framework import serializers

class TimetableSlotSerializer(serializers.BaseSerializer):
    """
    Serialize slot data for timetable endpoint output.

    This serialization converts the spectrum information of slot, which
    consist of start and end time, to only slot information, which only
    consist of start time in discrete hour.

    If a slot has a non-zero minute start time, two slots are created,
    one with the hour equals to the hour in start time, and another
    with the hour for the previous slot created, incremented by one.

    This serializer has to be instantiated with a Slot instance, or a
    list of Slot instances when serializing multiple instances of Slot.

    Additional context is required:
        - 'tz': The ``tzinfo`` object representing local timezone.
        - 'current_day': The ``datetime`` object representing local
            date.

    Data returns:

        dict -  'id': ID of slot.
                'hour': Numeric string representation of hour.
                'machine': ID of machine which the slot belongs to.
                'user': Name of the user who owns the slot.
    """

    def to_representation(self, instance):
        _data_template = {
            'id': instance.id,
            'hour': -1,
            'machine': instance.machine.id,
            'user': instance.user,
        }
        tz = self.context.get('tz')
        current_day = self.context.get('current_day')
        current_day = current_day.replace(tzinfo=tz)

        # Localize UTC start time to local datetime
        s_start = instance.start.replace(tzinfo=None)
        s_start += tz.utcoffset(s_start)
        s_start = s_start.replace(tzinfo=tz)

        # When start time has non-zero minute, two slots will be
        # created. This requires additional check when the second slot
        # to be created is 00, which is the start of the next day.
        # If the slot is on previous day, return only slot with hour
        # '00', instead of '23' and '00'.
        # If the slot is on current day, return only slot with hour
        # '23'.
        if s_start.minute != 0 and (s_start.hour + 1) == 24:
            _data = deepcopy(_data_template)

            if s_start < current_day:
                _data['hour'] = '00'
            else:
                _data['hour'] = '23'

        # Check only minute, since Slot creation is guaranteed to be
        # made discretely on an hour. This only changes when slot is
        # created on region with non-zero minute timezone offset, hence
        # the check here.
        elif s_start.minute != 0:
            _data = []

            for i in range(2):
                d = deepcopy(_data_template)
                d['hour'] = str(s_start.hour + i).zfill(2)
                _data.append(d)

        else:
            _data = deepcopy(_data_template)
            _data['hour'] = str(s_start.hour).zfill(2)

        return _data


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sun May 27 15:25:11 2018

@author: arent
"""

import numpy as np
import pickle
from sklearn.model_selection import train_test_split

from keras.layers import Input, Dense
from keras.models import Model
from keras.optimizers import SGD, Adam

class Dataset:

    def __init__(self, path_train, path_test):
        self.path_train = path_train
        self.path_test = path_test

        self.x_train, self.y_train = self.load_data_train()
        self.x_test, self.y_test = self.load_data_test()

        self.x_train, self.x_val, self.y_train, self.y_val = self.split()

    def load_data_train(self):
        (T_inf_train,
         T_obs_train,
         beta_MAP_train,
         std_lst_train) = pickle.load(open(self.path_train, 'rb'))

        T_inf_train = np.vstack(T_inf_train).flatten()
        T_obs_train = np.vstack(T_obs_train).flatten()
        beta_MAP_train = np.vstack(beta_MAP_train).flatten()

        x_train = np.concatenate([T_obs_train[:,np.newaxis], T_inf_train[:,np.newaxis]],
                         axis=1)
        y_train = beta_MAP_train
        return x_train, y_train

    def load_data_test(self):
        (T_inf_test,
         T_true_test,
         beta_true_test) = pickle.load(open(self.path_test, 'rb'))

        T_inf_test = np.vstack(T_inf_test).flatten()
        T_true_test = np.vstack(T_true_test).flatten()
        beta_true_test = np.vstack(beta_true_test).flatten()

        x_test = np.concatenate([T_true_test[:,np.newaxis],
                                 T_inf_test[:,np.newaxis]], axis=1)
        y_test = beta_true_test
        return x_test, y_test

    def split(self):
        x_train, x_val, y_train, y_val = train_test_split(self.x_train, self.y_train)
        return x_train, x_val, y_train, y_val

class Network:

    def __init__(self, data, N_epochs, lr, N_hidden, N_nodes):
        self.data = data
        self.N_epochs = N_epochs
        self.lr = lr
        self.N_hidden = N_hidden
        self.N_nodes = N_nodes

        self.model = self.build_model()

    def build_model(self):
        inputs = Input(shape=(2,))
        x = Dense(self.N_nodes, activation='relu')(inputs)
        for i in range(self.N_hidden-1):
            x = Dense(self.N_nodes, activation='relu')(x)
        outputs = Dense(1, activation='linear')(x)

        model = Model(inputs=inputs, outputs=outputs)
        opt = SGD(lr=self.lr)
        model.compile(optimizer=opt, loss='mse')

        return model

    def train(self):
        self.model.fit(self.data.x_train, self.data.y_train,
                       epochs=self.N_epochs,
                       verbose=0,
                       validation_data=(self.data.x_val, self.data.y_val))

        mse = self.model.history.history['loss']
        mse_val = self.model.history.history['val_loss']

        return mse, mse_val

#class GaussianProcess:
#    '''
#    kernel = Kernel class with method kernel_func
#    '''
#
#    def __init__(self, sigma_y, kernel):
#        self.sigma_y = sigma_y
#        self.kernel = kernel
#
#
#
#    def train(self, x_train, y_train):
#        '''
#        Computes the matrices K and K_y for the Gaussian process.
#        '''
#        N = len(x_train)
#
#        self.K = self.compute_K(x_train, x_train)
#        self.K_y = self.K+self.sigma_y**2*np.eye(N)
#        self.invK_y = np.linalg.inv(self.K_y)
#
#        self.x_train = x_train
#        self.y_train = y_train
#
#
#    def compute_K(self, x, xp):
#        '''
#        Returns a matrix of the kernel output of all possible combinations of
#        an observation in array x and an observation in array xp.
#
#        x has shape (N, D)
#        xp has shape (Np, D)
#        '''
#        N = x.shape[0]
#        Np = xp.shape[0]
#
#        K = np.ones((N, Np))
#
#        for i in range(N):
#            for j in range(Np):
#                K[i,j] = self.kernel.kernel_func(x[i,:], xp[j,:])
#
#        return K
#
#
#    def predict(self, x_test):
#        K_s = self.compute_K(self.x_train, x_test)
#        K_ss = self.compute_K(x_test, x_test)
#
#        mu_s = K_s.T.dot(self.invK_y).dot(self.y_train)
#        Sigma_s = K_ss-K_s.T.dot(self.invK_y).dot(K_s)
#
#        return mu_s, Sigma_s
#
#
#class RBF1d:
#
#    def __init__(self, sigma_f, l):
#        self.sigma_f = sigma_f
#        self.l = l
#
#
#    def kernel_func(self, x, xp):
#        return self.sigma_f**2*np.exp(-(x-xp)**2/(2*self.l**2)) # RBF1d
#
#
#
#class RBFnd:
#
#    def __init__(self, h):
#        self.h = h
#
#
#    def kernel_func(self, x, xp):
#        return np.exp(-np.linalg.norm(x-xp)**2/self.h**2)
#
#


from __future__ import print_function, division
import argparse
import codecs
import os
from glob import glob
import ntpath

import numpy as np
import pandas as pd
import warnings

from sets import Set
import re

from sets import Set
import re

from bs4 import BeautifulSoup
from bokeh.plotting import figure, show, save, output_notebook, output_file
from bokeh.models import ColumnDataSource, Range1d

def build_figure_extraction_patterns():
    bf = "\s*f(igs|igs\.|ig|ig\.|igure|\.|ig\:){0,1}"
    d =  "\s*(\d+\s*[\.\;\,]{0,1}\s*[a-z]*)\s*\.{0,1}\s*"
    d_split =  "\s*(\d*)\s*[\.\;\,]{0,1}\s*([a-z]*)"
    interval = "\s*(\d+)([a-z]+)\\-([a-z]+)"
    pattHash = {}

    figPatt = []
    pattHash['figPatt'] = figPatt

    # 0. No alphanumeric codes at all: 'Figure. 1; more text'
    figPatt.append(re.compile("^" + bf + d + "$"))
    figPatt.append(re.compile("^" + bf + "\s*(\d+\s*[\.\;\,]{0,1}\s*[a-z]*)[\,\;\.]{0,1}\s*t"))
    figPatt.append(re.compile("^" + bf + "\s*(\d+\s*[\.\;\,]{0,1}\s*[a-z]*)[\,\;\.]{0,1}\s*s"))
    figPatt.append(re.compile("^" + bf + "\s*(\d+\s*[\.\;\,]{0,1}\s*[a-z]*)[\,\;\.]{0,1}\s+and\s+s"))

    # [1]
    simplePatt = re.compile("^" + d + "$")
    pattHash['simplePatt'] = simplePatt

    # [2,4]
    space2Patt = re.compile("^" + bf + d + "\s+" + bf + d + "$")
    pattHash['space2Patt'] = space2Patt

    # [2,4,6]
    space3Patt = re.compile("^"+bf+d+"\s+"+bf+d+"\s+"+bf+d+"$")
    pattHash['space3Patt'] = space3Patt

    # [2,4]
    fullComma2Patt = re.compile("^" + bf + d + "[\;\,]" + bf + d + "$")
    pattHash['fullComma2Patt'] = fullComma2Patt

    # [2,3]
    comma2Patt = re.compile("^" + bf + d + "[\;\,]" + d + "$")
    pattHash['comma2Patt'] = comma2Patt

    # [1,2]
    simpleComma2Patt = re.compile("^" + d + "[\;\,]" + d + "$")
    pattHash['simpleComma2Patt'] = simpleComma2Patt

    # [2,3,4]
    comma3Patt = re.compile("^" + bf + d + "[\;\,]" + d + "[\;\,]" + d + "$")
    pattHash['comma3Patt'] = comma3Patt

    # [1,2,3]
    simpleComma3Patt = re.compile("^" + d + "[\;\,]" + d + "[\;\,]" + d + "$")
    pattHash['simpleComma3Patt'] = simpleComma3Patt

    # [2,3,4,5]
    comma4Patt = re.compile("^"+bf+d+"[\;\,]"+d+"[\;\,]"+d+"[\;\,]"+d+"$")
    pattHash['comma4Patt'] = comma4Patt

    # [2,3,4,5,6]
    comma5Patt = re.compile("^"+bf+d+"[\;\,]"+d+"[\;\,]"+d+"[\;\,]"+d+"[\;\,]"+d+"$")
    pattHash['comma5Patt'] = comma5Patt

    # [1,2,3,4]
    simpleComma4Patt = re.compile("^"+d+"[\;\,]"+d+"[\;\,]"+d+"[\;\,]"+d+"$")
    pattHash['simpleComma4Patt'] = simpleComma4Patt

    # [2,3]
    and2Patt = re.compile("^" + bf + d + "\s+and\s+" + d + "$")
    pattHash['and2Patt'] = and2Patt

    # [1,2]
    simpleAnd2Patt = re.compile("^" + d + "\s+and\s+" + d + "$")
    pattHash['simpleAnd2Patt'] = simpleAnd2Patt

    # [1,2,3]
    simple_a_and_b_patt = re.compile("^" + d_split + "\s+and\s+([a-z])$")
    pattHash['simple_a_and_b_patt'] = simple_a_and_b_patt

    # [2,3,4]
    a_and_b_patt = re.compile("^" + bf + d_split + "\s+and\s+([a-z])$")
    pattHash['a_and_b_patt'] = a_and_b_patt

    # [1,2,3]
    simple_a_comma_b_patt = re.compile("^" + d_split + "[\;\,]\s*([a-z])$")
    pattHash['simple_a_comma_b_patt'] = simple_a_comma_b_patt

    # [2,3,4]
    a_comma_b_patt = re.compile("^"+bf+d_split+"[\;\,]\s*([a-z])$")
    pattHash['a_comma_b_patt'] = a_comma_b_patt

    # [1,2,3]
    simple_a_comma_b_comma_c_patt = re.compile("^" + d_split + "[\;\,]\s*([a-z])\s*[\;\,]\s*([a-z])$")
    pattHash['simple_a_comma_b_comma_c_patt'] = simple_a_comma_b_comma_c_patt

    # [2,3,4]
    a_comma_b_comma_c_patt = re.compile("^"+bf+d_split+"[\;\,]\s*([a-z])\s*[\;\,]\s*([a-z])$")
    pattHash['a_comma_b_comma_c_patt'] = a_comma_b_comma_c_patt

    # [2,3,4,5]
    a_b_and_c_patt = re.compile("^" + bf + d_split + "[\;\,]\s+([a-z])\s+and\s+([a-z])$")
    pattHash['a_b_and_c_patt'] = a_b_and_c_patt

    # [1,2,3,4]
    simple_a_b_and_c_patt = re.compile("^" + d_split + "[\;\,]\s+([a-z])\s+and\s+([a-z])$")
    pattHash['simple_a_b_and_c_patt'] = simple_a_b_and_c_patt

    tableFigPatt = re.compile("^t(ab\.|ab|able){0,1}.*" + bf + d + "$")
    pattHash['tableFigPatt'] = tableFigPatt

    intervalPatt = re.compile("^" + bf + interval + "$");
    pattHash['intervalPatt'] = intervalPatt

    # simple single table (table 1, t1, tab. 1a)
    # returned value is second group
    tablePatt = re.compile("^t(ab\.|ab|able){0,1}\s*([\di]+[a-z]{0,1})[\,\;\.]{0,1}$")
    pattHash['tablePatt'] = tablePatt

    # simple supplemental table (table 1, t1, tab. 1a)
    # returned value is third group
    suppTablePatt = re.compile("^s(upp|upp.|lementary){0,1}\s*t(ab\.|ab|able){0,1}\s*([i\d]+[a-z]{0,1})[\,\;\.]{0,1}$")
    pattHash['suppTablePatt'] = suppTablePatt

    return pattHash

def run_simple_matcher(fig_text, patt_hash, patt_code, groups=[1]):
    match = re.search(patt_hash.get(patt_code), fig_text)
    results = []
    if( match ) :
        for g in groups:
            results.append(match.group(g))
        return results
    else:
        return None

def build_matched_string(matched_list,code):
    matched_str = ""
    for mf in matched_list:
        if len(matched_str) > 0 :
            matched_str += '|'
        matched_str += code + mf.replace(" ", "").replace(".", "")
    return matched_str

def run_matcher(fig_text, patt_hash):

    if(fig_text == 'nfa' ):
        return None

    # strip out all parentheses.
    paren_patt = re.compile("(\(.+?\))")
    fig_text = re.sub(paren_patt, "", fig_text)

    # covert & to 'and'.
    fig_text = fig_text.replace("&", "and")

    fig_patt = patt_hash.get('figPatt')
    for p in fig_patt:
        match = re.search(p, fig_text)
        if match:
            return 'f' + match.group(2).replace(" ","").replace(".","").replace(",","")

    # [1] simplePatt
    # [2,4] space2Patt
    # [2,4,6] space3Patt
    # [2,4] fullComma2Patt
    # [2,3] comma2Patt
    # [1,2] simpleComma2Patt
    # [2,3,4] comma3Patt
    # [1,2,3] simpleComma3Patt
    # [2,3,4,5] comma4Patt
    # [1,2,3,4] simpleComma4Patt
    # [1,2] simpleAnd2Patt
    # [1,2,3] simple_a_comma_b_patt
    # [2,3,4] a_comma_b_patt
    # [2,3,4,5]   a_b_and_c_patt
    # [1,2,3,4] simple_a_b_and_c_patt

    matched_figs = run_simple_matcher(fig_text, patt_hash, 'simplePatt', [1])
    if( matched_figs is None ):
        matched_figs = run_simple_matcher(fig_text, patt_hash, 'tableFigPatt', [3])
    if( matched_figs is None ):
        matched_figs = run_simple_matcher(fig_text, patt_hash, 'comma2Patt', [2,3])
    if( matched_figs is None ):
        matched_figs = run_simple_matcher(fig_text, patt_hash, 'fullComma2Patt', [2,4])
    if( matched_figs is None ):
        matched_figs = run_simple_matcher(fig_text, patt_hash, 'simpleComma2Patt', [1,2])
    if( matched_figs is None ):
        matched_figs = run_simple_matcher(fig_text, patt_hash, 'comma3Patt', [2,3,4])
    if( matched_figs is None ):
        matched_figs = run_simple_matcher(fig_text, patt_hash, 'simpleComma3Patt', [1,2,3])
    if( matched_figs is None ):
        matched_figs = run_simple_matcher(fig_text, patt_hash, 'comma4Patt', [2,3,4,5])
    if( matched_figs is None ):
        matched_figs = run_simple_matcher(fig_text, patt_hash, 'simpleComma4Patt', [1,2,3,4])
    if( matched_figs is None ):
        matched_figs = run_simple_matcher(fig_text, patt_hash, 'comma5Patt', [2,3,4,5,6])
    if( matched_figs is None ):
        matched_figs = run_simple_matcher(fig_text, patt_hash, 'space2Patt', [2,4])
    if( matched_figs is None ):
        matched_figs = run_simple_matcher(fig_text, patt_hash, 'space3Patt', [2,4,6])
    if( matched_figs is None ):
        matched_figs = run_simple_matcher(fig_text, patt_hash, 'simpleAnd2Patt', [1,2])
    if( matched_figs is None ):
        matched_figs = run_simple_matcher(fig_text, patt_hash, 'and2Patt', [2,3])
    if( matched_figs is None ):
        match = re.search(patt_hash.get('simple_a_comma_b_patt'), fig_text)
        if( match ):
            f =  match.group(1)
            a = match.group(2)
            b = match.group(3)
            return 'f'+f+a+'|'+'f'+f+b
    if( matched_figs is None ):
        match = re.search(patt_hash.get('a_comma_b_patt'), fig_text)
        if( match ):
            f =  match.group(2)
            a = match.group(3)
            b = match.group(4)
            return 'f'+f+a+'|'+'f'+f+b
    if( matched_figs is None ):
        match = re.search(patt_hash.get('simple_a_and_b_patt'), fig_text)
        if( match ):
            f =  match.group(1)
            a = match.group(2)
            b = match.group(3)
            return 'f'+f+a+'|'+'f'+f+b
    if( matched_figs is None ):
        match = re.search(patt_hash.get('a_and_b_patt'), fig_text)
        if( match ):
            f =  match.group(2)
            a = match.group(3)
            b = match.group(4)
            return 'f'+f+a+'|'+'f'+f+b
    if( matched_figs is None ):
        match = re.search(patt_hash.get('a_b_and_c_patt'), fig_text)
        if( match ):
            f =  match.group(2)
            a = match.group(3)
            b = match.group(4)
            c = match.group(5)
            return 'f'+f+a+'|'+'f'+f+b+'|'+'f'+f+c
    if( matched_figs is None ):
        match = re.search(patt_hash.get('simple_a_b_and_c_patt'), fig_text)
        if( match ):
            f =  match.group(1)
            a = match.group(2)
            b = match.group(3)
            c = match.group(4)
            return 'f'+f+a+'|'+'f'+f+b+'|'+'f'+f+c
    if( matched_figs is None ):
        match = re.search(patt_hash.get('simple_a_comma_b_comma_c_patt'), fig_text)
        if( match ):
            f =  match.group(1)
            a = match.group(2)
            b = match.group(3)
            c = match.group(4)
            return 'f'+f+a+'|'+'f'+f+b+'|'+'f'+f+c
    if( matched_figs is None ):
        match = re.search(patt_hash.get('a_comma_b_comma_c_patt'), fig_text)
        if( match ):
            f =  match.group(2)
            a = match.group(3)
            b = match.group(4)
            c = match.group(5)
            return 'f'+f+a+'|'+'f'+f+b+'|'+'f'+f+c
    if( matched_figs is None ):
        match = re.search(patt_hash.get('intervalPatt'), fig_text)
        if( match ):
            fig_number =  match.group(2)
            start = match.group(3)
            end = match.group(4)
            if( len(start) > 1 or len(end)>1 ):
                return None
            matched_str = ""
            subfigs = [chr(i) for i in range(ord(start),ord(end)+1)]
            for subfig in subfigs :
                if len(matched_str) > 0 :
                    matched_str += '|'
                matched_str += 'f' + fig_number + subfig
            return matched_str

    if(matched_figs is not None):
        return build_matched_string(matched_figs, 'f')

    matched_tab = run_simple_matcher(fig_text, patt_hash, 'tablePatt', [2])
    if(matched_tab is not None):
        return build_matched_string(matched_tab, 't')

    matched_tab = run_simple_matcher(fig_text, patt_hash, 'suppTablePatt', [3])
    if(matched_tab is not None):
        return build_matched_string(matched_tab, 'st')

    return None

def extract_simple_intact_data(input, title, tsv_output):

    with open(input, 'r') as input_file:
        xml = input_file.read()

    # Check if the figure legends are specified
    if "\"figure legend\"" not in xml:
        return

    soup = BeautifulSoup(xml, 'lxml')

    intact_headings = ['pmid','i_id','primary_ref','secondary_ref','orig_fig','fig','kinetics','kinetics_conditions','type','type_xref','p1_name','p1_xref','p1_site','p2_name','p2_xref','p2_site','p3_name','p3_xref','p3_site','i_meth','p_meth']
    intact_rows = []

    patt_hash = build_figure_extraction_patterns()

    # EXPERIMENTS
    all_expt_dict = {}
    for e in soup.select('experimentlist experimentdescription'):
        ex_dict = {}
        ex_dict['i_meth'] = e.interactiondetectionmethod.names.shortlabel.text
        ex_dict['i_meth_code'] = e.interactiondetectionmethod.xref.primaryref.attrs['id']
        ex_dict['p_meth'] = e.participantidentificationmethod.names.shortlabel.text
        ex_dict['p_meth_code'] = e.participantidentificationmethod.xref.primaryref.attrs['id']
        all_expt_dict[e.get('id')] = ex_dict

    # INTERACTORS
    all_int_dict = {}
    for i1 in soup.select('interactorlist interactor'):
        int_dict = {}
        int_dict['name'] = i1.names.shortlabel.text
        urls = []
        for t in i1.select('primaryref[db="uniprotkb"]'):
            if( t.get('reftype') == 'identity' ) :
                urls.append(t.get('id'))
        for t in i1.select('secondaryref[db="uniprotkb"]'):
            if( t.get('reftype') == 'identity' ) :
                urls.append(t.get('id'))
        int_dict['xref'] = urls
        all_int_dict[i1.get('id')] = int_dict

    # INTERACTIONS
    for i in soup.select('interactionlist interaction'):
        int_dict = {}
        int_dict['pmid'] = title
        int_dict['i_id'] = i.get('id')
        int_dict['type'] = i.interactiontype.names.shortlabel.text
        int_dict['type_xref'] = i.interactiontype.xref.primaryref.get('id')
        p_count = 1
        if i.primaryref is not None:
            int_dict['primary_ref'] = i.primaryref.attrs['id']
        else:
            int_dict['primary_ref'] = '-'
        if i.secondaryref is not None:
            int_dict['secondary_ref']  = i.secondaryref.attrs['id']
        else:
            int_dict['secondary_ref'] = '-'

        for p_tag in i.select('participantlist participant'):
            p_id = p_tag.interactorref.text
            p = all_int_dict[p_id]
            int_dict['p'+str(p_count)+"_name"] = p.get('name')
            int_dict['p'+str(p_count)+"_xref"] = '|'.join(p.get('xref'))
            p_count += 1
        int_dict['fig'] = '-'
        int_dict['orig_fig'] = '-'
        int_dict['kinetics'] = '-'
        int_dict['kinetics_conditions'] = '-'
        for a in i.select('attributelist attribute[name]'):
            if( a.get('name') == "figure legend" ):
                fig_text = a.text.lower()
                fig_text = run_matcher(fig_text, patt_hash)
                if( fig_text is None):
                    print(a.text.lower() + "  :  None")
                int_dict['orig_fig'] = a.text
                int_dict['fig'] = fig_text
            if( a.get('name') == "kinetics" ):
                int_dict['kinetics'] = a.text.replace('\t', ' ').replace('\n', ' ')
            if( a.get('name') == "kinetics_conditions" ):
                int_dict['kinetics_conditions'] = a.text.replace('\t', ' ').replace('\n', ' ')
        e_id = i.experimentlist.experimentref.text
        e = all_expt_dict.get(e_id)
        if( e is not None ):
            int_dict['i_meth'] = e.get('i_meth', '-')
            int_dict['i_meth_code'] = e.get('i_meth_code', '-')
            int_dict['p_meth'] = e.get('p_meth', '-')
            int_dict['p_meth_code'] = e.get('p_meth_code', '-')
        else:
            int_dict['i_meth'] = '-'
            int_dict['i_meth_code'] = '-'
            int_dict['p_meth'] = '-'
            int_dict['p_meth_code'] = '-'

        r = []
        for h in intact_headings:
            r.append(int_dict.get(h,'-'))
        intact_rows.append(r)

    intact_df = pd.DataFrame.from_records(intact_rows, columns=intact_headings)
    intact_df.to_csv(tsv_output, sep='\t', encoding='utf-8')

if __name__ == '__main__':

    parser = argparse.ArgumentParser()
    parser.add_argument('-i', '--inDir', help='Directory for input files')
    parser.add_argument('-o', '--outDir', help='Directory for output files')
    parser.add_argument('-p', '--pmid', help='PMID file')
    args = parser.parse_args()

    #pmids = []
    #with open(args.pmid, 'r') as pmid_file:
    #    for l in pmid_file.readlines():
    #        pmids.append(l.rstrip())

    for x in os.walk(args.inDir):
        for infile in glob(os.path.join(x[0], '*.xml')):
            fn = ntpath.basename(infile)
            if( os.path.isfile(infile) and fn.endswith('.xml') ):
                title = fn.replace(".xml", "")
                #if( title not in pmids ):
                #    continue

                print(infile)

                outfile = args.outDir + "/" + title + ".tsv"
                if( not os.path.isfile(outfile) ):
                    try:
                        extract_simple_intact_data(infile, title, outfile)
                    except KeyError:
                        print("KeyError for " + infile)


# import the libraries
import cv2
import imutils

# initializing the HOGDescriptor for person detection
HOGCV = cv2.HOGDescriptor()
HOGCV.setSVMDetector(cv2.HOGDescriptor_getDefaultPeopleDetector())

#get the video file by specified path
video = cv2.VideoCapture('human_vid.mp4')

# open the video and check its running
while video.isOpened():
    check, frame = video.read()
    if check:
        # resize the output of video file
        frame = imutils.resize(frame, width=min(800, frame.shape[1]))
        bounding_box_cordinates, weights = HOGCV.detectMultiScale(frame, winStride=(4, 4), padding=(8, 8),
                                                              scale=1.03)
        # create person var for count of people
        person = 0
        # define the dimensions and text of rectangular boxes identifying humans
        for x, y, w, h in bounding_box_cordinates:
            cv2.rectangle(frame, (x, y), (x + w, y + h), (0, 255, 0), 2)
            cv2.putText(frame, f'person {person}', (x, y), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255), 1)
            person += 1
            cv2.putText(frame, 'Status : Detecting', (40, 40), cv2.FONT_HERSHEY_SIMPLEX, 0.5,
                                                        (255, 255, 0), 1)
            cv2.putText(frame, f'Total People : {person}', (40, 70), cv2.FONT_HERSHEY_SIMPLEX,
                        0.5, (255, 255, 0), 1)
            # get the output video
            cv2.imshow('output', frame)
            key = cv2.waitKey(1)
            if key == ord('q'):
                break
    else:
        break
video.release()
cv2.destroyAllWindows()


import os
import gdown
import tarfile
import imageio
from pathlib import Path
from shutil import copyfile

import numpy as np
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
from matplotlib.backends.backend_agg import FigureCanvasAgg

from torchviz import make_dot
from torch.utils.data import DataLoader, dataset

import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F

import torchvision.models as models
import torchvision.datasets as ds
import torchvision.transforms as trans

from poutyne import Callback, Dict, CSVLogger, Experiment

os.environ["CUDA_VISIBLE_DEVICES"]="0"

__result_dir__ = 'results'

def save_model(model, file_path):
    """save_model saves the trained model in the specified path

    Args:
        model (nn.Module): the model to be saved.
        file_path (str): file path.
    """

    torch.save(model, file_path)

def load_model(file_path):
    """load_model loads the saved model from specified path.

    Args:
        file_path (str): file path to the trained model.

    Returns:
        nn.Module: the trained model.
    """
    return torch.load(file_path)

def get_torch_device(): return torch.device(
    'cuda:0' if torch.cuda.is_available() else 'cpu')

is_cuda_available = lambda: torch.cuda.is_available()

def get_cub200(dir_path: str):
    # Dowload the dataset
    url = 'https://drive.google.com/uc?id=1GDr1OkoXdhaXWGA8S3MAq3a522Tak-nx'
    dataset_file = 'images.tgz'

    print('Create required directories ...')
    # Make the original dataset directories
    Path(dir_path).mkdir(parents=True, exist_ok=True)
    ds_zipfile = os.path.join(dir_path, dataset_file)

    if not os.path.exists(ds_zipfile):
        print('Download the dataset zip file ...')
        gdown.download(url, output=ds_zipfile, quiet=False)

    # Extract the tgz file into the dataset directory
    print('Extract the zip file ...')
    tar = tarfile.open(ds_zipfile, "r:gz")
    tar.extractall(path=dir_path)
    tar.close()

    return os.path.join(dir_path, 'images')

def prepare_subsets(orig_ds_dir : str, target_dir : str):
    # Make the dataset directory
    training_ds_dir = os.path.join(target_dir, 'training')
    testing_ds_dir = os.path.join(target_dir, 'testing')

    if os.path.isdir(orig_ds_dir):
        print('Make required folder for the dataset ...')
        Path(training_ds_dir).mkdir(parents=True, exist_ok=True)
        Path(testing_ds_dir).mkdir(parents=True, exist_ok=True)
        # Prepare the training and testing subsets
        print('Preparing the training and testing subsets ...')
        separate_train_test(orig_ds_dir, training_ds_dir, testing_ds_dir)

    return training_ds_dir, testing_ds_dir

def make_dir(file_path):
    if not os.path.exists(file_path):
        os.makedirs(file_path)

def separate_train_test(dataset_path, train_path, test_path):
    class_index = 1
    for classname in sorted(os.listdir(dataset_path)):
        if classname.startswith('.'):
            continue
        make_dir(os.path.join(train_path, classname))
        make_dir(os.path.join(test_path, classname))
        i = 0
        for file in sorted(os.listdir(os.path.join(dataset_path, classname))):
            if file.startswith('.'):
                continue
            file_path = os.path.join(dataset_path, classname, file)
            if i < 15:
                copyfile(file_path, os.path.join(test_path, classname, file))
            else:
                copyfile(file_path, os.path.join(train_path, classname, file))
            i += 1

        class_index += 1

ds_filename_valid = lambda filename : not os.path.basename(filename).startswith('.')

def create_model(pretrained=True):
    model = models.resnet18(pretrained=pretrained)
    model = model.cuda() if is_cuda_available() else model
    num_features = model.fc.in_features
    model.fc = nn.Linear(num_features, 200)
    model.fc = model.fc.cuda() if is_cuda_available() else model.fc

    return model

def calculate_normalization_params(dataset_dir : str):
    transforms = trans.Compose([
        trans.Resize((224,224)),
        trans.ToTensor()
    ])

    dataset = ds.ImageFolder(root=dataset_dir, transform=transforms, is_valid_file=ds_filename_valid)
    dl = DataLoader(dataset, batch_size=64, num_workers=2)

    count = 0
    first_moment = torch.empty(3)
    second_moment = torch.empty(3)

    for images, _ in dl:
        b, _, h, w = images.shape
        nump = b * h * w
        sumv = torch.sum(images, dim=[0, 2, 3])
        sum_of_square = torch.sum(images ** 2, dim=[0, 2, 3])
        first_moment = (count * first_moment + sumv) / (count + nump)
        second_moment = (count * second_moment + sum_of_square) / (count + nump)
        count += nump

    mean_ = first_moment
    std_ = torch.sqrt(second_moment - first_moment ** 2)

    return mean_, std_

def get_datasets(ds_path, transforms, batch_size=32, num_workers=2, shuffle=True):
    dataset = ds.ImageFolder(root=ds_path, transform=transforms,
        is_valid_file=ds_filename_valid)
    data_loader = DataLoader(dataset, batch_size=batch_size,
        num_workers=num_workers, shuffle=shuffle)

    return dataset, data_loader

def run_experiment(
    name,
    model,
    transforms,
    training_dir,
    testing_dir,
    learning_rate=0.01,
    momentum=0.9,
    num_epochs=10,
    batch_size=32
):
    _, training_loader = get_datasets(training_dir, transforms, batch_size=batch_size)
    _, testing_loader = get_datasets(testing_dir, transforms)

    # Optimizer Initialization
    optimizer = optim.SGD(model.parameters(), lr=learning_rate, momentum=momentum)
    # Loss Function Initialization
    loss_function = nn.CrossEntropyLoss()

    result_dir = os.path.join(__result_dir__,name)
    Path(result_dir).mkdir(parents=True, exist_ok=True)

    res_photo = os.path.join(result_dir, 'result.png')
    if os.path.isfile(res_photo):
        img = mpimg.imread(res_photo)
        plt.imshow(img)
        return

    callbacks = [
        PlottingCallback(result_dir=result_dir),
        # Save the losses and accuracies for each epoch in a TSV.
        CSVLogger(os.path.join(result_dir, 'log_callback.tsv'), separator='\t'),
    ]

    # Create & Initialize Experiment
    exp_obj = Experiment(
        result_dir, model,
        device=get_torch_device(),
        optimizer=optimizer,
        loss_function=loss_function,
        batch_metrics=['accuracy'],
        epoch_metrics=['f1'],
        task='classif'
    )

    # Use Poutyne experiment to train the model.
    exp_obj.train(train_generator=training_loader, valid_generator=testing_loader,
                epochs=num_epochs, callbacks=callbacks)

    print(exp_obj.test(testing_loader))

    # Save the trained model
    save_model(model, os.path.join(result_dir, name + '.phm'))

def freeze_conv_params(model):
    for name, param in model.named_parameters():
        if name.startswith('conv'):
            param.requires_grad = False
    return model

def freeze_layer1_params(model):
    for name, param in model.named_parameters():
        if name.startswith('layer1'):
            param.requires_grad = False
    return model
    # for param in model.layer1.parameters():
    #     param.requires_grad = False

    # return model

class PlottingCallback(Callback):
    """ PhmPlottingCallback is the class containing the codes for visualizing the model's results.
        The class also extends Callback class makes it able to attach to Poutyne experiment and register for the raised events.
    """

    def __init__(self, result_dir='./'):
        super().__init__()
        self.report = list()
        self.fig_list = list()
        self.fig, (self.loss_ax, self.acc_ax) = plt.subplots(2, 1)
        self.fig.tight_layout(pad=1.0)
        self.fig.suptitle('Statistics of training model')
        self.result_dir = result_dir
        mng = plt.get_current_fig_manager()
        mng.full_screen_toggle()
        plt.ion()
        # plt.show()

    def on_train_begin(self, logs: Dict):
        """ on_train_begin is a method invoked in the begining of each train. """
        pass

    def _plot(self, axis, epoch, train_data, valid_data, label, maxv=1.0, minv=0.0, stepv=0.05):
        axis.clear()
        plt.yticks(fontsize=11)
        axis.plot(epoch, train_data, '.-', label='Train')
        axis.plot(epoch, valid_data, '.-', label='Validation')
        axis.set_xlabel('epoch')
        axis.set_ylabel(label)
        # axis.set_yticks(np.arange(minv, maxv, stepv), minor = True)
        axis.set_yticks(np.arange(min(train_data + valid_data),
                                  max(train_data + valid_data), stepv), minor=True)
        axis.legend()
        axis.grid(True, which='both', axis='both', linestyle='--')

    def on_epoch_end(self, epoch_number: int, logs: Dict):
        """ on_epoch_end is a method invoked in the end of each epoch. """

        self.report.append(logs)

        epoch = [item['epoch'] for item in self.report]
        train_loss = [item['loss'] for item in self.report]
        val_loss = [item['val_loss'] for item in self.report]
        train_acc = [item['acc'] for item in self.report]
        val_acc = [item['val_acc'] for item in self.report]

        canvas = FigureCanvasAgg(self.fig)
        # Loss
        if train_loss is not None and val_loss is not None:
            self._plot(self.loss_ax, epoch, train_loss, val_loss, 'Loss')
        # Accuracy
        if train_acc is not None and val_acc is not None:
            self._plot(self.acc_ax, epoch, train_acc,
                    val_acc, 'Accuracy', 100, 0, 5)

        # plt.show()
        # plt.pause(0.05)

        canvas.draw()
        fimg = np.array(canvas.renderer.buffer_rgba())
        self.fig_list.append(fimg)

    def on_train_end(self, logs: Dict):
        """ on_train_end is the method invoked in the end of each training. """

        if len(self.fig_list) > 0:
            imageio.mimsave(os.path.join(self.result_dir, 'result.gif'), self.fig_list, 'GIF')
            imageio.imwrite(os.path.join(self.result_dir, 'result.png'), self.fig_list[-1])


from discord.ext import commands
import random
import json
import os

def load():
    if not os.path.isfile('cogs/guess.json'):
        guess = {'integer': {'min': 0, 'max': 99}, 'float': {'min': 0, 'max': 10}}
        store(guess)
    with open('cogs/guess.json') as f:
        return json.load(f)

def store(config):
    with open('cogs/guess.json','w') as f:
        json.dump(config, f)

class Guess:
    def reload_numbers(self):
        config = load()
        self.int_min = config['integer']['min']
        self.int_max = config['integer']['max']
        self.f_min = config['float']['min']
        self.f_max = config['float']['max']

    def __init__(self, bot):
        self.bot = bot
        self.reload_numbers()
        self.integer_to_guess = self.new_int()
        self.decimal_to_guess = self.new_decimal()

    def new_int(self):
        return random.randint(self.int_min, self.int_max)

    def new_decimal(self):
        return random.random() * self.f_max + self.f_min

    @commands.group(help='Do you know what I want?', aliases=['g'], pass_context=True)
    async def guess(self, ctx):
        if ctx.invoked_subcommand is None:
            await self.bot.say('Welcome to the unknown. Guess your way!')

    @guess.command(hidden=True, help='reload the numbers', aliases=['r', 'rld'])
    async def reload(self):
        self.reload_numbers()
        await self.bot.say('Fresh numbers coming right in!')

    @guess.command(help='Guess the correct integer', aliases=['i', 'int'])
    async def integer(self, *, guess:int=-1):
        if guess > self.int_max or guess < self.int_min:
            await self.bot.say("Choose an integer between " + str(self.int_min) + " and " + str(self.int_max) + "!")
        elif guess == self.integer_to_guess:
            await self.bot.say("Correct!")
            integer_to_guess = random.randint(self.int_min, self.int_max)
        elif guess < self.integer_to_guess:
            await self.bot.say("Too small!")
        else:
            await self.bot.say("Too big!")

    @integer.error
    async def integer_error(self, error, ctx):
        await self.bot.say("Choose an integer between " + str(self.int_min) + " and " + str(self.int_max) + "!")

    @guess.command(help='Guess the correct decimal', aliases=['f'])
    async def float(self, decimal:float=-1):
        if decimal > self.f_max or decimal < self.f_min:
            await self.bot.say("Choose an decimal between " + str(self.f_min) + " and " + str(self.f_max) + "!")
        elif decimal == self.decimal_to_guess:
            await self.bot.say("Correct!")
            self.decimal_to_guess = random.random() * self.f_max + self.f_min
        elif decimal < self.decimal_to_guess:
            await self.bot.say("Too small!")
        else:
            await self.bot.say("Too big!")

    @float.error
    async def float_error(self, error, ctx):
        await self.bot.say("Choose an decimal between " + str(self.f_min) + " and " + str(self.f_max) + "!")

def setup(bot):
    bot.add_cog(Guess(bot))


#!/usr/bin/python3

import requests
import json
import os
import datetime

PARENT_DIR = "/home/opc/World-Wide-API/Datasets/India"
HEADER_STRING = "Day,Date,Confirmed,Recovered,Deaths\n"

def modifyDate(date_str):
    # date_string 14-Mar-20
    # output: 2020-04-14
    months = {
        "Jan": '01',
        "Feb": '02',
        "Mar": '03',
        "Apr": '04',
        "May": '05',
        "Jun": '06',
        "Jul": '07',
        "Aug": '08',
        "Sep": '09',
        "Oct": '10',
        "Nov": '11',
        "Dec": '12'
    }
    date, month, year = date_str.split('-')

    year += '20'
    new_date = "-".join([year, months[month], date])
    return new_date

def parseCasesInfo(casesInfo):
    try:
        confirmed = casesInfo["confirmed"]
    except:
        confirmed = 0
    try:
        recovered = casesInfo["recovered"]
    except:
        recovered = 0
    try:
        deaths = casesInfo["deceased"]
    except:
        deaths = 0

    return confirmed, recovered, deaths

def stateCodes():
    API_URL = "https://api.covid19india.org/state_district_wise.json"

    filename = "state_codes.json"

    response = requests.get(API_URL)
    states = response.json()

    state_codes = dict()
    for state in states:
        # name = state.replace(' ', '-')
        code = states[state]['statecode']
        state_codes[code.lower()] = state

    fhand = open(filename, 'w')
    json.dump(state_codes, fhand)
    fhand.close()

def statesData():
    STATE_API_URL = "https://api.covid19india.org/states_daily.json"
    codes_file = "/home/opc/World-Wide-API/state_codes.json"
    fhand = open(codes_file)

    all_states = dict()

    state_codes = json.load(fhand)
    fhand.close()

    for code in state_codes:
        state_name = state_codes[code]
        all_states[state_name] = list()

    response = requests.get(STATE_API_URL)
    data = response.json()

    states_info = data["states_daily"]

    for day_data in states_info:
        date = day_data['date']
        status = day_data['status']
        for key in day_data:
            if key in ["date", "status", "dd", "tt"]:
                continue

            state_name = state_codes[key]
            if status == "Confirmed":
                all_states[state_name].append(dict())

            new_date = modifyDate(date)
            all_states[state_name][-1]["Date"] = new_date
            try:
                previous = all_states[state_name][-2][status]
            except:
                previous = 0
            try:
                current = int(day_data[key])
            except:
                current = 0
            all_states[state_name][-1][status] = (current + previous)

    # print(all_states)

    json_states_file = "all_states_time_series.json"
    fhand = open(json_states_file, 'w')

    json.dump(all_states, fhand)
    fhand.close()

    fhand = open(json_states_file)
    data = json.load(fhand)
    day = 1
    for state in data:
        dir_name = state.replace(' ', '-')
        filename = dir_name + "/total.csv"
        try:
            os.mkdir(PARENT_DIR + '/' + dir_name)
        except FileExistsError:
            pass
        except Exception as e:
            print(e, "occured while creating ", PARENT_DIR + '/' + dir_name)
        filename = PARENT_DIR + '/' + filename
        fhand = open(filename, 'w')
        fhand.write(HEADER_STRING)

        for info in data[state]:
            date = info["Date"]
            recovered = info["Recovered"]
            confirmed = info["Confirmed"]
            deaths = info["Deceased"]

            line = ', '.join(
                list(map(str, [day, date, confirmed, recovered, deaths])))
            line += '\n'
            fhand.write(line)
            day += 1
        fhand.close()

def districtsData():
    DISTRICT_API_URL = "https://raw.githubusercontent.com/covid19india/api/gh-pages/v4/data-all.json"
    startDate = "2020-04-21"
    codes_file = "/home/opc/World-Wide-API/state_codes.json"
    fhand = open(codes_file)

    state_codes = json.load(fhand)
    fhand.close()

    response = requests.get(DISTRICT_API_URL)
    jsonData = response.json()

    filenameToDataMap = dict()

    startDateFound = False
    for date in jsonData:
        if startDate == date:
            startDateFound = True
        if startDateFound:
            print(date)
            dayData = jsonData[date]
            for stateCode in dayData:
                if stateCode in ["TT", "tt"]:
                    continue
                stateName = state_codes[stateCode.lower()]
                print("\t", stateName)
                dir_name = stateName.replace(' ', '-')
                try:
                    os.mkdir(PARENT_DIR + '/' + dir_name)
                except FileExistsError:
                    # print("Dir Exists")
                    pass
                except Exception as e:
                    print(e, "occured while creating ",
                          PARENT_DIR + '/' + dir_name, "directory")
                    quit()

                # print(stateName, stateCode, dayData[stateCode])
                if "districts" not in dayData[stateCode]:
                    print(
                        f"\x1b[37mDistricts Data on {date} not available for {stateName}\x1b[0m")
                    continue
                districtsInfo = dayData[stateCode]["districts"]
                for district in districtsInfo:
                    print("\t\t", district)
                    if "total" not in districtsInfo[district]:
                        print(
                            f"Cases info for {stateName} => {district} on {date} isn't available")
                        continue
                    casesInfo = districtsInfo[district]["total"]
                    confirmed, recovered, deaths = parseCasesInfo(casesInfo)
                    itemsToWrite = [0, date, confirmed, recovered, deaths]
                    lineToWrite = ', '.join(list(map(str, itemsToWrite)))
                    print("\t\t\t", lineToWrite)

                    filename = district
                    filename = filename.replace('*', '')
                    filename = filename.replace(' ', '-')
                    filename = dir_name + '/' + filename
                    filename = PARENT_DIR + '/' + filename + ".csv"

                    if filename not in filenameToDataMap:
                        filenameToDataMap[filename] = [lineToWrite + '\n']
                    else:
                        filenameToDataMap[filename].append(lineToWrite + '\n')

    for filename in filenameToDataMap:
        data = filenameToDataMap[filename]
        lastLine = data[-1]
        lastDate = lastLine.split(',')[1].strip()
        today = str(datetime.datetime.today()).split()[0]
        if lastDate == today:
            data.pop()
        fhand = open(filename, 'w')
        fhand.write(HEADER_STRING)
        fhand.writelines(data)

def retrieve():
    stateCodes()
    statesData()
    districtsData()

retrieve()


import face_alignment
from skimage import io
from mpl_toolkits.mplot3d import axes3d
import matplotlib.pyplot as plt
from mpldatacursor import datacursor

fa = face_alignment.FaceAlignment(face_alignment.LandmarksType._3D, device='cpu', flip_input=False)

input = io.imread('test/assets/aflw-test.jpg')
preds = fa.get_landmarks(input)

fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')

def func():
    for i,spot in enumerate(preds[0]):
        print(spot)
        spotX=spot[0]
        spotY=spot[1]
        spotZ=spot[2]
        ax.scatter(spotX,spotY,spotZ,marker='x')
        ax.text(spotX,spotY,spotZ,i)

func()
ax.set_xlabel('X')
ax.set_ylabel('Y')
ax.set_zlabel('Z')
plt.show()


import numpy as np
from matplotlib import pyplot as plt

def plot_overlay(test_accuracy,
                 training_accuracy,
                 num_epochs,
                 xmin,
                 training_set_size):

    fig = plt.figure()
    ax = fig.add_subplot(111)
    ax.plot(np.arange(xmin, num_epochs),
            [accuracy/100.0 for accuracy in test_accuracy],
            color='#2A6EA6',
            label="Accuracy on the test data")
    ax.plot(np.arange(xmin, num_epochs),
            [accuracy*100.0/training_set_size
             for accuracy in training_accuracy],
            color='#FFA933',
            label="Accuracy on the training data")
    ax.grid(True)
    ax.set_xlim([xmin, num_epochs])
    ax.set_xlabel('Epoch')
    ax.set_ylim([90, 100])
    plt.legend(loc="lower right")
    plt.show()


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import json
from django.core.serializers.json import DjangoJSONEncoder
from django.db import models, migrations
import wagtail.wagtailimages.blocks
import wagtail.wagtailcore.fields
import wagtail.wagtailcore.blocks
import ellen.blocks

def convert_referral_links_to_struct_with_url(apps, schema_editor):
    LandingPage = apps.get_model('ellen', 'LandingPage')

    for page in LandingPage.objects.all():
        new_page_body = []

        for child in page.body:
            prep_value = None

            if child.block_type == 'as_seen_in':
                prep_value = [{
                    'thumbnail': image.pk,
                    'url': 'http://example.com',
                } for image in child.value ]
            else:
                prep_value = child.block.get_prep_value(child.value)

            new_page_body.append({
                'type': child.block_type,
                'value': prep_value
            })

        page.body = wagtail.wagtailcore.blocks.StreamValue(page.body.stream_block, [], raw_text=json.dumps(new_page_body, cls=DjangoJSONEncoder))
        page.save()

class Migration(migrations.Migration):

    dependencies = [
        ('ellen', '0001_initial'),
    ]

    operations = [
        migrations.RunPython(
            convert_referral_links_to_struct_with_url,
        ),
        migrations.AlterField(
            model_name='landingpage',
            name='body',
            field=wagtail.wagtailcore.fields.StreamField((('paragraph', wagtail.wagtailcore.blocks.RichTextBlock()), ('featured_stories', ellen.blocks.FeaturedStoriesBlock(ellen.blocks.StoryChooserBlock())), ('as_seen_in', ellen.blocks.ReferralLinksBlock(wagtail.wagtailcore.blocks.StructBlock((('thumbnail', wagtail.wagtailimages.blocks.ImageChooserBlock()), ('url', wagtail.wagtailcore.blocks.URLBlock())))))), blank=True, null=True),
        ),
    ]


import re
import json
import time

from django.conf import settings
from django.contrib.staticfiles.storage import staticfiles_storage
from django.utils.module_loading import import_string

__all__ = ('get_assets', 'get_config', 'get_bundle',)

DEFAULT_CONFIG = {
    'DEFAULT': {
        'BUNDLE_DIR_NAME': 'webpack_bundles/',
        'STATS_FILE': 'webpack-stats.json',
        # FIXME: Explore usage of fsnotify
        'POLL_INTERVAL': 0.1,
        'IGNORE': ['.+\.hot-update.js', '.+\.map'],
        'GET_CHUNK_URL': 'webpack_loader.utils.default_chunk_url',
    }
}

user_config = getattr(settings, 'WEBPACK_LOADER', DEFAULT_CONFIG)

user_config = {
    name: dict(DEFAULT_CONFIG['DEFAULT'], **cfg)
    for name, cfg in user_config.items()
}

for entry in user_config.values():
    entry['ignores'] = [re.compile(I) for I in entry['IGNORE']]

class WebpackError(Exception):
    pass

class WebpackLoaderBadStatsError(Exception):
    pass

def get_config(config_name):
    config = user_config[config_name]
    if 'get_chunk_url' not in config:
        config['get_chunk_url'] = config['GET_CHUNK_URL'] if callable(config['GET_CHUNK_URL']) else import_string(config['GET_CHUNK_URL'])
    return config

def get_assets(config):
    try:
        with open(config['STATS_FILE']) as f:
            return json.load(f)
    except IOError:
        raise IOError(
            'Error reading {}. Are you sure webpack has generated the file '
            'and the path is correct?'.format(config['STATS_FILE']))

def filter_files(files, config):
    for F in files:
        filename = F['name']
        ignore = any(regex.match(filename) for regex in config['ignores'])
        if not ignore:
            relpath = '{}{}'.format(config['BUNDLE_DIR_NAME'], filename)
            F['url'] = staticfiles_storage.url(relpath)
            yield F

def get_bundle(bundle_name, config):
    assets = get_assets(config)

    if settings.DEBUG:
        # poll when debugging and block request until bundle is compiled
        # TODO: support timeouts
        while assets['status'] == 'compiling':
            time.sleep(config['POLL_INTERVAL'])
            assets = get_assets(config)

    if assets.get('status') == 'done':
        files = assets['chunks'][bundle_name]
        return filter_files(files, config)

    elif assets.get('status') == 'error':
        if 'file' not in assets:
            assets['file'] = ''
        error = u"""
        {error} in {file}
        {message}
        """.format(**assets)
        raise WebpackError(error)

    raise WebpackLoaderBadStatsError(
        "The stats file does not contain valid data. Make sure "
        "webpack-bundle-tracker plugin is enabled and try to run "
        "webpack again.")

def default_chunk_url(chunk, config):
    url = chunk.get('publicPath') or chunk['url']
    return url


# pylint:disable=unused-variable
# pylint:disable=unused-argument
# pylint:disable=redefined-outer-name

import sys
from pathlib import Path
from typing import Tuple

import pytest

current_dir = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent

@pytest.fixture(scope="session")
def project_slug_dir() -> Path:
    raise NotImplementedError("Override fixture 'project_slug_dir' REQUIRED")

@pytest.fixture(scope="session")
def project_name(project_slug_dir) -> str:
    # Override if it does not apply
    return project_slug_dir.name

@pytest.fixture(scope="session")
def src_dir(project_slug_dir: Path) -> Path:
    _src_dir = project_slug_dir / "src"
    assert _src_dir.exists()
    return _src_dir

@pytest.fixture(scope="session")
def tests_dir(project_slug_dir) -> Path:
    _tests_dir = project_slug_dir / "tests"
    assert _tests_dir.exists()
    return _tests_dir

@pytest.fixture(scope="session")
def validation_dir(project_slug_dir: Path) -> Path:
    validation_dir = project_slug_dir / "validation"
    assert validation_dir.exists()
    return validation_dir

@pytest.fixture(scope="session")
def tools_dir(project_slug_dir: Path) -> Path:
    tools_dir = project_slug_dir / "tools"
    assert tools_dir.exists()
    return tools_dir

@pytest.fixture(scope="session")
def docker_dir(project_slug_dir: Path) -> Path:
    docker_dir = project_slug_dir / "docker"
    assert docker_dir.exists()
    return docker_dir

@pytest.fixture(scope="session")
def metadata_file(project_slug_dir: Path) -> Path:
    metadata_file = project_slug_dir / "metadata" / "metadata.yml"
    assert metadata_file.exists()
    return metadata_file

# HELPERS -----------

def get_expected_files(docker_name: str) -> Tuple[str, ...]:
    return (
        ".cookiecutterrc",
        ".dockerignore",
        ".gitignore",
        ".pylintrc",
        "metadata:metadata.yml",
        f"docker/{docker_name}:entrypoint.sh",
        f"docker/{docker_name}:Dockerfile",
        "service.cli:execute.sh",
        "versioning:integration.cfg",
        "versioning:service.cfg",
        "requirements.in",
        "requirements.txt",
        "Makefile",
        "VERSION",
        "VERSION_INTEGRATION",
        "README.md",
        "docker-compose-build.yml",
        "docker-compose-meta.yml",
        "docker-compose.devel.yml",
        "docker-compose.yml",
    )

def assert_path_in_repo(expected_path: str, project_slug_dir: Path):

    if ":" in expected_path:
        folder, glob = expected_path.split(":")
        folder_path = project_slug_dir / folder
        assert folder_path.exists(), f"folder {folder_path} is missing!"
        assert any(folder_path.glob(glob)), f"no {glob} in {folder_path}"
    else:
        assert (
            project_slug_dir / expected_path
        ).exists(), f"{expected_path} is missing from {project_slug_dir}"


from __future__ import (print_function,
                        unicode_literals,
                        division)
from future.builtins import str, open, range, dict

import matplotlib
# matplotlib.use('Agg')
import matplotlib.pyplot as plt
import numpy as np
import os
import sys
import argparse
from spikevo import *
from spikevo.pynn_transforms import PyNNAL
from pprint import pprint

parser = argparse.ArgumentParser()
parser.add_argument('-b', '--backend', help='available %s' % supported_backends,
                    default=GENN)

args = parser.parse_args()
backend = args.backend
pynn = backend_setup(backend)

print(pynn.__name__)

pynnx = PyNNAL(pynn, max_subpop_size=2)

N_NEURONS = 10
w = 0.025
syn_delay = 1.
sim_time = 1000.

neuron_parameters = {
    'v_thresh': -35.0,
    'tau_m': 20.,
    'tau_syn_E': 10.0,
    # 'e_rev_E':     0.,
    'tau_refrac': 0.1,
    'v_reset': -50.0,  # hdbrgs
    'tau_syn_I': 5.,
    'i_offset': 0.0,
    # ESS - BrainScaleS
    'cm': 0.2,
    'v_rest': -50.0,
    # 'e_rev_I':    -100.,
}

pynn.setup(timestep=1.0, min_delay=1.0)

neurons = pynnx.Pop(N_NEURONS, pynn.IF_cond_exp, neuron_parameters,
                    label='neurons')
neurons.record('spikes')

inputs = pynnx.Pop(N_NEURONS, pynn.SpikeSourcePoisson,
                   {'rate': [10.0] * N_NEURONS},
                   label='poisson'
                   )
inputs.record('spikes')

proj = pynnx.Proj(inputs, neurons, pynn.FixedProbabilityConnector,
                  weights=w, delays=syn_delay,
                  conn_params={
                      'p_connect': 0.1,
                      'rng': pynnx.NumpyRNG()})
# proj = pynnx.Proj(inputs, neurons, pynn.AllToAllConnector,
#                   weights=w, delays=syn_delay)
# proj = pynnx.Proj(inputs, neurons, pynn.OneToOneConnector,
#                   weights=w, delays=syn_delay)

graph = pynnx._graph.clone()
pprint(graph)

pynnx.run(sim_time)

out_spikes = pynnx.get_spikes(neurons)
in_spikes = pynnx.get_spikes(inputs)

if backend != GENN:
    weights = pynnx.get_weights(proj)
else:
    weights = np.zeros((N_NEURONS, N_NEURONS))

pynnx.end()

fig = plt.figure(figsize=(10, 5))
ax = plt.subplot(1, 1, 1)
for nid, times in enumerate(in_spikes):
    plt.plot(times, np.ones_like(times) * nid, '+g', markersize=5)
for nid, times in enumerate(out_spikes):
    plt.plot(times, np.ones_like(times) * nid, 'xb', markersize=5)

ax.set_xlim(0, sim_time)
ax.set_ylim(0, N_NEURONS)
ax.set_xlabel('Time [ms]')
ax.set_ylabel('Neuron id')

# ax = plt.subplot(1, 2, 2)
# plt.imshow(weights)
# ax.set_xlabel('Post id')
# ax.set_ylabel('Pre id')

plt.savefig("output.pdf")
plt.show()
#
#
#
#


from typing import Set, List, Union

import networkx
from flloat.parser.ltlf import LTLfParser
from flloat.semantics.pl import _PLInterpretation
from pythomata.base import DFA

import autograph.lib.hoa as hoa

def style_agraph(network: networkx.MultiDiGraph, currentstates: List[int], layout="dot"):
    """
    Turn an automaton network into a nicely styled agraph (graphviz graph).
    :param network: The network to format
    :param currentstates: A list of states to highlight in the output
    :return: The formatted graph
    """
    graph = networkx.drawing.nx_agraph.to_agraph(network)

    for state in currentstates:
        node = graph.get_node(state)
        node.attr["style"] = "filled"
        node.attr["fillcolor"] = "yellow"

    for state in network:
        acc = network.nodes[state]["acceptance"]
        if acc and len(acc) > 0:
            # graph.get_node(state).attr["label"] = str(state) + "\n" + str(acc)
            graph.get_node(state).attr["shape"] = "doublecircle"
        else:
            graph.get_node(state).attr["shape"] = "circle"

    for u, v, key, data in network.edges(keys=True, data=True):
        graph.get_edge(u, v, key).attr["label"] = " " + data["label"] + " "

    # graph.add_node("start")
    # graph.get_node("start").attr["style"] = "invis"
    # graph.add_edge("start", network.graph["start"])

    if layout:
        graph.layout(prog=layout)

    return graph

class AutomatonSet:
    """
    An automaton, along with one or more states that the automaton is currently in.
    """

    def __init__(self, graph: "AutomatonGraph", states: Set[int]):
        self.states = states
        self.graph = graph

    @staticmethod
    def from_hoa(automaton: hoa.HOA):
        """
        Create an automaton graph from a parsed HOA file
        :param automaton: The HOA file
        :return: An automaton in the start state
        """
        graph, start = AutomatonGraph.from_hoa(automaton)
        return AutomatonSet(graph, {start})

    @staticmethod
    def from_ltlf(ltl: str, apnames: List[str]):
        graph, start = AutomatonGraph.from_ltlf(ltl, apnames)
        return AutomatonSet(graph, {start})

    def _repr_svg_(self):
        return self.graph.to_image(self.states)

    def transition(self, aps):
        """
        Transition an automaton from all the states it might be in now to all the states that
        :param aps: A bitmap of atomic propositions to determine which transitions to take
        """
        states = [self.graph.transition_states(state, aps) for state in self.states]

        return AutomatonSet(self.graph, set().union(*states))

    def acceptance(self):
        """
        Get a set of all acceptances which are currently satisfied by any state that the automaton may be in
        """
        accs = [self.graph.acceptance(state) for state in self.states]

        return set().union(*accs)

    def reaches_acceptance(self):
        return any(map(self.graph.reaches_acceptance, self.states))

    def __eq__(self, other):
        return isinstance(other, AutomatonSet) and self.states == other.states and self.graph == other.graph

def alike(tup1, tup2):
    diff = sum(0 if e1 == e2 else 1 for e1, e2 in zip(tup1, tup2))
    return diff <= 1

def combined_tuple(tup1, tup2):
    return tuple(e1 if e1 == e2 else "X" for e1, e2 in zip(tup1, tup2))

def simplify_conds(conds):
    """
    Combine multiple conditions into as few as possible.
    For example, (1, 0, 1) and (1, 0, 0) become (1, 0, X).
    This process is repeated until it doesn't do anything.
    Note that this may not be a perfect solution (np-complete problem), but it provides a decent approximation.
    :param conds:
    :return:
    """
    conds = sorted(conds)

    lastconds = None
    while lastconds != conds:
        lastconds = conds.copy()
        conds = set()
        for cond in lastconds:
            added = False

            for comp in conds:
                if alike(cond, comp):
                    conds.remove(comp)
                    conds.add(combined_tuple(cond, comp))
                    added = True
                    break

            if not added:
                conds.add(cond)

    return conds

class evaluator():
    def __init__(self, c):
        self.c = c

    def __call__(self, aps):
        return tuple(aps) in self.c

class AutomatonGraph:
    """
    Stateless representation of an automaton which may be non-deterministic
    """

    def __init__(self, network: networkx.MultiDiGraph):
        self.network = network
        accepted_states = [node for node in network.nodes if len(self.acceptance(node)) > 0]
        path_lengths = {fr: to for fr, to in networkx.all_pairs_shortest_path_length(network)}
        self.reachable = {node for node in network.nodes if
                          any(map(lambda dest: dest in path_lengths[node], accepted_states))}

    @staticmethod
    def from_hoa(automaton: hoa.HOA):
        """
        Turn a parse tree of a hoa file into an automaton and extract the starting state
        :return: The graph, and the starting state of the graph
        """
        graph = networkx.MultiDiGraph()

        start = None
        alias = dict()
        apnames = None

        # Process headers
        for headeritem in automaton.header.items:
            if isinstance(headeritem, hoa.StartHeader):  # Starting state of automaton
                start = int(headeritem.startstate)
            elif isinstance(headeritem, hoa.AliasHeader):  # Take care of aliases
                alias[headeritem.name] = headeritem.label
            elif isinstance(headeritem, hoa.APHeader):  # Names of atomic propositions
                apnames = [name.value for name in headeritem.props]

        assert start is not None, "Automaton needs a start value"
        assert apnames is not None, "Automaton needs to have at least one atomic proposition"

        # Add each individual state to graph
        for state in automaton.body:
            id = int(state.id)
            accsig = state.accsig if state.accsig else []
            accsig: Set[int] = {int(condition) for condition in accsig}
            graph.add_node(id, acceptance=accsig)

            for edge in state.edges:
                assert not edge.accsig, "Automaton must use state-based acceptance"
                to = int(edge.dest_state)
                lbl: hoa.LabelExpr = edge.label.value

                # Generate evaluator for edge (accepts array of APs and outputs bool)
                graph.add_edge(id, to, evaluator=lbl.generate_evaluator(alias), label=lbl.str_with_ap_names(apnames))

        graph.graph["start"] = start

        return AutomatonGraph(graph), start

    @staticmethod
    def from_ltlf(ltlf: str, ap_names: List[str]):
        """
        Construct an automaton graph from a DFA
        :param ltlf: The ltlf formula
        :param ap_names: An ordered list of names for the atomic propositions
        """
        ltl_parser = LTLfParser()

        # Parse to DFA
        parsed_formula = ltl_parser(ltlf)
        dfa: DFA = parsed_formula.to_automaton(determinize=True)
        assert 2 ** len(ap_names) == len(dfa.alphabet.symbols), "Alphabet size mismatch- make sure your ap_names are " \
                                                                "the right length"
        graph = networkx.MultiDiGraph()

        for state in dfa.states:
            trans_to = dfa.transition_function[state]
            states_to = dict()

            accsig = {0} if state in dfa.accepting_states else set()
            graph.add_node(state, acceptance=accsig)

            # Go from {set(atomprop):state} to {state:f_set([atomprop-bitmap])}

            # Collect {state:list[set(atomprop)]}
            for cond, dest in trans_to.items():
                if dest not in states_to:
                    states_to[dest] = [cond]
                else:
                    states_to[dest].append(cond)

            # set(atomprop) -> [atomprop-bitmap]
            def bitmap_for_conditions(cond: _PLInterpretation):
                return tuple([name in cond.true_propositions for name in ap_names])

            # Put it all together
            states_to_bitmap = {to: frozenset(map(bitmap_for_conditions, conds)) for to, conds in states_to.items()}

            for to, conds in states_to_bitmap.items():
                graph.add_edge(state, to,
                               evaluator=evaluator(conds),
                               label=str(simplify_conds(conds)).replace("\'X\'", "X").replace("True", "T").replace(
                                   "False", "F"))

        graph.graph["start"] = dfa.initial_state
        return AutomatonGraph(graph), dfa.initial_state

    def transition_states(self, state: int, aps: List[bool]):
        """
        What states can be the direct result of a given transition?
        :param state: The state that the transition starts at
        :param aps: A bitmap of atomic propositions that are True and False
        :return: A set of states that the automaton transitions to
        """
        outstates = set()

        for u, v, data in self.network.edges(state, data=True):
            label: hoa.Evaluator = data["evaluator"]
            if label(aps):
                outstates.add(int(v))

        return outstates

    def reaches_acceptance(self, state):
        return state in self.reachable

    def acceptance(self, state):
        """
        Get the acceptance conditions for a given state
        :return: A set of acceptance conditions
        """
        return self.network.nodes[state]["acceptance"]

    def to_image(self, current_state: Union[int, Set[int]] = None):
        """
        Render an AutomationGraph to an image, used for public access to _repr_svg_
        :param current_state: Either a state number or set of states that the automaton is in
        :return: An svg image
        """
        return self._repr_svg_(current_state)

    def _repr_svg_(self, current_state: Union[int, Set[int]] = None):
        if current_state is None:
            current_state = set()

        if isinstance(current_state, int):
            current_state = [current_state]

        return style_agraph(self.network, current_state).draw(format='svg').decode("utf-8")