import logging
import threading
import uuid

from django import forms
from django.db import models
from django.utils.text import Truncator, slugify
from modelcluster.fields import ParentalManyToManyField
from modelcluster.models import ClusterableModel
from wagtail.admin.forms import WagtailAdminPageForm
from wagtail.admin.panels import FieldPanel, FieldRowPanel, MultiFieldPanel
from wagtail.api import APIField
from wagtail.models import Page
from wagtail.search import index

from mycarehub.common.models import Organisation, Program
from mycarehub.content.models.snippets import CustomSnippetForm

LOGGER = logging.getLogger(__name__)

class SMSContentItemPageForm(WagtailAdminPageForm):
    def __init__(
        self, data=None, files=None, parent_page=None, subscription=None, *args, **kwargs
    ):  # pragma: no cover
        super().__init__(data, files, parent_page, subscription, *args, **kwargs)
        self.fields["category"].queryset = self.fields["category"].queryset.filter(
            organisation=self.for_user.organisation, programs=parent_page.specific.program_id
        )
        self.fields["tag"].queryset = self.fields["tag"].queryset.filter(
            organisation=self.for_user.organisation, programs=parent_page.specific.program_id
        )

class SMSContentItemCategory(ClusterableModel):
    """Category associated with an SMS."""

    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
    code = models.CharField(
        max_length=32,
        unique=True,
        help_text="A code that will be used to uniquely identify a category",
    )
    name = models.CharField(
        max_length=64, help_text="This name will be displayed while creating sms content"
    )
    sequence_key = models.IntegerField(
        help_text="A pre-determined value to be used while sequencing content under this category"
    )
    programs = ParentalManyToManyField(
        Program, help_text="Select a program to assign this category"
    )
    organisation = models.ForeignKey(
        Organisation,
        on_delete=models.PROTECT,
        null=True,
        blank=True,
        related_name="%(app_label)s_%(class)s_related",
    )
    base_form_class = CustomSnippetForm

    def __str__(self):
        return f"{self.name}"

    class Meta:
        unique_together = (
            "sequence_key",
            "code",
        )
        verbose_name_plural = "sms content item categories"

    panels = [
        FieldPanel("code"),
        FieldPanel("name"),
        FieldPanel("sequence_key"),
        FieldPanel("programs", widget=forms.CheckboxSelectMultiple),
    ]

    def get_programs(self):
        return "\n".join([p.name for p in self.programs.all()])

class SMSContentItemTag(ClusterableModel):
    """Tag associated with an SMS."""

    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
    name = models.CharField(
        max_length=64, help_text="This name will be displayed when creating sms content"
    )
    code = models.IntegerField(
        help_text="A pre-determined value that will be used to uniquely identify a tag"
    )
    programs = ParentalManyToManyField(Program, help_text="Select a program to assign this tag")
    organisation = models.ForeignKey(
        Organisation,
        on_delete=models.PROTECT,
        null=True,
        blank=True,
        related_name="%(app_label)s_%(class)s_related",
    )
    base_form_class = CustomSnippetForm

    def __str__(self):
        return f"{self.name}"

    class Meta:
        verbose_name_plural = "sms content item tags"
        unique_together = (
            "name",
            "code",
        )

    panels = [
        FieldPanel("name"),
        FieldPanel("code"),
        FieldPanel("programs", widget=forms.CheckboxSelectMultiple),
    ]

    def get_programs(self):
        return "\n".join([p.name for p in self.programs.all()])

class SMSContentItem(Page):
    """
    An `SMSContentItem` represents the content that
    is sent to sms subscribers.
    """

    organisation = models.ForeignKey(
        Organisation,
        on_delete=models.PROTECT,
        null=True,
        blank=True,
        related_name="%(app_label)s_%(class)s_related",
    )

    program = models.ForeignKey(
        Program,
        on_delete=models.PROTECT,
        null=True,
        blank=True,
        related_name="%(app_label)s_%(class)s_related",
    )

    category = models.ForeignKey(
        SMSContentItemCategory,
        on_delete=models.PROTECT,
        related_name="categories",
        help_text="Category to associate content with",
    )

    tag = models.ForeignKey(
        SMSContentItemTag,
        on_delete=models.PROTECT,
        related_name="tags",
        help_text="Tag to associate content with",
    )
    sequence_number = models.IntegerField(null=True, blank=True)
    sequence = models.CharField(max_length=10, null=True, blank=True)
    body = models.TextField(
        max_length=160, help_text="Write out the body of the content to be sent to subscribers"
    )

    base_form_class = SMSContentItemPageForm

    content_panels = Page.content_panels + [
        MultiFieldPanel(
            [
                FieldRowPanel(
                    [
                        FieldPanel("category", widget=forms.Select),
                        FieldPanel("tag", widget=forms.Select),
                    ]
                ),
            ],
            heading="About",
        ),
        FieldPanel("body"),
    ]

    # these fields determine the content that is indexed for search purposes
    search_fields = Page.search_fields + [
        index.SearchField("category"),
        index.SearchField("body"),
    ]

    # this configuration allows these custom fields to be available over the API
    api_fields = [
        APIField("live"),
        APIField("organisation"),
        APIField("program"),
        APIField("category"),
        APIField("tag"),
        APIField("sequence"),
        APIField("sequence_number"),
        APIField("body"),
    ]

    # limit the parent page types
    parent_page_type = [
        "content.ContentItemIndexPage",
    ]

    subpage_types = []  # type: ignore

    def __str__(self):
        """Human readable model representation."""
        return f"{self.sequence} {self.title}"

    def generate_sequence_number(self):
        """Generate content sequence and sequence numbers."""
        if self.__class__.objects.filter(pk=self.pk, sequence__isnull=False):
            msg = "Sequence has already been set"
            LOGGER.info(msg)
            return

        filters = {
            "organisation": self.organisation,
            "program": self.program,
            "category": self.category,
        }
        seq = (
            self.__class__.objects.filter(**filters)
            .order_by("translation_key")
            .distinct("translation_key")
            .count()
        )
        lock = threading.Lock()

        def increment_seq(lock):
            lock.acquire()
            nonlocal seq
            seq = seq + 1
            lock.release()

        thread = threading.Thread(target=increment_seq, args=(lock,))
        thread.start()
        thread.join()

        self.sequence_number = seq
        self.sequence = f"{self.category.sequence_key}.{self.sequence_number}"

    def save(self, *args, **kwargs):
        """Override save method.

        We replace the title to make it easier for users to
        keep track of their sms's.
        """
        new_title = Truncator(self.body).chars(30)
        slug = slugify(new_title)

        unique_slug = slug
        offset = 1
        while self.__class__.objects.filter(slug=unique_slug).exists():
            msg = "Duplicate slug found, generating unique slug"
            LOGGER.info(msg)

            unique_slug = "{}-{}".format(slug, offset)  # type: ignore
            offset += 1

        self.slug = unique_slug

        self.title = new_title
        super().save(*args, **kwargs)


from typing import Iterable
from itertools import pairwise
from more_itertools import triplewise

def number_of_increases(values: Iterable[int]) -> int:
    """
    Given a list of numbers, return the number of consecutive pairs in which a
    number is larger than the one that precedes it.
    """
    return sum(a < b for a, b in pairwise(values))

def triple_sums(values: Iterable[int]) -> Iterable[int]:
    for vals in triplewise(values):
        yield sum(vals)

if __name__ == '__main__':
    content = open('input.txt').read()
    values = [int(line) for line in content.strip().splitlines()]

    print('Part1:')
    n = number_of_increases(values)
    print(n)

    print('Part2:')
    s = triple_sums(values)
    n = number_of_increases(s)
    print(n)


# coding=utf-8
"""
Django settings for tansparency project.

For more information on this file, see
https://docs.djangoproject.com/en/dev/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/dev/ref/settings/
"""

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os
import dj_database_url
from local_settings import *
import dropbox

BASE_DIR = os.path.dirname(os.path.dirname(__file__))
PROJECT_ROOT = os.path.dirname(os.path.abspath(__file__))

# Application definition
INSTALLED_APPS = (
    # 'admin_tools',
    # 'admin_tools.theming',
    # 'admin_tools.menu',
    # 'admin_tools.dashboard',

    # django-autocomplete-light (2017/03/29)
    'dal',
    'dal_select2',

    # Social OAuth logins (2017/03/10)
    'social_django',

    # Forms usability improvements (2017/03/10)  TODO: fix this date (2017/03/15)
    'widget_tweaks',

    # CPF and CEP form fields (2017/03/10)  TODO: fix this date (2017/03/15)
    'localflavor',

    # Main app, handles internal management (2017/03/10)  TODO: fix this date (2017/03/15)
    'website',

    # Website at https://purubemalves.com.br (2017/03/10)  TODO: fix this date (2017/03/15)
    'home',

    # Applications for students performance tracking (2017/03/10)  TODO: fix this date (2017/03/15)
    'projeto_redacao',
    'controle_de_frequencia',

    # Feedback meta applications (2017/03/10)  TODO: fix this date (2017/03/15)
    'issues',
    'issues.templatetags',

    # Affirmative actions management (2017/03/10)
    'politicas_afirmativas',

    # Students views  (2017/03/15)
    'alunos',

    # Reports views  (2017/03/21)
    'reports',

    # Static content distribution (2017/03/10)
    'storages',

    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
)

MIDDLEWARE_CLASSES = (
    # 'sslify.middleware.SSLifyMiddleware',  # TODO: Decomment this line before pushing (2017/03/14)
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
    'django.middleware.security.SecurityMiddleware',

    # Social OAuth login handler
    'social_django.middleware.SocialAuthExceptionMiddleware',
)

ROOT_URLCONF = 'transparency.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [
            os.path.join(PROJECT_ROOT, '../static/templates/'),
            os.path.join(BASE_DIR, '/static/templates/'),
        ],
        'APP_DIRS': True,
        'OPTIONS': {
            # 'loaders': [
            #     'admin_tools.template_loaders.Loader',
            # ],
            'libraries': {
                # Templete tools for dictionaries (2017/03/15)
                'dict_filters': 'issues.templatetags.dict_filters',
            },
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',

                # Social OAuth login context processors
                'social_django.context_processors.backends',
                'social_django.context_processors.login_redirect',
            ],
        },
    },
]

# Social & Internal logins (2017/03/10)
AUTHENTICATION_BACKENDS = (
    # 'social_core.backends.github.GithubOAuth2',
    # 'social_core.backends.twitter.TwitterOAuth',
    'social_core.backends.facebook.FacebookOAuth2',

    'django.contrib.auth.backends.ModelBackend',
)

# Login information for the Alunos application (2017/03/15)
LOGIN_URL = 'login'
LOGOUT_URL = 'alunos/logout'
LOGIN_REDIRECT_URL = '/alunos'

# Django's main web application address
WSGI_APPLICATION = 'transparency.wsgi.application'

# Password validation
# https://docs.djangoproject.com/en/1.10/ref/settings/#auth-password-validators
AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
    },
]

# Internationalization
# https://docs.djangoproject.com/en/dev/topics/i18n/

LANGUAGE_CODE = 'pt-BR'

TIME_ZONE = 'America/Sao_Paulo'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Update database configuration with $DATABASE_URL.
db_from_env = dj_database_url.config(conn_max_age=500)
DATABASES['default'].update(db_from_env)

# Honor the 'X-Forwarded-Proto' header for request.is_secure()
SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/dev/howto/static-files/

STATIC_URL = '/static/'
STATIC_ROOT = os.path.join(PROJECT_ROOT, 'staticfiles')
ENCRYPT_URL = '/.well-known/'
# noinspection PyUnresolvedReferences
ENCRYPT_ROOT = os.path.join(STATIC_ROOT, '.well-known')

# Extra places for collectstatic to find static files.
STATICFILES_DIRS = (
    os.path.join(PROJECT_ROOT, '../static'),
)

# STATICFILES_FINDERS = (
#     'django.contrib.staticfiles.finders.AppDirectoriesFinder',
# )

# Simplified static file serving.
# https://warehouse.python.org/project/whitenoise/
STATICFILES_STORAGE = 'whitenoise.django.GzipManifestStaticFilesStorage'

# Dropbox storage settings (2017/03/15)
dbx = dropbox.Dropbox(DROPBOX_OAUTH2_TOKEN)

SECURE_SSL_REDIRECT = False
ADMIN_SITE_HEADER = "Pré-Universitário Comunitário Rubem Alves"
ADMIN_INDEX_TITLE = "Administração do Projeto"


#+---------------------------------------------------------------------
#+ Python Script that creates CMAQ OCEAN files form MCIP GRICRO2d file
#+ Check the 'CHANGE' comments to define your directories for the run
#+ Author: Camilo Moreno
#+ Email = cama9709@gmail.com
#+---------------------------------------------------------------------
#%%
import numpy as np
import glob
from netCDF4 import Dataset
from datetime import datetime, date, timedelta
from os import listdir, scandir, getcwd

def create_forced_var(ds_cro):
    """Create array where cells that wanted to be forced are equal to 1
    and the rest are equal to 0 for multiple locations.

    Keyword arguments:
    ds_cro -- Dataset of the latlon netCDF file of the grid
    dic_coords -- dictionary of latlon coordinates for lower left and
                  upper right corners of each specified kay-value location
    """
    open_var = ds_cro.variables['LWMASK'][:][:][:][:]
    open_var[open_var > 0] = 2
    open_var[open_var == 0] = 1
    open_var[open_var > 1] = 0

    return open_var

def create_ncfile(save_dir, spc_name, open_var, ds_ocean):
    """Create Final NETCDF file.

    Keyword arguments:
    save_dir -- string of the location for saving the netCDF files
    ds_cro -- Dataset of the latlon netCDF file of the grid
    spc_name -- string of the forced species name
    open_var -- array containing the forced values by location for the species
    ds_ocean -- conc file dataset
    """
    hr = 0
    num_vars = 1
    lays = 1
    ltime = 1
    cols = len(ds_ocean.dimensions['COL'])
    rows = len(ds_ocean.dimensions['ROW'])
    datetimes = len(ds_ocean.dimensions['DATE-TIME'])

    #* Create new netCDF
    new_cmaq_file = f'{save_dir}/ocean_file_2018_NSthAm_CROS.ncf'
    ds_new_cmaq = Dataset(new_cmaq_file, open = True, mode = 'w', format=  "NETCDF3_64BIT")

    #* Create dimenssions
    TSTEP = ds_new_cmaq.createDimension("TSTEP", None)
    DATE_TIME = ds_new_cmaq.createDimension("DATE-TIME", datetimes)
    LAY = ds_new_cmaq.createDimension("LAY", lays)
    VAR = ds_new_cmaq.createDimension("VAR", num_vars)
    ROW = ds_new_cmaq.createDimension("ROW", rows)
    COL = ds_new_cmaq.createDimension("COL", cols)

    ds_new_cmaq.sync()

    #* Creatae attributes
    attrs=["IOAPI_VERSION", "EXEC_ID", "FTYPE", "CDATE", "CTIME", "WDATE", "WTIME",
           "SDATE", "STIME", "TSTEP", "NTHIK", "NCOLS", "NROWS", "GDTYP", "P_ALP",
           "P_BET", "P_GAM", "XCENT", "YCENT", "XORIG", "YORIG", "XCELL", "YCELL",
           "VGTYP", "VGTOP", "VGLVLS", "GDNAM", "HISTORY"]

    for attr in attrs:
        if hasattr(ds_ocean, attr):
            attrVal = getattr(ds_ocean, attr)
            setattr(ds_new_cmaq, attr, attrVal)

    varlist = spc_name + ' '*(16 - len(spc_name))
    cmaq_attrs = {'NLAYS': np.int32(lays),
                  'NVARS': np.int32(num_vars),
                  'UPNAM': "OCEAN_FILE",
                  'VAR-LIST': varlist,
                  'FILEDESC': "Created by Camilo Moreno"
                  }

    for attr in cmaq_attrs:
        ds_new_cmaq.setncattr(attr, cmaq_attrs[attr])

    #* Create variables
    tflag = ds_new_cmaq.createVariable('TFLAG', 'i4', ('TSTEP', 'VAR', 'DATE-TIME'))
    fill_attrs(ds_ocean, tflag, 'TFLAG')

    var_temp = ds_new_cmaq.createVariable(spc_name,"f4",("TSTEP", "LAY", "ROW", "COL"))
    fill_attrs(ds_ocean, var_temp, 'OPEN')

    #* Fill variables
    ds_new_cmaq.variables[spc_name][:, :, :] = open_var
    dattim = np.squeeze(ds_ocean.variables['TFLAG'][:][:])
    tflag[:] = np.zeros([ltime,lays,datetimes])
    tflag[:] = np.reshape(dattim,[ltime,lays,datetimes])

    #* Close new netcdf file
    ds_new_cmaq.close()

    print(f" OCEAN file DONE")

def fill_attrs(ds_ocean, nc_var, var_name):
    """Fill atribute values for variables as appear in conc file.

    Keyword arguments:
    ds_ocean -- conc file dataset
    nc_var -- netCDF variable of the file
    var_name -- variable name
    """
    varattrs=["long_name","units","var_desc"]
    for varattr in varattrs:
        if hasattr(ds_ocean.variables[var_name], varattr):
            varattrVal = getattr(ds_ocean.variables[var_name], varattr)
            setattr(nc_var, varattr, varattrVal)

def get_concfiels(ocean_file):
    """Get all files that begins whith 'CONC' from the given directory.

    Keyword arguments:
    ocean_file -- string of the directory whre conc day files
    """
    all_files = [f for f in glob.glob(f'{ocean_file}/CONC.*')]
    all_files.sort()
    return all_files

# %%
if __name__ == "__main__":
    #CHANGE: save_dir: directory path where the forced files will be saved
    #        latlon_file: path of the latlon netCDF of the run grid
    #        ocean_file: path where CONC files are located
    #        spc_name: cbo5_aero5_aq name of the one species to be forced
    #        dic_coord: dictionary containing a list of latlon coords of lower left
    #                   and upper right corners for each key-value location.
    #                   Format fo dic_coord must be:
    #                   dic_coord = {
    #                               'name_of_location_1': [lower_left_lat_1, lower_left_lon_1, upper_left_lat_1, upper_left_lon_1]
    #                               'name_of_location_2': [lower_left_lat_2, lower_left_lon_2, upper_left_lat_2, upper_left_lon_2]
    #                                 }
    save_dir = '/Volumes/Avispa/OCEAN'
    cro_file = '/Users/camilo/Downloads/ocean/GRIDCRO2D_20180208.nc'
    ocean_file = '/Users/camilo/Downloads/ocean_file_2018_NSthAm_CROS.ncf'
    spc_name = 'OPEN'

    ds_cro = Dataset(cro_file, mode = 'r',  open = True)
    ds_ocean = Dataset(ocean_file, mode = 'r',  open = True)

    open_var = create_forced_var(ds_cro)
    create_ncfile(save_dir, spc_name, open_var, ds_ocean)

    ds_ocean.close()
    ds_cro.close()

# %%


############################################################
# -*- coding: utf-8 -*-
#
#       #   #  #   #   #    #
#      ##  ##  #  ##  #    #
#     # # # #  # # # #    #  #
#    #  ##  #  ##  ##    ######
#   #   #   #  #   #       #
#
# Python-based Tool for interaction with the 10micron mounts
# GUI with PyQT5 for python
#
# written in python3, (c) 2019-2023 by mworion
# Licence APL2.0
#
###########################################################
# standard libraries
import os
from pathlib import Path

# external packages
from PyQt5.QtWidgets import QListView, QApplication
from astropy.io import fits

# local import

class Rename:
    """
    """

    def __init__(self):
        self.selectorsDropDowns = {'rename1': self.ui.rename1,
                                   'rename2': self.ui.rename2,
                                   'rename3': self.ui.rename3,
                                   'rename4': self.ui.rename4,
                                   'rename5': self.ui.rename5,
                                   'rename6': self.ui.rename6,
                                   }
        self.fitsHeaderKeywords = {'None': [''],
                                   'Datetime': ['DATE-OBS'],
                                   'Frame': ['FRAME', 'IMAGETYP'],
                                   'Filter': ['FILTER'],
                                   'Binning': ['XBINNING'],
                                   'Exp Time': ['EXPTIME'],
                                   'CCD Temp': ['CCD-TEMP'],
                                   }
        self.setupGuiTools()
        self.ui.renameStart.clicked.connect(self.renameRunGUI)
        self.ui.renameInputSelect.clicked.connect(self.chooseDir)

    def initConfig(self):
        """
        initConfig read the key out of the configuration dict and stores it to the gui
        elements. if some initialisations have to be proceeded with the loaded persistent
        data, they will be launched as well in this method.

        :return: True for test purpose
        """
        config = self.app.config['mainW']
        defaultDir = self.app.mwGlob['imageDir']
        self.ui.renameDir.setText(config.get('renameDir', defaultDir))
        self.ui.newObjectName.setText(config.get('newObjectName', ''))
        self.ui.includeSubdirs.setChecked(config.get('includeSubdirs', False))
        for name, ui in self.selectorsDropDowns.items():
            ui.setCurrentIndex(config.get(name, 0))

        self.ui.renameProgress.setValue(0)
        return True

    def storeConfig(self):
        """
        storeConfig writes the keys to the configuration dict and stores. if some
        saving has to be proceeded to persistent data, they will be launched as
        well in this method.

        :return: True for test purpose
        """
        config = self.app.config['mainW']
        config['renameDir'] = self.ui.renameDir.text()
        config['newObjectName'] = self.ui.newObjectName.text()
        config['includeSubdirs'] = self.ui.includeSubdirs.isChecked()
        for name, ui in self.selectorsDropDowns.items():
            config[name] = ui.currentIndex()

        return True

    def setupGuiTools(self):
        """
        :return: success for test
        """
        for name, selectorUI in self.selectorsDropDowns.items():
            selectorUI.clear()
            selectorUI.setView(QListView())
            for headerEntry in self.fitsHeaderKeywords:
                selectorUI.addItem(headerEntry)
        return True

    @staticmethod
    def getNumberFiles(pathDir='', search=''):
        """
        getNumberFiles counts the number of files to be valid for the renaming.

        :param pathDir: path to root directory to be scanned
        :param search: search string
        :return: number of files found
        """
        if not pathDir:
            return 0
        if not os.path.isdir(pathDir):
            return 0
        if not search:
            return 0

        number = sum(1 for _ in Path(pathDir).glob(search))
        return number

    @staticmethod
    def convertHeaderEntry(entry='', fitsKey=''):
        """
        convertHeaderEntry takes the fitsHeader entry and reformat it to a
        reasonable string.

        :param entry:
        :param fitsKey:
        :return:
        """
        if not fitsKey:
            return ''
        if not entry:
            return ''

        if fitsKey == 'DATE-OBS':
            chunk = entry.replace(':', '-')
            chunk = chunk.replace('T', '_')
            chunk = chunk.split('.')[0]
        elif fitsKey == 'XBINNING':
            chunk = f'Bin{entry:1.0f}'
        elif fitsKey == 'CCD-TEMP':
            chunk = f'Temp{entry:03.0f}'
        elif fitsKey == 'FRAME':
            chunk = f'{entry}'
        elif fitsKey == 'FILTER':
            chunk = f'{entry}'
        elif fitsKey == 'EXPTIME':
            chunk = f'Exp{entry:1.0f}s'
        else:
            chunk = ''

        return chunk

    def processSelectors(self, fitsHeader=None, selection=''):
        """
        processSelectors takes the selection for a fileName chunk and runs through the
        possible list of valid fits header keys. if there is more than one valid fitsKey,
        it automatically selects only the first on for conversion.

        :param fitsHeader:
        :param selection: str entry from the drop-down selector
        :return: nameChunk: part of the entry
        """
        if fitsHeader is None:
            return ''
        if not selection:
            return ''

        nameChunk = ''
        fitsKeywords = self.fitsHeaderKeywords[selection]
        for fitsKey in fitsKeywords:
            if fitsKey not in fitsHeader:
                continue
            nameChunk = self.convertHeaderEntry(entry=fitsHeader[fitsKey],
                                                fitsKey=fitsKey)
            break
        return nameChunk

    def renameFile(self, fileName=''):
        """
        renameFile opens the given FITS file and retrieves its header. if valid it
        runs through selectors of the drop-down lists and checks all header keys to
        get the new filename build. afterwards it renames the given file.

        :param fileName: fits file to be renamed
        :return: success
        """
        if not fileName:
            return False
        if not os.path.isfile(fileName):
            return False

        fileName = os.path.normpath(fileName)
        with fits.open(name=fileName) as fd:
            fitsHeader = fd[0].header

            # object should be in lower case. if not, it will be set
            newObjectName = self.ui.newObjectName.text().upper()
            if newObjectName:
                newFilename = newObjectName

            else:
                if 'OBJECT' in fitsHeader:
                    newFilename = fitsHeader['OBJECT'].upper()
                else:
                    newFilename = 'UNKNOWN'

            for _, selector in self.selectorsDropDowns.items():
                selection = selector.currentText()
                chunk = self.processSelectors(fitsHeader=fitsHeader,
                                              selection=selection
                                              )
                if chunk:
                    newFilename += f'_{chunk}'

            newFilename += '.fits'
            dirName = os.path.dirname(fileName)
            newFilename = os.path.normpath(f'{dirName}/{newFilename}')
            os.rename(fileName, newFilename)

        return True

    def renameRunGUI(self):
        """
        renameRunGUI retrieves a full list of files to be renamed and renames
        them on by one.

        :return: True for test purpose
        """
        pathDir = self.ui.renameDir.text()
        includeSubdirs = self.ui.includeSubdirs.isChecked()
        if not os.path.isdir(pathDir):
            self.msg.emit(2, 'Tools', 'Rename error',
                          'No valid input directory given')
            return False

        if includeSubdirs:
            search = '**/*.fit*'
        else:
            search = '*.fit*'

        numberFiles = self.getNumberFiles(pathDir, search=search)
        if not numberFiles:
            self.msg.emit(2, 'Tools', 'Rename error',
                          'No files to rename')
            return False

        for i, fileName in enumerate(Path(pathDir).glob(search)):
            self.ui.renameProgress.setValue(int(100 * (i + 1) / numberFiles))
            QApplication.processEvents()
            suc = self.renameFile(fileName=fileName)
            if not suc:
                self.msg.emit(2, 'Tools', 'Rename error',
                              f'{fileName} could not be renamed')

        self.msg.emit(0, 'Tools', 'Rename',
                      f'{numberFiles:d} images were renamed')

        return True

    def chooseDir(self):
        """
        chooseDir selects the input directory and sets the default value for the
        output directory as well

        :return: True for test purpose
        """
        folder = self.ui.renameDir.text()
        pathDir, _, _ = self.openDir(self, 'Choose Input Dir', folder,)
        if pathDir:
            self.ui.renameDir.setText(pathDir)
            self.ui.renameProgress.setValue(0)
        return True


import time
from types import MappingProxyType
from collections import namedtuple
import math

VdmInfo = namedtuple('VdmInfo', 'index data_type dimension')

class Vdm:
    """
    call Vdm.init_vdm_mapping() once before any vdm operation
    # VDM Config pages: 20h, 21h, 22h, 23h
    # VDM Real Value pages: 24h, 25h, 26h, 27h
    """
    DATA_TYPE_U16 = 1
    DATA_TYPE_S16 = 2
    DATA_TYPE_F16 = 3

    CONFIG_CODE_MAPPING = MappingProxyType({
        # config_code: [data_type, key, dimension]
        9:   [DATA_TYPE_F16, "Media Pre-FEC Minimum", 1],
        11:  [DATA_TYPE_F16, "Media Pre-FEC Maximum", 1],
        13:  [DATA_TYPE_F16, "Media Pre-FEC Average", 1],
        15:  [DATA_TYPE_F16, "Media Pre-FEC Current", 1],
        17:  [DATA_TYPE_F16, "Media Post-FEC Minimum", 1],
        19:  [DATA_TYPE_F16, "Media Post-FEC Maximum", 1],
        21:  [DATA_TYPE_F16, "Media Post-FEC Average", 1],
        23:  [DATA_TYPE_F16, "Media Post-FEC Current", 1],

        10:  [DATA_TYPE_F16, "Host Pre-FEC Minimum", 1],
        12:  [DATA_TYPE_F16, "Host Pre-FEC Maximum", 1],
        14:  [DATA_TYPE_F16, "Host Pre-FEC Average", 1],
        16:  [DATA_TYPE_F16, "Host Pre-FEC Current", 1],
        18:  [DATA_TYPE_F16, "Host Post-FEC Minimum", 1],
        20:  [DATA_TYPE_F16, "Host Post-FEC Maximum", 1],
        22:  [DATA_TYPE_F16, "Host Post-FEC Average", 1],
        24:  [DATA_TYPE_F16, "Host Post-FEC Current", 1],

        134: [DATA_TYPE_S16, "CD", 1],
        136: [DATA_TYPE_U16, "DGD", 0.01],
        138: [DATA_TYPE_U16, "PDL", 0.1],
        141: [DATA_TYPE_S16, "CFO", 1],
        142: [DATA_TYPE_U16, "EVM", 100/65535],
        139: [DATA_TYPE_U16, "OSNR", 0.1],
        140: [DATA_TYPE_U16, "ESNR", 0.1],
        147: [DATA_TYPE_U16, "MER", 0.1],

        4:   [DATA_TYPE_S16, "Laser Temp", 1/256],
        143: [DATA_TYPE_S16, "Tx Power", 0.01],
        144: [DATA_TYPE_S16, "Rx Total Power", 0.01],
        145: [DATA_TYPE_S16, "Rx Signal Power", 0.01],
        128: [DATA_TYPE_U16, "BIAS_XI", 100/65535],
        129: [DATA_TYPE_U16, "BIAS_XQ", 100/65535],
        132: [DATA_TYPE_U16, "BIAS_XP", 100/65535],
        130: [DATA_TYPE_U16, "BIAS_YI", 100/65535],
        131: [DATA_TYPE_U16, "BIAS_YQ", 100/65535],
        133: [DATA_TYPE_U16, "BIAS_YP", 100/65535],
        2:   [DATA_TYPE_S16, "TEC_CUR", 100/32767]
    })

    def __init__(self, trx):
        self.__trx = trx
        self.__vdm_mapping = None

    def __getitem__(self, key):
        return self.__get_vdm(key)

    def init_vdm_mapping(self):
        config_pages = [0x20, 0x21, 0x22, 0x23]
        vdm_mapping = {}
        for idx_page, page in enumerate(config_pages):
            self.__trx.select_bank_page(0, page)
            b_full_page = self.__trx[128:255]
            config_codes = [int.from_bytes(b_full_page[2*i+1: 2*i+2], 'big') for i in range(64)]
            for idx_config, i_code in enumerate(config_codes):
                if i_code in self.CONFIG_CODE_MAPPING:
                    d_type, key, dim = self.CONFIG_CODE_MAPPING[i_code]
                    vdm_mapping[key] = VdmInfo(
                        index=idx_config+idx_page*64+1,
                        data_type=d_type,
                        dimension=dim
                    )
        self.__vdm_mapping = vdm_mapping

    @property
    def mapping(self):
        if self.__vdm_mapping is None:
            raise ValueError('Please call init_vdm_mapping before any vdm operation.')
        else:
            return self.__vdm_mapping

    @property
    def keys(self):
        return self.mapping.keys()

    @property
    def FreezeRequest(self):
        flag = self.__trx[0, 0x2F, 144][7]
        freeze = bool(flag)
        return freeze

    @FreezeRequest.setter
    def FreezeRequest(self, freezed):
        if not freezed in (True, False):
            raise ValueError('Parameter freezed should be bool.')
        self.__trx[0, 0x2F, 144][7] = int(freezed)

    @property
    def FreezeDone(self):
        flag = self.__trx[0, 0x2F, 145][7]
        done = bool(flag)
        return done

    @property
    def UnfreezeDone(self):
        flag = self.__trx[0, 0x2F, 145][6]
        done = bool(flag)
        return done

    def freezeUntilDone(self):
        self.FreezeRequest = True
        while not self.FreezeDone:
            time.sleep(0.05)

    def unfreezeUntilDone(self):
        self.FreezeRequest = False
        while not self.UnfreezeDone:
            time.sleep(0.05)

    def __parse_data(self, raw, data_type, dimension):
        if data_type == self.DATA_TYPE_U16:
            val = raw.to_unsigned()
        elif data_type == self.DATA_TYPE_S16:
            val = raw.to_signed()
        elif data_type == self.DATA_TYPE_F16:
            u_int_val = raw.to_unsigned()
            val = (u_int_val & 0x7FF) * 10**((u_int_val >> 11) - 24)
        else:
            raise ValueError('Invalid data_type: {:!r}'.format(data_type))
        if val != 1:
            val *= dimension
        return val

    def __calc_page_reg_from_vdm_index(self, index):
        page = math.ceil(index/64) + 0x24 - 1
        reg_addr = ((index-1) % 64) * 2 + 128
        return page, reg_addr

    def __get_vdm(self, key):
        if key not in self.keys:
            raise KeyError('Invalid key for VDM: {key}. Not configured in VDM Configuration pages.'.format(key=key))
        vdm_info = self.mapping[key]
        index, data_type, dimension = vdm_info
        page, reg_addr = self.__calc_page_reg_from_vdm_index(index)
        raw = self.__trx[0, page, reg_addr: reg_addr+1]
        val = self.__parse_data(raw, data_type, dimension)
        return val


'''
9. Write a program which implements master-worker mode:
    1 host must create N worker processes by itself
    2 when worker is created, he writes his rank in the console
    3 each worker sends his rank to host
    4 host shows the messages received in the console
    For this program you need to write two scripts - host.py and worker.py, each will implement its own logic.
'''

'''
9. Напишите программу, которая реализует режим master-worker:
    1 хост должен сам создать N рабочих процессов
    2 когда работник создан, он записывает свой ранг в консоль
    3 каждый работник отправляет свой ранг хозяину
    4 хост показывает сообщения, полученные в консоли
    Для этой программы вам нужно написать два скрипта - host.py и worker.py , каждый будет реализовывать свою собственную логику.
'''

import logging as log
from mpi4py import MPI as mpi

comm = mpi.Comm.Get_parent()
rank = comm.Get_rank()
host = 0

# 2 when worker is created, he writes his rank in the console
log.error("Host created {} worker!".format(rank))
#print("Host created {} worker!".format(rank))

# 3 each worker sends his rank to host
comm.send(rank, dest=host)

comm.Disconnect()


from django.conf.urls import patterns, include, url
from techstack_app.views import *

# Uncomment the next two lines to enable the admin:
from django.contrib import admin
admin.autodiscover()

urlpatterns = patterns('',
    # Examples:
    # url(r'^$', 'techstack.views.home', name='home'),
    url(r'^admin/doc/', include('django.contrib.admindocs.urls')),
    url(r'^admin', include(admin.site.urls)),
    url(r'^companies', companies_page),
    url(r'^search', search_controller),
    url(r'^new', add_company_page),
    url(r'^company/(\d+)$', company_page),
    url(r'^tagcloud', tagcloud_page),
    url(r'^parse', parse_blogs),
    url(r'', home_page),
)


def maxPerformance(n, speed, efficiency, k):
    _speed = [x for x, _ in sorted(zip(speed, efficiency), key=lambda p:p[1])]
    efficiency.sort()
    print(_speed)
    print(efficiency)
    from functools import lru_cache
    @lru_cache
    def util(index, total_speed, min_eff, count):
        if index >= n:
            return total_speed * min_eff
        if count == k:
            return total_speed * min_eff
        r1 = util(index + 1, total_speed + _speed[index], min(min_eff, efficiency[index]), count + 1)
        r2 = util(index + 1, total_speed, min_eff, count)
        return max(r1, r2)
    return util(0, 0, float('inf'), 0)

def maxPerformanceOp(n, speed, efficiency, k):
    from heapq import heappop, heappush
    engineers = list(zip(efficiency, speed))
    engineers.sort(reverse=True)
    min_heap = []
    speedSum = 0
    ans = 0
    for e, s in engineers:
        speedSum += s
        heappush(min_heap, s)
        if len(min_heap) > k:
            speedSum -= heappop(min_heap)
        ans = max(ans, speedSum*e)
    return ans % 1_000_000_007

n = 6
speed =  [2,10,3,1,5,8]
efficiency = [5,4,3,9,7,2]
k =4
result = maxPerformanceOp(n, speed, efficiency, k)
print(result)


import joblib
import numpy as np

if __name__ == '__main__':
    print('loading models...')
    sub_cat = joblib.load('sub_model.pkl')
    sub_categories = joblib.load('sub_slugs.pkl')

    while(True):
        text = input("input (or ctrl+c to stop):")

        a = sub_cat.predict_proba([text])
        z = int(np.argmax(a, axis=1))
        r = sub_categories[z],max(a[0])
        print(r)


import os, pyowm, subprocess, tweepy
#This contains the API keys
from credentials import *

class ExternalConnection(object):
    __instance = None
    @staticmethod
    def getInstance():
        if ExternalConnection.__instance == None:
            ExternalConnection()
        return ExternalConnection.__instance
    def __init__(self):
        if ExternalConnection.__instance !=None:
            raise Exception ("This class is a singleton")
        else:
            ExternalConnection.__instance = self
            self.api = ""
            self.TweetAPIConnect()
            self.pyOWMConnect()
    def TweetAPIConnect(self):
        auth = tweepy.OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
        auth.set_access_token(ACCESS_TOKEN, ACCESS_SECRET)
        self.api = tweepy.API(auth)
        print("\nSuccessfully connected to Twitter API.\n")
        #needs a disconnect
    def pyOWMConnect(self):
        #initialize OWM object with API key
        self.owm = pyowm.OWM(OWM_KEY)
        print("\nSuccessfully connected to PYOWM API.")
    def pingTest(self):
        #this will ping infinitely on linux without the -c parameter
        hostname = "twitter.com"
        with open(os.devnull, 'w') as DEVNULL:
            try:
                subprocess.check_call(
                    ['ping', hostname],
                    stdout=DEVNULL,  # suppress output
                    stderr=DEVNULL
                )
                isUp = True
            except subprocess.CalledProcessError:
                isUp = False
        return isUp
    def WANtest(self):
        #run pingTest up to maxTries. return result of pingTest
        WANup = False
        tryCount = 1
        maxTries = 10
        waitTime = 5
        while(WANup == False and tryCount <= maxTries):
            if(self.pingTest()):
                #test connectivity
                WANup = True
                if(tryCount > 1):
                    print("\nWAN was down, but was restored.")
                return WANup
            else:
                print("\nWAN down. Try " + str(tryCount) + " failed. Waiting " + str(waitTime) + " seconds to retry.")
                time.sleep(waitTime)
                tryCount +=1
                if(tryCount > maxTries):
                    print("\nMax tries attempted.")
                    return WANup
    def returnTwitterConnection(self):
        return self.api
    def returnOWMConnection(self):
        return self.owm


from django.shortcuts import render, redirect
from django.db.models import Max
from .models import News, CountryData, CountryNews, DailyData

import requests
from os import path
from bs4 import BeautifulSoup
from datetime import datetime
from fbprophet import Prophet
import pandas as pd

country_dict = {"USA": "United States of America", "UK": "United Kingdom", "UAE": "United Arab Emirates",
                "S. Korea": "Korea South", "Czechia": "Czech Republic", "North Macedonia": "Macedonia",
                "Ivory Coast": "Cote d Ivoire", "DRC": "Democratic Republic of the Congo", "Taiwan": "Republic of China",
                "Réunion": "France", "Palestine": "Palestinian territory", "Congo": "Republic of the Congo",
                "Guinea-Bissau": "Guinea Bissau", "Faeroe Islands": "Faroe Islands", "Cabo Verde": "Cape Verde",
                "Eswatini": "Swaziland", "CAR": "Central African Republic", "Timor-Leste": "East Timor", "Curaçao": "Curacao",
                "St. Vincent Grenadines": "Saint Vincent and the Grenadines", "Turks and Caicos": "Turks and Caicos Islands",
                "British Virgin Islands": "Virgin Islands British", "St. Barth": "Saint Barthelemy", "Caribbean Netherlands": "Netherlands",
                "Saint Pierre Miquelon": "Saint Pierre and Miquelon"}

def scrape_world():
    source = requests.get('https://www.worldometers.info/coronavirus/').text
    soup = BeautifulSoup(source, 'lxml')

    totals = soup.find_all('div', class_='maincounter-number')
    world_data = []
    for total in totals:
        world_data.append(total.span.text)

    return world_data

def india_state_data():
    url_state = 'https://api.covid19india.org/data.json'
    request_state = requests.get(url_state)
    data_state = request_state.json()

    url_district = 'https://api.covid19india.org/v2/state_district_wise.json'
    request_district = requests.get(url_district)
    data_district = request_district.json()

    states = data_state['statewise'][1:]

    for state in states:
        district_raw = [obj for obj in data_district if obj['state'] == state['state']]
        districts = district_raw[0]['districtData']

        state['districts'] = districts

    return states

def state_data(state):
    url_state = 'https://api.covid19india.org/data.json'
    request_state = requests.get(url_state)
    data_state = request_state.json()

    states = data_state['statewise'][1:]
    x = [obj for obj in states if obj['state'] == state]
    return x

def district_data(state):
    url_district = 'https://api.covid19india.org/v2/state_district_wise.json'
    request_district = requests.get(url_district)
    data_district = request_district.json()

    district_raw = [obj for obj in data_district if obj['state'] == state]
    districts = district_raw[0]['districtData']

    return districts

def state_daily_data(state_code):
    url_daily = 'https://api.covid19india.org/states_daily.json'
    request_daily = requests.get(url_daily)
    data_daily = request_daily.json()['states_daily']

    date      = [x["date"] for x in data_daily if x["status"] == 'Confirmed']
    confirmed = [x[state_code] for x in data_daily if x['status'] == 'Confirmed']
    recovered = [x[state_code] for x in data_daily if x['status'] == 'Recovered']
    deceased  = [x[state_code] for x in data_daily if x['status'] == 'Deceased']

    return {'date': date, 'confirmed': confirmed, 'recovered': recovered, 'deceased': deceased}

def district_daily_data(state, district):
    url_daily = 'https://api.covid19india.org/districts_daily.json'
    request_daily = requests.get(url_daily)
    data_daily = request_daily.json()['districtsDaily'][state][district]

    date      = [x["date"] for x in data_daily]
    active    = [x["active"] for x in data_daily]
    confirmed = [x["confirmed"] for x in data_daily]
    recovered = [x["recovered"] for x in data_daily]
    deceased  = [x["deceased"] for x in data_daily]

    return {'date': date, 'active': active, 'confirmed': confirmed, 'recovered': recovered, 'deceased': deceased}

def predictions2(country):
    if country == 'World':
        world = DailyData.objects.filter(country='World')
        world_values = world.values('date','totalcase')
        country_data = pd.DataFrame.from_records(world_values)
        country = 'World'
    else:
        if country == 'United States':
            country='US'
        data = requests.get('https://pomber.github.io/covid19/timeseries.json').json()
        country_data = data[country]
        country = 'Other'

    df = pd.DataFrame(country_data)

    if country == 'World':
        df.rename(columns={'date': 'ds', 'totalcase': 'y'}, inplace=True)
    else:
        df.rename(columns={'date': 'ds', 'confirmed': 'y'}, inplace=True)

    m = Prophet(changepoint_prior_scale=5, interval_width=1)
    m.fit(df)

    future = m.make_future_dataframe(periods=15)
    forecast = m.predict(future)
    updated_forecast = forecast[['ds', 'yhat_upper']]

    tail_updated_forecast = updated_forecast.tail(15)

    tail_updated_forecast.set_index('ds', inplace=True)

    return tail_updated_forecast

def predict_country(country):
    url = "https://pomber.github.io/covid19/timeseries.json"
    countries = requests.get(url).json()
    countries['World'] = True
    countries['US'] = True

    x = country
    r = [country for country in countries if country == x]

    if len(r) != 0:
        df = predictions2(country)
        df.rename(columns={'ds':'dates','yhat_upper':'predictions'},inplace=True)
        df.reset_index(inplace=True)
        df = df.to_json()
        return {'data':df}

    else:
        return {'data':'failed'}

def home(request):
    world_data = scrape_world()

    recoveries=int(world_data[2].replace(',', ''))
    deaths = int(world_data[1].replace(',', ''))

    total_outcome = int(recoveries) + int(deaths)
    recovery_percent = round((int(recoveries) / int(total_outcome)) * 100, 2)
    death_percent = round(100 - recovery_percent, 2)

    worlddailydata = DailyData.objects.filter(country='World')
    return render(request, 'home.html',
                  {'world_total': world_data[0], 'world_death': world_data[1], 'world_recovery': world_data[2],
                   'world_daily_data': worlddailydata, 'closed_cases':total_outcome, 'recovery_percent':recovery_percent,
                   'death_percent':death_percent})

def graphs(request):
    #daily_data()
    world_daily_data = DailyData.objects.filter(country='World')

    countries_to_exclude = ['International']
    max_date = DailyData.objects.aggregate(Max('date'))['date__max']
    percentage_data = DailyData.objects.exclude(country__in=countries_to_exclude).filter(date__exact=str(max_date)).order_by('-totalcase')

    for data in world_daily_data:
        data.date = str(data.date)

    return render(request,'graphs.html', {'world_daily_data' : world_daily_data[32:], 'percentage_data': percentage_data})

def news(request):
    #scrape_news()
    latest_news = News.objects.all().order_by('-id')
    return render(request, 'news.html', {'latest_news': latest_news})

def country(request):
    #scrape()
    countries = CountryData.objects.order_by('-totalcase')
    return render(request, 'country.html', {'countries': countries})

def country_detail(request, country_name):

    if country_name == 'India':
        return redirect('india')

    news = CountryNews.objects.filter(country=country_name).order_by('-id')[:9]
    #country_daily_data(country_name)

    dailydata = DailyData.objects.filter(country=country_name)

    country_data = CountryData.objects.get(name__iexact=f'{country_name}')
    return render(request, 'country_detail.html', {'country': country_data, 'latest_news': news, 'countrydailydata':dailydata[32:]})

def world(request):
    world_data = scrape_world()
    worlddailydata = DailyData.objects.filter(country='World')
    return render(request, 'world.html', {'world_total': world_data[0], 'world_death': world_data[1], 'world_recovery': world_data[2], 'worlddailydata' : worlddailydata})

def search(request):

    if request.method == "POST":
        data = request.POST['data']
        news = News.objects.filter(heading__contains=data).order_by('-id')
        return render(request,'news.html', {'latest_news':news})
    else:
        pass

    return redirect('/')

def news_detail(request, news_id):
    news = News.objects.filter(id=news_id)
    return render(request, 'news-detail.html', {'news': news})

def about(request):
    return render(request,'about.html')

def tips(request):
    return render(request,'tips.html')

def india(request):
    news = CountryNews.objects.filter(country="India").order_by('-id')
    # country_daily_data(country_name)

    states = india_state_data()

    dailydata = DailyData.objects.filter(country="India")

    country_data = CountryData.objects.get(name__iexact="India")
    return render(request, 'india.html',
                  {'country': country_data, 'latest_news': news, 'countrydailydata': dailydata[32:], 'states':states})

def state(request, state_name):
    states_data = state_data(state_name)

    state_code = states_data[0]['statecode'].lower()

    state_wise_data = state_daily_data(state_code)
    state_date = state_wise_data['date']
    state_data_confirmed = state_wise_data['confirmed']
    state_data_recovered = state_wise_data['recovered']
    state_data_deceased = state_wise_data['deceased']
    #print(state_wise_data)
    districts = district_data(state_name)

    '''df = pd.DataFrame(state_data_confirmed)
    print(df)
    df = df.cumsum()
    print(df)'''

    total_confirmed = []

    counter = 0

    for confirmed in state_data_confirmed:
        confirmed = int(confirmed)
        if counter == 0:
            total_confirmed.append(confirmed)
        else:
            day_total_confirmed = total_confirmed[counter-1] + confirmed
            total_confirmed.append(day_total_confirmed)
        counter+=1

    return render(request,'state.html',{'state':state_name , 'districts' : districts, 'states_data':states_data , 'state_confirmed':state_data_confirmed, 'state_date':state_date, 'state_recovered':state_data_recovered, 'total_confirmed':total_confirmed})

def predictions(request):
    countries = []

    for country in CountryData.objects.all():
        countries.append(country.name)

    return render(request,'predict.html',{'countries':countries})

def resources(request):
    return render(request, 'resource.html')


import torch.nn as nn
from util import dataloader as loader
from util.dataloader import process_tdpts
import torch.nn.functional as F
from util import config
import torch
from opacus.layers.dp_rnn import DPLSTMCell as LSTMCell

class Model(nn.Module):
    def __init__(self, z_dim=16, t_dim=48, te_dim=32, xe_dim=128, h_dim=128, o_dim=32, device=None):
        super(Model, self).__init__()
        self.z_dim = z_dim
        self.register_buffer('adj', loader.load_edgeadjs())
        self.register_buffer('eprop', loader.load_edgeproperties())
        if config.USE_TGRAPH:
            self.t_emb = nn.Embedding(t_dim * 7, te_dim)
            # self.t_emb = nn.Linear(t_dim * 7, te_dim)
            self.t_emb.weight = nn.Parameter(loader.load_time_emb(), True)  # load pre-trained temporal graph values
            self.tcost_emb = nn.Embedding(self.adj.size(0), te_dim)  # New
        else:
            self.t_emb = nn.Embedding(t_dim, te_dim)
            self.tcost_emb = nn.Embedding(self.adj.size(0), te_dim)  # New

        self.x_emb = nn.Embedding(self.adj.size(0), xe_dim)  # embed edges, 0 for padding

        self.lstms = nn.ModuleList()  # nn.LSTMCell(h_dim, h_dim) for _ in range(LSTMS_NUM)

        bias = True
        self.lstms.append(LSTMCell(z_dim + te_dim + xe_dim + self.eprop.size(1), h_dim, bias))
        for _ in range(1, config.LSTMS_NUM - 1):
            self.lstms.append(LSTMCell(h_dim, h_dim, bias))
        self.lstms.append(LSTMCell(h_dim, o_dim, bias)) if config.L2_NUM == 0 else self.lstms.append(
            LSTMCell(h_dim, h_dim, bias))

        self.linear_afterx = []  # nn.Sequential()
        self.linear_aftert = []
        self.linear_aftert += [nn.Linear(te_dim + te_dim, o_dim), nn.ReLU()]
        if config.L2_NUM != 0:
            self.linear_afterx += [nn.Linear(h_dim, o_dim), nn.ReLU()]
        for _ in range(1, config.L2_NUM):
            self.linear_afterx += [nn.Linear(o_dim, o_dim), nn.ReLU()]
            self.linear_aftert += [nn.Linear(o_dim, o_dim), nn.ReLU()]
        self.linear_afterx = nn.Sequential(*self.linear_afterx)
        self.linear_aftert = nn.Sequential(*self.linear_aftert)
        self.mlp_meta_x = nn.Sequential(nn.Linear(xe_dim + te_dim + self.eprop.size(1), o_dim))
        self.mlp_meta_t = nn.Sequential(nn.Linear(xe_dim + te_dim + self.eprop.size(1), o_dim))
        self.to(device)

    def adj_idx(self, input, output):  # return adj idx
        return (self.adj[input] == output.unsqueeze(-1)).nonzero()[:, -1]  # (B*L, )

    def init_h(self, batch_size, device):
        hs, cs = [], []
        for lstm in self.lstms:
            hs.append(torch.zeros(batch_size, lstm.hidden_size, device=device))
            cs.append(torch.zeros(batch_size, lstm.hidden_size, device=device))
        return hs, cs

    def forward(self, x, dpt):  # B * L, traj sequences
        x, dpt = x.t(), dpt.t()
        L, B = x.size()
        pred_x, pred_t = [], []
        hs, cs = self.init_h(B, x.device)
        for i in range(L):
            hidden, hs, cs = self.step(x[i], dpt[i], hs, cs)
            pred_x.append(F.log_softmax(hidden[0], dim=-1))  # B*adj_dim
            pred_t.append(torch.sigmoid(hidden[1]))

        pred_x = torch.stack(pred_x, 1).view(-1, self.adj.size(1))  # (B*L) * adj_dim
        pred_t = torch.stack(pred_t, 1).view(-1, self.adj.size(1))
        return pred_x, pred_t

    def step(self, xt, dptslot, hs, cs):  # B
        # print(xt.shape)
        B = xt.size(0)
        zt = torch.randn(B, self.z_dim, device=xt.device)
        xt_emb = self.x_emb(xt)  # B * emb_dim
        # print(dptslot.max(), dptslot.min())
        tt_emb = self.t_emb(dptslot)  # B * temb_dim
        xt_eprop = self.eprop[xt]  # B * eprop_dim
        hidden = torch.cat([zt, xt_emb, tt_emb, xt_eprop], dim=-1)
        for i, lstm in enumerate(self.lstms):  # LSTMs
            hs[i], cs[i] = lstm(hidden, (hs[i], cs[i]))
            hidden = hs[i]
        afterx = self.linear_afterx(hidden)
        adjs = self.adj[xt]  # B * adj_dim
        tt_emb = tt_emb.unsqueeze(1).expand(-1, self.adj.size(1), -1)
        aftert = self.linear_aftert(torch.cat([self.tcost_emb(adjs), tt_emb], dim=-1))
        props = self.eprop[adjs]  # .view(B, self.adj_dim, -1)
        mk = torch.cat([self.x_emb(adjs), tt_emb, props], -1)  # B * adj_size * know_emb_len
        softmax_w = self.mlp_meta_x(mk)
        sigmoid_w = self.mlp_meta_t(mk)

        pred_xnext = afterx.view(B, 1, 1, -1).matmul(softmax_w.view(B, self.adj.size(1), -1, 1)).view(B, -1)  # B * adj_dim
        pred_xnext[adjs == self.adj.size(0) + config.ADJ_MASK] = float('-inf')
        pred_tnext = aftert.unsqueeze(2).matmul(sigmoid_w.unsqueeze(-1)).view(B, self.adj.size(1))
        return (pred_xnext, pred_tnext), hs, cs

    def sample(self, x, dpts, ts):  # B * L, parts of sequences
        with torch.no_grad():
            if x is None or len(x) == 0: return torch.LongTensor().to(config.device), torch.FloatTensor().to(config.device)
            # assert x is not None and len(x) > 0
            x, dpts, ts = x.t(), dpts.t(), ts.t()
            given_len, B = x.size()
            samples_x, samples_t = [], []
            hs, cs = self.init_h(B, x.device)
            for i in range(given_len):
                hidden, hs, cs = self.step(x[i], dpts[i], hs, cs)
                if i + 1 < given_len: samples_x.append(x[i + 1]), samples_t.append(ts[i])
            sample_batch_idx = F.softmax(hidden[0], -1).multinomial(1).view(-1)  # .argmax(-1)
            samples_x.append(self.adj[x[-1]][range(B), sample_batch_idx])
            samples_t.append(torch.sigmoid(hidden[1][range(B), sample_batch_idx]))
            dpt = (dpts[-1] + (samples_t[-1] * config.TCOST_MAX).round().long()) % config.T_LOOP
            dpt = process_tdpts(dpt)
            for i in range(given_len, config.TRAJ_FIX_LEN - 1):
                hidden, hs, cs = self.step(samples_x[-1], dpt, hs, cs)
                sample_batch_idx = F.softmax(hidden[0], -1).multinomial(1).view(-1)  # .multinomial(1).view(-1)
                samples_x.append(self.adj[samples_x[-1]][range(B), sample_batch_idx])
                samples_t.append(torch.sigmoid(hidden[1][range(B), sample_batch_idx]))
                dpt = (dpt + (samples_t[-1] * config.TCOST_MAX).round().long()) % config.T_LOOP
                dpt = process_tdpts(dpt)
            samples_x, samples_t = torch.stack(samples_x, 1), torch.stack(samples_t, 1)  # B*L
            return samples_x, samples_t

    def sample_t(self, x, yidx, dpts):
        with torch.no_grad():
            if x is None or len(x) == 0: return torch.FloatTensor().to(config.device)
            x, yidx, dpts_slots = x.t(), yidx.t(), dpts.t()
            L, B = x.size()
            assert L == config.TRAJ_FIX_LEN - 1
            samples_costs = []
            hs, cs = self.init_h(B, x.device)
            for i in range(L):
                hidden, hs, cs = self.step(x[i], dpts_slots[i], hs, cs)
                pred_t = torch.sigmoid(hidden[1][range(B), yidx[i]])
                pred_t[yidx[i] == config.STOP_EDGE] = 0
                samples_costs.append(pred_t)
            return torch.stack(samples_costs, 1)  # B*L


from datetime import datetime, timedelta
from functools import wraps

from hamcrest import assert_that, equal_to, none, is_
from httmock import urlmatch, HTTMock
from mock import patch

from django.conf import settings
from django.http import HttpRequest, HttpResponse
from django.test import TestCase

from stagecraft.apps.datasets.models import OAuthUser
from stagecraft.apps.datasets.tests.support.test_helpers import has_header
from stagecraft.libs.authorization.http import (
    check_permission, authorize, _get_resource_role_permissions
)

def govuk_signon_mock(**kwargs):
    # So that this behaves as expected, when we use
    # it be sure to clean out any existing users.
    # This will prevent loading them from the database
    # instead of using the signon mock.
    OAuthUser.objects.all().delete()

    @urlmatch(netloc=r'.*signon.*')
    def func(url, request):
        if request.headers['Authorization'] == 'Bearer correct-token':
            status_code = 200
            user = {
                "user": {
                    "email": kwargs.get("email", "foobar.lastname@gov.uk"),
                    "name": kwargs.get("name", "Foobar"),
                    "organisation_slug": kwargs.get(
                        "organisation_slug", "cabinet-office"),
                    "permissions": kwargs.get(
                        "permissions", ["signin", "anon"]),
                    "uid": "a-long-uid",
                }
            }

        else:
            status_code = 401
            user = {}

        return {'status_code': status_code, 'content': user}

    return func

def with_govuk_signon(**signon_kwargs):

    def decorator(func):
        @wraps(func)
        def wrapped(*args, **kwargs):
            use_development_users = settings.USE_DEVELOPMENT_USERS
            settings.USE_DEVELOPMENT_USERS = False
            signon_mock = HTTMock(
                govuk_signon_mock(**signon_kwargs))
            try:
                with signon_mock:
                    result = func(*args, **kwargs)
            finally:
                settings.USE_DEVELOPMENT_USERS = use_development_users
            return result
        return wrapped

    return decorator

class CheckPermissionTestCase(TestCase):

    def setUp(self):
        self.use_development_users = settings.USE_DEVELOPMENT_USERS

    def tearDown(self):
        settings.USE_DEVELOPMENT_USERS = self.use_development_users

    def test_use_development_users_gets_from_dictionary(self):
        (user, has_permission) = check_permission(
            'development-oauth-access-token', set(['signin']))
        assert_that(user['name'], equal_to('Some User'))
        assert_that(has_permission, equal_to(True))

    def test_user_with_permission_from_signon_returns_object_and_true(self):
        settings.USE_DEVELOPMENT_USERS = False
        with HTTMock(govuk_signon_mock()):
            (user, has_permission) = check_permission(
                'correct-token', set(['signin']))
        assert_that(user['name'], equal_to('Foobar'))
        assert_that(has_permission, equal_to(True))

    @patch('requests.get')
    def test_signon_with_client_id(self, get_patch):
        settings.USE_DEVELOPMENT_USERS = False

        with HTTMock(govuk_signon_mock()):
            (user, has_permission) = check_permission(
                'correct-token', set(['signin']))
        get_patch.assert_called_with(
            'http://signon.dev.gov.uk/user.json?client_id=clientid',
            headers={
                'Authorization': 'Bearer correct-token',
            })

    def test_user_without_permission_from_signon_returns_none_and_false(self):
        settings.USE_DEVELOPMENT_USERS = False
        with HTTMock(govuk_signon_mock()):
            (user, has_permission) = check_permission(
                'bad-auth', set(['signin']))
        assert_that(user, none())
        assert_that(has_permission, equal_to(False))

    def test_user_with_permission_from_database_returns_object_and_true(self):
        settings.USE_DEVELOPMENT_USERS = False

        OAuthUser.objects.create(access_token='correct-token',
                                 uid='my-uid',
                                 email='joe@example.com',
                                 permissions=['signin'],
                                 expires_at=datetime.now() + timedelta(days=1))

        (user, has_permission) = check_permission(
            'correct-token', set(['signin']))

        assert_that(user['email'], equal_to('joe@example.com'))
        assert_that(has_permission, equal_to(True))

    def test_user_with_returns_object_and_true_when_permissions_is_list(self):
        settings.USE_DEVELOPMENT_USERS = False

        OAuthUser.objects.create(access_token='correct-token',
                                 uid='my-uid',
                                 email='joe@example.com',
                                 permissions=['signin'],
                                 expires_at=datetime.now() + timedelta(days=1))

        (user, has_permission) = check_permission(
            'correct-token', set(['signin', 'bob']))

        assert_that(user['email'], equal_to('joe@example.com'))
        assert_that(has_permission, equal_to(True))

    def test_user_without_permission_from_database_returns_false(self):
        settings.USE_DEVELOPMENT_USERS = False
        OAuthUser.objects.create(access_token='correct-token',
                                 uid='my-uid',
                                 email='joe@example.com',
                                 permissions=['signin'],
                                 expires_at=datetime.now() + timedelta(days=1))

        (user, has_permission) = check_permission(
            'correct-token', set(['admin']))

        assert_that(has_permission, equal_to(False))

    def test_check_permission_returns_object_and_true_when_permissions_list(
            self):
        settings.USE_DEVELOPMENT_USERS = False

        OAuthUser.objects.create(access_token='correct-token',
                                 uid='my-uid',
                                 email='jon@example.com',
                                 permissions=['signin'],
                                 expires_at=datetime.now() + timedelta(days=1))

        (user, has_permission) = check_permission(
            'correct-token', set(['signin', 'bob']))

        assert_that(user['email'], equal_to('jon@example.com'))
        assert_that(has_permission, equal_to(True))

    def test_user_from_database_should_not_be_returned_if_expired(self):
        settings.USE_DEVELOPMENT_USERS = False
        OAuthUser.objects.create(access_token='correct-token-2',
                                 uid='my-uid',
                                 email='joe@example.com',
                                 permissions=['signin'],
                                 expires_at=datetime.now() - timedelta(days=1))

        with HTTMock(govuk_signon_mock()):
            (user, has_permission) = check_permission('correct-token-2',
                                                      set(['admin']))

        assert_that(user, none())
        assert_that(has_permission, equal_to(False))
        assert_that(OAuthUser.objects.count(), equal_to(0))

    def test_user_is_written_to_database_after_successful_auth(self):
        settings.USE_DEVELOPMENT_USERS = False

        with HTTMock(govuk_signon_mock()):
            (user, has_permission) = check_permission('correct-token',
                                                      set(['signin']))

        assert_that(OAuthUser.objects.count(), equal_to(1))

        (user, has_permission) = check_permission(
            'correct-token', set(['signin']))

        assert_that(has_permission, equal_to(True))

    def test_if_permission_is_none_and_user_then_not_ok(self):
        settings.USE_DEVELOPMENT_USERS = False

        with HTTMock(govuk_signon_mock()):
            (user, has_permission) = check_permission('correct-token',
                                                      set())

            assert_that(has_permission, equal_to(False))

    def test_if_permission_is_anon_and_user_then_ok(self):
        settings.USE_DEVELOPMENT_USERS = False

        with HTTMock(govuk_signon_mock()):
            (user, has_permission) = check_permission('correct-token',
                                                      set(['anon']))

            assert_that(has_permission, equal_to(True))

    def test_if_permission_is_none_and_no_user_then_fail(self):
        settings.USE_DEVELOPMENT_USERS = False

        with HTTMock(govuk_signon_mock()):
            (user, has_permission) = check_permission('incorrect-token',
                                                      set())

            assert_that(has_permission, equal_to(False))

    def test_anon_user_if_no_token(self):
        settings.USE_DEVELOPMENT_USERS = False

        (user, has_permission) = check_permission(None, set(['anon']), True)

        assert_that(has_permission, equal_to(True))
        assert_that(user.get('name'), equal_to('Anonymous'))

    def test_no_access_without_role(self):
        settings.USE_DEVELOPMENT_USERS = False

        (user, has_permission) = check_permission(None, set(), True)

        assert_that(has_permission, equal_to(False))

    def test_no_user_if_no_token_and_anon_user_not_allowed(self):
        settings.USE_DEVELOPMENT_USERS = False

        (user, has_permission) = check_permission(None, set(), False)

        assert_that(has_permission, equal_to(False))
        assert_that(user, is_(None))

    def test_anon_user_if_permission_requested(self):
        settings.USE_DEVELOPMENT_USERS = False

        (user, has_permission) = check_permission(
            None, set(['permission']), True)

        assert_that(has_permission, equal_to(False))
        assert_that(user.get('name'), equal_to('Anonymous'))

    def test_role_permissions_for_a_resource(self):
        permissions_set = [
            {
                "role": "admin",
                "permissions": {
                    "Dashboard": ["get"],
                    "Transform": ["get"]
                },
            },
            {
                "role": "dashboard-editor",
                "permissions": {
                    "Dashboard": ["get", "post"]
                }
            }
        ]
        permissions = _get_resource_role_permissions(
            'Dashboard', permissions_set)
        assert_that(permissions, equal_to(
            {
                'get': set(['admin', 'dashboard-editor']),
                'post': set(['dashboard-editor']),
                'put': set(),
                'delete': set()
            }
        ))
        permissions = _get_resource_role_permissions(
            'Transform', permissions_set)
        assert_that(permissions, equal_to(
            {
                'get': set(['admin']),
                'post': set(),
                'put': set(),
                'delete': set()
            }
        ))

class AuthorizeTestCase(TestCase):

    def setUp(self):
        self.use_development_users = settings.USE_DEVELOPMENT_USERS

    def tearDown(self):
        settings.USE_DEVELOPMENT_USERS = self.use_development_users

    def test_authorize(self):
        settings.USE_DEVELOPMENT_USERS = False

        request = HttpRequest()
        request.META['HTTP_AUTHORIZATION'] = 'Bearer correct-token'

        with HTTMock(govuk_signon_mock()):
            user, err = authorize(request, set(['signin']))
            assert_that(err, is_(None))
            assert_that(user['uid'], is_('a-long-uid'))

    def test_authorize_no_user(self):
        settings.USE_DEVELOPMENT_USERS = False

        request = HttpRequest()
        request.META['HTTP_AUTHORIZATION'] = 'Bearer incorrect-token'

        with HTTMock(govuk_signon_mock()):
            user, err = authorize(request, set(['signin']))
            assert_that(err.status_code, is_(401))
            assert_that(user, is_(None))

    def test_authorize_bad_permission(self):
        settings.USE_DEVELOPMENT_USERS = False

        request = HttpRequest()
        request.META['HTTP_AUTHORIZATION'] = 'Bearer correct-token'

        with HTTMock(govuk_signon_mock()):
            user, err = authorize(
                request, set(['super-high-level-permission']))
            assert_that(err.status_code, is_(403))
            assert_that(user['uid'], is_('a-long-uid'))


"""
Extracts a BERT archive from an existing model
"""

import logging
import argparse

from udify import util

logging.basicConfig(format='%(asctime)s - %(levelname)s - %(name)s - %(message)s',
                    level=logging.INFO)
logger = logging.getLogger(__name__)

parser = argparse.ArgumentParser()
parser.add_argument("archive_dir", type=str, help="The directory where model.tar.gz resides")
parser.add_argument("--output_path", default=None, type=str, help="The path to output the file")

args = parser.parse_args()

bert_config = "config/archive/bert-base-multilingual-cased/bert_config.json"
util.archive_bert_model(args.archive_dir, bert_config, args.output_path)


import os
import time
import math

import numpy as np
import torch
from metrics import averageMeter
# from evaluation.evaluation_tools import eval_log_likelihood

class BaseTrainer:
    """Trainer for a conventional iterative training of model for classification"""
    def __init__(self, optimizer, training_cfg, device):
        self.training_cfg = training_cfg
        self.device = device
        self.optimizer = optimizer

    def train(self, model, d_dataloaders, logger=None, logdir=""):
        cfg = self.training_cfg

        time_meter = averageMeter()
        train_loader, val_loader = (d_dataloaders["training"], d_dataloaders["validation"])
        kwargs = {'dataset_size': len(train_loader.dataset)}
        i_iter = 0
        best_val_loss = np.inf

        for i_epoch in range(1, cfg['n_epoch'] + 1):
            for x, w in train_loader:
                i_iter += 1
                if train_loader.dataset.augmentation:
                    x, w = train_loader.dataset.rand_aug(x, w)
                model.train()
                start_ts = time.time()
                if i_epoch < cfg['n_epoch'] * 0.2:
                    reg = False
                else:
                    reg = True
                d_train = model.train_step(x.to(self.device), w.to(self.device), optimizer=self.optimizer, reg=reg, **kwargs)
                time_meter.update(time.time() - start_ts)
                if logger is not None:
                    logger.process_iter_train(d_train)

                if i_iter % cfg.print_interval == 0:
                    if logger is not None:
                        d_train = logger.summary_train(i_iter)
                    else:
                        d_train['loss/train_loss_'] = d_train['loss']
                    print(
                        f"Epoch [{i_epoch:d}] \nIter [{i_iter:d}]\tAvg Loss: {d_train['loss/train_loss_']:.6f}\tElapsed time: {time_meter.sum:.4f}"
                    )
                    time_meter.reset()
                if i_epoch % cfg.save_epochs == 0:
                    self.save_model(model, logdir, i_epoch=i_epoch)
                model.eval()
                if i_iter % cfg.val_interval == 0:
                    # log_likelihood = eval_log_likelihood(model, train_loader, split=50)
                    # d_ll= {'log_likelihood_': log_likelihood.sum()}
                    # logger.add_val(i_iter, d_ll)
                    for x, w in val_loader:
                        d_val = model.validation_step(x.to(self.device), w.to(self.device))
                        if logger is not None:
                            logger.process_iter_val(d_val)
                    if logger is not None:
                        d_val = logger.summary_val(i_iter)
                        print(d_val['print_str'])
                    else:
                        d_val['loss/val_loss_'] = d_val['loss']
                        print(
                            f"Iter [{i_iter:d}]\tAvg Loss: {d_val['loss/val_loss_']:.6f}"
                        )
                    val_loss = d_val['loss/val_loss_']
                    best_model = val_loss < best_val_loss

                    if best_model:
                        print(f'Iter [{i_iter:d}] best model saved {val_loss:.6f} <= {best_val_loss:.6f}')
                        best_val_loss = val_loss
                        self.save_model(model, logdir, best=True)

                if cfg.visualization.type == 'none' or cfg.visualization.type == []:
                    pass
                else:
                    if i_iter % cfg.visualize_interval == 0:
                        d_vis = model.visualization_step(train_loader, val_loader, device=self.device, **cfg.visualization)
                        if logger is not None:
                            logger.add_val(i_iter, d_vis)

        self.save_model(model, logdir, i_iter="last")
        return model, best_val_loss

    def save_model(self, model, logdir, best=False, i_iter=None, i_epoch=None):
        if best:
            pkl_name = "model_best.pkl"
        else:
            if i_iter is not None:
                pkl_name = f"model_iter_{i_iter}.pkl"
            else:
                pkl_name = f"model_epoch_{i_epoch}.pkl"
        state = {"epoch": i_epoch, "iter": i_iter, "model_state": model.state_dict()}
        save_path = os.path.join(logdir, pkl_name)
        torch.save(state, save_path)
        print(f"Model saved: {pkl_name}")

class LatentTrainer(BaseTrainer):
    def __init__(self, optimizer, training_cfg, device):
        super().__init__(optimizer, training_cfg, device)