#!/usr/bin/python
# Compile the cognate assignment result (JSON) to the LaTeX source file of correspondence pattern (Doug-style) with comparison of two different transducers
# Fixed-location files:
#   source.tsv: lexical data
# Standard input:
#   json containing:
#   langsUnderStudy: something like ['Maru', 'Old_Burmese', 'Bola']
#   oldTransducer: (Old version)
#   newTransducer: (New version)
#   board: (Board file)

## Single # = Xun Gong's notes
## Double ## = Seth Knights's notes

## Deleting lines 504-5 in the default transducer of the site leaves terms (i.e. 'forehead') where
## for changing between languages if one changes for the worse, it leaves blank the 'old' version..
## No clue at all why this happens...

## For disabling eprints that are unecessary
production = True

# Constants
language_title = {'Old_Burmese': 'OBurm', 'Achang_Longchuan': 'Acha-LC', 'Xiandao': 'Acha-XD', 'Maru': 'Maru', 'Bola': 'Bola', 'Atsi': 'Atsi', 'Lashi': 'Lashi'}
fst_index = {'Old_Burmese': 'burmese', 'Achang_Longchuan': 'ngochang', 'Xiandao': 'xiandao', 'Maru': 'maru', 'Bola': 'bola', 'Atsi': 'atsi', 'Lashi': 'lashi'}

# Basic imports
from hashlib import new
import sys
import re
import json
import csv

from numpy import equal
from merge_phonemes import merge_phonemes
from functools import reduce
from collections import Counter
from tabulate import tabulate

##### ROUTINES #####

def eprint(*args, **kwargs):
    print(*args, file=sys.stderr, **kwargs)
    sys.stderr.flush()

# Replace unicode diacritics with ASCII equivalents for sending to the transducers
UNICODE_MACRON_UNDER = " ̱ "[1]
UNICODE_TILDE_OVER = " ̃"[1]
def replace_diacritics(s):
    return s.replace(UNICODE_MACRON_UNDER, '_').replace(UNICODE_TILDE_OVER, '~').replace('_~', '~_')

def replace_diacritics_forward(s):
    return s.replace('_', UNICODE_MACRON_UNDER).replace('~', UNICODE_TILDE_OVER)

# Convert a piece of text to its component syllables
# If there is alrady "◦" or a space, use it to separate them
# Otherwise, separate the tone letters other letters
def syllabize(text):
    split_result = re.split(r'([ ◦¹²³⁴⁵˩˨˧˦˥]+)', text)
    l = len(split_result)

    # There are two possibilities
    # 'a◦b◦' → ['a', '◦', 'b', '◦', '']
    # 'a◦b' → ['a', '◦', 'b']
    # Both of them are odd, but should be treated differently
    # At least, everything but the last can be done in the same way
    # (5-1)/2 = 2

    result = []
    for i in range((l-1) // 2):
        result.append(split_result[i*2] + split_result[i*2+1].strip('◦ '))
    # If last non-zero, then copy to result
    if split_result[l-1]:
        result.append(split_result[l-1])

    return result

def emphasize_syllable_raw(syllables, syllable_index, usual_formatting = None, emphatic_formatting = None):
    l = len(syllables)
    result = ''
    for i in range(l):
        if i == syllable_index:
            result = result + (emphatic_formatting or '<%s>') % syllables[i]
        else:
            result = result + (usual_formatting or '%s') % syllables[i]
    return result

# emphasize_syllable("mi ma mu", 1) → "mi<ma>mu"
def emphasize_syllable(text, syllable_index, usual_formatting = None, emphatic_formatting = None):
    return emphasize_syllable_raw(syllabize(text), syllable_index, usual_formatting, emphatic_formatting)

# fetch_syllable("mi ma mu", 1) → "ma"
def fetch_syllable(text, syllable_index):
    syllables = syllabize(text)
    return syllables[syllable_index]

# Convert internal language name to printed name
def print_language_name(name):
    if name == 'Old_Burmese':
        return 'OBur.'
    else:
        return name.replace('_', r'\_')

# Compute the projected reconstruction from a bunch of syllable_id's and a certain fst
# syllable_ids: syllable_id's of the syllables in a class, as defined contra "words"
# fsts: different sets of fst
# words: given here in order not to abuse global variables
# Return values: (inferred_reconstructions, strict)
def back_reconstruct_list(syllable_ids, fsts, words):
    reconsts = {}
    at_least_one = False
    first_form = False

    for syllable_id in syllable_ids:
        word_id, _, n = syllable_id.rpartition('-')
        n = int(n)
        ipa = words[word_id]['syllables'][n]
        doculect = words[word_id]['doculect']

        if not first_form:
            first_form = ipa
        if doculect in fsts:
            the_syl = replace_diacritics(ipa)
            rec = list(fsts[doculect].apply_up(the_syl))
            if rec:
                # only record reconstructions when something *is* reconstructed
                at_least_one = True
                if doculect not in reconsts:
                    reconsts[doculect] = set(rec)
                else:
                    reconsts[doculect] = reconsts[doculect].union(set(rec))

    strict = True
    inferred_reconstructions = []

    if at_least_one:
        strict = True
        inferred_reconstructions = list(set.intersection(*reconsts.values()))

        if not inferred_reconstructions:
            strict = False
            # kinda lenient way of generating reconstructions
            # try and make intersection of every pair of doculects
            pairwise_intersections = [set.intersection(reconsts[a], reconsts[b])
                    for a in reconsts for b in reconsts if a != b]
            if pairwise_intersections:
                inferred_reconstructions = list(set.union(*pairwise_intersections))
            else:
                inferred_reconstructions = []

    return (inferred_reconstructions, strict)

##### MAIN PROGRAM #####

# eprint('Program starts')

# Compile transducers
from foma import FST
import tempfile
import os
import subprocess

def read_transducer(input_json, old_new, errors):
    ret = {}
    script_path = os.path.dirname(os.path.realpath(__file__))
    with tempfile.TemporaryDirectory() as tmpdirname:
        os.chdir(tmpdirname)
        eprint(f'Compiling FSTs ({old_new})')
        with open('transducer.foma', 'w') as fp:
            fp.write(input_json[f'{old_new}Transducer'])
        eprint("---------")
        output = subprocess.run(['foma', '-f', 'transducer.foma'],
                                capture_output=True, check=True, text=True)

        # Add errors from stderr
        if (len(output.stderr) > 0):
            for err in output.stderr.split("\n"):
                if len(err) > 0:
                    errors.append(f"Error loading {old_new} transducer: {err}")
                    eprint(err)

        # Check for errors in stdout, to be passed back to the program
        if '***' in output.stdout:
            error = f"Error loading {old_new} transducer: " + output.stdout.split('***')[1]
            eprint(error)
            errors.append(error)

        for doculect_name in fst_index:
            if os.path.isfile(fst_index[doculect_name] + '.bin'):
                ret[doculect_name] = FST.load(fst_index[doculect_name] + '.bin')
        os.chdir(script_path)
        eprint('FSTs loaded:', ', '.join(ret))
    return ret

def process_row(row):
    if row['ID'].startswith('#'):
        # internal to lingpy
        return {}

    if row['CROSSIDS']:
        crossids = row['CROSSIDS'].split(' ')
    else:
        if not production:
            eprint('SANITY: empty crossid: ', row['ID'])
        return {}

    word_id = 'word-' + row['ID']
    syllables = syllabize(row['IPA'])
    syllables_parsed = [merge_phonemes(str(sch), str(tk), 'i m r t',
                                       {'i': 'im', 'm':'m',
                                        'r':'mnNc', 't':'t'})
                        for sch, tk in zip(row['STRUCTURE'].split(' + '),
                                           row['TOKENS'].split(' + '))]

    word_json = {
            'id':word_id,
            'doculect': row['DOCULECT'],
            'syllables': syllables,
            'syllables_parsed': syllables_parsed,
            'gloss': row['CONCEPT'],
            'glossid': row['GLOSSID']}
    return {word_id: word_json}

def compare_fst(input_json):
    # decode the input JSON
    # input_json = json.load(open('input-correspondence.json', 'r+'))

    # First field from JSON: langsUnderStudy
    langs_under_study = input_json['langsUnderStudy']

    # Board from JSON
    errors = []

    fsts_old = read_transducer(input_json, 'old', errors)
    fsts_new = read_transducer(input_json, 'new', errors)

    both_missing = []
    if not all(b in fsts_new for b in langs_under_study) and not all(b in fsts_old for b in langs_under_study):
        old_missing = set(langs_under_study).difference(fsts_old)
        new_missing = set(langs_under_study).difference(fsts_new)
        eprint('Missing old FST for languages:', ', '.join(old_missing))
        eprint('Missing new FST for languages:', ', '.join(new_missing))
        both_missing = list((old_missing) & (new_missing))
        langs_under_study = [lang for lang in langs_under_study if lang not in both_missing]

    # read the word CSV
    # import fileinput
    csvreader = csv.DictReader(filter(lambda row: row.strip() and row[0]!='#', open('./lexicon.tsv', 'r')), dialect='excel-tab')

    eprint('Processing TSV rows...')
    words = {}
    for row in csvreader:
        words.update(process_row(row))

    eprint('Processing boards...')
    input_board = input_json['board']

    columns = input_board['columns']
    boards = input_board['boards']

    # list indexed by languages and initials of columns containing a certain initial in a certain language
    # column_index['i']['Old_Burmese']['p'] -> [...]
    # column_index = {}

    # column_index['i']['p:p:p'] -> [...]
    column_index = {pos:{} for pos in 'imrt'}

    for board_id in boards:
        for column_id in boards[board_id]['columnIds']:
            if not columns[column_id]['syllableIds']:
                continue

            # First, guess the reconstruction
            # print(columns, flush=True)
            print('id', column_id, flush=True)
            print('words', len(words), flush=True)
            print('words', words, flush=True)
            inferred_reconstructions, strict_reconstructions = back_reconstruct_list(columns[column_id]['syllableIds'], fsts_old, words)

            # cnt[pos][doculect] → Counter of possibilities
            cnt = {}
            sylls = {}
            senses = {}

            for syllable_id in columns[column_id]['syllableIds']:
                word_id, _, n = syllable_id.rpartition('-')
                n = int(n)

                ipa = words[word_id]['syllables'][n]
                doculect = words[word_id]['doculect']
                gloss = words[word_id]['gloss']
                syllable_parsed = words[word_id]['syllables_parsed'][n]
                syllable_fields = list(zip(syllable_parsed[0].split(' '), syllable_parsed[1].split(' ')))

                for position, sound in syllable_fields:
                    if position not in cnt:
                        cnt[position] = {}
                        sylls[position] = {}
                        senses[position] = {}
                    if doculect not in cnt[position]:
                        cnt[position][doculect] = Counter()
                        sylls[position][doculect] = {}
                        senses[position][doculect] = {}
                    if sound not in sylls[position][doculect]:
                        sylls[position][doculect][sound] = []
                        senses[position][doculect][sound] = []
                    cnt[position][doculect][sound] += 1
                    sylls[position][doculect][sound].append(ipa)
                    senses[position][doculect][sound].append(gloss)

            for position in cnt:
                # generate the information needed for displaying a corr. chart
                description = []
                most_common_ipas = []
                shared_senses = []

                # Some flags that impact display
                last_doculect_present = False
                any_non_last_doculect_present = False

                for doculect in langs_under_study:
                    if doculect in cnt[position]:
                        if doculect == langs_under_study[-1]:
                            last_doculect_present = True
                        else:
                            any_non_last_doculect_present = True

                        most_common_sound = cnt[position][doculect].most_common()[0][0]
                        most_common_ipa = Counter(sylls[position][doculect][most_common_sound]).most_common()[0][0]
                        description.append(most_common_sound)
                        most_common_ipas.append(most_common_ipa)
                        shared_senses.extend(senses[position][doculect][most_common_sound])
                    else:
                        description.append('-')
                        most_common_ipas.append('--')
                description = ':'.join(description)

                most_common_gloss = '?'
                if shared_senses:
                    most_common_gloss = Counter(shared_senses).most_common()[0][0]

                column_info = {'column_id': column_id,
                        'most_common_gloss': most_common_gloss,
                        'most_common_ipas': most_common_ipas,
                        'last_doculect_present': last_doculect_present,
                        'any_non_last_doculect_present': any_non_last_doculect_present,
                        'old_fst_reconstructions': (inferred_reconstructions, strict_reconstructions),
                        'new_fst_reconstructions': back_reconstruct_list(columns[column_id]['syllableIds'], fsts_new, words)}
                if description not in column_index[position]:
                    column_index[position][description] = []
                column_index[position][description].append(column_info)

    text = ''

    pos_name = {'i': 'Initial', 'm': 'Medial', 'r': 'Rime', 't': 'Tone'}
    json_chapters = {}

    for pos in column_index:
        text += '=====' + pos_name[pos] + ' Correspondence =====\n'
        json_chapters[pos] = []
        for description in sorted(column_index[pos]):
            if not column_index[pos][description] or not column_index[pos][description][0]['last_doculect_present'] or not column_index[pos][description][0]['any_non_last_doculect_present']:
                continue

            text += "=== %s: %s ===\n" % (pos_name[pos], description)

            this_section = {'title': "=== %s: %s ===\n" % (pos_name[pos], description)}
            rows = []

            for column in column_index[pos][description]:
                this_row = {}

                # first row: gloss & IPA's
                this_row['gloss'] = column['most_common_gloss']
                this_row['ipas'] = column['most_common_ipas']

                # second row: old reconstructions
                row = []
                inferred_reconstructions, strict_reconstructions = column['old_fst_reconstructions']
                if inferred_reconstructions:
                    rec_str = ', '.join(['*' + w for w in inferred_reconstructions])
                    if not strict_reconstructions:
                        rec_str += '?'
                    this_row['old_reconstruction'] = rec_str

                old_reconstruction_matched = False
                new_reconstruction_matched = False

                for i in range(len(langs_under_study)):
                    doculect = langs_under_study[i]
                    under_study = i == len(langs_under_study) - 1
                    rec = []
                    if doculect in fsts_old and column['most_common_ipas'][i] != '--':
                        the_syl = replace_diacritics(column['most_common_ipas'][i])
                        rec = list(set(fsts_old[doculect].apply_up(the_syl)))

                    if rec:
                        rec_strs = []
                        for w in rec:
                            if w in inferred_reconstructions:
                                rec_strs.append(r'_*' + w + '_')
                                if under_study:
                                    old_reconstruction_matched = True
                            else:
                                rec_strs.append('*' + w)
                        rec_str = ', '.join(rec_strs)
                        row.append(rec_str)
                    else:
                        row.append('')
                        if doculect in fsts_old and inferred_reconstructions and i == len(langs_under_study) - 1:
                            # forward projection for the language under study
                            fwd_recs = []
                            for w in inferred_reconstructions:
                                fwd_recs.extend(list(fsts_old[doculect].apply_down(w)))
                            fwd_recs = [replace_diacritics_forward(w) for w in set(fwd_recs)]
                            row[-1] = r'≠ †%s' % (', '.join(fwd_recs))

                this_row['old_reconstructions'] = row

                # third row: new reconstructions
                row = []
                inferred_reconstructions, strict_reconstructions = column['new_fst_reconstructions']
                if inferred_reconstructions:
                    rec_str = ', '.join(['*' + w for w in inferred_reconstructions])
                    if not strict_reconstructions:
                        rec_str += '?'
                    this_row['new_reconstruction'] = rec_str

                for i in range(len(langs_under_study)):
                    doculect = langs_under_study[i]
                    under_study = i == len(langs_under_study) - 1
                    rec = []
                    if doculect in fsts_new and column['most_common_ipas'][i] != '--':
                        the_syl = replace_diacritics(column['most_common_ipas'][i])
                        rec = list(set(fsts_new[doculect].apply_up(the_syl)))

                    if rec:
                        rec_strs = []
                        for w in rec:
                            if w in inferred_reconstructions:
                                rec_strs.append(r'_*' + w + '_')
                                if i == len(langs_under_study) - 1:
                                    new_reconstruction_matched = True
                            else:
                                rec_strs.append('*' + w)
                        rec_str = ', '.join(rec_strs)
                        row.append(rec_str)
                    else:
                        row.append('')
                        if doculect in fsts_new and inferred_reconstructions and i == len(langs_under_study) - 1:
                            # forward projection for the language under study
                            fwd_recs = []
                            for w in inferred_reconstructions:
                                fwd_recs.extend(list(fsts_new[doculect].apply_down(w)))
                            fwd_recs = [replace_diacritics_forward(w) for w in set(fwd_recs)]
                            row[-1] = r'≠ †%s' % (', '.join(fwd_recs))

                this_row['new_reconstructions'] = row

                this_row['status'] = ''
                if old_reconstruction_matched and not new_reconstruction_matched:
                    this_row['status'] = 'frowning'
                elif (not old_reconstruction_matched) and new_reconstruction_matched:
                    this_row['status'] = 'smiling'

                rows.append(this_row)

            this_section['rows'] = rows
            json_chapters[pos].append(this_section)

    eprint("Successful comparison.")
    return {'chapters': json_chapters, 'missing_transducers': both_missing, 'errors': errors}


#!/usr/bin/env python
''' TODO
add a way to update the shell file instead of overwriting it
'''

from argparse import ArgumentParser
from datetime import datetime
import json
import os
import sys

here = os.path.abspath(os.path.dirname(__file__))
ALIAS_PATH = os.path.join(here, 'aliases.json')
EDITOR_CONFIG_PATH = os.path.join(here, 'editorconfig')
VIM_DIR_PATH = os.path.join(here, 'vim')
VIMRC_PATH = os.path.join(VIM_DIR_PATH, 'vimrc')
TMUX_PATH = os.path.join(here, 'tmux.conf')

PS1 = '\\[\033[1;34m\\]\\u\\[\033[1;36m\\]@\\[\033[1;35m\\]\\W\\[\033[1;32m\\] $ \\[\033[0m\\]'
with open(ALIAS_PATH) as f:
    ALIASES = json.load(f)

VUNDLE_REPO = 'https://github.com/VundleVim/Vundle.vim.git'
PYENV_REPO = 'https://github.com/yyuu/pyenv.git'
PYENV_VIRTUALENV_REPO = 'https://github.com/yyuu/pyenv-virtualenv.git'

def configure_shell_file(install_dir, shell_file):
    with open(shell_file, 'a') as f:
        # set vim keybindings
        f.write('\nset -o vi\n')

        # install terminal prompt (PS1)
        f.write('\nPS1="{}"\n'.format(PS1))

        # install aliases
        for alias_type, aliases in ALIASES.items():
            f.write('\n# {} aliases\n'.format(alias_type))
            for alias in aliases:
                f.write('alias {}\n'.format(alias))

def install_vim(install_dir, shell_file):
    vim_dir_target = os.path.join(install_dir, '.vim')
    os.system('ln -s %s %s' % (VIM_DIR_PATH, vim_dir_target))

    vimrc_target = os.path.join(install_dir, '.vimrc')
    os.system('ln -s %s %s' % (VIMRC_PATH, vimrc_target))

    vundle_target = os.path.join(vim_dir_target, 'bundle', 'Vundle.vim')
    if not os.path.exists(vundle_target):
        os.system('git clone %s %s' % (VUNDLE_REPO, vundle_target))

def install_tmux(install_dir, shell_file):
    tmux_target = os.path.join(install_dir, '.tmux.conf')
    os.system('ln -s %s %s' % (TMUX_PATH, tmux_target))

def install_pyenv(install_dir, shell_file):
    pyenv_target = os.path.join(install_dir, '.pyenv')
    os.system('git clone %s %s' % (PYENV_REPO, pyenv_target))

    pyenv_virtualenv_target = os.path.join(pyenv_target, 'plugins', 'pyenv-virtualenv')
    os.system('git clone %s %s' % (PYENV_VIRTUALENV_REPO, pyenv_virtualenv_target))

    with open(shell_file, 'a') as f:
        f.write('\n# pyenv + pyenv-virtualenv\n')
        f.write('export PYENV_ROOT="%s"\n' % pyenv_target)
        f.write('export PATH="$PYENV_ROOT/bin:$PATH"\n')
        f.write('eval "$(pyenv init -)"\n')
        f.write('eval "$(pyenv virtualenv-init -)"\n')

def install(install_dir, shell_file):
    ''' Install .vim/ + .vimrc
    Install editorconfig
    Install .tmux.conf
    Setup pyenv + pyenv-virtualenv
    '''

    if not os.path.isdir(install_dir):
        print('install directory %s does not exist. exiting' % install_dir)
        sys.exit(1)

    install_dir = os.path.realpath(install_dir)
    shell_file = os.path.join(install_dir, shell_file)
    if not os.path.isfile(shell_file):
        with open(shell_file, 'w+') as f:
            now = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
            f.write('# file generated by denv at %s\n' % now)

    editor_config_target = os.path.join(install_dir, '.editorconfig')
    os.system('ln -s %s %s' % (EDITOR_CONFIG_PATH, editor_config_target))

    configure_shell_file(install_dir, shell_file)
    install_vim(install_dir, shell_file)
    install_tmux(install_dir, shell_file)
    install_pyenv(install_dir, shell_file)

if __name__ == '__main__':
    arg_parser = ArgumentParser()
    arg_parser.add_argument('-i', '--install-dir', type=str,
        help='specify the install directory')
    arg_parser.add_argument('-s', '--shell-file', type=str,
        help='specify the shell file')
    args = arg_parser.parse_args()

    install_dir = args.install_dir or os.environ['HOME']
    shell_file = args.shell_file or '.bashrc'
    install(install_dir, shell_file)


import nltk # Import all the libraries need to build the seq2seq model
from nltk.stem.lancaster import LancasterStemmer
stemmer = LancasterStemmer()

import numpy as np
import tflearn
import tensorflow as tf
import random

import pickle
import json

ERROR_THRESHOLD = 0.25 # The error should not be greater than this value

def clean_up_sentence(sentence):
    # tokenize the pattern
    sentence_words = nltk.word_tokenize(sentence)
    # stem each word
    sentence_words = [stemmer.stem(word.lower()) for word in sentence_words]
    return sentence_words

# return bag of words array: 0 or 1 for each word in the bag that exists in the sentence
def bow(sentence, words, show_details=False): # and to pass that numpy array to the model.predict
    # tokenize the pattern
    sentence_words = clean_up_sentence(sentence)
    # bag of words
    bag = [0]*len(words)
    for s in sentence_words:
        for i, w in enumerate(words):
            if w == s:
                bag[i] = 1
                if show_details:
                    print("found in bag %s" % w)
    return np.array(bag)

def classify(sentence):
    # generate probabilities from the model
    results = model.predict([bow(sentence, words)])[0]
    # filter out predictions below a threshold
    results = [[i, r] for i, r in enumerate(results) if r > ERROR_THRESHOLD]
    # sort by strength of probability
    results.sort(key = lambda x: x[1], reverse=True)
    return_list = []
    for r in results:
        return_list.append((classes[r[0]], r[1]))
    # return tuple of intent and probability
    return return_list

# ===
data = pickle.load(open("training_data", "rb"))
words = data['words']
classes = data['classes']
train_x = data['train_x']
train_y = data['train_y']

# import our chat-bot intents file
with open('intents.json') as json_data:
    intents = json.load(json_data)

# load saved model
net = tflearn.input_data(shape=[None, len(train_x[0])])
net = tflearn.fully_connected(net, 8)
net = tflearn.fully_connected(net, 8)
net = tflearn.fully_connected(net, len(train_y[0]), activation='softmax')
net = tflearn.regression(net)
model = tflearn.DNN(net, tensorboard_dir='tflearn_logs')
model.load('./model.tflearn')

print (classify('What is the cutoff for computer science in your college?'))
print (classify('What kind of faculty does your college have?'))
print (classify('Can I take this book?'))
print (classify('When was COEP founded ?'))
print (classify('What is your name ?'))


import torch
import unittest

from torch_blade.version import cuda_available
from tests.disc.testing_base import DiscTestCase

class TestDiscGroupNorm(DiscTestCase):
    def _test_group_norm(self, groupnorm):
        test_data = torch.randn([2, 320, 64, 64], device=self.device)
        annotation = ([-1, -1, -1, -1], torch.float)
        self._test_disc(groupnorm, [annotation], (test_data,), rtol=1e-3)

    def test_groupnorm_module(self):
        groupnorm = torch.nn.GroupNorm(32, 320, affine=False)
        self._test_group_norm(groupnorm)

    def test_groupnorm_module_has_affine(self):
        groupnorm = torch.nn.GroupNorm(32, 320, affine=True)
        self._test_group_norm(groupnorm)

if __name__ == "__main__":
    unittest.main()


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models

class Migration(migrations.Migration):

    dependencies = [
        ('partner', '0007_auto_20160417_0253'),
    ]

    operations = [
        migrations.AddField(
            model_name='shop',
            name='description',
            field=models.TextField(verbose_name='Description', blank=True),
        ),
        migrations.AddField(
            model_name='shop',
            name='image',
            field=models.ImageField(upload_to=b'images/products/%Y/%m/', max_length=255, verbose_name=b'image', blank=True),
        ),
    ]


from collections import deque
from copy import deepcopy
import time
from template import Agent, GameRule
import random
import heapq
from Azul.azul_model import AzulGameRule as GameRule
from Azul.azul_model import AzulState as GameState
from itertools import count

THINKTIME   = 0.95
NUM_PLAYERS = 2

class myAgent(Agent):
    def __init__(self,_id):
        super().__init__(_id)
        self.game_rule = GameRule(NUM_PLAYERS)
        self.game_state = GameState(NUM_PLAYERS)

    def isGoal(self, state, action):
        numRowsCompleted = 0
        numRowsCompletedSucc = 0
        for i in range(state.agents[self.id].GRID_SIZE):
            if (state.agents[self.id].lines_number[i] == i+1):
                numRowsCompleted += 1

        state = self.game_rule.generateSuccessor(state, action, self.id)

        for j in range(state.agents[self.id].GRID_SIZE):
            if (state.agents[self.id].lines_number[j] == j+1):
                numRowsCompletedSucc += 1

        return numRowsCompletedSucc > numRowsCompleted

    def GetActions(self, state):
        return self.game_rule.getLegalActions(state, self.id)

    def heuristic(self, state):
        arrayOnFloor = state.agents[self.id].floor
        numOnFloor = 0
        for val in arrayOnFloor:
            if val == 1:
                numOnFloor += 1
        return -numOnFloor

    def SelectAction(self,actions,root_state):
        unique = count()
        start_time = time.time()
        myPQ = []
        node = (deepcopy(root_state), [])
        hValue = self.heuristic(root_state)
        heapq.heappush(myPQ, (hValue, next(unique),  node))
        ##myPQ = deque([node])
        closedStates = []
        while len(myPQ) and time.time()-start_time < THINKTIME:
            whole = heapq.heappop(myPQ)
            _,_, currNode = whole
            state, path = currNode
            newActions = self.GetActions(state)
            if (state not in closedStates):
                closedStates = closedStates + [state]
                for a in newActions:
                    next_path = path + [a]
                    next_state = deepcopy(state)
                    if(self.isGoal(next_state, a)):
                        return next_path[0]
                    else:
                        hValue = self.heuristic(root_state)
                        newNode = (next_state, next_path)
                        heapq.heappush(myPQ, (hValue, next(unique), newNode))
                        ##myPQ.append(newNode)
        return random.choice(actions)


import os
import sys
import cv2
import time
import numpy as np
import easyocr

image_dir = sys.argv[1]
out_dir = sys.argv[2]

debug_dir = None
if (len(sys.argv) == 4):
    print("Debug out on")
    debug_dir = sys.argv[3]

reader = easyocr.Reader(['en'])

start_time = time.time()

ims = os.listdir(image_dir)
im_num = len(ims)

failed = []

remaining = -1

for i, im in enumerate(ims):
    outpath = os.path.join(out_dir, im + "_mask.npy")
    if (os.path.exists(outpath)) or not("jpg" in im.lower() or "png" in im.lower() or "jpeg" in im.lower()):
        continue

    filename = os.path.join(image_dir, im)

    try:
        results = reader.readtext(filename)
        im = cv2.imread(filename)
        if im is None:
            failed.append("Not found:")
            failed.append(filename)
            continue
        mask = np.zeros_like(im)
        for result in results:
            a, b, c, d = result[0]
            min_x = round(min(a[0], b[0], c[0], d[0]))
            min_y = round(min(a[1], b[1], c[1], d[1]))
            max_x = round(max(a[0], b[0], c[0], d[0]))
            max_y = round(max(a[1], b[1], c[1], d[1]))
            mask[min_y:max_y,min_x:max_x,:] = 1

        if not(debug_dir is None):
            debug_out = os.path.join(debug_dir, filename)
            cv2.imwrite(debug_out + "_masked.png", im*mask)
            cv2.imwrite(debug_out + "_un_masked.png", im*(1-mask))
        np.save(outpath, mask)

        elapsed = time.time() - start_time
        speed = (i+1) / elapsed
        remaining = (im_num - i - 1) / speed
        remaining = int(remaining * 100) / 100.0
    except Exception as e:
        failed.append(e)
        failed.append(filename)

    print(str(i+1) + " / " + str(im_num) + ", about " + str(remaining) + "s left.")
    sys.stdout.write("\033[F") # Cursor up one line

print()
print()
print()
print(failed)
print("~~~~~~~~~~~~~")
print(len(failed))
print(im_num)


import asyncio
import grpc
import logging
import time

from payment_pb2 import (
    ProcessPaymentRequest,
    ProcessPaymentReply,
)
from payment_pb2_grpc import (
    PaymentService,
    add_PaymentServiceServicer_to_server,
)
from tasks import process_payment

class PaymentServer(PaymentService):

    async def ProcessPayment(
        self,
        request: ProcessPaymentRequest,
        context: grpc.aio.ServicerContext,
    ) -> ProcessPaymentReply:
        logging.info("[ProcessPayment] Serving request %s", request)
        for payment in request.payments:
            initiated_at = time.strftime('%Y-%m-%d %H:%M:%S')
            task = process_payment.s(
                payment.payment_id,
                payment.card_holder_name,
                payment.card_number,
                payment.price,
                initiated_at,
            ).apply_async()
            task.forget()
        return ProcessPaymentReply(
            message=f"[ProcessPayment] Sent task-process_payment to celery-worker"
        )

SERVER_PORT = 50051

async def serve() -> None:
    server = grpc.aio.server()
    add_PaymentServiceServicer_to_server(PaymentServer(), server)
    listen_addr = f"[::]:{SERVER_PORT}"
    server.add_insecure_port(listen_addr)
    logging.info("Starting server on %s", listen_addr)
    await server.start()
    await server.wait_for_termination()

if __name__ == "__main__":
    logging.basicConfig(level=logging.INFO)
    asyncio.run(serve())


import datetime, numpy, os, sys, pathlib
from PyQt5 import QtGui, QtWidgets, QtCore
import pyqtgraph as pg
import numpy as np

smallfont = QtGui.QFont()
smallfont.setPointSize(7)
verysmallfont = QtGui.QFont()
verysmallfont.setPointSize(5)

def remove_size_props(o, fcol=False, button=False, image=False):
    o.setMinimumHeight(0)
    o.setMinimumWidth(0)
    o.setMaximumHeight(int(1e5))
    if fcol:
        o.setMaximumWidth(250) # just a max width for the first column
    elif button:
        o.setMaximumWidth(250/5) # just a max width for the first column
    elif image:
        # o.setMaximumWidth(500) # just a max width for the first column
        o.setMaximumHeight(500) # just a max width for the first column
    else:
        o.setMaximumWidth(int(1e5))

def create_calendar(self, Layout, min_date=(2020, 8, 1)):

    self.cal = QtWidgets.QCalendarWidget(self)
    self.cal.setFont(verysmallfont)
    self.cal.setMinimumHeight(160)
    self.cal.setMaximumHeight(160)
    self.cal.setMinimumWidth(265)
    self.cal.setMaximumWidth(265)
    self.cal.setMinimumDate(QtCore.QDate(datetime.date(*min_date)))
    self.cal.setMaximumDate(QtCore.QDate(datetime.date.today()+datetime.timedelta(1)))
    self.cal.adjustSize()
    self.cal.clicked.connect(self.pick_date)
    Layout.addWidget(self.cal)

    # setting an highlight format
    self.highlight_format = QtGui.QTextCharFormat()
    self.highlight_format.setBackground(self.cal.palette().brush(QtGui.QPalette.Link))
    self.highlight_format.setForeground(self.cal.palette().color(QtGui.QPalette.BrightText))

    self.cal.setSelectedDate(datetime.date.today())

def reinit_calendar(self, min_date=(2020, 8, 1), max_date=None):

    day, i = datetime.date(*min_date), 0
    while day!=datetime.date.today():
        self.cal.setDateTextFormat(QtCore.QDate(day),
                                   QtGui.QTextCharFormat())
        day = day+datetime.timedelta(1)
    day = day+datetime.timedelta(1)
    self.cal.setDateTextFormat(QtCore.QDate(day),
                               QtGui.QTextCharFormat())
    self.cal.setMinimumDate(QtCore.QDate(datetime.date(*min_date)))

    if max_date is not None:
        self.cal.setMaximumDate(QtCore.QDate(datetime.date(*max_date)+datetime.timedelta(1)))
        self.cal.setSelectedDate(datetime.date(*max_date)+datetime.timedelta(1))
    else:
        self.cal.setMaximumDate(QtCore.QDate(datetime.date.today()+datetime.timedelta(1)))
        self.cal.setSelectedDate(datetime.date.today())

def add_buttons(self, Layout):

    self.styleUnpressed = ("QPushButton {Text-align: left; "
                           "background-color: rgb(200, 200, 200); "
                           "color:white;}")
    self.stylePressed = ("QPushButton {Text-align: left; "
                         "background-color: rgb(100,50,100); "
                         "color:white;}")
    self.styleInactive = ("QPushButton {Text-align: left; "
                          "background-color: rgb(200, 200, 200); "
                          "color:gray;}")

    iconSize = QtCore.QSize(20, 20)

    self.playButton = QtWidgets.QToolButton()
    self.playButton.setIcon(self.style().standardIcon(QtGui.QStyle.SP_MediaPlay))
    self.playButton.setIconSize(iconSize)
    self.playButton.setToolTip("Play   -> [Space]")
    self.playButton.setCheckable(True)
    self.playButton.setEnabled(True)
    self.playButton.clicked.connect(self.play)

    self.pauseButton = QtWidgets.QToolButton()
    self.pauseButton.setCheckable(True)
    self.pauseButton.setIcon(self.style().standardIcon(QtGui.QStyle.SP_MediaPause))
    self.pauseButton.setIconSize(iconSize)
    self.pauseButton.setToolTip("Pause   -> [Space]")
    self.pauseButton.clicked.connect(self.pause)

    btns = QtWidgets.QButtonGroup(self)
    btns.addButton(self.playButton,0)
    btns.addButton(self.pauseButton,1)
    btns.setExclusive(True)

    self.playButton.setEnabled(False)
    self.pauseButton.setEnabled(True)
    self.pauseButton.setChecked(True)

    self.refreshButton = QtWidgets.QToolButton()
    self.refreshButton.setCheckable(True)
    self.refreshButton.setIcon(self.style().standardIcon(QtGui.QStyle.SP_BrowserReload))
    self.refreshButton.setIconSize(iconSize)
    self.refreshButton.setToolTip("Refresh   -> [r]")
    self.refreshButton.clicked.connect(self.refresh)

    self.quitButton = QtWidgets.QToolButton()
    # self.quitButton.setCheckable(True)
    self.quitButton.setIcon(self.style().standardIcon(QtGui.QStyle.SP_DialogCloseButton))
    self.quitButton.setIconSize(iconSize)
    self.quitButton.setToolTip("Quit")
    self.quitButton.clicked.connect(self.quit)

    self.backButton = QtWidgets.QToolButton()
    # self.backButton.setCheckable(True)
    self.backButton.setIcon(self.style().standardIcon(QtGui.QStyle.SP_FileDialogBack))
    self.backButton.setIconSize(iconSize)
    self.backButton.setToolTip("Back to initial view   -> [i]")
    self.backButton.clicked.connect(self.back_to_initial_view)

    self.settingsButton = QtWidgets.QToolButton()
    # self.settingsButton.setCheckable(True)
    self.settingsButton.setIcon(self.style().standardIcon(QtGui.QStyle.SP_FileDialogDetailedView))
    self.settingsButton.setIconSize(iconSize)
    # self.settingsButton.setToolTip("Settings")
    # self.settingsButton.clicked.connect(self.change_settings)
    self.settingsButton.setToolTip("Metadata")
    self.settingsButton.clicked.connect(self.see_metadata)

    Layout.addWidget(self.quitButton)
    Layout.addWidget(self.playButton)
    Layout.addWidget(self.pauseButton)
    Layout.addWidget(self.refreshButton)
    Layout.addWidget(self.backButton)
    Layout.addWidget(self.settingsButton)

def build_slider(self, Layout):
    self.frameSlider = QtWidgets.QSlider(QtCore.Qt.Horizontal)
    self.frameSlider.setMinimum(0)
    self.frameSlider.setMaximum(self.settings['Npoints'])
    self.frameSlider.setTickInterval(1)
    self.frameSlider.setTracking(False)
    self.frameSlider.valueChanged.connect(self.update_frame)
    self.frameSlider.setMaximumHeight(20)
    Layout.addWidget(self.frameSlider)

class NewWindow(QtWidgets.QMainWindow):

    def __init__(self,
                 parent=None, i=0,
                 title='New Window',
                 size=(1000,600)):

        super(NewWindow, self).__init__()

        self.script = os.path.join(str(pathlib.Path(__file__).resolve().parents[1]),
                                   'script.sh') # for batch processing

        self.setGeometry(600+20*i, 250+20*i, size[0], size[1])

        ##############################
        ##### keyboard shortcuts #####
        ##############################

        # adding a few general keyboard shortcut
        self.openSc = QtWidgets.QShortcut(QtGui.QKeySequence('O'), self)
        self.openSc.activated.connect(self.open_file)

        self.spaceSc = QtWidgets.QShortcut(QtGui.QKeySequence('Space'), self)
        self.spaceSc.activated.connect(self.hitting_space)

        self.saveSc = QtWidgets.QShortcut(QtGui.QKeySequence('S'), self)
        self.saveSc.activated.connect(self.save)

        self.add2Bash = QtWidgets.QShortcut(QtGui.QKeySequence('B'), self)
        self.add2Bash.activated.connect(self.add_to_bash_script)

        self.quitSc = QtWidgets.QShortcut(QtGui.QKeySequence('Q'), self)
        self.quitSc.activated.connect(self.quit)

        self.refreshSc = QtWidgets.QShortcut(QtGui.QKeySequence('R'), self)
        self.refreshSc.activated.connect(self.refresh)

        self.homeSc = QtWidgets.QShortcut(QtGui.QKeySequence('I'), self)
        self.homeSc.activated.connect(self.back_to_initial_view)

        self.maxSc = QtWidgets.QShortcut(QtGui.QKeySequence('M'), self)
        self.maxSc.activated.connect(self.showwindow)

        self.processSc = QtWidgets.QShortcut(QtGui.QKeySequence('P'), self)
        self.processSc.activated.connect(self.process)

        self.fitSc = QtWidgets.QShortcut(QtGui.QKeySequence('F'), self)
        self.fitSc.activated.connect(self.fit)

        ########################
        ##### building GUI #####
        ########################

        pg.setConfigOptions(imageAxisOrder='row-major')

        self.setWindowTitle(title)
        self.minView = False

        self.cwidget = QtWidgets.QWidget(self)
        self.setCentralWidget(self.cwidget)

        self.statusBar = QtWidgets.QStatusBar()
        self.setStatusBar(self.statusBar)
        self.statusBar.setFont(smallfont)

        self.showwindow()

    def quit(self):
        sys.exit()

    def print_datafile(self):
        if hasattr(self, 'datafile'):
            print('current datafile:\n', self.datafile)

    def process(self):
        print(' "process" function not implemented')
        print(' --> should be implemented in child class !')
        self.print_datafile()

    def fit(self):
        print(' "fit" function not implemented')
        print(' --> should be implemented in child class !')
        self.print_datafile()

    def add_to_bash_script(self):
        print(' "add_to_bash_script" function not implemented')
        print(' --> should be implemented in child class !')

    def refresh(self):
        print(' "refresh" function not implemented')
        print(' --> should be implemented in child class !')

    def open_file(self):
        print(' "open_file" function not implemented')
        print(' --> should be implemented in child class !')

    def hitting_space(self):
        print(' "hitting_space" function not implemented')
        print(' --> should be implemented in child class !')

    def back_to_initial_view(self):
        print(' "back_to_initial_view" function not implemented')
        print(' --> should be implemented in child class !')

    def showwindow(self):
        if self.minView:
            self.minView = self.maxview()
        else:
            self.minView = self.minview()

    def maxview(self):
        self.showFullScreen()
        return False

    def minview(self):
        self.showNormal()
        return True

    def save(self):
        print(' "save" function not implemented')
        print(' --> should be implemented in child class !')

    ###########################################
    ################ Widget tools #############
    ###########################################

    def init_basic_widget_grid(self,
                               wdgt_length=3,
                               Ncol_wdgt=20,
                               Nrow_wdgt=20):

        self.i_wdgt = 0 # initialize widget counter

        self.wdgt_length = wdgt_length

        self.cwidget = QtWidgets.QWidget(self)
        self.setCentralWidget(self.cwidget)

        # grid layout
        self.layout = QtWidgets.QGridLayout()
        self.cwidget.setLayout(self.layout)

        self.graphics_layout = pg.GraphicsLayoutWidget()
        self.layout.addWidget(self.graphics_layout, 0, self.wdgt_length,
                              Nrow_wdgt, Ncol_wdgt)

    def add_widget(self, wdgt, spec='None'):
        if 'small' in spec:
            wdgt.setFixedWidth(70)

        # if spec=='shift-right':
        #     self.layout.addWidget(wdgt, self.i_wdgt-1, self.wdgt_length,
        #                           1, self.wdgt_length+1)
        if spec=='small-left':
            self.layout.addWidget(wdgt, self.i_wdgt, 0, 1, 1)
        elif spec=='small-middle':
            self.layout.addWidget(wdgt, self.i_wdgt, 1, 1, 1)
        elif spec=='large-left':
            self.layout.addWidget(wdgt, self.i_wdgt, 0, 1, self.wdgt_length-1)
        elif spec=='small-right':
            self.layout.addWidget(wdgt, self.i_wdgt, self.wdgt_length-1, 1, 1)
            self.i_wdgt += 1
        elif spec=='large-right':
            self.layout.addWidget(wdgt, self.i_wdgt, 1, 1, self.wdgt_length-1)
            self.i_wdgt += 1
        else:
            self.layout.addWidget(wdgt, self.i_wdgt, 0, 1, self.wdgt_length)
            self.i_wdgt += 1

    ###########################################
    ########## Data-specific tools ############
    ###########################################

    def select_ROI_from_pick(self, data):

        if self.roiPick.text() in ['sum', 'all']:
            roiIndices = np.arange(data.nROIs)

        elif len(self.roiPick.text().split('-'))>1:
            try:
                roiIndices = np.arange(int(self.roiPick.text().split('-')[0]), int(self.roiPick.text().split('-')[1]))
            except BaseException as be:
                print(be)
                roiIndices = None

        elif len(self.roiPick.text().split(','))>1:
            try:
                roiIndices = np.array([int(ii) for ii in self.roiPick.text().split(',')])
            except BaseException as be:
                print(be)
                roiIndices = None
        else:
            try:
                i0 = int(self.roiPick.text())
                if (i0<0) or (i0>=data.nROIs):
                    roiIndices = [0]
                    self.statusBar.showMessage(' "%i" not a valid ROI index, roiIndices set to [0]'  % i0)
                else:
                    roiIndices = [i0]

            except BaseException as be:
                print(be)
                roiIndices = [0]
                self.statusBar.showMessage(' /!\ Problem in setting indices /!\ ')

        return roiIndices

    def keyword_update(self, string=None, parent=None):

        if string is None:
            string = self.guiKeywords.text()

        cls = (parent if parent is not None else self)

        if string in ['Stim', 'stim', 'VisualStim', 'Stimulation', 'stimulation']:
            cls.load_VisualStim()
        elif string in ['no_stim', 'no_VisualStim']:
            cls.visual_stim = None
        elif string in ['scan_folder', 'scanF', 'scan']:
            cls.scan_folder()
        elif string in ['meanImg', 'meanImg_chan2', 'meanImgE', 'Vcorr', 'max_proj']:
            cls.CaImaging_bg_key = string
        elif 'plane' in string:
            cls.planeID = int(string.split('plane')[1])
        elif string=='no_subsampling':
            cls.no_subsampling = True
        elif string in ['F', 'Fluorescence', 'Neuropil', 'Deconvolved', 'Fneu', 'dF/F', 'dFoF'] or ('F-' in string):
            if string=='F':
                cls.CaImaging_key = 'Fluorescence'
            elif string=='Fneu':
                cls.CaImaging_key = 'Neuropil'
            else:
                cls.CaImaging_key = string
        elif string=='subsampling':
            cls.no_subsampling = False
        elif string=='subjects':
            cls.compute_subjects()
        else:
            self.statusBar.showMessage('  /!\ keyword "%s" not recognized /!\ ' % string)

    # Layout11 = QtWidgets.QVBoxLayout()
    # Layout1.addLayout(Layout11)
    # create_calendar(self, Layout11)
    # self.notes = QtWidgets.QLabel(63*'-'+5*'\n', self)
    # self.notes.setMinimumHeight(70)
    # self.notes.setMaximumHeight(70)
    # Layout11.addWidget(self.notes)

    # self.pbox = QtWidgets.QComboBox(self)
    # self.pbox.activated.connect(self.display_quantities)
    # self.pbox.setMaximumHeight(selector_height)
    # if self.raw_data_visualization:
    #     self.pbox.addItem('')
    #     self.pbox.addItem('-> Show Raw Data')
    #     self.pbox.setCurrentIndex(1)

#     def __init__(self, parent=None,
#                  fullscreen=False):

#         super(TrialAverageWindow, self).__init__()

#         # adding a "quit" keyboard shortcut
#         self.quitSc = QtWidgets.QShortcut(QtGui.QKeySequence('Q'), self) # or 'Q'
#         self.quitSc.activated.connect(self.close)
#         self.refreshSc = QtWidgets.QShortcut(QtGui.QKeySequence('R'), self) # or 'Q'
#         self.refreshSc.activated.connect(self.refresh)
#         self.maxSc = QtWidgets.QShortcut(QtGui.QKeySequence('M'), self)
#         self.maxSc.activated.connect(self.showwindow)

#         ####################################################
#         # BASIC style config
#         self.setWindowTitle('Analysis Program -- Physiology of Visual Circuits')

#     def close(self):
#         pass

class Slider(QtWidgets.QSlider):
    def __init__(self, bid, parent=None):
        super(self.__class__, self).__init__()
        self.bid = bid
        self.setOrientation(QtCore.Qt.Horizontal)
        self.setMinimum(0)
        self.setMaximum(255)
        self.setValue(255)
        self.setTickInterval(1)
        self.valueChanged.connect(lambda: self.level_change(parent,bid))
        self.setTracking(False)

    def level_change(self, parent, bid):
        parent.saturation = float(self.value())
        if parent.ROI is not None:
            parent.ROI.plot(parent)
        parent.win.show()

# ### custom QDialog which makes a list of items you can include/exclude
# class ListChooser(QtWidgets.QDialog):
#     def __init__(self, title, parent):
#         super(ListChooser, self).__init__(parent)
#         self.setGeometry(300,300,320,200)
#         self.setWindowTitle(title)
#         self.win = QtWidgets.QWidget(self)
#         layout = QtWidgets.QGridLayout()
#         self.win.setLayout(layout)
#         #self.setCentralWidget(self.win)
#         layout.addWidget(QtWidgets.QLabel('click to select videos (none selected => all used)'),0,0,1,1)
#         self.list = QtWidgets.QListWidget(parent)
#         for f in parent.filelist:
#             self.list.addItem(f)
#         layout.addWidget(self.list,1,0,7,4)
#         #self.list.resize(450,250)
#         self.list.setSelectionMode(QtGui.QAbstractItemView.MultiSelection)
#         done = QtGui.QPushButton('done')
#         done.clicked.connect(lambda: self.exit_list(parent))
#         layout.addWidget(done,8,0,1,1)

#     def exit_list(self, parent):
#         parent.filelist = []
#         items = self.list.selectedItems()
#         for i in range(len(items)):
#             parent.filelist.append(str(self.list.selectedItems()[i].text()))
#         self.accept()

# class TextChooser(QtWidgets.QDialog):
#     def __init__(self,parent=None):
#         super(TextChooser, self).__init__(parent)
#         self.setGeometry(300,300,350,100)
#         self.setWindowTitle('folder path')
#         self.win = QtWidgets.QWidget(self)
#         layout = QtWidgets.QGridLayout()
#         self.win.setLayout(layout)
#         self.qedit = QtWidgets.QLineEdit('')
#         layout.addWidget(QtWidgets.QLabel('folder name (does not have to exist yet)'),0,0,1,3)
#         layout.addWidget(self.qedit,1,0,1,3)
#         done = QtGui.QPushButton('OK')
#         done.clicked.connect(self.exit)
#         layout.addWidget(done,2,1,1,1)

#     def exit(self):
#         self.folder = self.qedit.text()
#         self.accept()

# class RGBRadioButtons(QtGui.QButtonGroup):
#     def __init__(self, parent=None, row=0, col=0):
#         super(RGBRadioButtons, self).__init__()
#         parent.color = 0
#         self.parent = parent
#         self.bstr = ["image", "flowsX", "flowsY", "flowsZ", "cellprob"]
#         #self.buttons = QtGui.QButtonGroup()
#         self.dropdown = []
#         for b in range(len(self.bstr)):
#             button = QtGui.QRadioButton(self.bstr[b])
#             button.setStyleSheet('color: white;')
#             if b==0:
#                 button.setChecked(True)
#             self.addButton(button, b)
#             button.toggled.connect(lambda: self.btnpress(parent))
#             self.parent.l0.addWidget(button, row+b,col,1,1)
#         self.setExclusive(True)
#         #self.buttons.

#     def btnpress(self, parent):
#        b = self.checkedId()
#        self.parent.view = b
#        if self.parent.loaded:
#            self.parent.update_plot()

# class ViewBoxNoRightDrag(pg.ViewBox):
#     def __init__(self, parent=None, border=None, lockAspect=False, enableMouse=True, invertY=False, enableMenu=True, name=None, invertX=False):
#         pg.ViewBox.__init__(self, parent, border, lockAspect, enableMouse,
#                             invertY, enableMenu, name, invertX)

#     def mouseDragEvent(self, ev, axis=None):
#         ## if axis is specified, event will only affect that axis.
#         ev.accept()  ## we accept all buttons

#         pos = ev.pos()
#         lastPos = ev.lastPos()
#         dif = pos - lastPos
#         dif = dif * -1

#         ## Ignore axes if mouse is disabled
#         mouseEnabled = np.array(self.state['mouseEnabled'], dtype=np.float)
#         mask = mouseEnabled.copy()
#         if axis is not None:
#             mask[1-axis] = 0.0

#         ## Scale or translate based on mouse button
#         if ev.button() & (QtCore.Qt.LeftButton | QtCore.Qt.MidButton):
#             if self.state['mouseMode'] == pg.ViewBox.RectMode:
#                 if ev.isFinish():  ## This is the final move in the drag; change the view scale now
#                     #print "finish"
#                     self.rbScaleBox.hide()
#                     ax = QtCore.QRectF(Point(ev.buttonDownPos(ev.button())), Point(pos))
#                     ax = self.childGroup.mapRectFromParent(ax)
#                     self.showAxRect(ax)
#                     self.axHistoryPointer += 1
#                     self.axHistory = self.axHistory[:self.axHistoryPointer] + [ax]
#                 else:
#                     ## update shape of scale box
#                     self.updateScaleBox(ev.buttonDownPos(), ev.pos())
#             else:
#                 tr = dif*mask
#                 tr = self.mapToView(tr) - self.mapToView(Point(0,0))
#                 x = tr.x() if mask[0] == 1 else None
#                 y = tr.y() if mask[1] == 1 else None

#                 self._resetTarget()
#                 if x is not None or y is not None:
#                     self.translateBy(x=x, y=y)
#                 self.sigRangeChangedManually.emit(self.state['mouseEnabled'])

# class ImageDraw(pg.ImageItem):
#     """
#     **Bases:** :class:`GraphicsObject <pyqtgraph.GraphicsObject>`
#     GraphicsObject displaying an image. Optimized for rapid update (ie video display).
#     This item displays either a 2D numpy array (height, width) or
#     a 3D array (height, width, RGBa). This array is optionally scaled (see
#     :func:`setLevels <pyqtgraph.ImageItem.setLevels>`) and/or colored
#     with a lookup table (see :func:`setLookupTable <pyqtgraph.ImageItem.setLookupTable>`)
#     before being displayed.
#     ImageItem is frequently used in conjunction with
#     :class:`HistogramLUTItem <pyqtgraph.HistogramLUTItem>` or
#     :class:`HistogramLUTWidget <pyqtgraph.HistogramLUTWidget>` to provide a GUI
#     for controlling the levels and lookup table used to display the image.
#     """

#     sigImageChanged = QtCore.Signal()

#     def __init__(self, image=None, viewbox=None, parent=None, **kargs):
#         super(ImageDraw, self).__init__()
#         #self.image=None
#         #self.viewbox=viewbox
#         self.levels = np.array([0,255])
#         self.lut = None
#         self.autoDownsample = False
#         self.axisOrder = 'row-major'
#         self.removable = False

#         self.parent = parent
#         #kernel[1,1] = 1
#         self.setDrawKernel(kernel_size=self.parent.brush_size)
#         self.parent.current_stroke = []
#         self.parent.in_stroke = False

#     def mouseClickEvent(self, ev):
#         if self.parent.masksOn:
#             if ev.button()==QtCore.Qt.RightButton and self.parent.loaded and self.parent.nmasks < 2:
#                 if not self.parent.in_stroke:
#                     ev.accept()
#                     self.parent.in_stroke = True
#                     self.create_start(ev.pos())
#                     self.parent.stroke_appended = False
#                     self.drawAt(ev.pos(), ev)
#                 else:
#                     ev.accept()
#                     self.end_stroke()
#                     self.parent.in_stroke = False
#             else:
#                 ev.ignore()
#                 return
#         else:
#             ev.ignore()
#             return

#     def mouseDragEvent(self, ev):
#         ev.ignore()
#         return

#     def hoverEvent(self, ev):
#         QtWidgets.QApplication.setOverrideCursor(QtCore.Qt.CrossCursor)
#         if self.parent.in_stroke:
#             if self.parent.in_stroke:
#                 # continue stroke if not at start
#                 self.drawAt(ev.pos())
#                 if self.is_at_start(ev.pos()):
#                     self.end_stroke()
#                     self.parent.in_stroke = False
#         else:
#             ev.acceptClicks(QtCore.Qt.RightButton)
#             #ev.acceptClicks(QtCore.Qt.LeftButton)

#     def create_start(self, pos):
#         self.scatter = pg.ScatterPlotItem([pos.x()], [pos.y()], pxMode=False,
#                                         pen=pg.mkPen(color=(255,0,0), width=self.parent.brush_size),
#                                         size=max(3*2, self.parent.brush_size*1.8*2), brush=None)
#         self.parent.p0.addItem(self.scatter)

#     def is_at_start(self, pos):
#         thresh_out = max(6, self.parent.brush_size*3)
#         thresh_in = max(3, self.parent.brush_size*1.8)
#         # first check if you ever left the start
#         if len(self.parent.current_stroke) > 3:
#             stroke = np.array(self.parent.current_stroke)
#             dist = (((stroke[1:] - stroke[:1][np.newaxis,:,:])**2).sum(axis=-1))**0.5
#             dist = dist.flatten()
#             #print(dist)
#             has_left = (dist > thresh_out).nonzero()[0]
#             if len(has_left) > 0:
#                 first_left = np.sort(has_left)[0]
#                 has_returned = (dist[max(4,first_left+1):] < thresh_in).sum()
#                 if has_returned > 0:
#                     return True
#                 else:
#                     return False
#             else:
#                 return False

#     def end_stroke(self):
#         self.parent.p0.removeItem(self.scatter)
#         if not self.parent.stroke_appended:
#             self.parent.stroke = np.array(self.parent.current_stroke)
#             self.parent.current_stroke = []
#             self.parent.stroke_appended = True
#             ioutline = self.parent.stroke[:,-1]==1
#             self.parent.point_set = list(self.parent.stroke[ioutline])
#             self.parent.add_set()

#     def tabletEvent(self, ev):
#         pass
#         #print(ev.device())
#         #print(ev.pointerType())
#         #print(ev.pressure())

#     def drawAt(self, pos, ev=None):
#         mask = self.greenmask
#         set = self.parent.current_point_set
#         stroke = self.parent.current_stroke
#         pos = [int(pos.y()), int(pos.x())]
#         dk = self.drawKernel
#         kc = self.drawKernelCenter
#         sx = [0,dk.shape[0]]
#         sy = [0,dk.shape[1]]
#         tx = [pos[0] - kc[0], pos[0] - kc[0]+ dk.shape[0]]
#         ty = [pos[1] - kc[1], pos[1] - kc[1]+ dk.shape[1]]
#         kcent = kc.copy()
#         if tx[0]<=0:
#             sx[0] = 0
#             sx[1] = kc[0] + 1
#             tx    = sx
#             kcent[0] = 0
#         if ty[0]<=0:
#             sy[0] = 0
#             sy[1] = kc[1] + 1
#             ty    = sy
#             kcent[1] = 0
#         if tx[1] >= self.parent.Ly-1:
#             sx[0] = dk.shape[0] - kc[0] - 1
#             sx[1] = dk.shape[0]
#             tx[0] = self.parent.Ly - kc[0] - 1
#             tx[1] = self.parent.Ly
#             kcent[0] = tx[1]-tx[0]-1
#         if ty[1] >= self.parent.Lx-1:
#             sy[0] = dk.shape[1] - kc[1] - 1
#             sy[1] = dk.shape[1]
#             ty[0] = self.parent.Lx - kc[1] - 1
#             ty[1] = self.parent.Lx
#             kcent[1] = ty[1]-ty[0]-1

#         ts = (slice(tx[0],tx[1]), slice(ty[0],ty[1]))
#         ss = (slice(sx[0],sx[1]), slice(sy[0],sy[1]))
#         self.image[ts] = mask[ss]

#         for ky,y in enumerate(np.arange(ty[0], ty[1], 1, int)):
#             for kx,x in enumerate(np.arange(tx[0], tx[1], 1, int)):
#                 iscent = np.logical_and(kx==kcent[0], ky==kcent[1])
#                 stroke.append([x, y, iscent])
#         self.updateImage()

#     def setDrawKernel(self, kernel_size=3):
#         bs = kernel_size
#         kernel = np.ones((bs,bs), np.uint8)
#         self.drawKernel = kernel
#         self.drawKernelCenter = [int(np.floor(kernel.shape[0]/2)),
#                                  int(np.floor(kernel.shape[1]/2))]
#         onmask = 255 * kernel[:,:,np.newaxis]
#         offmask = np.zeros((bs,bs,1))
#         opamask = 100 * kernel[:,:,np.newaxis]
#         self.redmask = np.concatenate((onmask,offmask,offmask,onmask), axis=-1)
#         self.greenmask = np.concatenate((onmask,offmask,onmask,opamask), axis=-1)

# class RangeSlider(QtWidgets.QSlider):
#     """ A slider for ranges.

#         This class provides a dual-slider for ranges, where there is a defined
#         maximum and minimum, as is a normal slider, but instead of having a
#         single slider value, there are 2 slider values.

#         This class emits the same signals as the QSlider base class, with the
#         exception of valueChanged

#         Found this slider here: https://www.mail-archive.com/pyqt@riverbankcomputing.com/msg22889.html
#         and modified it
#     """
#     def __init__(self, parent=None, *args):
#         super(RangeSlider, self).__init__(*args)

#         self._low = self.minimum()
#         self._high = self.maximum()

#         self.pressed_control = QtGui.QStyle.SC_None
#         self.hover_control = QtGui.QStyle.SC_None
#         self.click_offset = 0

#         self.setOrientation(QtCore.Qt.Vertical)
#         self.setTickPosition(QtWidgets.QSlider.TicksRight)
#         self.setStyleSheet(\
#                 "QSlider::handle:horizontal {\
#                 background-color: white;\
#                 border: 1px solid #5c5c5c;\
#                 border-radius: 0px;\
#                 border-color: black;\
#                 height: 8px;\
#                 width: 6px;\
#                 margin: -8px 2; \
#                 }")

#         #self.opt = QtGui.QStyleOptionSlider()
#         #self.opt.orientation=QtCore.Qt.Vertical
#         #self.initStyleOption(self.opt)
#         # 0 for the low, 1 for the high, -1 for both
#         self.active_slider = 0
#         self.parent = parent

#     def level_change(self):
#         if self.parent is not None:
#             if self.parent.loaded:
#                 self.parent.saturation = [self._low, self._high]
#                 self.parent.update_plot()

#     def low(self):
#         return self._low

#     def setLow(self, low):
#         self._low = low
#         self.update()

#     def high(self):
#         return self._high

#     def setHigh(self, high):
#         self._high = high
#         self.update()

#     def paintEvent(self, event):
#         # based on http://qt.gitorious.org/qt/qt/blobs/master/src/gui/widgets/qslider.cpp
#         painter = QtGui.QPainter(self)
#         style = QtGui.QApplication.style()

#         for i, value in enumerate([self._low, self._high]):
#             opt = QtGui.QStyleOptionSlider()
#             self.initStyleOption(opt)

#             # Only draw the groove for the first slider so it doesn't get drawn
#             # on top of the existing ones every time
#             if i == 0:
#                 opt.subControls = QtGui.QStyle.SC_SliderHandle#QtGui.QStyle.SC_SliderGroove | QtGui.QStyle.SC_SliderHandle
#             else:
#                 opt.subControls = QtGui.QStyle.SC_SliderHandle

#             if self.tickPosition() != self.NoTicks:
#                 opt.subControls |= QtGui.QStyle.SC_SliderTickmarks

#             if self.pressed_control:
#                 opt.activeSubControls = self.pressed_control
#                 opt.state |= QtGui.QStyle.State_Sunken
#             else:
#                 opt.activeSubControls = self.hover_control

#             opt.sliderPosition = value
#             opt.sliderValue = value
#             style.drawComplexControl(QtGui.QStyle.CC_Slider, opt, painter, self)

#     def mousePressEvent(self, event):
#         event.accept()

#         style = QtGui.QApplication.style()
#         button = event.button()
#         # In a normal slider control, when the user clicks on a point in the
#         # slider's total range, but not on the slider part of the control the
#         # control would jump the slider value to where the user clicked.
#         # For this control, clicks which are not direct hits will slide both
#         # slider parts
#         if button:
#             opt = QtGui.QStyleOptionSlider()
#             self.initStyleOption(opt)

#             self.active_slider = -1

#             for i, value in enumerate([self._low, self._high]):
#                 opt.sliderPosition = value
#                 hit = style.hitTestComplexControl(style.CC_Slider, opt, event.pos(), self)
#                 if hit == style.SC_SliderHandle:
#                     self.active_slider = i
#                     self.pressed_control = hit

#                     self.triggerAction(self.SliderMove)
#                     self.setRepeatAction(self.SliderNoAction)
#                     self.setSliderDown(True)

#                     break

#             if self.active_slider < 0:
#                 self.pressed_control = QtGui.QStyle.SC_SliderHandle
#                 self.click_offset = self.__pixelPosToRangeValue(self.__pick(event.pos()))
#                 self.triggerAction(self.SliderMove)
#                 self.setRepeatAction(self.SliderNoAction)
#         else:
#             event.ignore()

#     def mouseMoveEvent(self, event):
#         if self.pressed_control != QtGui.QStyle.SC_SliderHandle:
#             event.ignore()
#             return

#         event.accept()
#         new_pos = self.__pixelPosToRangeValue(self.__pick(event.pos()))
#         opt = QtGui.QStyleOptionSlider()
#         self.initStyleOption(opt)

#         if self.active_slider < 0:
#             offset = new_pos - self.click_offset
#             self._high += offset
#             self._low += offset
#             if self._low < self.minimum():
#                 diff = self.minimum() - self._low
#                 self._low += diff
#                 self._high += diff
#             if self._high > self.maximum():
#                 diff = self.maximum() - self._high
#                 self._low += diff
#                 self._high += diff
#         elif self.active_slider == 0:
#             if new_pos >= self._high:
#                 new_pos = self._high - 1
#             self._low = new_pos
#         else:
#             if new_pos <= self._low:
#                 new_pos = self._low + 1
#             self._high = new_pos

#         self.click_offset = new_pos
#         self.update()

#     def mouseReleaseEvent(self, event):
#         self.level_change()

#     def __pick(self, pt):
#         if self.orientation() == QtCore.Qt.Horizontal:
#             return pt.x()
#         else:
#             return pt.y()

#     def __pixelPosToRangeValue(self, pos):
#         opt = QtGui.QStyleOptionSlider()
#         self.initStyleOption(opt)
#         style = QtGui.QApplication.style()

#         gr = style.subControlRect(style.CC_Slider, opt, style.SC_SliderGroove, self)
#         sr = style.subControlRect(style.CC_Slider, opt, style.SC_SliderHandle, self)

#         if self.orientation() == QtCore.Qt.Horizontal:
#             slider_length = sr.width()
#             slider_min = gr.x()
#             slider_max = gr.right() - slider_length + 1
#         else:
#             slider_length = sr.height()
#             slider_min = gr.y()
#             slider_max = gr.bottom() - slider_length + 1

#         return style.sliderValueFromPosition(self.minimum(), self.maximum(),
#                                              pos-slider_min, slider_max-slider_min,
#                                              opt.upsideDown)

# if __name__=='__main__':

#     # app = QtWidgets.QApplication(sys.argv)
#     # # build_dark_palette(app)
#     # # window = TrialAverageWindow(app)
#     # # window.show()
#     # sys.exit(app.exec_())


import numpy as np
import matplotlib.pyplot as plt

from games import NormalFormGame
from tools import *
import math

# %% Player cell
class Player:
    def __init__(self, model, playerIndex, verbose=False, strategy='UCB'):
        self.verbose = verbose
        self.strategy = strategy
        self.model = model
        self.nActions = model.nActions[playerIndex]
        self.playerIndex = playerIndex
        self.actions = []
        self.regrets = []
        self.probabilityDistibution = np.random.random(self.nActions)
        self.probabilityDistibution /= np.sum(self.probabilityDistibution)

        # UCB attributes
        self.explorationTime = self.nActions
        self.S = np.zeros(self.nActions)
        self.N = np.zeros(self.nActions)
        self.rho = 0.2

        #Exp3 attributes
        self.gamma = 0.01
        self.eta = self.gamma/self.nActions
        self.beta = 0.01
        self.weights = np.ones(self.nActions)

    def chooseAction(self):
        action = np.random.randint(self.nActions)
        if self.strategy == 'UCB':
            t = len(self.actions)
            action = t if (t < self.nActions) else np.argmax(self.S / self.N + self.rho * np.sqrt(np.log(t) / (2*self.N)))
        elif self.strategy == 'Exp3':
            action = choose(self.probabilityDistibution)
        elif self.strategy == 'Thompson sampling':
            t = len(self.actions)
            action = t if (t < self.nActions) else np.argmax(np.random.beta((self.S+4*self.N)/8+1, self.N - (self.S+4)/8 + 1))
        elif self.strategy == 'Naive':
            t = len(self.actions)
            action = t if (t < self.nActions) else np.argmax(self.S/self.N)
        else :
            print("strategy not implemented : random policy applied -- choose 'UCB', 'Exp3', 'Thompson sampling' or 'Naive'")
        return action

    def step(self):
        action = self.chooseAction()
        self.actions.append(action)
        self.N[action] += 1
        return action

    def analyzeStep(self, playerAction, opponentAction, rewards):
        reward = rewards[self.playerIndex]
        regret = self.model.bestReward(self.playerIndex, opponentAction) - reward
        self.regrets.append(regret)
        self.S[playerAction] += reward

        if self.strategy == 'Exp3':
            maxreward = np.max(self.model.matrix[:,:,self.playerIndex])
            minreward = np.min(self.model.matrix[:,:,self.playerIndex])

            estimatedRewards = self.beta/self.probabilityDistibution
            estimatedRewards[playerAction] += (reward - minreward)/(maxreward-minreward)/self.probabilityDistibution[playerAction]
            self.weights = self.weights * np.exp(self.eta * estimatedRewards)
            self.probabilityDistibution = (1 - self.gamma) * self.weights/ np.sum(self.weights) + self.gamma/self.nActions

    def plotAnalysis(self, player):
        plt.figure(1)
        plt.plot(np.cumsum(self.regrets))
        plt.title("Regrets for %s"%(player))
        plt.show()
        plt.figure(1)
        plt.plot(self.actions)
        plt.title("Actions for %s"%(player))
        plt.show()


# Wydruk polskich świąt z datami - z pliku csv.

import pandas as pd
import numpy as np
from tabulate import tabulate

path = ('holidays_csv.csv')

df = pd.read_csv(path)
# kasuj kolumnę province code
df.__delitem__('Province Code')
#print(df.loc[0:3, lambda x: ['Country Code']])
#print(df.loc[0:3, ['Country Code', 'Holiday Name']])

print('Kolumny w pliku csv:',list(df))
#  tylko PL z kolumny Country Code:
pl = df.loc[df['Country Code'] == 'PL']
# ilosc wierszy w zbiorze PL = ilosc swiat :
print('Ilosc swiat w PL:', pl.shape[0])

# sprawdzamy jakie sa typy swiat:
print('Typy swiat:', ((df['Type of Holiday']).unique()))

# ilosc swiat posortowane:
#print(df['Country Code'].value_counts())
#  ilosc swiat nieposortowane:
#print(df.groupby('Country Code').size())
wydruk = df.groupby('Country Code').size()

# święta - tylko public:
swieta_optional  = df.loc[(df['Country Code'] == 'PL') & (df['Type of Holiday'] == 'public')]

swieta_optional.drop_duplicates('Holiday Name',  keep='last')
polskie_swieta = swieta_optional.loc[:, ['Date','Type of Holiday','Holiday Name']]
sq = pd.DataFrame(polskie_swieta)

rok_2019 = sq['Date']>'2019-00-00 00:00:00'
print(tabulate(sq[rok_2019],headers='keys',tablefmt='psql'))


"""
Module containing classes that facilitate the writing of data to file
"""

import logging
import numpy as np
from netCDF4 import Dataset
from cftime import num2pydate
from cftime import date2num
import pathlib

try:
    import configparser
except ImportError:
    import ConfigParser as configparser

from pylag.data_types_python import DTYPE_INT, DTYPE_FLOAT
from pylag.data_types_python import INT_INVALID, FLOAT_INVALID

from pylag import variable_library
from pylag.exceptions import PyLagValueError

class NetCDFLogger(object):
    """ NetCDF data logger.

    Objects of type NetCDFLogger can be used to write particle data to file.

    Parameters
    ----------
    config : ConfigParser
        Configuration object

    file_name : str
        Name of the `*.nc` output file to be generated. If the `.nc` extension
        is not present it is automatically added.

    start_datetime : str
        String giving the simulation start date and time.

    n_particles : int
        The number of particles.

    grid_names : list[str]
        List of grids on which input data is defined.
    """
    def __init__(self, config, file_name, start_datetime, n_particles,
                 grid_names):

        logger = logging.getLogger(__name__)

        self.config = config

        if file_name[-3:] == '.nc':
            self.file_name = file_name
        else:
            self.file_name = ''.join([file_name, '.nc'])

        pathlib.Path(self.file_name).unlink(missing_ok=True)

        logger.info(f'Creating output file: {self.file_name}.')
        self._ncfile = Dataset(self.file_name, mode='w', format='NETCDF4')

        # Compression options for the netCDF variables.
        self._ncopts = {'zlib': True, 'complevel': 7}

        # Time units
        self._simulation_time_units = f'seconds since {start_datetime}'

        # Read in the coordinate system
        coordinate_system = self.config.get("SIMULATION",
                                            "coordinate_system").strip().lower()
        if coordinate_system in ["cartesian", "geographic"]:
            self.coordinate_system = coordinate_system
        else:
            raise PyLagValueError(f"Unsupported model coordinate "
                                  f"system `{coordinate_system}'")

        # Are we saving biological variables
        try:
            self.with_biology = self.config.getboolean("BIO_MODEL",
                                                       "use_bio_model")
        except (configparser.NoSectionError, configparser.NoOptionError) as e:
            self.with_biology = False

        # Set precision for diagnostic variables
        try:
            precision = self.config.get("OUTPUT", "precision")
        except (configparser.NoSectionError, configparser.NoOptionError) as e:
            logger.info(f'The precision of outputs has not been specified. '
                        f'Defaulting to single precision.')
            precision = "single"

        if precision == "single":
            self._precision = 's'
        elif precision == "double":
            self._precision = 'd'
        else:
            raise PyLagValueError(f'Unknown precision parameter `{precision}')

        # Local environmental variables
        self._env_vars = {}

        # Add environmental variables, as requested
        try:
            var_names = self.config.get("OUTPUT",
                    "environmental_variables").strip().split(',')
            self.environmental_variables = \
                    [var_name.strip() for var_name in var_names]
        except (configparser.NoSectionError, configparser.NoOptionError) as e:
            self.environmental_variables = []

        # Save a list of extra grid variables to be returned as diagnostics
        try:
            var_names = self.config.get("OUTPUT",
                    "extra_grid_variables").strip().split(',')
            self.extra_grid_variables = \
                    [var_name.strip() for var_name in var_names]
        except (configparser.NoSectionError, configparser.NoOptionError) as e:
            self.extra_grid_variables = []

        # Grid names
        self.grid_names = grid_names

        # Host horizontal elements
        self._host_vars = {}

        # Create coordinate variables etc.
        self._create_file_structure(n_particles)

    def _create_file_structure(self, n_particles):

        self._ncfile.title = 'PyLag -- Plymouth Marine Laboratory'

        # Create coordinate dimensions
        self._ncfile.createDimension('particles', n_particles)
        self._ncfile.createDimension('time', None)

        # Add time variable
        self._time = self._ncfile.createVariable('time', np.int64, ('time',))
        self._time.units = 'seconds since 1960-01-01 00:00:00'
        self._time.calendar = 'standard'
        self._time.long_name = 'Time'

        # Add particle group ids
        self._group_id = self._ncfile.createVariable('group_id',
                variable_library.get_integer_type(self._precision),
                ('particles',))
        self._group_id.long_name = 'Particle group ID'

        # x1
        x1_var_name = variable_library.get_coordinate_variable_name(
                self.coordinate_system, 'x1')
        self._x1 = self._ncfile.createVariable(x1_var_name,
                variable_library.get_data_type(x1_var_name, self._precision),
                ('time', 'particles',), **self._ncopts)
        self._x1.units = variable_library.get_units(x1_var_name)
        self._x1.long_name = variable_library.get_long_name(x1_var_name)

        # x2
        x2_var_name = variable_library.get_coordinate_variable_name(
                self.coordinate_system, 'x2')
        self._x2 = self._ncfile.createVariable(x2_var_name,
                variable_library.get_data_type(x2_var_name, self._precision),
                ('time', 'particles',), **self._ncopts)
        self._x2.units = variable_library.get_units(x2_var_name)
        self._x2.long_name = variable_library.get_long_name(x2_var_name)

        # x3
        x3_var_name = variable_library.get_coordinate_variable_name(
                self.coordinate_system, 'x3')
        self._x3 = self._ncfile.createVariable(x3_var_name,
                variable_library.get_data_type(x3_var_name, self._precision),
                ('time', 'particles',), **self._ncopts)
        self._x3.units = variable_library.get_units(x3_var_name)
        self._x3.long_name = variable_library.get_long_name(x3_var_name)

        # Add hosts
        for grid_name in self.grid_names:
            self._host_vars[grid_name] = self._ncfile.createVariable(
                    f'host_{grid_name}',
                    variable_library.get_integer_type(self._precision),
                    ('time', 'particles',), **self._ncopts)
            self._host_vars[grid_name].units = 'None'
            self._host_vars[grid_name].long_name = (f"Host horizontal element "
                                                    f"on grid {grid_name}")
            self._host_vars[grid_name].invalid = f"{INT_INVALID}"

        # Add status variables
        data_type = variable_library.get_data_type('error_status',
                                                   self._precision)
        self._status = self._ncfile.createVariable('error_status', data_type,
                ('time', 'particles',), **self._ncopts)
        self._status.units = variable_library.get_units('error_status')
        self._status.long_name = variable_library.get_long_name('error_status')

        data_type = variable_library.get_data_type('in_domain', self._precision)
        self._in_domain = self._ncfile.createVariable('in_domain', data_type,
                ('time', 'particles',), **self._ncopts)
        self._in_domain.units = variable_library.get_units('in_domain')
        self._in_domain.long_name = variable_library.get_long_name('in_domain')

        data_type = variable_library.get_data_type('is_beached',
                                                   self._precision)
        self._is_beached = self._ncfile.createVariable('is_beached', data_type,
                ('time', 'particles',), **self._ncopts)
        self._is_beached.units = variable_library.get_units('is_beached')
        self._is_beached.long_name = \
                variable_library.get_long_name('is_beached')

        # Extra grid variables
        if 'h' in self.extra_grid_variables:
            data_type = variable_library.get_data_type('h', self._precision)
            self._h = self._ncfile.createVariable('h', data_type,
                    ('time', 'particles',), **self._ncopts)
            self._h.units = variable_library.get_units('h')
            self._h.long_name = variable_library.get_long_name('h')
            self._h.invalid = f'{variable_library.get_invalid_value(data_type)}'

        if 'zeta' in self.extra_grid_variables:
            data_type = variable_library.get_data_type('zeta', self._precision)
            self._zeta = self._ncfile.createVariable('zeta', data_type,
                    ('time', 'particles',), **self._ncopts)
            self._zeta.units = variable_library.get_units('zeta')
            self._zeta.long_name = variable_library.get_long_name('zeta')
            self._zeta.invalid = \
                    f'{variable_library.get_invalid_value(data_type)}'

        # Add number of land boundary encounters
        data_type = variable_library.get_data_type('land_boundary_encounters',
                                                   self._precision)
        self._land_boundary_encounters = self._ncfile.createVariable(
                'land_boundary_encounters', data_type,
                ('time', 'particles',), **self._ncopts)
        self._land_boundary_encounters.units = \
            variable_library.get_units('land_boundary_encounters')
        self._land_boundary_encounters.long_name = \
            variable_library.get_long_name('land_boundary_encounters')

        # Add environmental variables
        for var_name in self.environmental_variables:
            data_type = variable_library.get_data_type(var_name,
                                                       self._precision)
            units = variable_library.get_units(var_name)
            long_name = variable_library.get_long_name(var_name)
            invalid = variable_library.get_invalid_value(data_type)

            self._env_vars[var_name] = self._ncfile.createVariable(var_name,
                    data_type, ('time', 'particles',), **self._ncopts)
            self._env_vars[var_name].units = units
            self._env_vars[var_name].long_name = long_name
            self._env_vars[var_name].invalid = f'{invalid}'

        # Bio model variables
        if self.with_biology:
            data_type = variable_library.get_data_type('age', self._precision)
            self._age = self._ncfile.createVariable('age',
                    data_type, ('time', 'particles',), **self._ncopts)
            self._age.units = variable_library.get_units('age')
            self._age.long_name = variable_library.get_long_name('age')
            self._age.invalid = \
                    f"{variable_library.get_invalid_value(data_type)}"

            data_type = variable_library.get_data_type('is_alive',
                                                       self._precision)
            self._is_alive = self._ncfile.createVariable('is_alive',
                    data_type, ('time', 'particles',), **self._ncopts)
            self._is_alive.units = variable_library.get_units('is_alive')
            self._is_alive.long_name = \
                    variable_library.get_long_name('is_alive')

    def write_group_ids(self, group_ids):
        """ Write particle group IDs to file

        Parameters
        ----------
        group_ids : array_like
            Particle group IDs.

        Returns
        -------
        None
        """
        self._group_id[:] = group_ids

    def write(self, time, particle_data):
        """ Write particle data to file

        Parameters
        ----------
        time : float
            The current time

        particle_data : dict
            Dictionary containing particle data with format
            ['attribute': [...]].

        Returns
        -------
        None
        """
        # Next time index
        tidx = self._time.shape[0]

        # Rebase time units and save
        dt = num2pydate(time, units=self._simulation_time_units)
        self._time[tidx] = date2num(dt, units=self._time.units)

        self._x1[tidx, :] = particle_data['x1']
        self._x2[tidx, :] = particle_data['x2']
        self._x3[tidx, :] = particle_data['x3']
        self._is_beached[tidx, :] = particle_data['is_beached']
        self._in_domain[tidx, :] = particle_data['in_domain']
        self._status[tidx, :] = particle_data['error_status']
        self._land_boundary_encounters[tidx, :] = \
            particle_data['land_boundary_encounters']

        # Add host horizontal elements
        for grid_name in self.grid_names:
            self._host_vars[grid_name][tidx, :] = \
                    particle_data[f'host_{grid_name}']

        # Add extra grid variables
        if 'h' in self.extra_grid_variables:
            self._h[tidx, :] = particle_data['h']
        if 'zeta' in self.extra_grid_variables:
            self._zeta[tidx, :] = particle_data['zeta']

        # Add environmental variables
        for var_name in self.environmental_variables:
            self._env_vars[var_name][tidx, :] = particle_data[var_name]

        # Add bio model variables
        if self.with_biology:
            self._age[tidx, :] = particle_data['age']
            self._is_alive[tidx, :] = particle_data['is_alive']

    def sync(self):
        """ Sync data to disk

        Returns
        -------
        None
        """
        # Sync data to disk
        self._ncfile.sync()

    def close(self):
        """ Close the logger

        Returns
        -------
        None
        """
        logger = logging.getLogger(__name__)
        logger.info('Closing data logger.')
        self._ncfile.close()

__all__ = ['NetCDFLogger']