# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# 'License'); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# coding: utf-8

from __future__ import absolute_import
from __future__ import print_function

from collections import Counter
import unittest

from common import assertRaises
from mxnet import ndarray as nd
from mxnet.test_utils import *
from mxnet.text import utils
from mxnet.text.glossary import Glossary
from mxnet.text.indexer import TokenIndexer
from mxnet.text.embedding import TokenEmbedding, CustomEmbedding

def _get_test_str_of_tokens(token_delim, seq_delim):
    seq1 = token_delim + token_delim.join(['Life', 'is', 'great', '!']) \
           + token_delim + seq_delim
    seq2 = token_delim + token_delim.join(['life', 'is', 'good', '.']) \
           + token_delim + seq_delim
    seq3 = token_delim + token_delim.join(['life', "isn't", 'bad', '.']) \
           + token_delim + seq_delim
    seqs = seq1 + seq2 + seq3
    return seqs

def _test_count_tokens_from_str_with_delims(token_delim, seq_delim):
    source_str = _get_test_str_of_tokens(token_delim, seq_delim)

    cnt1 = utils.count_tokens_from_str(source_str, token_delim, seq_delim,
                                       to_lower=False)
    assert cnt1 == Counter(
        {'is': 2, 'life': 2, '.': 2, 'Life': 1, 'great': 1, '!': 1, 'good': 1,
         "isn't": 1, 'bad': 1})

    cnt2 = utils.count_tokens_from_str(source_str, token_delim, seq_delim,
                                       to_lower=True)
    assert cnt2 == Counter(
        {'life': 3, 'is': 2, '.': 2, 'great': 1, '!': 1, 'good': 1,
         "isn't": 1, 'bad': 1})

    counter_to_update = Counter({'life': 2})

    cnt3 = utils.count_tokens_from_str(
        source_str, token_delim, seq_delim, to_lower=False,
        counter_to_update=counter_to_update.copy())
    assert cnt3 == Counter(
        {'is': 2, 'life': 4, '.': 2, 'Life': 1, 'great': 1, '!': 1, 'good': 1,
         "isn't": 1, 'bad': 1})

    cnt4 = utils.count_tokens_from_str(
        source_str, token_delim, seq_delim, to_lower=True,
        counter_to_update=counter_to_update.copy())
    assert cnt4 == Counter(
        {'life': 5, 'is': 2, '.': 2, 'great': 1, '!': 1, 'good': 1,
         "isn't": 1, 'bad': 1})

def test_count_tokens_from_str():
    _test_count_tokens_from_str_with_delims(' ', '\n')
    _test_count_tokens_from_str_with_delims('IS', 'LIFE')

def test_tokens_to_indices():
    counter = Counter(['a', 'b', 'b', 'c', 'c', 'c', 'some_word$'])

    indexer = TokenIndexer(counter, most_freq_count=None, min_freq=1,
                           unknown_token='<unk>', reserved_tokens=None)

    i1 = indexer.to_indices('c')
    assert i1 == 1

    i2 = indexer.to_indices(['c'])
    assert i2 == [1]

    i3 = indexer.to_indices(['<unk>', 'non-exist'])
    assert i3 == [0, 0]

    i4 = indexer.to_indices(['a', 'non-exist', 'a', 'b'])
    assert i4 == [3, 0, 3, 2]

def test_indices_to_tokens():
    counter = Counter(['a', 'b', 'b', 'c', 'c', 'c', 'some_word$'])

    indexer = TokenIndexer(counter, most_freq_count=None, min_freq=1,
                           unknown_token='<unknown>', reserved_tokens=None)

    i1 = indexer.to_tokens(1)
    assert i1 == 'c'

    i2 = indexer.to_tokens([1])
    assert i2 == ['c']

    i3 = indexer.to_tokens([0, 0])
    assert i3 == ['<unknown>', '<unknown>']

    i4 = indexer.to_tokens([3, 0, 3, 2])
    assert i4 == ['a', '<unknown>', 'a', 'b']

    assertRaises(ValueError, indexer.to_tokens, 100)

def test_glove():
    glove_6b_50d = TokenEmbedding.create(
        'glove', pretrained_file_name='glove.6B.50d.txt')

    assert len(glove_6b_50d) == 400001
    assert glove_6b_50d.vec_len == 50
    assert glove_6b_50d.token_to_idx['hi'] == 11084
    assert glove_6b_50d.idx_to_token[11084] == 'hi'

    first_vec_sum = glove_6b_50d.idx_to_vec[0].sum().asnumpy()[0]
    assert_almost_equal(first_vec_sum, 0)

    unk_vec_sum = glove_6b_50d.get_vecs_by_tokens(
        '<unk$unk@unk>').sum().asnumpy()[0]
    assert_almost_equal(unk_vec_sum, 0)

    unk_vecs_sum = glove_6b_50d.get_vecs_by_tokens(
        ['<unk$unk@unk>', '<unk$unk@unk>']).sum().asnumpy()[0]
    assert_almost_equal(unk_vecs_sum, 0)

def test_fasttext():
    fasttext_simple = TokenEmbedding.create(
        'fasttext', pretrained_file_name='wiki.simple.vec',
        init_unknown_vec=nd.ones)

    assert len(fasttext_simple) == 111052
    assert fasttext_simple.vec_len == 300
    assert fasttext_simple.token_to_idx['hi'] == 3241
    assert fasttext_simple.idx_to_token[3241] == 'hi'

    first_vec_sum = fasttext_simple.idx_to_vec[0].sum().asnumpy()[0]
    assert_almost_equal(first_vec_sum, fasttext_simple.vec_len)

    unk_vec_sum = fasttext_simple.get_vecs_by_tokens(
        '<unk$unk@unk>').sum().asnumpy()[0]
    assert_almost_equal(unk_vec_sum, fasttext_simple.vec_len)

    unk_vecs_sum = fasttext_simple.get_vecs_by_tokens(
        ['<unk$unk@unk>', '<unk$unk@unk>']).sum().asnumpy()[0]
    assert_almost_equal(unk_vecs_sum, fasttext_simple.vec_len * 2)

def _mk_my_pretrain_file(path, token_delim, pretrain_file):
    path = os.path.expanduser(path)
    if not os.path.exists(path):
        os.makedirs(path)
    seq1 = token_delim.join(['a', '0.1', '0.2', '0.3', '0.4', '0.5']) + '\n'
    seq2 = token_delim.join(['b', '0.6', '0.7', '0.8', '0.9', '1.0']) + '\n'
    seqs = seq1 + seq2
    with open(os.path.join(path, pretrain_file), 'w') as fout:
        fout.write(seqs)

def _mk_my_pretrain_file2(path, token_delim, pretrain_file):
    path = os.path.expanduser(path)
    if not os.path.exists(path):
        os.makedirs(path)
    seq1 = token_delim.join(['a', '0.01', '0.02', '0.03', '0.04',
                             '0.05']) + '\n'
    seq2 = token_delim.join(['c', '0.06', '0.07', '0.08', '0.09', '0.1']) + '\n'
    seqs = seq1 + seq2
    with open(os.path.join(path, pretrain_file), 'w') as fout:
        fout.write(seqs)

def _mk_my_pretrain_file3(path, token_delim, pretrain_file):
    path = os.path.expanduser(path)
    if not os.path.exists(path):
        os.makedirs(path)
    seq1 = token_delim.join(['a', '0.1', '0.2', '0.3', '0.4', '0.5']) + '\n'
    seq2 = token_delim.join(['b', '0.6', '0.7', '0.8', '0.9', '1.0']) + '\n'
    seq3 = token_delim.join(['<unk1>', '1.1', '1.2', '1.3', '1.4',
                             '1.5']) + '\n'
    seqs = seq1 + seq2 + seq3
    with open(os.path.join(path, pretrain_file), 'w') as fout:
        fout.write(seqs)

def _mk_my_pretrain_file4(path, token_delim, pretrain_file):
    path = os.path.expanduser(path)
    if not os.path.exists(path):
        os.makedirs(path)
    seq1 = token_delim.join(['a', '0.01', '0.02', '0.03', '0.04',
                             '0.05']) + '\n'
    seq2 = token_delim.join(['c', '0.06', '0.07', '0.08', '0.09',
                             '0.1']) + '\n'
    seq3 = token_delim.join(['<unk2>', '0.11', '0.12', '0.13', '0.14',
                             '0.15']) + '\n'
    seqs = seq1 + seq2 + seq3
    with open(os.path.join(path, pretrain_file), 'w') as fout:
        fout.write(seqs)

def _mk_my_invalid_pretrain_file(path, token_delim, pretrain_file):
    path = os.path.expanduser(path)
    if not os.path.exists(path):
        os.makedirs(path)
    seq1 = token_delim.join(['a', '0.1', '0.2', '0.3', '0.4', '0.5']) + '\n'
    seq2 = token_delim.join(['b', '0.6', '0.7', '0.8', '0.9', '1.0']) + '\n'
    seq3 = token_delim.join(['c']) + '\n'
    seqs = seq1 + seq2 + seq3
    with open(os.path.join(path, pretrain_file), 'w') as fout:
        fout.write(seqs)

def _mk_my_invalid_pretrain_file2(path, token_delim, pretrain_file):
    path = os.path.expanduser(path)
    if not os.path.exists(path):
        os.makedirs(path)
    seq1 = token_delim.join(['a', '0.1', '0.2', '0.3', '0.4', '0.5']) + '\n'
    seq2 = token_delim.join(['b', '0.6', '0.7', '0.8', '0.9', '1.0']) + '\n'
    seq3 = token_delim.join(['c', '0.6', '0.7', '0.8']) + '\n'
    seqs = seq1 + seq2 + seq3
    with open(os.path.join(path, pretrain_file), 'w') as fout:
        fout.write(seqs)

def test_custom_embed():
    embed_root = '~/.mxnet/embeddings/'
    embed_name = 'my_embed'
    elem_delim = '/t'
    pretrain_file = 'my_pretrain_file.txt'

    _mk_my_pretrain_file(os.path.join(embed_root, embed_name), elem_delim,
                         pretrain_file)

    pretrain_file_path = os.path.join(embed_root, embed_name, pretrain_file)

    my_embed = CustomEmbedding(pretrain_file_path, elem_delim)

    assert len(my_embed) == 3
    assert my_embed.vec_len == 5
    assert my_embed.token_to_idx['a'] == 1
    assert my_embed.idx_to_token[1] == 'a'

    first_vec = my_embed.idx_to_vec[0]
    assert_almost_equal(first_vec.asnumpy(), np.array([0, 0, 0, 0, 0]))

    unk_vec = my_embed.get_vecs_by_tokens('A')
    assert_almost_equal(unk_vec.asnumpy(), np.array([0, 0, 0, 0, 0]))

    a_vec = my_embed.get_vecs_by_tokens('A', lower_case_backup=True)
    assert_almost_equal(a_vec.asnumpy(), np.array([0.1, 0.2, 0.3, 0.4, 0.5]))

    unk_vecs = my_embed.get_vecs_by_tokens(['<unk$unk@unk>', '<unk$unk@unk>'])
    assert_almost_equal(unk_vecs.asnumpy(),
                        np.array([[0, 0, 0, 0, 0],
                                  [0, 0, 0, 0, 0]]))

    # Test loaded unknown vectors.
    pretrain_file2 = 'my_pretrain_file2.txt'
    _mk_my_pretrain_file3(os.path.join(embed_root, embed_name), elem_delim,
                          pretrain_file2)
    pretrain_file_path = os.path.join(embed_root, embed_name, pretrain_file2)
    my_embed2 = CustomEmbedding(pretrain_file_path, elem_delim,
                                init_unknown_vec=nd.ones,
                                unknown_token='<unk>')
    unk_vec2 = my_embed2.get_vecs_by_tokens('<unk>')
    assert_almost_equal(unk_vec2.asnumpy(), np.array([1, 1, 1, 1, 1]))
    unk_vec2 = my_embed2.get_vecs_by_tokens('<unk$unk@unk>')
    assert_almost_equal(unk_vec2.asnumpy(), np.array([1, 1, 1, 1, 1]))

    my_embed3 = CustomEmbedding(pretrain_file_path, elem_delim,
                                init_unknown_vec=nd.ones,
                                unknown_token='<unk1>')
    unk_vec3 = my_embed3.get_vecs_by_tokens('<unk1>')
    assert_almost_equal(unk_vec3.asnumpy(), np.array([1.1, 1.2, 1.3, 1.4, 1.5]))
    unk_vec3 = my_embed3.get_vecs_by_tokens('<unk$unk@unk>')
    assert_almost_equal(unk_vec3.asnumpy(), np.array([1.1, 1.2, 1.3, 1.4, 1.5]))

    # Test error handling.
    invalid_pretrain_file = 'invalid_pretrain_file.txt'
    _mk_my_invalid_pretrain_file(os.path.join(embed_root, embed_name),
                                 elem_delim, invalid_pretrain_file)
    pretrain_file_path = os.path.join(embed_root, embed_name,
                                      invalid_pretrain_file)
    assertRaises(AssertionError, CustomEmbedding, pretrain_file_path,
                 elem_delim)

    invalid_pretrain_file2 = 'invalid_pretrain_file2.txt'
    _mk_my_invalid_pretrain_file2(os.path.join(embed_root, embed_name),
                                  elem_delim, invalid_pretrain_file2)
    pretrain_file_path = os.path.join(embed_root, embed_name,
                                      invalid_pretrain_file2)
    assertRaises(AssertionError, CustomEmbedding, pretrain_file_path,
                 elem_delim)

def test_token_indexer():
    counter = Counter(['a', 'b', 'b', 'c', 'c', 'c', 'some_word$'])

    g1 = TokenIndexer(counter, most_freq_count=None, min_freq=1,
                      unknown_token='<unk>', reserved_tokens=None)
    assert len(g1) == 5
    assert g1.token_to_idx == {'<unk>': 0, 'c': 1, 'b': 2, 'a': 3,
                               'some_word$': 4}
    assert g1.idx_to_token[1] == 'c'
    assert g1.unknown_token == '<unk>'
    assert g1.reserved_tokens is None

    g2 = TokenIndexer(counter, most_freq_count=None, min_freq=2,
                      unknown_token='<unk>', reserved_tokens=None)
    assert len(g2) == 3
    assert g2.token_to_idx == {'<unk>': 0, 'c': 1, 'b': 2}
    assert g2.idx_to_token[1] == 'c'
    assert g2.unknown_token == '<unk>'
    assert g2.reserved_tokens is None

    g3 = TokenIndexer(counter, most_freq_count=None, min_freq=100,
                      unknown_token='<unk>', reserved_tokens=None)
    assert len(g3) == 1
    assert g3.token_to_idx == {'<unk>': 0}
    assert g3.idx_to_token[0] == '<unk>'
    assert g3.unknown_token == '<unk>'
    assert g3.reserved_tokens is None

    g4 = TokenIndexer(counter, most_freq_count=2, min_freq=1,
                      unknown_token='<unk>', reserved_tokens=None)
    assert len(g4) == 3
    assert g4.token_to_idx == {'<unk>': 0, 'c': 1, 'b': 2}
    assert g4.idx_to_token[1] == 'c'
    assert g4.unknown_token == '<unk>'
    assert g4.reserved_tokens is None

    g5 = TokenIndexer(counter, most_freq_count=3, min_freq=1,
                      unknown_token='<unk>', reserved_tokens=None)
    assert len(g5) == 4
    assert g5.token_to_idx == {'<unk>': 0, 'c': 1, 'b': 2, 'a': 3}
    assert g5.idx_to_token[1] == 'c'
    assert g5.unknown_token == '<unk>'
    assert g5.reserved_tokens is None

    g6 = TokenIndexer(counter, most_freq_count=100, min_freq=1,
                      unknown_token='<unk>', reserved_tokens=None)
    assert len(g6) == 5
    assert g6.token_to_idx == {'<unk>': 0, 'c': 1, 'b': 2, 'a': 3,
                               'some_word$': 4}
    assert g6.idx_to_token[1] == 'c'
    assert g6.unknown_token == '<unk>'
    assert g6.reserved_tokens is None

    g7 = TokenIndexer(counter, most_freq_count=1, min_freq=2,
                      unknown_token='<unk>', reserved_tokens=None)
    assert len(g7) == 2
    assert g7.token_to_idx == {'<unk>': 0, 'c': 1}
    assert g7.idx_to_token[1] == 'c'
    assert g7.unknown_token == '<unk>'
    assert g7.reserved_tokens is None

    assertRaises(AssertionError, TokenIndexer, counter, most_freq_count=None,
                 min_freq=0, unknown_token='<unknown>',
                 reserved_tokens=['b'])

    assertRaises(AssertionError, TokenIndexer, counter, most_freq_count=None,
                 min_freq=1, unknown_token='<unknown>',
                 reserved_tokens=['b', 'b'])

    assertRaises(AssertionError, TokenIndexer, counter, most_freq_count=None,
                 min_freq=1, unknown_token='<unknown>',
                 reserved_tokens=['b', '<unknown>'])

    g8 = TokenIndexer(counter, most_freq_count=None, min_freq=1,
                      unknown_token='<unknown>', reserved_tokens=['b'])
    assert len(g8) == 5
    assert g8.token_to_idx == {'<unknown>': 0, 'b': 1, 'c': 2, 'a': 3,
                               'some_word$': 4}
    assert g8.idx_to_token[1] == 'b'
    assert g8.unknown_token == '<unknown>'
    assert g8.reserved_tokens == ['b']

    g9 = TokenIndexer(counter, most_freq_count=None, min_freq=2,
                      unknown_token='<unk>', reserved_tokens=['b', 'a'])
    assert len(g9) == 4
    assert g9.token_to_idx == {'<unk>': 0, 'b': 1, 'a': 2, 'c': 3}
    assert g9.idx_to_token[1] == 'b'
    assert g9.unknown_token == '<unk>'
    assert g9.reserved_tokens == ['b', 'a']

    g10 = TokenIndexer(counter, most_freq_count=None, min_freq=100,
                       unknown_token='<unk>', reserved_tokens=['b', 'c'])
    assert len(g10) == 3
    assert g10.token_to_idx == {'<unk>': 0, 'b': 1, 'c': 2}
    assert g10.idx_to_token[1] == 'b'
    assert g10.unknown_token == '<unk>'
    assert g10.reserved_tokens == ['b', 'c']

    g11 = TokenIndexer(counter, most_freq_count=1, min_freq=2,
                       unknown_token='<unk>', reserved_tokens=['<pad>', 'b'])
    assert len(g11) == 4
    assert g11.token_to_idx == {'<unk>': 0, '<pad>': 1, 'b': 2, 'c': 3}
    assert g11.idx_to_token[1] == '<pad>'
    assert g11.unknown_token == '<unk>'
    assert g11.reserved_tokens == ['<pad>', 'b']

    g12 = TokenIndexer(counter, most_freq_count=None, min_freq=2,
                       unknown_token='b', reserved_tokens=['<pad>'])
    assert len(g12) == 3
    assert g12.token_to_idx == {'b': 0, '<pad>': 1, 'c': 2}
    assert g12.idx_to_token[1] == '<pad>'
    assert g12.unknown_token == 'b'
    assert g12.reserved_tokens == ['<pad>']

    g13 = TokenIndexer(counter, most_freq_count=None, min_freq=2,
                       unknown_token='a', reserved_tokens=['<pad>'])
    assert len(g13) == 4
    assert g13.token_to_idx == {'a': 0, '<pad>': 1, 'c': 2, 'b': 3}
    assert g13.idx_to_token[1] == '<pad>'
    assert g13.unknown_token == 'a'
    assert g13.reserved_tokens == ['<pad>']

    counter_tuple = Counter([('a', 'a'), ('b', 'b'), ('b', 'b'),
                             ('c', 'c'), ('c', 'c'), ('c', 'c'),
                             ('some_word$', 'some_word$')])

    g14 = TokenIndexer(counter_tuple, most_freq_count=None, min_freq=1,
                       unknown_token=('<unk>', '<unk>'), reserved_tokens=None)
    assert len(g14) == 5
    assert g14.token_to_idx == {('<unk>', '<unk>'): 0, ('c', 'c'): 1,
                                ('b', 'b'): 2, ('a', 'a'): 3,
                                ('some_word$', 'some_word$'): 4}
    assert g14.idx_to_token[1] == ('c', 'c')
    assert g14.unknown_token == ('<unk>', '<unk>')
    assert g14.reserved_tokens is None

def test_glossary_with_one_embed():
    embed_root = '~/.mxnet/embeddings/'
    embed_name = 'my_embed'
    elem_delim = '/t'
    pretrain_file = 'my_pretrain_file1.txt'

    _mk_my_pretrain_file(os.path.join(embed_root, embed_name), elem_delim,
                         pretrain_file)

    pretrain_file_path = os.path.join(embed_root, embed_name, pretrain_file)

    my_embed = CustomEmbedding(pretrain_file_path, elem_delim,
                               init_unknown_vec=nd.ones)

    counter = Counter(['a', 'b', 'b', 'c', 'c', 'c', 'some_word$'])

    g1 = Glossary(counter, my_embed, most_freq_count=None, min_freq=1,
                  unknown_token='<unk>', reserved_tokens=['<pad>'])

    assert g1.token_to_idx == {'<unk>': 0, '<pad>': 1, 'c': 2, 'b': 3, 'a': 4,
                               'some_word$': 5}
    assert g1.idx_to_token == ['<unk>', '<pad>', 'c', 'b', 'a', 'some_word$']

    assert_almost_equal(g1.idx_to_vec.asnumpy(),
                        np.array([[1, 1, 1, 1, 1],
                                  [1, 1, 1, 1, 1],
                                  [1, 1, 1, 1, 1],
                                  [0.6, 0.7, 0.8, 0.9, 1],
                                  [0.1, 0.2, 0.3, 0.4, 0.5],
                                  [1, 1, 1, 1, 1]])
                        )

    assert g1.vec_len == 5
    assert g1.reserved_tokens == ['<pad>']

    assert_almost_equal(g1.get_vecs_by_tokens('c').asnumpy(),
                        np.array([1, 1, 1, 1, 1])
                        )

    assert_almost_equal(g1.get_vecs_by_tokens(['c']).asnumpy(),
                        np.array([[1, 1, 1, 1, 1]])
                        )

    assert_almost_equal(g1.get_vecs_by_tokens(['a', 'not_exist']).asnumpy(),
                        np.array([[0.1, 0.2, 0.3, 0.4, 0.5],
                                  [1, 1, 1, 1, 1]])
                        )

    assert_almost_equal(g1.get_vecs_by_tokens(['a', 'b']).asnumpy(),
                        np.array([[0.1, 0.2, 0.3, 0.4, 0.5],
                                  [0.6, 0.7, 0.8, 0.9, 1]])
                        )

    assert_almost_equal(g1.get_vecs_by_tokens(['A', 'b']).asnumpy(),
                        np.array([[1, 1, 1, 1, 1],
                                  [0.6, 0.7, 0.8, 0.9, 1]])
                        )

    assert_almost_equal(g1.get_vecs_by_tokens(['A', 'b'],
                                              lower_case_backup=True).asnumpy(),
                        np.array([[0.1, 0.2, 0.3, 0.4, 0.5],
                                  [0.6, 0.7, 0.8, 0.9, 1]])
                        )

    g1.update_token_vectors(['a', 'b'],
                            nd.array([[2, 2, 2, 2, 2],
                                      [3, 3, 3, 3, 3]])
                            )

    assert_almost_equal(g1.idx_to_vec.asnumpy(),
                        np.array([[1, 1, 1, 1, 1],
                                  [1, 1, 1, 1, 1],
                                  [1, 1, 1, 1, 1],
                                  [3, 3, 3, 3, 3],
                                  [2, 2, 2, 2, 2],
                                  [1, 1, 1, 1, 1]])
                        )

    assertRaises(ValueError, g1.update_token_vectors, 'unknown$$$',
                 nd.array([0, 0, 0, 0, 0]))

    assertRaises(AssertionError, g1.update_token_vectors, '<unk>',
                 nd.array([[0, 0, 0, 0, 0], [0, 0, 0, 0, 0]]))

    assertRaises(AssertionError, g1.update_token_vectors, '<unk>',
                 nd.array([0]))

    g1.update_token_vectors(['<unk>'],
                            nd.array([0, 0, 0, 0, 0])
                            )
    assert_almost_equal(g1.idx_to_vec.asnumpy(),
                        np.array([[0, 0, 0, 0, 0],
                                  [1, 1, 1, 1, 1],
                                  [1, 1, 1, 1, 1],
                                  [3, 3, 3, 3, 3],
                                  [2, 2, 2, 2, 2],
                                  [1, 1, 1, 1, 1]])
                        )
    g1.update_token_vectors(['<unk>'],
                            nd.array([[10, 10, 10, 10, 10]])
                            )
    assert_almost_equal(g1.idx_to_vec.asnumpy(),
                        np.array([[10, 10, 10, 10, 10],
                                  [1, 1, 1, 1, 1],
                                  [1, 1, 1, 1, 1],
                                  [3, 3, 3, 3, 3],
                                  [2, 2, 2, 2, 2],
                                  [1, 1, 1, 1, 1]])
                        )
    g1.update_token_vectors('<unk>',
                            nd.array([0, 0, 0, 0, 0])
                            )
    assert_almost_equal(g1.idx_to_vec.asnumpy(),
                        np.array([[0, 0, 0, 0, 0],
                                  [1, 1, 1, 1, 1],
                                  [1, 1, 1, 1, 1],
                                  [3, 3, 3, 3, 3],
                                  [2, 2, 2, 2, 2],
                                  [1, 1, 1, 1, 1]])
                        )
    g1.update_token_vectors('<unk>',
                            nd.array([[10, 10, 10, 10, 10]])
                            )
    assert_almost_equal(g1.idx_to_vec.asnumpy(),
                        np.array([[10, 10, 10, 10, 10],
                                  [1, 1, 1, 1, 1],
                                  [1, 1, 1, 1, 1],
                                  [3, 3, 3, 3, 3],
                                  [2, 2, 2, 2, 2],
                                  [1, 1, 1, 1, 1]])
                        )

def test_glossary_with_two_embeds():
    embed_root = '.'
    embed_name = 'my_embed'
    elem_delim = '/t'
    pretrain_file1 = 'my_pretrain_file1.txt'
    pretrain_file2 = 'my_pretrain_file2.txt'

    _mk_my_pretrain_file(os.path.join(embed_root, embed_name), elem_delim,
                         pretrain_file1)
    _mk_my_pretrain_file2(os.path.join(embed_root, embed_name), elem_delim,
                          pretrain_file2)

    pretrain_file_path1 = os.path.join(embed_root, embed_name, pretrain_file1)
    pretrain_file_path2 = os.path.join(embed_root, embed_name, pretrain_file2)

    my_embed1 = CustomEmbedding(pretrain_file_path1, elem_delim,
                                init_unknown_vec=nd.ones)
    my_embed2 = CustomEmbedding(pretrain_file_path2, elem_delim)

    counter = Counter(['a', 'b', 'b', 'c', 'c', 'c', 'some_word$'])

    g1 = Glossary(counter, [my_embed1, my_embed2], most_freq_count=None,
                  min_freq=1, unknown_token='<unk>', reserved_tokens=None)

    assert g1.token_to_idx == {'<unk>': 0, 'c': 1, 'b': 2, 'a': 3,
                               'some_word$': 4}
    assert g1.idx_to_token == ['<unk>', 'c', 'b', 'a', 'some_word$']

    assert_almost_equal(g1.idx_to_vec.asnumpy(),
                        np.array([[1, 1, 1, 1, 1, 0, 0, 0, 0, 0],
                                  [1, 1, 1, 1, 1, 0.06, 0.07, 0.08, 0.09, 0.1],
                                  [0.6, 0.7, 0.8, 0.9, 1, 0, 0, 0, 0, 0],
                                  [0.1, 0.2, 0.3, 0.4, 0.5,
                                   0.01, 0.02, 0.03, 0.04, 0.05],
                                  [1, 1, 1, 1, 1, 0, 0, 0, 0, 0]])
                        )

    assert g1.vec_len == 10
    assert g1.reserved_tokens is None
    assert_almost_equal(g1.get_vecs_by_tokens('c').asnumpy(),
                        np.array([1, 1, 1, 1, 1, 0.06, 0.07, 0.08, 0.09, 0.1])
                        )

    assert_almost_equal(g1.get_vecs_by_tokens(['b', 'not_exist']).asnumpy(),
                        np.array([[0.6, 0.7, 0.8, 0.9, 1, 0, 0, 0, 0, 0],
                                  [1, 1, 1, 1, 1, 0, 0, 0, 0, 0]])
                        )

    g1.update_token_vectors(['a', 'b'],
                            nd.array([[2, 2, 2, 2, 2, 2, 2, 2, 2, 2],
                                      [3, 3, 3, 3, 3, 3, 3, 3, 3, 3]])
                            )
    assert_almost_equal(g1.idx_to_vec.asnumpy(),
                        np.array([[1, 1, 1, 1, 1, 0, 0, 0, 0, 0],
                                  [1, 1, 1, 1, 1, 0.06, 0.07, 0.08, 0.09, 0.1],
                                  [3, 3, 3, 3, 3, 3, 3, 3, 3, 3],
                                  [2, 2, 2, 2, 2, 2, 2, 2, 2, 2],
                                  [1, 1, 1, 1, 1, 0, 0, 0, 0, 0]])
                        )

    # Test loaded unknown tokens
    pretrain_file3 = 'my_pretrain_file3.txt'
    pretrain_file4 = 'my_pretrain_file4.txt'

    _mk_my_pretrain_file3(os.path.join(embed_root, embed_name), elem_delim,
                          pretrain_file3)
    _mk_my_pretrain_file4(os.path.join(embed_root, embed_name), elem_delim,
                          pretrain_file4)

    pretrain_file_path3 = os.path.join(embed_root, embed_name, pretrain_file3)
    pretrain_file_path4 = os.path.join(embed_root, embed_name, pretrain_file4)

    my_embed3 = CustomEmbedding(pretrain_file_path3, elem_delim,
                                init_unknown_vec=nd.ones,
                                unknown_token='<unk1>')
    my_embed4 = CustomEmbedding(pretrain_file_path4, elem_delim,
                                unknown_token='<unk2>')

    g2 = Glossary(counter, [my_embed3, my_embed4], most_freq_count=None,
                  min_freq=1, unknown_token='<unk>', reserved_tokens=None)
    assert_almost_equal(g2.idx_to_vec.asnumpy(),
                        np.array([[1.1, 1.2, 1.3, 1.4, 1.5,
                                   0.11, 0.12, 0.13, 0.14, 0.15],
                                  [1.1, 1.2, 1.3, 1.4, 1.5,
                                   0.06, 0.07, 0.08, 0.09, 0.1],
                                  [0.6, 0.7, 0.8, 0.9, 1,
                                   0.11, 0.12, 0.13, 0.14, 0.15],
                                  [0.1, 0.2, 0.3, 0.4, 0.5,
                                   0.01, 0.02, 0.03, 0.04, 0.05],
                                  [1.1, 1.2, 1.3, 1.4, 1.5,
                                   0.11, 0.12, 0.13, 0.14, 0.15]])
                        )

    g3 = Glossary(counter, [my_embed3, my_embed4], most_freq_count=None,
                  min_freq=1, unknown_token='<unk1>', reserved_tokens=None)
    assert_almost_equal(g3.idx_to_vec.asnumpy(),
                        np.array([[1.1, 1.2, 1.3, 1.4, 1.5,
                                   0.11, 0.12, 0.13, 0.14, 0.15],
                                  [1.1, 1.2, 1.3, 1.4, 1.5,
                                   0.06, 0.07, 0.08, 0.09, 0.1],
                                  [0.6, 0.7, 0.8, 0.9, 1,
                                   0.11, 0.12, 0.13, 0.14, 0.15],
                                  [0.1, 0.2, 0.3, 0.4, 0.5,
                                   0.01, 0.02, 0.03, 0.04, 0.05],
                                  [1.1, 1.2, 1.3, 1.4, 1.5,
                                   0.11, 0.12, 0.13, 0.14, 0.15]])
                        )

    g4 = Glossary(counter, [my_embed3, my_embed4], most_freq_count=None,
                  min_freq=1, unknown_token='<unk2>', reserved_tokens=None)
    assert_almost_equal(g4.idx_to_vec.asnumpy(),
                        np.array([[1.1, 1.2, 1.3, 1.4, 1.5,
                                   0.11, 0.12, 0.13, 0.14, 0.15],
                                  [1.1, 1.2, 1.3, 1.4, 1.5,
                                   0.06, 0.07, 0.08, 0.09, 0.1],
                                  [0.6, 0.7, 0.8, 0.9, 1,
                                   0.11, 0.12, 0.13, 0.14, 0.15],
                                  [0.1, 0.2, 0.3, 0.4, 0.5,
                                   0.01, 0.02, 0.03, 0.04, 0.05],
                                  [1.1, 1.2, 1.3, 1.4, 1.5,
                                   0.11, 0.12, 0.13, 0.14, 0.15]])
                        )

    counter2 = Counter(['b', 'b', 'c', 'c', 'c', 'some_word$'])

    g5 = Glossary(counter2, [my_embed3, my_embed4], most_freq_count=None,
                  min_freq=1, unknown_token='a', reserved_tokens=None)
    assert g5.token_to_idx == {'a': 0, 'c': 1, 'b': 2, 'some_word$': 3}
    assert g5.idx_to_token == ['a', 'c', 'b', 'some_word$']
    assert_almost_equal(g5.idx_to_vec.asnumpy(),
                        np.array([[1.1, 1.2, 1.3, 1.4, 1.5,
                                   0.11, 0.12, 0.13, 0.14, 0.15],
                                  [1.1, 1.2, 1.3, 1.4, 1.5,
                                   0.06, 0.07, 0.08, 0.09, 0.1],
                                  [0.6, 0.7, 0.8, 0.9, 1,
                                   0.11, 0.12, 0.13, 0.14, 0.15],
                                  [1.1, 1.2, 1.3, 1.4, 1.5,
                                   0.11, 0.12, 0.13, 0.14, 0.15]])
                        )

def test_get_embedding_names_and_pretrain_files():
    assert len(TokenEmbedding.get_embedding_and_pretrained_file_names(
        embedding_name='fasttext')) == 3

    assert len(TokenEmbedding.get_embedding_and_pretrained_file_names(
        embedding_name='glove')) == 10

    reg = TokenEmbedding.get_embedding_and_pretrained_file_names(
        embedding_name=None)

    assert len(reg['glove']) == 10
    assert len(reg['fasttext']) == 3

    assertRaises(KeyError,
                 TokenEmbedding.get_embedding_and_pretrained_file_names,
                 'unknown$$')

if __name__ == '__main__':
    import nose
    nose.runmodule()


def rot(input_string, rot_x):
    arr=[]

    for char in input_string:
        character = ord(char)
        shift_modulo = int(rot_x) % 25
        shift_modulo_numbers = int(rot_x) % 10
        character_shift = character + shift_modulo
        number_shift = character + shift_modulo_numbers

        #Capital letters
        if 65 <= character <= 90:
            if character_shift <= 90:
                arr.append(chr(character_shift))
            else:
                arr.append(chr(65 + (character_shift % 90) - 1))

        #Small letters
        elif 97 <= character <= 122:
            if character_shift <= 122:
                arr.append(chr(character_shift))
            else:
                arr.append(chr(97 + (character_shift % 122) - 1))

        #Numbers, because why not?
        elif 48 <= character <= 57:
            if number_shift <= 57:
                arr.append(chr(number_shift))
            else:
                arr.append(chr(48 + (number_shift % 57) - 1))

        #Space
        elif character == 32:
            arr.append(" ")

    return ''.join(arr)

# This was Cesar's Cipher 1.0, however, I figured it's bad practice to nest for loops, because it causes big O notation --> O(n²)

# alphabet = ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U",
#             "V", "W", "X", "Y", "Z"]

# def rot(input_string, rot_x):
#     arr = []
#     input_string = input_string.upper()
#     for character in input_string:
#         for i in range(len(alphabet)):
#             if alphabet[i] == character:
#                 arr.append(alphabet[(i + int(rot_x)) % len(alphabet)])
#                 continue

#             elif character == " ":
#                 arr.append(" ")
#                 break

#     return "".join(arr).capitalize()

# input_string = input(str("Enter character: "))
# shift = int(input("Shift: "))


from Vector import Vector

def main():

    my_vector1 = Vector([8.218,-9.341])
    my_vector2 = Vector([-1.129,2.111])
    my_vector3 = Vector([7.119, 8.215])
    my_vector4 = Vector([-8.223, 0.878])
    my_vector5 = Vector([1.671,-1.012,-0.318])

    my_vector = [1,2,3]
    scalar = 7.41
    # c = tuple([item1+item2 for item1, item2 in zip( a, b)])

    print(my_vector1 + my_vector2)
    print(my_vector3 - my_vector4)
    print( my_vector5.times_scalar(scalar))

    print(my_vector1.magnitude)
    print(my_vector4.normalized())

if __name__ == '__main__':
    main()


import pygame

#  Константи
FPS = 100                                           #   100 кадрів в секунду, або 1 кадр кожні 10 мілісекунд
WINDOW_BACKGROUND_COLOR_WHITE = (255, 255, 255)     #   білий колір, яким ми зафарбовуємо екран (в форматі RGB)
WINDOW_SIZE = (1000, 800)                           #   розміри вікна (розширення)
pygame.init()                                       #   початкові налаштування бібліотеки  pygame
window = pygame.display.set_mode(WINDOW_SIZE)       #   створити вікна програми з заданими розмірами
clock = pygame.time.Clock()                         #   створити годинник

class Panda:
    # створити панду з такими параметрами: картинка панди, координати x та y, ім'я панди
    def __init__(self, image_file, start_x, start_y, panda_name):
        self.image = pygame.image.load(image_file)          #   завантажуємо картинку в пам'ять програми
        self.rectangle = self.image.get_rect()              #   прямокутник навколо картинки
        self.name = panda_name                              #   ім'я панди
        self.rectangle.x = start_x                          #   початкова координата по ширині
        self.rectangle.y = start_y                          #   початкова координата по висоті
        print("Panda " + panda_name + " created")

    # вивести повідомлення, якщо інша панда поряд
    def show_message_if_another_panda_is_near(self, other_panda):
        is_near = self.rectangle.colliderect(other_panda.rectangle) #  colliderect - це вбудована функція бібліотеки pygame
                                                                    #  вона перевіряє, що прямокутники навколо двох панд накладаються один на інший
        if is_near:
            print("I am panda " + self.name + ". Another panda " + other_panda.name + " is near me!")

    # намалювати панду на вікні
    def draw_panda_on_window(self):
        global window
        window.blit(self.image, self.rectangle)     # функція blit - вбудована функція бібліотеки pygame

    # змінити координати панди
    def move(self, x_diff, y_diff, direction_text):
        print("Panda " + self.name + " is moved to: " + direction_text)
        self.rectangle.x = self.rectangle.x + x_diff
        self.rectangle.y = self.rectangle.y + y_diff

# Перевірити, чи треба завершити програму
# Подія pygame.QUIT відбудеться, якщо натиснути на хрестик закриття вікна програми
def need_to_close_proram():
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            return True
    return False

# набори параметрів для функції
# в них показано:  на скільки змінити координати панди по висоті, по ширині і як ця операція буде називатися

MOVE_UP =       (0, -10, "UP")
MOVE_DOWN =     (0, +10, "DOWN")
MOVE_LEFT =     (-10, 0, "LEFT")
MOVE_RIGHT =    (+10, 0, "RIGHT")

# функція яка перевіряє усі натиснуті кнопки і відповідно рухає панду
def move_panda_if_needed(panda):
    all_pressed_keys = pygame.key.get_pressed()

    if all_pressed_keys[pygame.K_LEFT]:
        panda.move(*MOVE_LEFT)

    if all_pressed_keys[pygame.K_RIGHT]:
        panda.move(*MOVE_RIGHT)

    if all_pressed_keys[pygame.K_UP]:
        panda.move(*MOVE_UP)

    if all_pressed_keys[pygame.K_DOWN]:
        panda.move(*MOVE_DOWN)

panda1 = Panda('panda.jpg', 100, 100, "Перша панда")
panda2 = Panda('panda.jpg', 100, 100, "Друга панда")
running = True                                             #   флаг завершення програми

while running:

    move_panda_if_needed(panda1)                            #   перемістити панду 1, якщо натиснуті кнопки вниз вверз вліво вправо

    # перемалювари повністю вікно
    window.fill(WINDOW_BACKGROUND_COLOR_WHITE)              #   намалювати білий екран
    panda1.draw_panda_on_window()                           #   намалювати панду 1
    panda2.draw_panda_on_window()                           #   намалювати панду 2
    pygame.display.update()                                 #   оновити екран
    panda1.show_message_if_another_panda_is_near(panda2)    #   вивести повідомлення, якщо поряд інша панда

    running = not need_to_close_proram()                    #   індикатор завершення програми (використовується в циклі while)

    clock.tick(FPS)                                         #   почекати 10 мілісекунд


import traceback

#CountingSort: takes as input a list of Student objects, and
#a character houseOrYear, which is 'h' if we are sorting the
#Students by house, or 'y' if sorting by year.
#Returns a list of Student objects, which is sorted by the
#appropriate attribute.
def CountingSort(studentList,houseOrYear):
    A = studentList
    if houseOrYear == 'h':
        B = [0] * len(A)        # Initialize output to 0 with length = input array
        C = [0] * 4             # Know that house ranges from 0-3 to set count array to length 4
        for j in range(0,len(A)):   # Add 1 to count of each occurence of each house
            C[A[j].getIndex("h")] += 1
        for i in range(1,4):    # Sum previous values in count array
            C[i] += C[i-1]
        for i in range(len(A)-1,-1,-1): # iterate down from right of array
            house = A[i].getIndex("h")  # get the house number
            B[C[house]-1] = A[i]    # set in appropriate position in output array
            C[house] -= 1       # decrement count in count array
        return B
    if houseOrYear == 'y':
        B = [0] * len(A)
        C = [0] * 8
        for j in range(0,len(A)):
            C[A[j].year-1] += 1
        for i in range(1,8):
            C[i] = C[i] + C[i-1]
        for i in range(len(A)-1,-1,-1):
            year = A[i].getIndex("y")
            B[C[year]-1] = A[i]
            C[year] -= 1
        return B

#  DO NOT EDIT BELOW THIS LINE

#SortStudents function. Takes as input a list of Student objects,
#sorted alphabetically by name, and outputs a list of Student objects,
#sorted by the following priority: house, then year, then name.
def SortStudents(studentList):
    yearSorted = CountingSort(studentList,'y')
    houseSorted = CountingSort(yearSorted,'h')
    return houseSorted

#Student class
#Each task has three instance variables:
#   self.name is a string representing the name of the student
#   self.house is a string representing which house the student is in
#   self.year is an integer representing what year the student is
class Student:
    def __init__(self,csvstring):
        csvdata = csvstring.split(",")
        self.name = csvdata[0]
        self.house = csvdata[1]
        self.year = int(csvdata[2])
    def __repr__(self):
        return "\n"+self.name + ": " + self.house + " " + str(self.year)
    def __eq__(self,other):
        return type(self) == type(other) and \
               self.name == other.name and \
               self.house == other.house and \
               self.year == other.year
    #A helpful method for dealing with Counting Sort.
    #Maps the houses onto integers from 0-3, and
    #years to integers from 0-7.
    def getIndex(self,houseOrYear):
        if houseOrYear == 'h':
            if self.house == 'Eagletalon':
                return 0
            elif self.house == 'Lannister':
                return 1
            elif self.house == 'Pufflehuff':
                return 2
            else:
                return 3
        else:
            return self.year-1

#Takes a string filename as an argument, and constructs a list
#  of Students from the information in the CSV file at filename
def getStudentList(filename):
    fp = open(filename)
    fp.readline()
    studentList = []
    for line in fp:
        studentList.append(Student(line))
    return studentList

tests = ['roster0.csv','roster1.csv','roster2.csv','roster3.csv']
correct = ['roster0sorted.csv','roster1sorted.csv',
           'roster2sorted.csv','roster3sorted.csv']

#Run test cases, check whether sorted list correct
count = 0

try:
    for i in range(len(tests)):
        ("\n---------------------------------------\n")
        print("TEST #",i+1)
        print("Reading student data from:",tests[i])
        roster = getStudentList(tests[i])
        print("Reading sorted student data from",correct[i])
        rosterSorted = getStudentList(correct[i])
        print("Running: SortStudents() on data list\n")
        output = SortStudents(roster)
        if i == 0:
            print("Expected:",rosterSorted,"\n\nGot:",output)
        assert len(output) == len(rosterSorted), "Output list length "\
               +str(len(output))+\
                  ", but should be "+str(len(rosterSorted))
        for j in range(len(output)):
            assert output[j] == rosterSorted[j],"Student #"\
                       +str(j+1)+" incorrect: "+\
                    str(output[j])+" \nshould be "+str(rosterSorted[i])
        print("Test Passed!\n")
        count += 1
except AssertionError as e:
    print("\nFAIL: ",e)

except Exception:
    print("\nFAIL: ",traceback.format_exc())

cursed = False
with open(__file__) as f:
    source = f.read()
    for ch in source:
        if ord(ch) == 60:
            print("Less than sign detected: Curse activated!")
            count = 0
            cursed = True
        if ord(ch) == 62:
            print("Greater than sign detected: Curse activated!")
            count = 0
            cursed = True

print()
if cursed:
    print("You are now a newt.  Don't worry, you'll get better.")
print(count,"out of",len(tests),"tests passed.")


import torch
from layers.invertible_layer import InvertibleInputLayer, InvertibleLeakyReluLayer, InvertibleLinearOutputLayer
from optimizers.optimizers import SGDInvertible
from networks.invertible_network import InvertibleNetwork
from utils.create_datasets import GenerateDatasetFromModel
from tensorboardX import SummaryWriter
from utils import helper_functions as hf
import matplotlib.pyplot as plt

iterations = 10000
nb_blocks = 10
block_size = 5

angles = torch.empty(iterations)
angles_random = torch.empty(iterations)
angles_bp_block = torch.empty(iterations)
angles_bp_gn = torch.empty(iterations)

for iter in range(iterations):

    error = torch.randn(block_size, 1)
    layer_jacobians = torch.empty(nb_blocks,block_size,block_size)
    J = torch.eye(block_size)
    for i in range(nb_blocks):
        J_new = torch.randn(block_size,block_size)
        J = torch.matmul(J,J_new)
        layer_jacobians[nb_blocks-i-1,:,:] = J

    # Construct GN hessian
    J_tot = torch.empty(block_size, nb_blocks*block_size)
    for i in range(nb_blocks):
        J_tot[:,i*block_size:(i+1)*block_size] = layer_jacobians[i,:,:]

    J_tot_pinv = torch.pinverse(J_tot)
    h_update = torch.matmul(J_tot_pinv, error)

    h_update_blockdiag = torch.empty(h_update.shape)
    for i in range(nb_blocks):
        J_i_pinv = torch.pinverse(layer_jacobians[i,:,:])
        h_update_blockdiag[i*block_size:(i+1)*block_size] = torch.matmul(J_i_pinv, error)

    h_update = h_update.unsqueeze(0)
    h_update_blockdiag = h_update_blockdiag.unsqueeze(0)
    angle = hf.get_angle(h_update,h_update_blockdiag)
    angles[iter] = angle

    h_backprop = torch.matmul(torch.transpose(J_tot,0,1), error)
    h_backprop = h_backprop.unsqueeze(0)

    angle_bp_block = hf.get_angle(h_update_blockdiag, h_backprop)
    angles_bp_block[iter] = angle_bp_block
    angle_bp_gn = hf.get_angle(h_update, h_backprop)
    angles_bp_gn[iter] = angle_bp_gn

    # Random control
    J_random1 = torch.randn(J_tot_pinv.shape)
    J_random2 = torch.randn(J_tot_pinv.shape)

    h_random1 = torch.matmul(J_random1, error)
    h_random2 = torch.matmul(J_random2, error)
    h_random1 = h_random1.unsqueeze(0)
    h_random2 = h_random2.unsqueeze(0)
    angle_random = hf.get_angle(h_random1,h_random2)
    angles_random[iter] = angle_random

plt.figure('angle block approx gn')
plt.hist(angles,bins=int(iterations/100))
plt.title('angle block approx gn')
plt.show()
plt.figure('angle BP block approx')
plt.hist(angles_bp_block, bins=int(iterations/100))
plt.title('angle BP block approx')
plt.show()
plt.figure()
plt.hist(angles_bp_gn, bins=int(iterations/100))
plt.title('angle BP GN')
plt.show()
plt.figure('angle random control')
plt.hist(angles_random, bins=int(iterations/100))
plt.title('angle random control')
plt.show()

# GN_hessian = torch.empty(nb_blocks*block_size, nb_blocks*block_size)
# GN_hessian_blockdiag = torch.zeros(nb_blocks*block_size, nb_blocks*block_size)
# for i in range(nb_blocks):
#     J_i = layer_jacobians[i, :, :]
#     GN_hessian_blockdiag[i * block_size:(i + 1) * block_size,
#     i * block_size:(i + 1) * block_size] = torch.matmul(torch.transpose(J_i,0,1),J_i)
#     for j in range(nb_blocks):
#         J_j = layer_jacobians[j,:,:]
#         GN_hessian[i*block_size:(i+1)*block_size,
#         j*block_size:(j+1)*block_size] = torch.matmul(torch.transpose(J_i,0,1),J_j)


from undine.client.command.base_command import BaseCommand

import re

class Task(BaseCommand):
    _STATE = ['all', 'ready', 'issued', 'done', 'canceled', 'failed']

    _WHERE_CLAUSE = ['mid', 'name', 'cid', 'iid', 'wid', 'host', 'reportable']

    _DESC = {
        'list': '{} tasks lookup commands for the specific mid.',
        'info': 'Detail task information lookup command for specific tid.',
        'cancel': 'Cancel all target tasks not yet started.',
        'remove': 'Remove all target tasks.',
        'rerun': 'Rerun all tasks in selected mission.'
    }

    def __init__(self, config, connector):
        super(self.__class__, self).__init__(config, connector)

    @staticmethod
    def help():
        return 'List-up specific tasks'

    @staticmethod
    def add_arguments(parser):
        subparsers = parser.add_subparsers(help='Task lookup commands',
                                           dest='sub_command')

        # Task list-up parser
        for state in Task._STATE:
            description = Task._DESC['list'].format(state.capitalize())

            _list = subparsers.add_parser(state, description=description)

            _list.add_argument('mission', help='Mission ID or name',
                               action='store')

            # Temporary blocked!
            # TODO Change the name of 'name' field to m_name, and t_name
            # _list.add_argument('-n', '--name', dest='name',
            #                    help='Task name', action='store')

            _list.add_argument('-c', '--cid', dest='cid',
                               help='Config ID (CID)', action='store')

            _list.add_argument('-i', '--iid', dest='iid',
                               help='Input ID (IID)', action='store')

            _list.add_argument('-w', '--wid', dest='wid',
                               help='Worker ID (WID)', action='store')

            _list.add_argument('-H', '--host', dest='host',
                               help='Hostname', action='store')

            _list.add_argument('-r', '--reportable', dest='reportable',
                               help='Reportable task', default='all',
                               choices=['all', 'true', 'false'])

        # Task detail information parser
        _info = subparsers.add_parser('info', description=Task._DESC['info'])
        _info.add_argument('tid', help='Task ID (TID)')

        # Cancel tasks
        _cancel = subparsers.add_parser('cancel',
                                        description=Task._DESC['cancel'])
        _cancel.add_argument('tid', nargs='+', help='Task ID (TID)')

        # Remove tasks
        _remove = subparsers.add_parser('remove',
                                        description=Task._DESC['remove'])
        _remove.add_argument('tid', nargs='+', help='Task ID (TID)')

        # rerun tasks
        _rerun = subparsers.add_parser('rerun',
                                       description=Task._DESC['rerun'])
        _rerun.add_argument('tid', nargs='+', help='Task ID (TID)')

    def run(self):
        if self._config.sub_command in self._STATE:
            where = {key: value
                     for key, value in vars(self._config).items()
                     if key in self._WHERE_CLAUSE and value}

            if not re.match(r'^(?:[0-9A-Fa-f]){32}$', self._config.mission):
                where.update({'name': self._config.mission})
            else:
                where.update({'mid': self._config.mission})

            if self._config.sub_command != 'all':
                where['state'] = self._config.sub_command

            if where['reportable'] == 'all':
                del where['reportable']
            else:
                where['reportable'] = (where['reportable'] == 'true')

            print(self._connector.task_list(**where))

        elif self._config.sub_command == 'info':
            print(self._connector.task_info(self._config.tid))

        elif self._config.sub_command == 'cancel':
            total_tasks = self._connector.cancel_tasks(*self._config.tid)
            print('{} tasks has been canceled.'.format(total_tasks))

        elif self._config.sub_command == 'remove':
            total_tasks = self._connector.drop_tasks(*self._config.tid)
            print('{} tasks has been removed.'.format(total_tasks))

        elif self._config.sub_command == 'rerun':
            total_tasks = self._connector.rerun_tasks(*self._config.tid)
            print('{} tasks has been restart.'.format(total_tasks))

class Config(BaseCommand):
    def __init__(self, config, connector):
        super(self.__class__, self).__init__(config, connector)

    @staticmethod
    def help():
        return 'Lookup configuration'

    @staticmethod
    def add_arguments(parser):
        parser.add_argument('cid', help='Config ID (CID)')

    def run(self):
        print(self._connector.config_info(self._config.cid))

class Input(BaseCommand):
    _DESC = {
        'list': 'Input list lookup command.',
        'info': 'Input information lookup command.'
    }

    def __init__(self, config, connector):
        super(self.__class__, self).__init__(config, connector)

    @staticmethod
    def help():
        return 'Lookup input parameters'

    @staticmethod
    def add_arguments(parser):
        subparsers = parser.add_subparsers(help='Input lookup commands',
                                           dest='sub_command')

        # Input list's parser
        subparsers.add_parser('list', description=Input._DESC['list'])

        # Input detail information parser
        input_info = subparsers.add_parser('info',
                                           description=Input._DESC['info'])

        input_info.add_argument('iid', help='Input ID (IID)')

    def run(self):
        if self._config.sub_command == 'list':
            print(self._connector.input_list())
        else:
            print(self._connector.input_info(self._config.iid))

class Worker(BaseCommand):
    _DESC = {
        'list': 'Input list lookup command.',
        'info': 'Input information lookup command.'
    }

    def __init__(self, config, connector):
        super(self.__class__, self).__init__(config, connector)

    @staticmethod
    def help():
        return 'Lookup worker information'

    @staticmethod
    def add_arguments(parser):
        subparsers = parser.add_subparsers(help='Worker lookup commands',
                                           dest='sub_command')

        # Worker list's parser
        subparsers.add_parser('list', description=Worker._DESC['list'])

        # Worker detail information parser
        worker_info = subparsers.add_parser('info',
                                            description=Worker._DESC['info'])

        worker_info.add_argument('wid', help='Worker ID (WID)')

    def run(self):
        if self._config.sub_command == 'list':
            print(self._connector.worker_list())
        else:
            print(self._connector.worker_info(self._config.wid))


import re
import sys
import os
sys.path.insert(1, os.path.join(sys.path[0], '..'))
import utils.utils as ut
from copy import deepcopy
import math
from functools import reduce

def get_active(grid, i, j, k, m):
    num = 0
    for dm in range(-1, 2):
        for di in range(-1, 2):
            for dj in range(-1, 2):
                for dk in range(-1, 2):
                    if grid[i+di][j+dj][k+dk][m+dm] == '#':
                        num += 1

    act  = 1 if grid[i][j][k][m] == '#' else 0
    num -= act
    return [act, num]

def run(grid, num):
    grid2 = [[[['.' for m in range(num)] for k in range(num)] for j in range(num)] for i in range(num)]
    for m in range(1, num-1):
        for i in range(1, num-1):
            for j in range(1, num-1):
                for k in range(1, num-1):
                    act = get_active(grid, i, j, k, m)
                    if (act[0] == 1) and (act[1] == 2 or act[1] == 3):
                        grid2[i][j][k][m] = '#'
                    if (act[0] == 0) and (act[1] == 3):
                        grid2[i][j][k][m] = '#'
    return grid2

def count_active(grid, num):
    count = 0
    for m in range(1, num-1):
        for i in range(1, num-1):
            for j in range(1, num-1):
                for k in range(1, num-1):
                    if grid[i][j][k][m] == '#':
                        count += 1
    return count

def part2(data, cycles):
    num  = (len(data) + (cycles + 3) * 2)
    off  = num // 2 - 1
    print(num, off)

    grid = [[[['.' for m in range(num)] for k in range(num)] for j in range(num)] for i in range(num)]
    for i in range(len(data)):
        for j in range(len(data[i])):
            grid[off][off][i+off][j+off] = data[i][j]

    for c in range(cycles):
        grid = run(grid, num)

    return count_active(grid, num)

def solution(file):
    print("Input: ", file)
    p2 = part2(ut.read_lines(file), 6)
    print("Part2: ", p2)

def main():
    solution("day17-test.txt")
    solution("day17-input.txt")

if __name__ == '__main__':
    main()


while True:
    print('Enter an integer.')
    try:
        number = int(input())
        break
    except ValueError:
        print('You must enter an integer!')

def collatz(number):
    while number != 1:
        if number % 2 == 0:
            number = number // 2
            print(number)
        else:
            number = 3 * number + 1
            print(number)

collatz(number)

Python/Automate the boring stuff/Practice program/Collatz sequence/app.py


from django.shortcuts import render
from .models import Information
import datetime

def information(request):
    now = datetime.datetime.now()
    nowDate = now.strftime('%Y-%m-%d')
    print(nowDate)
    info = Information.objects.all()
    return render(request, 'information/info.html', {
        'info': info,
        'today': nowDate,
    })


from openeye import oechem
from openeye import oequacpac
from simtk.openmm import app
import parmed
from LigPrepCubes import ff_utils
from OpenMMCubes import utils
import numpy as np
from oeommtools import utils as oeommutils
from pdbfixer import PDBFixer
from pkg_resources import resource_filename
from simtk import unit
from simtk.openmm import Vec3

proteinResidues = ['ALA', 'ASN', 'CYS', 'GLU', 'HIS',
                   'LEU', 'MET', 'PRO', 'THR', 'TYR',
                   'ARG', 'ASP', 'GLN', 'GLY', 'ILE',
                   'LYS', 'PHE', 'SER', 'TRP', 'VAL']

rnaResidues = ['A', 'G', 'C', 'U', 'I']
dnaResidues = ['DA', 'DG', 'DC', 'DT', 'DI']

def applyffProtein(protein, opt):
    """
    This function applies the selected force field to the
    protein

    Parameters:
    -----------
    protein: OEMol molecule
        The protein to parametrize
    opt: python dictionary
        The options used to parametrize the protein

    Return:
    -------
    protein_structure: Parmed structure instance
        The parametrized protein parmed structure
    """

    topology, positions = oeommutils.oemol_to_openmmTop(protein)

    forcefield = app.ForceField(opt['protein_forcefield'])
    unmatched_residues = forcefield.getUnmatchedResidues(topology)

    if unmatched_residues:
        # Extended ff99SBildn force field
        oechem.OEThrow.Info("The following protein residues are not recognized "
                            "by the selected FF: {} - {}"
                            "\n...Extended FF is in use".format(opt['protein_forcefield'], unmatched_residues))

        ffext_fname = utils.get_data_filename('ComplexPrepCubes', 'ffext/amber99SBildn_ext.xml')
        forcefield = app.ForceField()
        forcefield.loadFile(ffext_fname)

        unmatched_residues = forcefield.getUnmatchedResidues(topology)

        if unmatched_residues:
            oechem.OEThrow.Fatal("Error. The following protein residues are not recognized "
                                 "by the extended force field {}".format(unmatched_residues))

    omm_system = forcefield.createSystem(topology, rigidWater=False)
    protein_structure = parmed.openmm.load_topology(topology, omm_system, xyz=positions)

    return protein_structure

def applyffWater(water, opt):
    """
    This function applies the selected force field to the
    water

    Parameters:
    -----------
    water: OEMol molecule
        The water molecules to parametrize
    opt: python dictionary
        The options used to parametrize the water

    Return:
    -------
    water_structure: Parmed structure instance
        The parametrized water parmed structure
    """

    topology, positions = oeommutils.oemol_to_openmmTop(water)

    forcefield = app.ForceField(opt['solvent_forcefield'])
    unmatched_residues = forcefield.getUnmatchedResidues(topology)

    if unmatched_residues:
        oechem.OEThrow.Fatal("The following water molecules are not recognized "
                             "by the selected force field {}: {}".format(opt['solvent_forcefield'], unmatched_residues))

    omm_system = forcefield.createSystem(topology, rigidWater=False)
    water_structure = parmed.openmm.load_topology(topology, omm_system, xyz=positions)

    return water_structure

def applyffExcipients(excipients, opt):
    """
    This function applies the selected force field to the
    excipients

    Parameters:
    -----------
    excipients: OEMol molecule
        The excipients molecules to parametrize
    opt: python dictionary
        The options used to parametrize the excipients

    Return:
    -------
    excipient_structure: Parmed structure instance
        The parametrized excipient parmed structure
    """

    # OpenMM topology and positions from OEMol
    topology, positions = oeommutils.oemol_to_openmmTop(excipients)

    # Try to apply the selected FF on the excipients
    forcefield = app.ForceField(opt['protein_forcefield'])

    # List of the unrecognized excipients
    unmatched_res_list = forcefield.getUnmatchedResidues(topology)

    # Unique unrecognized excipient names
    templates = set()
    for res in unmatched_res_list:
        templates.add(res.name)

    if templates:  # Some excipients are not recognized
        oechem.OEThrow.Info("The following excipients are not recognized "
                            "by the protein FF: {}"
                            "\nThey will be parametrized by using the FF: {}".format(templates, opt['other_forcefield']))

        # Create a bit vector mask used to split recognized from un-recognize excipients
        bv = oechem.OEBitVector(excipients.GetMaxAtomIdx())
        bv.NegateBits()

        # Dictionary containing the name and the parmed structures of the unrecognized excipients
        unrc_excipient_structures = {}

        # Dictionary used to skip already selected unrecognized excipients and count them
        unmatched_excp = {}

        # Ordered list of the unrecognized excipients
        unmatched_res_order = []

        for r_name in templates:
            unmatched_excp[r_name] = 0

        hv = oechem.OEHierView(excipients)

        for chain in hv.GetChains():
            for frag in chain.GetFragments():
                for hres in frag.GetResidues():
                    r_name = hres.GetOEResidue().GetName()
                    if r_name not in unmatched_excp:
                        continue
                    else:
                        unmatched_res_order.append(r_name)
                        if unmatched_excp[r_name]:  # Test if we have selected the unknown excipient
                            # Set Bit mask
                            atms = hres.GetAtoms()
                            for at in atms:
                                bv.SetBitOff(at.GetIdx())
                            unmatched_excp[r_name] += 1
                        else:
                            unmatched_excp[r_name] = 1
                            #  Create AtomBondSet to extract from the whole excipient system
                            #  the current selected FF unknown excipient
                            atms = hres.GetAtoms()
                            bond_set = set()
                            for at in atms:
                                bv.SetBitOff(at.GetIdx())
                                bonds = at.GetBonds()
                                for bond in bonds:
                                    bond_set.add(bond)
                            atom_bond_set = oechem.OEAtomBondSet(atms)
                            for bond in bond_set:
                                atom_bond_set.AddBond(bond)

                            # Create the unrecognized excipient OEMol
                            unrc_excp = oechem.OEMol()
                            if not oechem.OESubsetMol(unrc_excp, excipients, atom_bond_set):
                                oechem.OEThrow.Fatal("Is was not possible extract the residue: {}".format(r_name))

                            # Charge the unrecognized excipient
                            if not oequacpac.OEAssignCharges(unrc_excp,
                                                             oequacpac.OEAM1BCCCharges(symmetrize=True)):
                                oechem.OEThrow.Fatal("Is was not possible to "
                                                     "charge the extract residue: {}".format(r_name))

                            # If GAFF or GAFF2 is selected as FF check for tleap command
                            if opt['other_forcefield'] in ['GAFF', 'GAFF2']:
                                ff_utils.ParamLigStructure(oechem.OEMol(), opt['other_forcefield']).checkTleap

                            if opt['other_forcefield'] == 'SMIRNOFF':
                                unrc_excp = oeommutils.sanitizeOEMolecule(unrc_excp)

                            # Parametrize the unrecognized excipient by using the selected FF
                            pmd = ff_utils.ParamLigStructure(unrc_excp, opt['other_forcefield'],
                                                             prefix_name=opt['prefix_name']+'_'+r_name)
                            unrc_excp_struc = pmd.parameterize()
                            unrc_excp_struc.residues[0].name = r_name
                            unrc_excipient_structures[r_name] = unrc_excp_struc

        # Recognized FF excipients
        pred_rec = oechem.OEAtomIdxSelected(bv)
        rec_excp = oechem.OEMol()
        oechem.OESubsetMol(rec_excp, excipients, pred_rec)

        if rec_excp.NumAtoms() > 0:
            top_known, pos_known = oeommutils.oemol_to_openmmTop(rec_excp)
            ff_rec = app.ForceField(opt['protein_forcefield'])
            try:
                omm_system = ff_rec.createSystem(top_known, rigidWater=False)
                rec_struc = parmed.openmm.load_topology(top_known, omm_system, xyz=pos_known)
            except:
                oechem.OEThrow.Fatal("Error in the recognised excipient parametrization")

        # Unrecognized FF excipients
        bv.NegateBits()
        pred_unrc = oechem.OEAtomIdxSelected(bv)
        unrc_excp = oechem.OEMol()
        oechem.OESubsetMol(unrc_excp, excipients, pred_unrc)

        # Unrecognized FF excipients coordinates
        oe_coord_dic = unrc_excp.GetCoords()
        unrc_coords = np.ndarray(shape=(unrc_excp.NumAtoms(), 3))
        for at_idx in oe_coord_dic:
            unrc_coords[at_idx] = oe_coord_dic[at_idx]

        # It is important the order used to assemble the structures. In order to
        # avoid mismatch between the coordinates and the structures, it is convenient
        # to use the unrecognized residue order
        unmatched_res_order_count = []
        i = 0
        while i < len(unmatched_res_order):
            res_name = unmatched_res_order[i]
            for j in range(i+1, len(unmatched_res_order)):
                if unmatched_res_order[j] == res_name:
                    continue
                else:
                    break
            if i == (len(unmatched_res_order) - 1):
                num = 1
                unmatched_res_order_count.append((res_name, num))
                break
            else:
                num = j - i
                unmatched_res_order_count.append((res_name, num))
                i = j

        # Merge all the unrecognized Parmed structure
        unrc_struc = parmed.Structure()

        for pair in unmatched_res_order_count:
            res_name = pair[0]
            nums = pair[1]
            unrc_struc = unrc_struc + nums*unrc_excipient_structures[res_name]

        # Set the unrecognized coordinates
        unrc_struc.coordinates = unrc_coords

        # Set the parmed excipient structure merging
        # the unrecognized and recognized parmed
        # structures together
        if rec_excp.NumAtoms() > 0:
            excipients_structure = unrc_struc + rec_struc
        else:
            excipients_structure = unrc_struc

        return excipients_structure
    else:  # All the excipients are recognized by the selected FF
        omm_system = forcefield.createSystem(topology, rigidWater=False)
        excipients_structure = parmed.openmm.load_topology(topology, omm_system, xyz=positions)

        return excipients_structure

def applyffLigand(ligand, opt):
    """
    This function applies the selected force field to the
    ligand

    Parameters:
    -----------
    ligand: OEMol molecule
        The ligand molecule to parametrize
    opt: python dictionary
        The options used to parametrize the ligand

    Return:
    -------
    ligand_structure: Parmed structure instance
        The parametrized ligand parmed structure
    """

    # Check TLeap
    if opt['ligand_forcefield'] in ['GAFF', 'GAFF2']:
        ff_utils.ParamLigStructure(oechem.OEMol(), opt['ligand_forcefield']).checkTleap

    # Parametrize the Ligand
    pmd = ff_utils.ParamLigStructure(ligand, opt['ligand_forcefield'], prefix_name=opt['prefix_name'])
    ligand_structure = pmd.parameterize()
    ligand_structure.residues[0].name = "LIG"
    opt['Logger'].info("Ligand parametrized by using: {}".format(opt['ligand_forcefield']))

    return ligand_structure

def hydrate(system, opt):
    """
    This function solvates the system by using PDBFixer

    Parameters:
    -----------
    system: OEMol molecule
        The system to solvate
    opt: python dictionary
        The parameters used to solvate the system

    Return:
    -------
    oe_mol: OEMol
        The solvated system
    """
    def BoundingBox(molecule):
        """
        This function calculates the Bounding Box of the passed
        molecule

        molecule: OEMol

        return: bb (numpy array)
            the calculated bounding box is returned as numpy array:
            [(xmin,ymin,zmin), (xmax,ymax,zmax)]
        """
        coords = [v for k, v in molecule.GetCoords().items()]
        np_coords = np.array(coords)
        min_coord = np_coords.min(axis=0)
        max_coord = np_coords.max(axis=0)
        bb = np.array([min_coord, max_coord])
        return bb

    # Create a system copy
    sol_system = system.CreateCopy()

    # Calculate system BoundingBox (Angstrom units)
    BB = BoundingBox(sol_system)

    # Estimation of the box cube length in A
    box_edge = 2.0 * opt['solvent_padding'] + np.max(BB[1] - BB[0])

    # BB center
    xc = (BB[0][0]+BB[1][0])/2.
    yc = (BB[0][1]+BB[1][1])/2.
    zc = (BB[0][2]+BB[1][2])/2.

    delta = np.array([box_edge/2., box_edge/2., box_edge/2.]) - np.array([xc, yc, zc])

    sys_coord_dic = {k: (v+delta) for k, v in sol_system.GetCoords().items()}

    sol_system.SetCoords(sys_coord_dic)

    # Load a fake system to initialize PDBfixer
    filename = resource_filename('pdbfixer', 'tests/data/test.pdb')
    fixer = PDBFixer(filename=filename)

    # Convert between OE and OpenMM topology
    omm_top, omm_pos = oeommutils.oemol_to_openmmTop(sol_system)

    chain_names = []

    for chain in omm_top.chains():
        chain_names.append(chain.id)

    # Set the correct topology to the fake system
    fixer.topology = omm_top
    fixer.positions = omm_pos

    # Solvate the system
    fixer.addSolvent(padding=unit.Quantity(opt['solvent_padding'], unit.angstroms),
                     ionicStrength=unit.Quantity(opt['salt_concentration'], unit.millimolar))

    # The OpenMM topology produced by the solvation fixer has missing bond
    # orders and aromaticity. The following section is creating a new openmm
    # topology made of just water molecules and ions. The new topology is then
    # converted in an OEMol and added to the passed molecule to produce the
    # solvated system

    wat_ion_top = app.Topology()

    # Atom dictionary between the the PDBfixer topology and the water_ion topology
    fixer_atom_to_wat_ion_atom = {}

    for chain in fixer.topology.chains():
        if chain.id not in chain_names:
            n_chain = wat_ion_top.addChain(chain.id)
            for res in chain.residues():
                n_res = wat_ion_top.addResidue(res.name, n_chain)
                for at in res.atoms():
                    n_at = wat_ion_top.addAtom(at.name, at.element, n_res)
                    fixer_atom_to_wat_ion_atom[at] = n_at

    for bond in fixer.topology.bonds():
        at0 = bond[0]
        at1 = bond[1]
        try:
            wat_ion_top.addBond(fixer_atom_to_wat_ion_atom[at0],
                                fixer_atom_to_wat_ion_atom[at1], type=None, order=1)
        except:
            pass

    wat_ion_pos = fixer.positions[len(omm_pos):]

    oe_mol = oeommutils.openmmTop_to_oemol(wat_ion_top, wat_ion_pos)

    # Setting the box vectors
    omm_box_vectors = fixer.topology.getPeriodicBoxVectors()
    box_vectors = utils.PackageOEMol.encodePyObj(omm_box_vectors)
    oe_mol.SetData(oechem.OEGetTag('box_vectors'), box_vectors)

    oechem.OEAddMols(oe_mol, sol_system)

    return oe_mol

def order_check(mol, fname):
    """
    TO REMOVE
    This function is used to debug
    """
    import logging
    logger = logging.getLogger('Testing')
    hdlr = logging.FileHandler(fname)
    formatter = logging.Formatter('%(message)s')
    hdlr.setFormatter(formatter)
    logger.addHandler(hdlr)
    logger.setLevel(logging.INFO)

    #hv = oechem.OEHierView(mol, oechem.OEAssumption_BondedResidue + oechem.OEAssumption_ResPerceived)
    hv = oechem.OEHierView(mol)

    for chain in hv.GetChains():
        logger.info('{}'.format(chain.GetChainID()))
        for frag in chain.GetFragments():
            for hres in frag.GetResidues():
                logger.info('\t{} {}'.format(hres.GetOEResidue().GetName(), hres.GetOEResidue().GetResidueNumber()))
                for oe_at in hres.GetAtoms():
                    logger.info('\t\t{} {}'.format(oe_at.GetName(), oe_at.GetIdx()))

    return


import pandas as pd
import os

#bit = 'sample'
bit = 'full'
info = ['title','topics','abstract']
languages =[]
#languages=['en']#,'en-en', 'en-nl','nl', 'lang']

sd = '/opt2/dissertaties/data_'+ bit
gd = '_'.join([bit]+info+languages)

#sd = '/opt2/dissertaties/data_full'
#gd = 'abstracts'

if not os.path.isdir(gd):
    os.mkdir(gd)
else:
    print('Warning: possibly overwriting content of ' + gd)

ggc = pd.read_excel('ggc_proefschriften_v4.xlsx',dtype=object)

brinkman = pd.read_csv('../brinkmanthesaurus_vocab.tsv', sep='\t', header = None, names = ['uri','label'])

oai = pd.read_csv('meta_oai.csv', sep=';', index_col = 0, dtype=object)
topic_en = [ col for col in oai.columns if 'topic_en' in col]
topic_lang = [ col for col in oai.columns if 'topic_lang' in col]

kinds = ['train','test']
for kind in kinds:
    if not os.path.isdir(os.path.join(gd,kind)): os.makedirs(os.path.join(gd,kind))
    identifiers = pd.read_csv(kind+'_identifiers.csv', sep=';', dtype='object')
    identifiers.dropna(inplace=True) # linked items only

    gold = ggc.merge(identifiers, left_on = 'ppn', right_on='ppn')

    for identifier in identifiers.identifier.unique(): # there are some duplicates in the gold data, ignore them

        # Collect document properties (as harvested from repo)

        print(identifier)

        oai_item = oai.loc[identifier]
        language = oai_item.language
        if languages:
            if language not in languages: continue

        # Collect abstract(s)
        dname = os.path.join(sd,identifier)
        if os.path.isdir(dname):  # The document does not occur in the source dir
            abstracts = {}
            for fname in os.listdir(os.path.join(dname, 'metadata')):
                if 'abstract' not in fname: continue
                try: lang = fname.split('_')[1].split('.')[0]
                except: lang = 'lang'

                try:
                    with open(os.path.join(dname, 'metadata', fname),'r') as f:
                         lines = f.readlines()
                except:
                    with open(os.path.join(dname, 'metadata', fname), 'r', encoding ='iso-8859-1') as f:
                         lines = f.readlines()
                abstracts[lang]=' '.join([line.strip() for line in lines]) # keep empty lines (separators)?
        else:
            print('No abstract??')
            abstracts = pd.Series()

        topics = oai_item[topic_en].dropna()
        if len(topics)==0:
            topics = oai_item[topic_lang].dropna()
        topics = topics.values

        title = oai_item.title
        assert type(title) == str
        if type(oai_item.subTitle) == str: title +=' '+ oai_item.subTitle

        with open(os.path.join(gd,kind,identifier)+'.txt', 'w', encoding='utf-8') as f:
            f.write(title+'\n\n')
            f.write(' '.join(topics)+'\n\n')
            f.write('\n\n'.join(abstracts.values()))

        # Collect labels (as assigned in KB catalogue)
        gold_item = gold.loc[gold.identifier==identifier]
        uris = [gold_item['ppn_brinkman_'+str(d)] for d in range(1,6)]
        uris = [uri for uri in uris if not uri.isna().values[0] ] # drop NaNs
        uris = ['<http://data.bibliotheken.nl/id/thes/p'+uri.values[0]+'>' for uri in uris] # convert to string (proper uri)

        try:
            with open(os.path.join(gd,kind,identifier)+'.tsv', 'w', encoding='utf-8') as f:
                f.write('\n'.join(['\t'.join([uri, brinkman.loc[brinkman.uri==uri,'label'].values[0]]) for uri in uris]))
        except:
            print('\t No labels?? Problem??')

# title
# subtitle
#
# abstract
# department (?)


from spack import *
from spack.stage import Stage
import os
class Sratoolkit(Package):

    homepage = "https://github.com/ncbi/sra-tools"
    url = "https://github.com/ncbi/sra-tools.git"

    version("2.4.5",git= "https://github.com/ncbi/sra-tools.git" )
    parallel = False
    #depends_on("libxml2")
    #depends_on("libmagic")
    #depends_on("hdf5")

    def install(self, spec, prefix):

        build_dir = join_path(self.stage.source_path, "builddir")
        os.mkdir(build_dir)

        stage1 = Stage("https://github.com/ncbi/ngs/archive/1.1.3.tar.gz") # Download tarball. fetch() doesn't work with git url?
        stage1.fetch()
        stage1.expand_archive()

        with working_dir(stage1.source_path):
            ngs_build = join_path(stage1.source_path,"builddir")
            os.mkdir(ngs_build)
            configure("--build-prefix="+ ngs_build, "--prefix=" + prefix)
            make("ngs-sdk")
            make("ngs-python")
            make("ngs-java")
            configure("--build-prefix=" + ngs_build, "--with-ngs-sdk=" + prefix, "--prefix=" + prefix)
            make("ngs-bam")
            make("install")

        stage2 = Stage("https://github.com/ncbi/ncbi-vdb/archive/2.5.2.tar.gz")
        stage2.fetch()
        stage2.expand_archive()

        with working_dir(stage2.source_path):
            vdb_build = join_path(stage2.source_path,"builddir")
            os.mkdir(vdb_build)
            configure("--build-prefix=" + vdb_build,"--with-ngs-sdk-prefix=" + prefix, "--with-ngs-java-prefix=" + prefix, "--prefix=" + prefix)
            make()
            make("install")

        # sra-tools build block
        with working_dir(self.stage.source_path):
            configure("--build-prefix=" + build_dir,
                    "--with-ngs-sdk-prefix=" + prefix, "--with-ncbi-vdb-build=" + vdb_build, "--with-ncbi-vdb-sources=" + stage2.source_path,
                    "--prefix=" + prefix)
            make()
            make("install")


import contextlib
import unittest

from apache.aurora.client.cli import AuroraCommandLine
from apache.aurora.client.hooks.hooked_api import HookedAuroraClientAPI
from apache.aurora.common.aurora_job_key import AuroraJobKey
from twitter.common.contextutil import temporary_file
from apache.aurora.client.cli.jobs import parse_instances
from apache.aurora.client.cli.util import AuroraClientCommandTest, FakeAuroraCommandContext

from gen.apache.aurora.ttypes import (
    Identity,
    TaskQuery,
)

from mock import Mock, patch

class TestInstancesParser(unittest.TestCase):
  def test_parse_instances(self):
    instances = '0,1-3,5'
    x = parse_instances(instances)
    assert x == [0, 1, 2, 3, 5]

  def test_parse_none(self):
    assert parse_instances(None) is None
    assert parse_instances("") is None

class TestClientKillCommand(AuroraClientCommandTest):
  @classmethod
  def setup_mock_api(cls):
    """Builds up a mock API object, with a mock SchedulerProxy.
    Returns the API and the proxy"""

    mock_scheduler = Mock()
    mock_scheduler.url = "http://something_or_other"
    mock_scheduler_client = Mock()
    mock_scheduler_client.scheduler.return_value = mock_scheduler
    mock_scheduler_client.url = "http://something_or_other"
    mock_api = Mock(spec=HookedAuroraClientAPI)
    mock_api.scheduler = mock_scheduler_client
    return (mock_api, mock_scheduler_client)

  @classmethod
  def get_kill_job_response(cls):
    return cls.create_simple_success_response()

  @classmethod
  def assert_kill_job_called(cls, mock_api):
    assert mock_api.kill_job.call_count == 1

  def test_kill_job(self):
    """Test kill client-side API logic."""
    mock_context = FakeAuroraCommandContext()
    mock_scheduler = Mock()
    with contextlib.nested(
        patch('apache.aurora.client.cli.jobs.Job.create_context', return_value=mock_context),
        patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS)):

      api = mock_context.get_api('west')
      api.kill_job.return_value = self.get_kill_job_response()
      mock_scheduler.scheduler.killTasks.return_value = self.get_kill_job_response()
      with temporary_file() as fp:
        fp.write(self.get_valid_config())
        fp.flush()
        cmd = AuroraCommandLine()
        cmd.execute(['job', 'kill', '--config=%s' % fp.name, 'west/mchucarroll/test/hello'])

      # Now check that the right API calls got made.
      assert api.kill_job.call_count == 1
      api.kill_job.assert_called_with(AuroraJobKey.from_path('west/mchucarroll/test/hello'), None)

  def test_kill_job_with_instances(self):
    """Test kill client-side API logic."""
    mock_context = FakeAuroraCommandContext()
    with contextlib.nested(
        patch('apache.aurora.client.cli.jobs.Job.create_context', return_value=mock_context),
        patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS)):
      api = mock_context.get_api('west')
      api.kill_job.return_value = self.get_kill_job_response()
      with temporary_file() as fp:
        fp.write(self.get_valid_config())
        fp.flush()
        cmd = AuroraCommandLine()
        cmd.execute(['job', 'kill', '--config=%s' % fp.name, '--instances=0,2,4-6',
           'west/mchucarroll/test/hello'])

      # Now check that the right API calls got made.
      assert api.kill_job.call_count == 1
      api.kill_job.assert_called_with(AuroraJobKey.from_path('west/mchucarroll/test/hello'),
          [0, 2, 4, 5, 6])

  def test_kill_job_with_instances_deep_api(self):
    """Test kill client-side API logic."""
    mock_context = FakeAuroraCommandContext()
    (mock_api, mock_scheduler) = self.setup_mock_api()
    with contextlib.nested(
        patch('apache.aurora.client.api.SchedulerProxy', return_value=mock_scheduler),
        patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS)):
      mock_scheduler.killTasks.return_value = self.get_kill_job_response()
      with temporary_file() as fp:
        fp.write(self.get_valid_config())
        fp.flush()
        cmd = AuroraCommandLine()
        cmd.execute(['job', 'kill', '--config=%s' % fp.name, '--instances=0,2,4-6',
           'west/mchucarroll/test/hello'])
      # Now check that the right API calls got made.
      assert mock_scheduler.killTasks.call_count == 1
      mock_scheduler.killTasks.assert_called_with(
        TaskQuery(jobName='hello', environment='test', instanceIds=frozenset([0, 2, 4, 5, 6]),
            owner=Identity(role='mchucarroll')), None)


import logging
import time
import os
import json
from mqtt import MQTTClient
from temp import Temp
from fan import Fan
from power import Power
from dotenv import load_dotenv
from math import floor

load_dotenv()

class HVACPIDController(object):
    logger = None
    mqtt = None
    temp = None
    fan = None
    power = None

    temp_outdoors = 0

    mode = 'auto'
    manual = False
    control_enable = False
    hvac_state = {}

    def __init__(self):
        self.logger = logging.getLogger('hvac-pid')
        self.logger.info('Starting hvac-pid')

        # PID options
        pid_options = {
            'Kp': float(os.getenv('PID_KP')),
            'Ki': float(os.getenv('PID_KI')),
            'Kd': float(os.getenv('PID_KD')),
        }

        temp_options = {
            'temp_min': float(os.getenv('SET_TEMP_MIN')),
            'temp_max': float(os.getenv('SET_TEMP_MAX')),
        }

        # Temp
        self.temp = Temp(**{**temp_options, **pid_options})

        # Fan
        self.fan = Fan()

        # Power
        self.power = Power()

        # MQTT
        self.topic_prefix = os.getenv('MQTT_PID_TOPIC_PREFIX')
        self.mqtt = MQTTClient(os.getenv('MQTT_CLIENT_ID'), os.getenv('MQTT_BROKER_HOST'))
        self.mqtt.connect()

        # subscribe
        self.mqtt.subscribe(os.getenv('MQTT_TEMP_TOPIC'), 0, self.temp_update_callback)
        self.mqtt.subscribe(os.getenv('MQTT_TEMP_OUTDOORS_TOPIC'), 0, self.temp_outdoors_update_callback)
        self.mqtt.subscribe(os.getenv('MQTT_HVAC_STATE_TOPIC'), 0, self.hvac_callback)
        self.mqtt.subscribe(self.topic_prefix + '/mode/set', 0, self.set_mode)
        self.mqtt.subscribe(self.topic_prefix + '/temperature/set', 0, self.set_temp)
        self.mqtt.subscribe(self.topic_prefix + '/fan/set', 0, self.set_fan)

        self.logger.info('MQTT connected')

        self.publish_temp()
        self.publish_mode()
        self.publish_fan()

        # wait a bit before enabling control
        time.sleep(3)
        self.control_enable = True

    def iterate(self):
        if self.manual:
            self.logger.info('Manual mode, skipping PID iteration')
        else:
            # temp hax
            # limit min temp when outdoors is < -10
            if self.temp_outdoors < -10:
                self.temp.setLimits(floor(self.temp.temp_request), float(os.getenv('SET_TEMP_MAX')))
                self.logger.debug('Limiting min temp to %g when outdoor temp is %g', self.temp.temp_min, self.temp_outdoors)
            else:
                self.temp.setLimits(float(os.getenv('SET_TEMP_MIN')), float(os.getenv('SET_TEMP_MAX')))

            self.temp.iteratePID()
            self.fan.calculate(self.temp.pid.output - self.temp.temp_request, self.mode)
            self.power.calculate(self.temp.temp_request, self.temp.temp_measure, self.mode, self.temp_outdoors)
            if not self.power.state:
                self.temp.pid.reset()
            self.publish_state()

    def temp_update_callback(self, client, userdata, message):
        payload_json = json.loads(message.payload.decode('utf-8'))
        self.temp.setMeasurement(payload_json['temperature'])

    def temp_outdoors_update_callback(self, client, userdata, message):
        payload_json = json.loads(message.payload.decode('utf-8'))
        self.temp_outdoors = float(payload_json['temperature'])

    def hvac_callback(self, client, userdata, message):
        payload_json = json.loads(message.payload.decode('utf-8'))
        self.logger.info('Received hvac state change %s', payload_json)
        self.hvac_state = payload_json

    def setHVAC(self):
        if self.control_enable:
            topic = os.getenv('MQTT_HVAC_TOPIC')
            new_state = {
                'power': self.power.state,
                'mode': self.mode.upper(),
                'temperature': self.temp.temp_set,
                'fan': self.fan.speed,
            }

            is_state_changed = (new_state['power'] and self.hvac_state != new_state)
            is_power_state_changed = (self.hvac_state and new_state['power'] != self.hvac_state['power'])
            old_state_doesnt_exists = (not self.hvac_state)

            if is_state_changed or is_power_state_changed or old_state_doesnt_exists:
                message = json.dumps(new_state)

                self.logger.debug('Controlling HVAC with command %s', message)
                self.mqtt.publish(topic, message, 1)
            else:
                self.logger.debug('HVAC state unchanged %s', self.hvac_state)
        else:
            self.logger.debug('Controlling HVAC disabled')

    def set_mode(self, client, userdata, message):
        mode = message.payload.decode('utf-8')
        previous_is_manual = self.manual

        if mode == 'off':
            self.manual = True
            self.mode = 'auto'
            self.power.state = False
            self.logger.info('Set mode to off')
        if mode == 'manual':
            self.manual = True
            self.power.state = True
            self.mode = 'auto'
            self.temp.temp_set = self.temp.temp_request
            self.logger.info('Set mode to manual')
        elif mode in ['heat', 'cool']:
            self.manual = False
            self.mode = mode

            self.logger.info('Set mode to %s', self.mode)

            # reset PID if switching between manual and pid modes
            if previous_is_manual != self.manual:
                self.temp.temp_set = self.temp.temp_request
                self.temp.pid.reset()

        self.publish_mode()
        self.setHVAC()

    def publish_mode(self):
        if not self.control_enable:
            return

        topic = self.topic_prefix + '/mode/state'

        if self.manual:
            if self.power.state == False:
                mode = 'off'
            else:
                mode = 'manual'
        elif self.mode == 'auto':
            mode = 'manual'
        else:
            mode = self.mode

        self.mqtt.publish(topic, mode, 1, True)

    def set_temp(self, client, userdata, message):
        temp = round(float(message.payload.decode('utf-8')), 2)

        if temp >= 17 and temp <= 30:
            self.temp.setRequest(temp)

            if self.manual:
                self.temp.temp_set = self.temp.temp_request
            else:
                self.temp.pid.reset()

            self.publish_temp()
            self.setHVAC()

    def publish_temp(self):
        if not self.control_enable:
            return

        self.mqtt.publish(self.topic_prefix + '/temperature/state', self.temp.temp_request, 1, True)
        self.mqtt.publish(self.topic_prefix + '/measured_temperature', self.temp.temp_measure, 1, True)

    def set_fan(self, client, userdata, message):
        fan = message.payload.decode('utf-8')

        if fan != "auto":
            fan_int = int(fan)

            if self.manual and fan_int >= 1 and fan_int <= 5:
                self.fan.speed = fan_int
                self.publish_fan()
                self.setHVAC()
                self.logger.info('Manually set fan speed to %s/5', self.fan.speed)

    def publish_fan(self):
        if not self.control_enable:
            return

        topic = self.topic_prefix + '/fan/state'

        if self.manual:
            fan = self.fan.speed
        else:
            fan = 'auto'

        self.mqtt.publish(topic, fan, 1, True)

    def publish_state(self):
        if not self.control_enable:
            return

        topic = os.getenv('MQTT_PID_TOPIC_PREFIX') + '/state'
        message = json.dumps({
            'mode': self.mode,
            'manual': self.manual,
            'temperature_request': float(self.temp.temp_request),
            'temperature_set': float(self.temp.temp_set),
            'temperature_measure': float(self.temp.temp_measure),
            'temperature_error': float(self.temp.pid.previous_error),
            'fan': int(self.fan.speed if self.power.state else 0),
            'power': self.power.state,
            'Kp': float(self.temp.pid.Kp),
            'Ki': float(self.temp.pid.Ki),
            'Kd': float(self.temp.pid.Kd),
            'integral': float(self.temp.pid.integral),
            'integral_max': float(self.temp.pid.integral_max),
            'pid_output': float(self.temp.pid.output)
        })
        self.mqtt.publish(topic, message, 1)

if __name__ == '__main__':
    logger = logging.getLogger('hvac-pid')
    logger.setLevel(logging.DEBUG)
    logger.propagate = False

    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')

    ch = logging.StreamHandler()
    ch.setLevel(logging.DEBUG)
    ch.setFormatter(formatter)

    logger.addHandler(ch)

    ctrl = HVACPIDController()
    interval = int(os.getenv('PID_INTERVAL'))

    while True:
        time.sleep(interval)

        if not ctrl.manual:
            ctrl.iterate()
            ctrl.setHVAC()
            ctrl.publish_mode()
            ctrl.publish_fan()

        ctrl.publish_temp()


import torch
import torch.nn as nn
import math
import numpy as np
class custom_model(nn.Module):
    def __init__(self):
        super(custom_model,self).__init__()
        self.a=torch.nn.Parameter(torch.randn(()))
        self.b = torch.nn.Parameter(torch.randn(()))
        self.c = torch.nn.Parameter(torch.randn(()))
        self.d = torch.nn.Parameter(torch.randn(()))
    def forward(self,x):
        return self.a+self.b*x+self.c*x**2+self.d*x**3
    def string(self):
        return "mango"
device=torch.device("cpu")
dtype=torch.float32 #the tensor dtype

x=torch.linspace(-math.pi,math.pi,2000,dtype=dtype,device=device).to(device)
y=torch.sin(x).to(device)

model=custom_model()

learn_rate=1e-6
loss_func=torch.nn.MSELoss(reduction='sum')
optimizer=torch.optim.SGD(model.parameters(),lr=learn_rate)
for i in range(2000):
    y_pred = model(x)#y_pred shape is [2000] because of the flatten is from 0 to 1
    loss=loss_func(y,y_pred)
    if(i+1)%10==0:
        print("{},loss:{}".format(i+1,loss))
    optimizer.zero_grad() #no use model.zero_grad() because the freezen part of the model
    loss.backward()
    optimizer.step()

x=torch.tensor([math.pi/6])
x=x.reshape(-1,1)
print(x.shape)
with torch.no_grad():
    res=model(x)
    print("sin(pi/6):{}".format(res[0]))