from enum import Enum
import unittest
from tags import TagAttribs
from tags import TagObjName
from tags import InvalidTagObjName
from words import WordAttribs
from words import WordObjName
from words import WordKind
from words import InvalidWordObjName
from bank_utils import VocabError
from bank_utils import BankUtils
import readline

class InvalidCommandError(VocabError):
    def __init__(self, message = ""):
        self.message = message

    def __str__(self):
        return self.message

class CommandKind(Enum):
    REVERSE_WORD_LOOKUP = 'reverse-word-lookup'

    WORD_DELETE = 'word-obj-delete'
    WORD_LOOKUP = 'word-obj-lookup'
    WORD_ASSIGN = 'word-obj-assign'
    WORD_REMOVE = 'word-obj-remove'
    WORD_APPEND = 'word-obj-append'
    WORD_OPPOSITE_SEX = 'word-opposite-sex'

    TAG_CLEAR = 'tag-obj-clear'
    TAG_DELETE = 'tag-obj-delete'
    TAG_LOOKUP = 'tag-obj-lookup'
    TAG_ASSIGN = 'tag-obj-assign'
    TAG_REMOVE = 'tag-obj-remove'
    TAG_APPEND = 'tag-obj-append'

    PHRASE_LOOKUP = 'phrase-lookup'
    REVERSE_PHRASE_LOOKUP = 'reverse-phrase-lookup'

    COURSE_LOOKUP = 'course-lookup'
    COURSE_DELETE = 'course-delete'
    COURSE_CREATE = 'course-create'
    COURSE_CHANGE = 'course-change'
    COURSE_CLEAR  = 'course-clear'
    COURSE_PUT    = 'course-put'
    COURSE_REMOVE = 'course-remove'

    SAVE = 'save'
    LOAD = 'load'
    EXIT = 'exit'
    CONTINUE = 'continue'
    LIST = 'list'

class CommandParser():
    def __init__(self, bank):
        self.bank = bank

    def complete(self, text, state):
        if state == 0:
            self.matches = []
            line = readline.get_line_buffer()
            sline = line.lstrip()
            if text and text.startswith('#'):
                if not line.startswith('cd '):
                    if not self.bank.tags.complete_tags(text, self.matches):
                        return None

            if sline.startswith('?'):
                if not self.bank.words.reverse_complete(sline, text, self.matches):
                    return None
            elif sline.startswith('more ') or sline.startswith('put '):
                if not self.bank.words.complete(sline, text, self.matches):
                    return None
            elif sline.startswith('cd '):
                if not self.bank.courses.complete(text, self.matches):
                    return None
        try:
            return self.matches[state]
        except IndexError:
            return None

    def list_command(self, original, cmd, options):
        return [CommandKind.LIST, options, cmd]

    def reverse_word_lookup(self, original, cmd, options):
        words = [a.strip() for a in cmd.split(',')]
        return [CommandKind.REVERSE_WORD_LOOKUP, options, words]

    def word_lookup(self, original, cmd, options):
        words = [a.strip() for a in cmd.split(',')]
        r = []
        for word in words:
            if not WordObjName.is_valid(word):
                raise InvalidWordObjName("Not a valid word name: " + word + " in '" + original + "'")
            r.append(WordObjName.parse_object_name(word))

        return [CommandKind.WORD_LOOKUP, options,  r]

    def word_delete(self, original, cmd, options):
        words = [a.strip() for a in cmd.split(',')]
        r = []
        for word in words:
            if not WordObjName.is_valid(word):
                raise InvalidWordObjName("Not a valid word name: " + word + " in '" + original + "'")
            r.append(WordObjName.parse_object_name(word))

        return [CommandKind.WORD_DELETE, options, r]

    def tag_lookup(self, original, cmd, options):
        tags = [a.strip() for a in cmd.split(',')]
        r = []
        for tag in tags:
            if not TagObjName.is_valid(tag):
                raise InvalidTagObjName("Not a valid tag name: " + tag + " in '" + original + "'")
            r.append(TagObjName.parse_object_name(tag))

        return [CommandKind.TAG_LOOKUP, options, r]

    def course_put(self, original, cmd, options):
        if not self.bank.course:
            print("WARNING: must first change to a course")
            return [CommandKind.CONTINUE]
        words = [a.strip() for a in cmd.split(',')]
        t = []
        w = []
        p = []
        for word in words:
            if word.startswith('#'):
                if not TagObjName.is_valid(word):
                    raise InvalidTagObjName("Not a valid tag name: " + word + " in '" + original + "'")
                t.append(word)
            else:
                if not WordObjName.is_valid(word):
                    raise InvalidWordObjName("Not a valid word name: " + word + " in '" + original + "'")
                w.append(word)

        return [CommandKind.COURSE_PUT, options, w, t, p]

    def course_remove(self, original, cmd, options):
        if not self.bank.course:
            print("WARNING: must first change to a course")
            return [CommandKind.CONTINUE]
        words = [a.strip() for a in cmd.split(',')]
        t = []
        w = []
        p = []
        for word in words:
            if word.startswith('#'):
                if not TagObjName.is_valid(word):
                    raise InvalidTagObjName("Not a valid tag name: " + word + " in '" + original + "'")
                t.append(word)
            else:
                if not WordObjName.is_valid(word):
                    raise InvalidWordObjName("Not a valid word name: " + word + " in '" + original + "'")
                w.append(word)

        return [CommandKind.COURSE_REMOVE, options, w, t, p]

    def tag_delete(self, original, cmd, options):
        if cmd.find(',') != -1:
            tags = [a.strip() for a in cmd.split(',')]
        else:
            tags = [a.strip() for a in cmd.split(' ')]
        r = []
        for tag in tags:
            if not TagObjName.is_valid(tag):
                raise InvalidTagObjName("Not a valid tag name: " + tag + " in '" + original + "'")
            r.append(TagObjName.parse_object_name(tag))

        return [CommandKind.TAG_DELETE, options, r]

    def tag_clear(self, original, cmd, options):
        if cmd.find(',') != -1:
            tags = [a.strip() for a in cmd.split(',')]
        else:
            tags = [a.strip() for a in cmd.split(' ')]
        r = []
        for tag in tags:
            if not TagObjName.is_valid(tag):
                raise InvalidTagObjName("Not a valid tag name: " + tag + " in '" + original + "'")
            s = TagObjName.parse_object_name(tag)
            if s[2] != 0 or s[4] != 0:
                raise InvalidCommandError("Can not clear an indexed item here: " + original)
            r.append(s)

        return [CommandKind.TAG_CLEAR, options, r]

    def reverse_phrase_lookup(self, original, cmd, options):
        return [CommandKind.REVERSE_PHRASE_LOOKUP, options, cmd]

    def phrase_lookup(self, original, cmd, options):
        return [CommandKind.PHRASE_LOOKUP, options, cmd]

    def phrase_delete(self, original, cmd, options):
        return [CommandKind.PHRASE_DELETE, options, cmd]

    def course_lookup(self, original, cmd, options):
        return [CommandKind.COURSE_LOOKUP, options, cmd]

    def course_delete(self, original, cmd, options):
        return [CommandKind.COURSE_DELETE, options, cmd]

    def course_clear(self, original, cmd, options):
        return [CommandKind.COURSE_CLEAR, options, cmd]

    def course_create(self, original, cmd, options):
        return [CommandKind.COURSE_CREATE, options, cmd]

    def course_change(self, original, cmd, options):
        return [CommandKind.COURSE_CHANGE, options, cmd]

    def word_opposite_sex(self, original, cmd, symbol, cmd_kind, kind = WordKind.Unknown):
        idx = cmd.find(symbol)
        symbol_len = len(symbol)

        if idx == 0 or idx == (len(cmd) - symbol_len):
            raise InvalidCommandError("Invalid command: " + original)

        value = cmd[idx+symbol_len:].strip()
        word = cmd[:idx].strip()

        if not WordObjName.is_valid(word):
            raise InvalidWordObjName("Not a valid word name: " + word + " in '" + original + "'")

        name, mkind, meaning_idx, attribute, attribute_idx = tuple(WordObjName.parse_object_name(word))
        if attribute:
            raise InvalidCommandError("Does not work with attributes: " + original)

        name2, mkind2, meaning_idx2, attribute, attribute_idx = tuple(WordObjName.parse_object_name(value))
        if attribute:
            raise InvalidCommandError("Does not work with attributes: " + original)

        return [cmd_kind, name, mkind, meaning_idx, name2, mkind2, meaning_idx2]

    def word_assignment(self, original, cmd, symbol, cmd_kind, kind = WordKind.Unknown):
        idx = cmd.find(symbol)
        symbol_len = len(symbol)

        if idx == 0 or idx == (len(cmd) - symbol_len):
            raise InvalidCommandError("Invalid command: " + original)

        value = cmd[idx+symbol_len:].strip()
        word = cmd[:idx].strip()

        if not WordObjName.is_valid(word):
            raise InvalidWordObjName("Not a valid word name: " + word + " in '" + original + "'")

        name, mkind, meaning_idx, attribute, attribute_idx = tuple(WordObjName.parse_object_name(word))

        if mkind != WordKind.Unknown:
            if kind == WordKind.Unknown:
                kind = mkind
            elif kind != mkind:
                raise VocabError("Mismatched kind for : " + word + " in " + original)

        if attribute:
            if WordAttribs.is_list(attribute):
                values = BankUtils.dequote_and_split(value)
            else:
                values = [value]
            if attribute_idx != 0 and cmd_kind != CommandKind.WORD_ASSIGN:
                raise InvalidCommandError("Can not remove or append to an attribute value: " + original)
        else:
            values = [a.strip() for a in value.split(',')]

        return [cmd_kind, name, kind, meaning_idx, attribute, attribute_idx, values]

    def tag_assignment(self, original, cmd, symbol, cmd_kind):
        idx = cmd.find(symbol)
        if idx == -1:
            raise InvalidCommandError("Invalid tag command " + original)

        symbol_len = len(symbol)

        if idx == 0 or idx == (len(cmd) - symbol_len):
            raise InvalidCommandError("Invalid command: " + original)

        value = cmd[idx+symbol_len:].strip()
        tag = cmd[:idx].strip()

        if not TagObjName.is_valid(tag):
            raise InvalidTagObjName("Not a valid tag name: " + tag + " in '" + original + "'")

        base_name, sub_name, word_idx, attrib, attrib_idx = tuple(TagObjName.parse_object_name(tag))

        if attrib:
            if TagAttribs.is_list(attrib):
                values = BankUtils.dequote_and_split(value)
            else:
                values = [value]
        else:
            values = [a.strip() for a in value.split(',')]

        return [cmd_kind, base_name, sub_name, word_idx, attrib, attrib_idx, values]

    def parse_command(self, original):
        # remove any double space
        cmd = ' '.join(original.strip().split())
        if cmd.startswith('#'):
            if cmd.find('-=') != -1:
                r = self.tag_assignment(original, cmd, '-=', CommandKind.TAG_REMOVE)
                if r[3] != 0:
                    raise InvalidCommandError("Can not remove something from a tag member")
                return r
            elif cmd.find('+=') != -1:
                r = self.tag_assignment(original, cmd, '+=', CommandKind.TAG_APPEND)
                if r[3] != 0:
                    raise InvalidCommandError("Can not append something to a tag member")
                return r
            elif cmd.find('=') != -1:
                return self.tag_assignment(original, cmd, '=', CommandKind.TAG_ASSIGN)
            else:
                idx = cmd.find(' ')
                if idx == -1:
                    return self.tag_lookup(original, cmd, "")
                else:
                    cmd = cmd[:idx] + ' = ' + cmd[idx+1:]
                    return self.tag_assignment(original, cmd, '=', CommandKind.TAG_ASSIGN)

        elif cmd.startswith('('):
            idx = cmd.find(')')
            if idx == -1:
                raise InvalidCommandError("Not closed parentheses: " + original)
            kind = WordKind.parse(cmd[1:idx].strip())
            cmd = cmd[idx+1:]
            if cmd.find('-=') != -1:
                return self.word_assignment(original, cmd, '-=', CommandKind.WORD_REMOVE, kind)
            elif cmd.find('+=') != -1:
                return self.word_assignment(original, cmd, '+=', CommandKind.WORD_APPEND, kind)
            else:
                return self.word_assignment(original, cmd, '=', CommandKind.WORD_ASSIGN, kind)

        elif cmd.find("==") != -1:

            return self.word_opposite_sex(original, cmd, '==', CommandKind.WORD_OPPOSITE_SEX)

        elif cmd.find("=") != -1:
            if cmd.find('-=') != -1:
                return self.word_assignment(original, cmd, '-=', CommandKind.WORD_REMOVE)
            elif cmd.find('+=') != -1:
                return self.word_assignment(original, cmd, '+=', CommandKind.WORD_APPEND)
            else:
                return self.word_assignment(original, cmd, '=', CommandKind.WORD_ASSIGN)

        elif cmd.startswith('load '):

            return [CommandKind.LOAD, cmd[5:].strip()]

        elif cmd == 'save':

            return [CommandKind.SAVE]

        elif cmd == 'exit':

            return [CommandKind.EXIT]

        elif cmd.startswith('mkdir '):

            return self.course_create(original, cmd[6:], "")

        elif cmd.startswith('cd '):

            return self.course_change(original, cmd[3:], "")

        elif cmd == 'cd':

            return self.course_change(original, '', "")

        elif cmd == 'clear' and self.bank.course:

            return self.course_clear(original, self.bank.course.name, "")

        elif cmd.startswith('clear '):
            cmd = cmd[6:]
            options = ""
            if cmd.startswith('-'):
                idx = cmd.find(' ')
                if idx == -1:
                    raise InvalidCommandError("More requires an argument: " + original)
                options = cmd[1:idx]
                cmd = cmd[idx+1]

            if cmd.startswith('#'):
                return self.tag_clear(original, cmd, options)
            elif cmd.startswith('$'):
                return self.course_clear(original, cmd[1:].strip(), options)

        elif cmd.startswith('put '):
            cmd = cmd[4:]
            options = ""
            if cmd.startswith('-'):
                idx = cmd.find(' ')
                if idx == -1:
                    raise InvalidCommandError("put requires an argument: " + original)
                options = cmd[1:idx]
                cmd = cmd[idx+1]

            return self.course_put(original, cmd, options)

        elif cmd.startswith('remove '):
            cmd = cmd[7:]
            options = ""
            if cmd.startswith('-'):
                idx = cmd.find(' ')
                if idx == -1:
                    raise InvalidCommandError("put requires an argument: " + original)
                options = cmd[1:idx]
                cmd = cmd[idx+1]

            return self.course_remove(original, cmd, options)

        elif cmd.startswith('del '):
            cmd = cmd[5:]
            options = ""
            if cmd.startswith('-'):
                idx = cmd.find(' ')
                if idx == -1:
                    raise InvalidCommandError("rm requires an argument: " + original)
                options = cmd[1:idx]
                cmd = cmd[idx+1]

            if cmd.startswith('#'):
                return self.tag_delete(original, cmd, options)
            elif cmd.startswith('$'):
                return self.course_delete(original, cmd[1:].strip(), options)
            elif cmd.startswith('>'):
                return self.phrase_delete(original, cmd[1:].strip(), options)
            else:
                return self.word_delete(original, cmd, options)

        elif cmd.startswith('ls'):
            cmd = cmd[2:]
            if cmd.startswith(' '):
                cmd = cmd[1:]
            options = ""
            if cmd.startswith('-'):
                idx = cmd.find(' ')
                if idx != -1:
                    options = cmd[1:idx]
                    cmd = cmd[idx+1:]
                else:
                    options = cmd[1:]
                    cmd = ""

            return self.list_command(original, cmd, options)
        elif cmd.startswith('more '):
            cmd = cmd[5:]
            options = ""
            if cmd.startswith('-'):
                idx = cmd.find(' ')
                if idx == -1:
                    raise InvalidCommandError("More requires an argument: " + original)
                options = cmd[1:idx]
                cmd = cmd[idx+1:]

            if cmd.startswith('#'):
                return self.tag_lookup(original, cmd, options)
            elif cmd.startswith('$'):
                return self.course_lookup(original, cmd[1:].strip(), options)
            elif cmd.startswith('>'):
                return self.phrase_lookup(original, cmd[1:].strip(), options)
            elif cmd.startswith('<'):
                return self.reverse_phrase_lookup(original, cmd[1:].strip(), options)
            elif cmd.startswith('?'):
                return self.reverse_word_lookup(original, cmd[1:].strip(), options)
            else:
                return self.word_lookup(original, cmd, options)

        elif cmd.startswith('?'):

            return self.reverse_word_lookup(original, cmd[1:].strip(), "")

        elif cmd.startswith('<'):

            return self.reverse_phrase_lookup(original, cmd[1:].strip(), "")

        elif cmd.startswith('>'):

            return self.phrase_lookup(original, cmd[1:].strip(), "")

        elif len(cmd) > 0:

            space_count = cmd.count(' ')

            if space_count > 3:
                return self.phrase_lookup(original, cmd, "")
            elif space_count == 3:
                if cmd.startswith('el ') or cmd.startswith('los ') or cmd.startswith('la ') or cmd.startswith('las '):
                    return self.word_lookup(original, cmd, "")
                else:
                    return self.phrase_lookup(original, cmd, "")
            else:
                return self.word_lookup(original, cmd, "")
        else:
            return [CommandKind.CONTINUE]

class Testing(unittest.TestCase):
    def check(self, commands):
        parser = CommandParser(None)
        for key in commands:
            r = parser.parse_command(key)
            try:
                assert r == commands[key]
            except:
                print(r)
                print(commands[key])
                assert False

    def test_lookups(self):
        commands = {
            "?mother": [CommandKind.REVERSE_WORD_LOOKUP, "", ["mother"]],
            "el padre": [CommandKind.WORD_LOOKUP, "", [["padre", WordKind.Male, 0, None, 0]]],
            "mi madre es inteligente": [CommandKind.PHRASE_LOOKUP, "", 'mi madre es inteligente'],
        }
        self.check(commands)

    def test_word_assignment(self):
        commands = {
            "el padre = father": [CommandKind.WORD_ASSIGN, "padre", WordKind.Male, 0, None, 0, ["father"]],
            "el padre=father": [CommandKind.WORD_ASSIGN, "padre", WordKind.Male, 0, None, 0, ["father"]],
            "(adj) inteligente = intelligent": [CommandKind.WORD_ASSIGN, "inteligente", WordKind.Adjective, 0, None, 0, ["intelligent"]],
            "el padre[1]=father": [CommandKind.WORD_ASSIGN, "padre", WordKind.Male, 1, None, 0, ["father"]],
            "el padre +=father": [CommandKind.WORD_APPEND, "padre", WordKind.Male, 0, None, 0, ["father"]],
            "el padre -=father": [CommandKind.WORD_REMOVE, "padre", WordKind.Male, 0, None, 0, ["father"]],
            "el padre-=father": [CommandKind.WORD_REMOVE, "padre", WordKind.Male, 0, None, 0, ["father"]],
            "el padre[1]:IMAGES = '/path/to/image.file'": [CommandKind.WORD_ASSIGN, "padre", WordKind.Male, 1, WordAttribs.IMAGES, 0, ['/path/to/image.file']],
            "el padre[1]:IMAGES -= '/path/to/image.file'": [CommandKind.WORD_REMOVE, "padre", WordKind.Male, 1, WordAttribs.IMAGES, 0, ['/path/to/image.file']],
            "el padre[1]:IMAGES += '/path/to/image.file'": [CommandKind.WORD_APPEND, "padre", WordKind.Male, 1, WordAttribs.IMAGES, 0, ['/path/to/image.file']],
        }
        self.check(commands)

        parser = CommandParser(None)
        try:
            r = parser.parse_command("el padre[1]:IMAGES[1] -= '/path/to/image.file'")
            assert r == [CommandKind.WORD_REMOVE, "padre", WordKind.Male, 1, WordAttribs.IMAGES, 0, ['/path/to/image.file']]
        except InvalidCommandError as e:
            assert e.message == "Can not remove or append to an attribute value: el padre[1]:IMAGES[1] -= '/path/to/image.file'"

        try:
            r = parser.parse_command("el padre&[1]:IMAGES[1] -= '/path/to/image.file'")
            assert r == [CommandKind.WORD_REMOVE, "padre&", 1, WordKind.Male, WordAttribs.IMAGES, 0, ['/path/to/image.file']]
        except InvalidWordObjName as e:
            assert e.message == "Not a valid word name: el padre&[1]:IMAGES[1] in 'el padre&[1]:IMAGES[1] -= '/path/to/image.file''"

        try:
            r = parser.parse_command("= '/path/to/image.file'")
            assert r == [CommandKind.WORD_REMOVE, "", WordKind.Unknown, 0, None, 0, ['/path/to/image.file']]
        except InvalidCommandError as e:
            assert e.message == "Invalid command: = '/path/to/image.file'"

        try:
            r = parser.parse_command("el padre =")
            assert r == [CommandKind.WORD_REMOVE, "padre", WordKind.Male, 0, None, 0, ['']]
        except InvalidCommandError as e:
            assert e.message == "Invalid command: el padre ="

    def test_tag_assignment(self):
        commands = {
            "#family el padre": [CommandKind.TAG_ASSIGN, "family", '*', 0, None, 0, ["el padre"]],
            "#family-beginner el padre": [CommandKind.TAG_ASSIGN, "family", 'beginner', 0, None, 0, ["el padre"]],
            "#family-beginner[1] el padre": [CommandKind.TAG_ASSIGN, "family", 'beginner', 1, None, 0, ["el padre"]],
            "#family-beginner = el padre, la madre": [CommandKind.TAG_ASSIGN, "family", 'beginner', 0, None, 0, ["el padre", "la madre"]],
            "#family-beginner += la madre": [CommandKind.TAG_APPEND, "family", 'beginner', 0, None, 0, ["la madre"]],
            "#family-beginner": [CommandKind.TAG_LOOKUP, "", [["family", 'beginner', 0, None, 0]]],
        }
        self.check(commands)

        parser = CommandParser(None)
        try:
            r = parser.parse_command("#family-beginner[1]:IMAGES el padre")
            assert r == [CommandKind.TAG_ASSIGN, "family", 'beginner', 1, TagAttribs.IMAGES, 0, ["el padre"]]
        except InvalidTagObjName as e:
            assert e.message == "Not a valid tag name: #family-beginner[1]:IMAGES in '#family-beginner[1]:IMAGES el padre'"

        try:
            r = parser.parse_command("#family-beginner[1]:IMAGES el padre")
            assert r == [CommandKind.TAG_ASSIGN, "family", 'beginner', 1, TagAttribs.IMAGES, 0, ["el padre"]]
        except InvalidTagObjName as e:
            assert e.message == "Not a valid tag name: #family-beginner[1]:IMAGES in '#family-beginner[1]:IMAGES el padre'"

    def test_tag_delete(self):
        commands = {
            "rm #family": [CommandKind.TAG_DELETE, '', [["family", '*', 0, None, 0]]],
            "rm #family-beginner": [CommandKind.TAG_DELETE, '', [["family", 'beginner', 0, None, 0]]],
            "rm #family-beginner:IMAGES": [CommandKind.TAG_DELETE, '', [["family", 'beginner', 0, TagAttribs.IMAGES, 0]]],
            "rm #family-beginner[2]": [CommandKind.TAG_DELETE, '', [["family", 'beginner', 2, None, 0]]],
            "rm #family-beginner[2], #family-advanced:IMAGES": [CommandKind.TAG_DELETE, '', [["family", 'beginner', 2, None, 0], ["family", 'advanced', 0, TagAttribs.IMAGES, 0]]],
            "rm #family-beginner[2] #family-advanced:IMAGES": [CommandKind.TAG_DELETE, '', [["family", 'beginner', 2, None, 0], ["family", 'advanced', 0, TagAttribs.IMAGES, 0]]],
        }
        self.check(commands)

    def test_word_delete(self):
        commands = {
            "rm el padre": [CommandKind.WORD_DELETE, '', [["padre", WordKind.Male, 0, None, 0]]],
            "rm el padre[1]": [CommandKind.WORD_DELETE, '', [["padre", WordKind.Male, 1, None, 0]]],
            "rm el padre[1]:IMAGES": [CommandKind.WORD_DELETE, '', [["padre", WordKind.Male, 1, WordAttribs.IMAGES, 0]]],
            "rm el padre[1]:IMAGES, la madre": [CommandKind.WORD_DELETE, '', [["padre", WordKind.Male, 1, WordAttribs.IMAGES, 0], ["madre", WordKind.Female, 0, None, 0]]],
        }
        self.check(commands)

if __name__ == 'main':
    test_suite = unittest.TestSuite()
    test_suite.addTest(Testing("test_word_assignment"))
    test_suite.addTest(Testing("test_tag_assignment"))
    test_suite.addTest(Testing("test_lookups"))
    test_suite.addTest(Testing("test_tag_delete"))
    test_suite.addTest(Testing("test_word_delete"))
    runner = unittest.TextTestRunner()
    runner.run(test_suite)


import numpy as np
import pandas as pd
import pickle

from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split

class FeaturePreparer:
    # when we call the Feature preparer for the first time
    # we initialise it with the training data set, which is
    # then stored as an attribute(raw_data)
    def __init__(self, *, raw_data):
        """
        Initialize the object with a file in the data/raw folder
        :param raw_data:
        """
        self.raw_data = pd.read_csv(f"../data/raw/{raw_data}")
        self.prepared_data = None
        self.X_train = None
        self.X_test = None
        self.y_train = None
        self.y_test = None
        self.categorical_variables = None
        self.low_card_vars = None
        self.high_card_vars = None
        self.label_encoder = LabelEncoder()

    def add_labels(self) -> None:
        """Adds data labels to the raw training set"""
        labels = pd.read_csv("../data/raw/train_labels.csv")

        self.prepared_data = pd.merge(self.prepared_data, labels, how="left", left_on="id", right_on="id")

    def filter_data(self) -> None:
        """Adds extra columns and filters out necessary columns"""

        self.prepared_data.drop("basin", axis=1)

        self.prepared_data["installed_DWE"] = np.where(self.prepared_data["installer"] == "DWE", 1, 0)

        self.prepared_data["has_wpt"] = np.where(self.prepared_data["wpt_name"] != "none", 1, 0)

        pickle_in = open("../parameters/model_features.pkl", "rb")

        columns_to_keep = pickle.load(pickle_in)

        print(f"Columns for modeling {columns_to_keep}")

        self.prepared_data = self.prepared_data[columns_to_keep]

    def separate_variables(self) -> None:
        self.categorical_variables = [var for var in self.prepared_data.columns if self.prepared_data[var].dtypes == "O"]

        self.low_card_vars = [var for var in self.categorical_variables if self.prepared_data[var].nunique() < 20]

        self.high_card_vars = [var for var in self.categorical_variables if var not in self.low_card_vars]

        print(f"There are {len(self.low_card_vars)} low cardinality (n < 20) categorical variables")

        print(f"There are {len(self.high_card_vars)} high cardinality (n > 20) categorical variables")

    def handle_na(self) -> None:

        #TODO: Should this be handled better?

        self.prepared_data.dropna()
        print(f"Shape of DataFrame after cleaning: {self.prepared_data.shape}")

    def encode_categorical_variables(self):
        self.low_card_vars.remove("status_group")

        # TODO: Remove once I find out how to encode high cardinality variables
        self.prepared_data.drop(self.high_card_vars, axis=1, inplace=True)

        dummies = pd.get_dummies(self.prepared_data[self.low_card_vars])

        self.prepared_data = pd.concat([self.prepared_data, dummies], axis=1)

        self.prepared_data.drop(self.low_card_vars, axis=1, inplace=True)

        print(f"Categorical variables encoded, new shape is: {self.prepared_data.shape}")

    def split_data(self) -> None:
        X = self.prepared_data.drop("status_group", axis=1)

        y = self.prepared_data["status_group"]

        self.X_train, self.X_test, self.y_train, self.y_test = train_test_split(
            X, y, test_size=0.2, random_state=42)

        print(f"X_train shape: {self.X_train.shape}, X_test shape: {self.X_test.shape}")

    def encode_label(self) -> None:
        self.y_train = self.label_encoder.fit_transform(self.y_train)

        self.y_test = self.label_encoder.transform(self.y_test)

        print("Labels encoded")

    def prepare_data(self):
        """Runs all data preparation steps"""
        self.prepared_data = self.raw_data.copy(deep=True)

        self.add_labels()
        self.filter_data()
        self.separate_variables()
        self.handle_na()
        self.encode_categorical_variables()
        self.split_data()
        self.encode_label()

        print("-" * 10, "Data prepared", "-" * 10)

        self.save_data()

    def save_data(self) -> None:
        self.prepared_data.to_csv("../data/interim/df_interim.csv", index = False)
        print("Data saved")

if __name__ == "__main__":
    test = FeaturePreparer(raw_data="train_data.csv")
    test.prepare_data()


import creature
import ability

class Player:
    def __init__(self, mode, **kwargs):
        if (mode == 'init'):
            self.name = kwargs["name"]
            deck = kwargs["deck"]
            self.creatures = []
            self.cards = []
            self.creature_index = 0
            self.finished = "false"
            self.discard = 0
            for index in range(0, 5):
                self.cards.append(deck.get_card())
        elif (mode == 'load'):
            json = kwargs["json"]
            self.name = json["name"]
            self.finished = json["finished"]
            self.discard = json["discard"]
            self.creature_index = json["creature_index"]
            self.cards = []
            self.creatures = []
            for card in json["cards"]:
                self.cards.append(card)
            for creature in json["creatures"]:
                self.creatures.append(Creature(creature))

    def add_creature(self, card):
        creature = Creature(self.name, card, self.creature_index)
        self.creatures.append(creature)
        self.creature_index = self.creature_index + 1

    def json(self):
        json = {}
        json["name"] = self.name
        json["creatures"] = {}
        json["finished"] = self.finished
        json["creature_index"] = self.creature_index
        for creature in self.creatures:
            json["creatures"][creature.id] = creature.json()
        json["cards"] = []
        for card in self.cards:
            json["cards"].append(card)
        json["discard"] = self.discard
        return json


from tezromach.layers import Normalization, Linear, Tanh, Sigmoid, ReLU
from tezromach.network import NeuralNet
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn import metrics
from sklearn import preprocessing

encoder = preprocessing.OneHotEncoder(sparse=False)

inputs = np.genfromtxt("mnist_data/inputs.csv", delimiter=',')
targets = np.genfromtxt("mnist_data/targets.csv", delimiter=',')
targets = encoder.fit_transform(np.array(targets).reshape(-1, 1))

inputs_train, inputs_test, targets_train, targets_test = train_test_split(inputs, targets, test_size=0.15)
print("inputs shape =", inputs.shape)
print("targets shape =", targets.shape)

net = NeuralNet([
    Normalization(),

    Linear(input_size=64, output_size=100),
    ReLU(),
    Linear(input_size=100, output_size=80),
    Sigmoid(),
    Linear(input_size=80, output_size=10),
    Sigmoid()
])

print("start fit NeuralNet")
net.fit(inputs_train, targets_train, verbose_print=10, num_epochs=200, learning_rate=0.1)
print("done fit NeuralNet")
print(net)

net.save_model("model_mnist")

predicted = net.predict(inputs_train)
predicted = np.argmax(predicted, axis=1)
predicted_targets_train = np.argmax(targets_train, axis=1)

print("accuracy metrics (train): ", metrics.accuracy_score(predicted, predicted_targets_train))

predicted = net.predict(inputs_test)
predicted = np.argmax(predicted, axis=1)
predicted_targets_test = np.argmax(targets_test, axis=1)

print("accuracy metrics (test): ", metrics.accuracy_score(predicted, predicted_targets_test))


# -*- coding: utf-8 -*-
from django.db import connection
from collections import namedtuple
from operator import itemgetter
from itertools import groupby

from django.db.models import Q

from common.lib.utils import dictfetchall
from common.lib.date_utils import create_date_range
from data_sources.models import YandexDirectStat, GoogleAdwordsStat, DirectPhraseStat, AdwordsPhraseStat, \
    DirectBannerStat, AdwordsBannerStat
from data_sources.serializers import YandexDirectStatQuerySetSerializer, GoogleAdwordsStatQuerySetSerializer, \
    DirectPhraseStatQuerySetSerializer, AdwordsPhraseStatQuerySetSerializer, DirectBannerStatQuerySetSerializer, \
    AdwordsBannerStatQuerySetSerializer
from statistics.lib.xls_stats_builder import StatsXlsTotalBuilder, StatsXlsCampaignBuilder, StatsXlsPhraseBuilder, \
    StatsXlsBannerBuilder

DIRECT_NAME = 'Yandex Direct'
ADWORDS_NAME = 'Google Adwords'

class StatsGetterTotal(object):

    ADWORDS_NAME = ADWORDS_NAME
    DIRECT_NAME = DIRECT_NAME

    def __init__(self, project, start_date, end_date, **kwargs):
        self.project = project
        self.range = create_date_range(start_date, end_date)

        self.previous_range = None
        prev_range = kwargs.get('prev_range')
        if prev_range:
            if prev_range == 'previous':
                self.previous_range = self.range.create_previous()
            else:
                dates = prev_range.split('_')
                self.previous_range = create_date_range(*dates)

    def get_stats_summary(self):
        populated_record_direct = self._create_stats_record(
            YandexDirectStat, self.DIRECT_NAME, DirectRecordUpdater
        )

        populated_record_adwords = self._create_stats_record(
            GoogleAdwordsStat, self.ADWORDS_NAME, AdwordsRecordUpdater
        )

        return [populated_record_direct, populated_record_adwords]

    def _create_stats_record(self, model_cls, source_name, record_updater_cls):
        stats = self._get_serialized_stats(model_cls, self.range)
        record = self._get_initial_record(source_name)
        populated_primary_record = self._get_record_populated_by_helper(record, stats, record_updater_cls)

        if self.previous_range:
            stats = self._get_serialized_stats(model_cls, self.previous_range)
            populated_record = self._get_record_populated_by_helper(populated_primary_record, stats, record_updater_cls.PREVIOUS)
        else:
            populated_record = populated_primary_record
        return populated_record

    def get_stats_summary_as_xls(self):
        stats = self.get_stats_summary()
        xls_builder = self._get_xls_builder()
        return xls_builder.get_xls_obj(stats)

    def _get_xls_builder(self):
        return StatsXlsTotalBuilder()

    def _get_serialized_stats(self, model_cls, _range):
        select_related_strings = [
            'campaign'
        ]
        stats = model_cls.objects.filter(
            campaign__data_resource__projects=self.project,
            date__range=_range.as_tuple
        ).select_related(*select_related_strings)

        qs_serializer = model_cls.get_qs_serializer()(
            stats, select_related_strings=select_related_strings
        )

        return qs_serializer.serialize()

    def _get_record_populated_by_helper(self, record, stats, helper_cls):
        updater = helper_cls(record)
        for st in stats:
            updater.update_by(st)
        updater.after_update()
        return record

    def _get_initial_record(self, source):
        return {
            'source': source,
            'sessions': 0,
            'shows': 0,
            'clicks': 0,
            'cost': 0,
            'clicks_search': 0,
            'shows_search': 0,
            'shows_context': 0,
            'shows_search_premium': 0,
            'shows_search_other': 0,
            'clicks_positions': 0,
            'shows_positions': 0,
            'income': 0,
            'leads': 0,
            'users': 0,
            'new_users': 0,
            'page_views': 0,
            'bounces': 0,
            'session_duration': 0,
            'transaction_revenue': 0,
            'prev_sessions': 0,
            'prev_shows': 0,
            'prev_clicks': 0,
            'prev_cost': 0,
            'prev_clicks_search': 0,
            'prev_shows_context': 0,
            'prev_shows_search': 0,
            'prev_shows_search_premium': 0,
            'prev_shows_search_other': 0,
            'prev_clicks_positions': 0,
            'prev_shows_positions': 0,
            'prev_income': 0,
            'prev_leads': 0,
            'prev_users': 0,
            'prev_new_users': 0,
            'prev_page_views': 0,
            'prev_bounces': 0,
            'prev_session_duration': 0,
            'prev_transaction_revenue': 0,
        }

class StatsGetterItem(StatsGetterTotal):
    DIRECT_STAT_MODEL = None
    ADWORDS_STAT_MODEL = None

    def __init__(self, project, start_date, end_date, **kwargs):
        super(StatsGetterItem, self).__init__(project, start_date, end_date, **kwargs)
        self.source_name = kwargs.get('source_name')

    def get_stats_summary(self):
        stats = self.get_direct_stats()
        stats.extend(self.get_adwords_stats())
        return stats

    def is_adwords(self):
        return self.source_name == self.ADWORDS_NAME.replace(' ', '_').lower()

    def get_direct_stats(self):
        return self._get_source_stats(self.DIRECT_STAT_MODEL, self.DIRECT_NAME, DirectRecordUpdater)

    def get_adwords_stats(self):
        return self._get_source_stats(self.ADWORDS_STAT_MODEL, self.ADWORDS_NAME, AdwordsRecordUpdater)

    def _get_source_stats(self, stat_model, source_name, record_updater):
        group_func = self._get_group_func()
        serialized_stats = self._get_stats(stat_model, self.range)

        prev_serialized_stats = []
        if self.previous_range:
            prev_serialized_stats = self._get_stats(stat_model, self.previous_range)

        result = self._get_result(
            serialized_stats, prev_serialized_stats, group_func, source_name, record_updater
        )
        return result

    def _get_result(self, serialized_stats, prev_serialized_stats, group_func, source_name, record_updater):
        serialized_stats.sort(key=group_func)
        prev_serialized_stats.sort(key=group_func)

        result = []
        prev_dict = {}
        for item, groupped_stats in groupby(prev_serialized_stats, key=group_func):
            stats_list = list(groupped_stats)
            ids_dict = self._get_items_ids_dict(item, stats_list)
            prev_dict[ids_dict['id']] = stats_list

        for item, groupped_stats in groupby(serialized_stats, key=group_func):
            stats_list = list(groupped_stats)
            record = self._get_initial_record(source_name, item, stats_list)

            ids_dict = self._get_items_ids_dict(item, stats_list)

            populated_primary_record = self._get_record_populated_by_helper(record, stats_list, record_updater)
            populated_record = self._get_record_populated_by_helper(
                populated_primary_record,
                prev_dict.get(ids_dict['id'], []),
                record_updater.PREVIOUS
            )

            result.append(populated_record)
        return result

    def _get_items_ids_dict(self, item, stats_list):
        return {'id': item['id']}

    def _get_stats(self, model_cls, _range):
        select_related_strings = self._get_select_related_strings()
        stats = model_cls.objects.filter(
            self._get_stat_filter_q(),
            date__range=_range.as_tuple
        ).select_related(*select_related_strings)

        serialized_stats = model_cls.get_qs_serializer()(
            stats, select_all_relations=False,
            select_related_strings=select_related_strings
        ).serialize()
        return serialized_stats

    def _get_initial_record(self, source, item, stats_list):
        return super(StatsGetterItem, self)._get_initial_record(source)

    def _get_stat_filter_q(self):
        raise NotImplementedError

    def _get_group_func(self):
        raise NotImplementedError

    def _get_select_related_strings(self):
        raise NotImplementedError

class StatsGetterCampaign(StatsGetterItem):

    DIRECT_STAT_MODEL = YandexDirectStat
    ADWORDS_STAT_MODEL = GoogleAdwordsStat

    def _get_stat_filter_q(self):
        return Q(campaign__data_resource__projects=self.project)

    def _get_group_func(self):
        return itemgetter('campaign')

    def _get_select_related_strings(self):
        return ['campaign']

    def _get_xls_builder(self):
        return StatsXlsCampaignBuilder()

    def _get_initial_record(self, source_name, campaign, stats_list):
        record = super(StatsGetterCampaign, self)._get_initial_record(source_name, campaign, stats_list)
        record.update({
            'campaign_id': campaign['id'],
            'campaign': campaign['name']
        })
        return record

class StatsGetterPhrase(StatsGetterItem):

    DIRECT_STAT_MODEL = DirectPhraseStat
    ADWORDS_STAT_MODEL = AdwordsPhraseStat

    def get_direct_stats(self):
        return self._get_source_stats(self._get_direct_sql, self.DIRECT_NAME, DirectRecordUpdater)

    def get_adwords_stats(self):
        return self._get_source_stats(self._get_adwords_sql, self.ADWORDS_NAME, AdwordsRecordUpdater)

    def _get_xls_builder(self):
        return StatsXlsPhraseBuilder()

    def _get_source_stats(self, sql_getter, source_name, record_updater):
        group_func = self._get_group_func()
        serialized_stats = self._get_stats(sql_getter, self.range)

        prev_serialized_stats = []
        if self.previous_range:
            prev_serialized_stats = self._get_stats(sql_getter, self.previous_range)

        result = self._get_result(
            serialized_stats, prev_serialized_stats, group_func, source_name, record_updater
        )
        return result

    def _get_initial_record(self, source_name, item, stats_list):
        record = super(StatsGetterPhrase, self)._get_initial_record(source_name, item, stats_list)

        ids_dict = self._get_items_ids_dict(item, stats_list)

        record.update({
            'phrase': item.phrase,
            'phrase_id': ids_dict['id'],
            'system_id': ids_dict['system_id'],
            'campaign': item.campaign_name,
            'campaign_id': item.campaign_id
        })
        return record

    def _get_stats(self, sql_getter, _range):
        with connection.cursor() as cursor:
            params = [self.project.pk, _range.start.isoformat(), _range.end.isoformat()]
            select_sql = sql_getter()
            cursor.execute(select_sql, params)
            return dictfetchall(cursor)

    def _get_direct_sql(self):
        return """
            SELECT
                dps.*,
                dp.id dp_phrase_id, dp.phrase_id phrase_external_id, dp.phrase, dp.banner_id,
                ydc.id campaign_id, ydc.name campaign_name
            FROM direct_phrase_stat dps
            JOIN direct_phrase dp ON dps.phrase_id=dp.id
            JOIN yandex_direct_campaign ydc ON dp.campaign_id=ydc.id
            JOIN projects_data_resources pdr ON ydc.data_resource_id=pdr.data_resource_id
            WHERE pdr.project_id=%s AND dps.date BETWEEN %s AND %s;
        """

    def _get_adwords_sql(self):
        return """
            SELECT
                aps.*,
                ap.id dp_phrase_id, ap.phrase_id phrase_external_id, ap.phrase,
                ac.id campaign_id, ac.name campaign_name
            FROM adwords_phrase_stat aps
            JOIN adwords_phrase ap ON aps.phrase_id=ap.id
            JOIN adwords_campaign ac ON ap.campaign_id=ac.id
            JOIN projects_data_resources pdr ON ac.data_resource_id=pdr.data_resource_id
            WHERE pdr.project_id=%s AND aps.date BETWEEN %s AND %s;
        """

    def _get_items_ids_dict(self, item, stats_list):
        item_id = stats_list[0]['dp_phrase_id']
        system_id = item.phrase_external_id
        if not system_id:
            item_id = self._get_unique_joined_string(stats_list, lambda st: str(st['dp_phrase_id']))
            system_id = self._get_unique_joined_string(stats_list, lambda st: str(st['banner_id']))

        return {
            'id': item_id,
            'system_id': system_id
        }

    def _get_unique_joined_string(self, stats_list, getter_fn):
        return ','.join(set(map(getter_fn, stats_list)))

    def _get_stat_filter_q(self):
        return Q(phrase__campaign__data_resource__projects=self.project)

    def _get_select_related_strings(self):
        return ['phrase__campaign']

    def _get_group_func(self):
        nt = namedtuple('GroupItem', ['phrase_external_id', 'phrase', 'campaign_id', 'campaign_name'])
        return lambda n: nt(n['phrase_external_id'], n['phrase'], n['campaign_id'], n['campaign_name'])

class StatsGetterBanner(StatsGetterItem):

    DIRECT_STAT_MODEL = DirectBannerStat
    ADWORDS_STAT_MODEL = AdwordsBannerStat

    def _get_initial_record(self, source_name, item, stats_list):
        record = super(StatsGetterBanner, self)._get_initial_record(source_name, item, stats_list)
        record.update({
            'banner': item['title'],
            'banner_id': item['id'],
            'campaign': item['campaign']['name'],
            'campaign_id': item['campaign']['id']
        })
        return record

    def _get_stat_filter_q(self):
        return Q(banner__campaign__data_resource__projects=self.project)

    def _get_xls_builder(self):
        return StatsXlsBannerBuilder()

    def _get_select_related_strings(self):
        return ['banner__campaign']

    def _get_group_func(self):
        return lambda n: {k: v for k, v in n['banner'].iteritems() if k != 'date_to'}

class RecordUpdaterBase(object):

    def __init__(self, record):
        self.handled_stats_count = 0
        self.record = record

    def update_by(self, st):
        self.handled_stats_count += 1

    def after_update(self):
        pass

class CommonRecordUpdater(RecordUpdaterBase):

    def update_by(self, st):
        super(CommonRecordUpdater, self).update_by(st)
        self.record['sessions'] += st['sessions']
        self.record['leads'] += st['leads']
        self.record['users'] += st['users']
        self.record['new_users'] += st['new_users']
        self.record['page_views'] += st['page_views']
        self.record['bounces'] += st['bounces']
        self.record['session_duration'] += st['session_duration']
        self.record['transaction_revenue'] += st['transaction_revenue']
        self.record['clicks_positions'] += st['clicks_positions']
        self.record['shows_positions'] += st['shows_positions']

class CommonPreviousRecordUpdater(RecordUpdaterBase):

    def update_by(self, st):
        super(CommonPreviousRecordUpdater, self).update_by(st)
        self.record['prev_sessions'] += st['sessions']
        self.record['prev_leads'] += st['leads']
        self.record['prev_users'] += st['users']
        self.record['prev_new_users'] += st['new_users']
        self.record['prev_page_views'] += st['page_views']
        self.record['prev_bounces'] += st['bounces']
        self.record['prev_session_duration'] += st['session_duration']
        self.record['prev_transaction_revenue'] += st['transaction_revenue']
        self.record['prev_clicks_positions'] += st['clicks_positions']
        self.record['prev_shows_positions'] += st['shows_positions']

class DirectPreviousRecordUpdater(CommonPreviousRecordUpdater):

    def update_by(self, st):
        super(DirectPreviousRecordUpdater, self).update_by(st)
        self.record['prev_shows'] += st['shows_search'] + st['shows_context']
        self.record['prev_cost'] += st['sum_search'] + st['sum_context']
        self.record['prev_clicks'] += st['clicks_search'] + st['clicks_context']
        self.record['prev_clicks_search'] += st['clicks_search']
        self.record['prev_shows_search'] += st['shows_search']
        self.record['prev_shows_search_premium'] += st['shows_search_premium']
        self.record['prev_shows_search_other'] += st['shows_search_other']

    def after_update(self):
        super(DirectPreviousRecordUpdater, self).after_update()
        self.record['prev_avg_shows_position'] = 0
        self.record['prev_avg_clicks_position'] = 0
        if self.record['prev_shows_search']:
            self.record['prev_avg_shows_position'] = self.record['prev_shows_positions'] / self.record['prev_shows_search']
        if self.record['prev_clicks_search']:
            self.record['prev_avg_clicks_position'] = self.record['prev_clicks_positions'] / self.record['prev_clicks_search']

class AdwordsPreviousRecordUpdater(CommonPreviousRecordUpdater):

    def update_by(self, st):
        super(AdwordsPreviousRecordUpdater, self).update_by(st)
        self.record['prev_shows'] += st['shows']
        self.record['prev_cost'] += st['amount']
        self.record['prev_clicks'] += st['clicks']

class DirectRecordUpdater(CommonRecordUpdater):
    PREVIOUS = DirectPreviousRecordUpdater

    def update_by(self, st):
        super(DirectRecordUpdater, self).update_by(st)
        self.record['shows'] += st['shows_search'] + st['shows_context']
        self.record['cost'] += st['sum_search'] + st['sum_context']
        self.record['clicks'] += st['clicks_search'] + st['clicks_context']
        self.record['clicks_search'] += st['clicks_search']
        self.record['shows_search'] += st['shows_search']
        self.record['shows_search_premium'] += st['shows_search_premium']
        self.record['shows_search_other'] += st['shows_search_other']

    def after_update(self):
        super(DirectRecordUpdater, self).after_update()
        self.record['avg_shows_position'] = 0
        self.record['avg_clicks_position'] = 0
        if self.record['shows_search']:
            self.record['avg_shows_position'] = self.record['shows_positions'] / self.record['shows_search']
        if self.record['clicks_search']:
            self.record['avg_clicks_position'] = self.record['clicks_positions'] / self.record['clicks_search']

class AdwordsRecordUpdater(CommonRecordUpdater):
    PREVIOUS = AdwordsPreviousRecordUpdater

    def update_by(self, st):
        super(AdwordsRecordUpdater, self).update_by(st)
        self.record['shows'] += st['shows']
        self.record['cost'] += st['amount']
        self.record['clicks'] += st['clicks']

class DetailStatsGetter(object):
    SOURCE_NAME = None
    STAT_MODEL = None
    SERIALIZER = None

    def __init__(self, project, start_date, end_date, **kwargs):
        self.project = project
        self.start_date = start_date
        self.end_date = end_date
        self.campaign_id = kwargs.get('campaign_id')
        self.phrase_ids = kwargs.get('phrase_ids')
        self.banner_id = kwargs.get('banner_id')
        self.select_related_strings = self._get_select_related_strings()

    def get_stats_by_date(self):
        stats = self._get_stats()
        result = []
        serialized_stats = self.SERIALIZER(
            stats, select_related_strings=self.select_related_strings
        ).serialize()
        serialized_stats.sort(key=lambda n: n['date'])
        for _date, groupped_stats in groupby(serialized_stats, key=itemgetter('date')):
            stats_list = list(groupped_stats)
            record = self._get_record(_date)
            for st in stats_list:
                self._update_record(record, st)
            result.append(record)
        return result

    def _get_stats(self):
        stats = self.STAT_MODEL.objects.filter(
            campaign__data_resource__projects=self.project,
            date__range=(self.start_date, self.end_date),
        ).select_related(*self.select_related_strings)
        if self.campaign_id:
            stats = stats.filter(campaign=self.campaign_id)
        return stats

    def _get_select_related_strings(self):
        return []

    def _get_record(self, date):
        rec = self._get_initial_record()
        rec.update({
            'date': date,
            'sessions': 0,
            'shows': 0,
            'clicks': 0,
            'shows_search': 0,
            'shows_context': 0,
            'shows_search_premium': 0,
            'shows_search_other': 0,
            'clicks_search': 0,
            'shows_positions': 0,
            'clicks_positions': 0,
            'cost': 0,
            'income': 0,
            'leads': 0,
            'users': 0,
            'new_users': 0,
            'page_views': 0,
            'bounces': 0,
            'session_duration': 0,
            'transaction_revenue': 0,
        })
        return rec

    def _update_record(self, record, st):
        self.RECORD_UPDATER(record).update_by(st)

    def _get_initial_record(self):
        return {
            'source': self.SOURCE_NAME
        }

class DetailPhrasesStatsGetter(DetailStatsGetter):

    def _get_stats(self):
        return self.STAT_MODEL.objects.filter(
            self._get_item_q(),
            date__range=(self.start_date, self.end_date)
        )

    def _get_item_q(self):
        return Q(phrase__in=self.phrase_ids)

class DetailBannersStatsGetter(DetailPhrasesStatsGetter):

    def _get_item_q(self):
        return Q(banner=self.banner_id)

class DirectDetailStatsGetter(DetailStatsGetter):
    SOURCE_NAME = DIRECT_NAME
    STAT_MODEL = YandexDirectStat
    SERIALIZER = YandexDirectStatQuerySetSerializer
    RECORD_UPDATER = DirectRecordUpdater

    def _get_select_related_strings(self):
        return ['campaign']

class AdwordsDetailStatsGetter(DetailStatsGetter):
    SOURCE_NAME = ADWORDS_NAME
    STAT_MODEL = GoogleAdwordsStat
    SERIALIZER = GoogleAdwordsStatQuerySetSerializer
    RECORD_UPDATER = AdwordsRecordUpdater

    def _get_select_related_strings(self):
        return ['campaign']

class DirectDetailPhrasesStatsGetter(DetailPhrasesStatsGetter):
    SOURCE_NAME = DIRECT_NAME
    STAT_MODEL = DirectPhraseStat
    SERIALIZER = DirectPhraseStatQuerySetSerializer
    RECORD_UPDATER = DirectRecordUpdater

    def _get_select_related_strings(self):
        return ['phrase__campaign']

class AdwordsDetailPhrasesStatsGetter(DetailPhrasesStatsGetter):
    SOURCE_NAME = ADWORDS_NAME
    STAT_MODEL = AdwordsPhraseStat
    SERIALIZER = AdwordsPhraseStatQuerySetSerializer
    RECORD_UPDATER = AdwordsRecordUpdater

    def _get_select_related_strings(self):
        return ['phrase__campaign']

class DirectDetailBannersStatsGetter(DetailBannersStatsGetter):
    SOURCE_NAME = DIRECT_NAME
    STAT_MODEL = DirectBannerStat
    SERIALIZER = DirectBannerStatQuerySetSerializer
    RECORD_UPDATER = DirectRecordUpdater

    def _get_select_related_strings(self):
        return ['banner__campaign']

class AdwordsDetailBannersStatsGetter(DetailBannersStatsGetter):
    SOURCE_NAME = ADWORDS_NAME
    STAT_MODEL = AdwordsBannerStat
    SERIALIZER = AdwordsBannerStatQuerySetSerializer
    RECORD_UPDATER = AdwordsRecordUpdater

    def _get_select_related_strings(self):
        return ['banner__campaign']


# -*- coding: utf-8 -*-

import logging

from odoo import models, fields, api
from odoo.addons.http_routing.models.ir_http import slugify

_logger = logging.getLogger(__name__)

class ActivityArea(models.Model):
    """
    Activity states of a company
    """
    _name = 'ar.activity_area'
    _description = 'Secteurs d\'activités d\'une entreprise'

    name = fields.Char(string='Nom du secteur', required=True)
    slug = fields.Char(compute='_slugify_name', store=True)

    @api.depends('name')
    def _slugify_name(self):
        self.slug = slugify(self.name)

class PartnerTypes(models.Model):
    """
    Types of contacts
    """
    _name = 'ar.partner_type'
    _description = 'Type de contacts'

    name = fields.Char(string='Nom', required=True)
    slug = fields.Char(compute='_slugify_name', store=True)

    @api.depends('name')
    def _slugify_name(self):
        self.slug = slugify(self.name)

class Partner(models.Model):
    """
    Add fields to contacts
    """
    _inherit = 'res.partner'

    def _get_defaultpartner_type(self):
        """
        :return: the default partner type
        """
        return self.env['ar.partner_type'].search([('slug', '=', 'autre')], limit=1)

    referent = fields.Many2one(
        'res.partner',
        string='Référent au sein d\'Aveyron Recrute',
        required=False,
        domain=[["user_ids.groups_id.name", "ilike", "internal"]]
    )

    partner_type = fields.Many2one(
        'ar.partner_type',
        string='Type de contact',
        required=False,
        default=_get_defaultpartner_type
    )

    presentation = fields.Text(
        string='Présentation',
        required=False
    )

    activity_areas = fields.Many2many(
        'ar.activity_area',
        string='Secteurs d\'Activités',
        required=False
    )

    employee_count = fields.Integer(
        string='Nombre d\'employés',
        required=False
    )

    needs = fields.Text(
        string='Mes besoins en recrutement les plus fréquents',
        required=False
    )

    year_birth = fields.Integer(
        string='Année de naissance',
        required=False,
        default=0
    )

    your_profession = fields.Text(
        string='Vos métiers / vos produits en synthèse',
        required=False
    )

    @api.model
    def send_contact_recruteur_email(self, partner_id, subject, message):
        partner_obj = self.env['res.partner'].browse(partner_id)

        if partner_obj is None:
            return False

        lang = partner_obj.user_id.lang
        template = self.env.ref('espace_recruteur.mail_template_contact_recruteur')
        template.with_context(dbname=self._cr.dbname, subject=subject, message=message, lang=lang).send_mail(
            partner_obj.id, force_send=True)

        return True

    @api.model
    def send_contact_candidat_email(self, partner_id, subject, message):
        partner_obj = self.env['res.partner'].browse(partner_id)

        if partner_obj is None:
            return False

        lang = partner_obj.user_id.lang
        template = self.env.ref('espace_candidat.mail_template_contact_candidat')
        template.with_context(dbname=self._cr.dbname, subject=subject, message=message, lang=lang).send_mail(
            partner_obj.id, force_send=True)

        return True

    @api.model
    def get_company(self, partner_id):
        partner = self.env['res.partner'].search_read(
            domain=[('id', '=', partner_id)],
            fields=('id', 'name', 'street', 'street2', 'zip', 'city', 'email', 'employee_count', 'website',
                    'activity_areas', 'presentation')
        )

        if not partner:
            return None

        areas = self.env['ar.activity_area'].search_read(
            domain=[('id', 'in', partner[0]['activity_areas'])],
            fields=('id', 'name')
        )

        partner[0]['activity_areas'] = areas

        return partner

    @api.model
    def get_all_companies(self):
        companies = self.env['res.company'].search_read(
            domain=[('partner_id', 'not ilike', 'aveyron recrute')],
            fields=('id', 'partner_id')
        )

        if len(companies) <= 0:
            return []

        partners = []

        for company in companies:
            if company['partner_id'] and company['partner_id'][0]:
                partners.append(self.get_company(company['partner_id'][0]))

        return partners

    @api.one
    def _prepare_model_for_send_website(self):
        partner = self.read(('id', 'name', 'street', 'street2', 'zip', 'city', 'email', 'employee_count', 'website',
                             'activity_areas', 'presentation'))

        if not partner:
            return None

        areas = self.env['ar.activity_area'].search_read(
            domain=[('id', 'in', partner['activity_areas'])],
            fields=('id', 'name')
        )

        partner['activity_areas'] = areas

        return partner

    @api.model
    def get_partner_with_profiles(self, email):
        base_url = self.env['ir.config_parameter'].sudo().get_param('web.base.url')
        partner = self.env['res.partner'].search([('email', 'ilike', email)], limit=1)

        if not partner:
            return False

        partner_dict = partner.read(['id', 'name', 'email'])
        partner_dict[0]['profiles'] = []
        profiles = self.env['hr.applicant'].search([('is_profile', '=', True), ('partner_id', '=', partner.id)])

        if len(profiles) > 0:
            partner_dict[0]['profiles'] = profiles.read([
                'id', 'name', 'description', 'formations', 'activities', 'attachment_ids',
                'diplomas', 'skills'
            ])

        for profile in partner_dict[0]['profiles']:
            if len(profile['formations']) > 0:
                profile['formations'] = self.env['ar.formation'].search_read(
                    domain=[('id', 'in', profile['formations'])],
                    fields=('id', 'name')
                )

            if len(profile['activities']) > 0:
                profile['activities'] = self.env['ar.activity_area'].search_read(
                    domain=[('id', 'in', profile['activities'])],
                    fields=('id', 'name')
                )

            if len(profile['attachment_ids']) > 0:
                profile['attachment_ids'] = self.env['ir.attachment'].search_read(
                    domain=[('id', 'in', profile['attachment_ids'])],
                    fields=('id', 'website_url')
                )

                for attachment in profile['attachment_ids']:
                    attachment['website_url'] = base_url + attachment['website_url']

            profile['diplomas'] = str(profile['diplomas']).split('|') if profile['diplomas'] else []
            profile['skills'] = str(profile['skills']).split('|') if profile['skills'] else []

        return partner_dict


import pandas as pd
import numpy as np
import io
import decimal as de
import sys

# import from system
windoww=sys.argv[1]
acttual=sys.argv[2]
predictedd =sys.argv[3]
outtput=sys.argv[4]

# import data
# actual_path=system_path+'/insight_testsuite/tests/my_own_test_1/input/actual.txt'
# predicted_path=system_path+'/insight_testsuite/tests/my_own_test_1/input/predicted.txt'
# system_path= os.getcwd()[:-4]

actual=pd.read_table(acttual, sep='|',header=None)
predicted=pd.read_table(predictedd, sep='|',header=None)

# pick up the right moving time window
# window_path=system_path+'/insight_testsuite/tests/my_own_test_1/input/window.txt'
f = io.open(windoww,'r')
message = f.read()
f.close()
sliding_window=int(message)

# add a header for easier manipulation
actual.columns=['hour','stock_code', 'stock_price']
predicted.columns=['hour','stock_code', 'stock_price']

# construct a moving window of elements to take average
predicted_time=predicted['hour'].tolist()
predicted_time=list(set(predicted_time))
q=len(predicted_time)-sliding_window
begin=[]
end=[]
i=0
while i<=q:
    begin.append(predicted_time[i])
    end.append(predicted_time[i+sliding_window-1])
    i=i+1
begin=np.array(begin).T
end=np.array(end).T

# construct a join pandas dataframe
mergedStuff = pd.merge(actual, predicted, on=['hour','stock_code'], how='outer')
mergedStuff['diff'] = mergedStuff.apply(lambda x: abs(x['stock_price_x'] - x['stock_price_y']) if not np.isnan(x['stock_price_y']) or np.isnan(x['stock_price_x']) else np.nan, axis=1)
mergedStuff.sort_values(by=['hour'])

# extract the difference and the time

desired=mergedStuff[['hour','diff']]
desire=desired.dropna(subset=['diff'])
desire.sort_values(by=['hour'])

# extract the sum of differences in each hour
L=[]
for time in predicted_time:
    net=desire.loc[desired['hour']==time]['diff'].sum()
    weight=desire[desire['hour']==time].shape[0]
    L.append([net,weight])

# extract the moving averages

output=[]
i=0
while i<=q:
    diff=0
    temp=L[i:i+sliding_window]
    weight=0
    for item in temp:
        diff=diff+item[0]
        weight=weight+item[1]
    a=de.Decimal(diff)
    if weight!=0:
        c=a/de.Decimal(weight)
        d=c.quantize(de.Decimal('.00'))
        output.append(d)
    else:
        output.append('NA')
    i=i+1
output=np.array(output)

# create the output dataframe
df_begin=pd.DataFrame(begin)
df_end=pd.DataFrame(end)
value=pd.DataFrame(output.T)
frames=[df_begin, df_end,value]
result=pd.concat(frames,axis=1)

# write the output_file
# output_path=system_path+'/output/comparison.txt'
result.to_csv(outtput, header=None, sep="|", index=False)


###############################################################################
#
# The MIT License (MIT)
#
# Copyright (c) Crossbar.io Technologies GmbH
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
###############################################################################

from __future__ import absolute_import

__all = (
    'sleep',
    'peer2str',
)

def peer2str(peer):
    if isinstance(peer, tuple):
        ip_ver = 4 if len(peer) == 2 else 6
        return u"tcp{2}:{0}:{1}".format(peer[0], peer[1], ip_ver)
    elif isinstance(peer, str):
        return u"unix:{0}".format(peer)
    else:
        return u"?:{0}".format(peer)

def get_serializers():
    from autobahn.wamp import serializer

    serializers = ['CBORSerializer', 'MsgPackSerializer', 'UBJSONSerializer', 'JsonSerializer']
    serializers = list(filter(lambda x: x, map(lambda s: getattr(serializer, s) if hasattr(serializer, s)
                                               else None, serializers)))
    return serializers


import typing
import re
import time
import os
import random
import datetime

import jk_utils
import jk_json
import jk_typing

from .ArchivedBackupInfoFile import ArchivedBackupInfoFile
from .BackupDataFile import BackupDataFile

def _toInt(s:str) -> int:
	while (len(s) > 1) and s[0] == "0":
		s = s[1:]
	return int(s)
#

#
# This class represents a backup within the archive.
#
class Backup(object):

	################################################################################################################################
	## Constructors
	################################################################################################################################

	@jk_typing.checkFunctionSignature()
	def __init__(self, dirPath:str):
		self.__dirPath = dirPath
		self.__infoFile = ArchivedBackupInfoFile.loadFromFile(os.path.join(dirPath, "_backup_info.json"))

		self.__files = []
		for fe in os.scandir(dirPath):
			if bool(re.match("[a-zA-Z0-9]", fe.name[0])) and fe.is_file():
				self.__files.append(BackupDataFile(fe))

		self.__files = tuple(self.__files)

		self.__cached_dtBackupStart = None
		self.__cached_sBackupStartDate = None
		self.__cached_sBackupStartDateTime = None
	#

	################################################################################################################################
	## Public Properties
	################################################################################################################################

	@property
	def files(self) -> typing.Tuple[BackupDataFile]:
		return self.__files
	#

	@property
	def dirPath(self) -> str:
		return self.__dirPath
	#

	@property
	def dtBackupStart(self) -> datetime.datetime:
		if self.__cached_dtBackupStart is None:
			#self.__cached_dtBackupStart = datetime.datetime.fromtimestamp(self.__infoFile.getValue("tBackupStart"))
			self.__cached_dtBackupStart = self.__infoFile.getValue("tBackupStart").dateTime
		return self.__cached_dtBackupStart
	#

	#
	# Returns a human readable representation of the start time stamp in the form:
	# "YYYY-MM-DD"
	#
	@property
	def sBackupStartDate(self) -> str:
		dt = self.dtBackupStart
		if self.__cached_sBackupStartDate is None:
			self.__cached_sBackupStartDate = "{:04d}-{:02d}-{:02d}".format(
				dt.year,
				dt.month,
				dt.day,
			)
		return self.__cached_sBackupStartDate
	#

	#
	# Returns a human readable representation of the start time stamp in the form:
	# "YYYY-MM-DD hh:mm:ss"
	#
	@property
	def sBackupStartDateTime(self) -> str:
		dt = self.dtBackupStart
		if self.__cached_sBackupStartDateTime is None:
			self.__cached_sBackupStartDateTime = "{:04d}-{:02d}-{:02d} {:02d}:{:02d}:{:02d}".format(
				dt.year,
				dt.month,
				dt.day,
				dt.hour,
				dt.minute,
				dt.second,
			)
		return self.__cached_sBackupStartDateTime
	#

	#
	# Returns a machine readable representation of the start time stamp in the form:
	# {
	#		"year": ...,
	#		"month": ...,
	#		"day": ...,
	#		"hour": ...,
	#		"minute": ...,
	#		"second": ...,
	# }
	#
	@property
	def jBackupStartDateTime(self) -> dict:
		dt = self.dtBackupStart
		return {
			"year": dt.year,
			"month": dt.month,
			"day": dt.day,
			"hour": dt.hour,
			"minute": dt.minute,
			"second": dt.second,
		}
	#

	#
	# Returns a machine readable representation of the start time stamp in the form:
	# ( ..year.., ..month.., ..day.., ..hour.., ..minute.., ..second.. )
	#
	@property
	def lBackupStartDateTime(self) -> tuple:
		dt = self.dtBackupStart
		return (
			dt.year,
			dt.month,
			dt.day,
			dt.hour,
			dt.minute,
			dt.second,
		)
	#

	@property
	def lBackupStartDate(self) -> tuple:
		dt = self.dtBackupStart
		return (
			dt.year,
			dt.month,
			dt.day,
		)
	#

	@property
	def timeStamp(self) -> jk_utils.TimeStamp:
		return self.__infoFile.getValue("tBackupStart")
	#

	@property
	def sizeInBytesUploaded(self) -> jk_utils.AmountOfBytes:
		return self.__infoFile.getValue("nTotalBytesUploaded")
	#

	@property
	def sizeInBytesArchived(self) -> jk_utils.AmountOfBytes:
		return self.__infoFile.getValue("nTotalBytesArchived")
	#

	@property
	def clientBackupDuration(self) -> float:
		return self.__infoFile.getValue("dClientBackupDuration")
	#

	@property
	def systemName(self) -> str:
		return self.__infoFile.getValue("systemName")
	#

	@property
	def backupUserName(self) -> str:
		return self.__infoFile.getValue("backupUserName")
	#

	@property
	def backupIdentifier(self) -> str:
		return self.__infoFile.getValue("backupIdentifier")
	#

	################################################################################################################################
	## Public Methods
	################################################################################################################################

	################################################################################################################################
	## Static Methods
	################################################################################################################################

	#
	# TODO: maybe move this to ArchiveDataStore? or a separate class?
	#
	@staticmethod
	@jk_typing.checkFunctionSignature()
	def buildRecommendedDirName(dt:datetime.datetime = None, systemName:str = None, backupUserName:str = None, backupIdentifier:str = None, backup = None) -> str:
		if backup is not None:
			assert isinstance(backup, Backup)
			dt = backup.dtBackupStart
			systemName = backup.systemName
			backupUserName = backup.backupUserName
			backupIdentifier = backup.backupIdentifier

		assert dt is not None
		Backup.verifySystemName(systemName)
		Backup.verifyBackupUserName(backupUserName)
		Backup.verifyBackupIdentifier(backupIdentifier)

		return "{}---{:04d}{:02d}{:02d}{:02d}{:02d}{:02d}---{}---{}".format(
			systemName,
			dt.year,
			dt.month,
			dt.day,
			dt.hour,
			dt.minute,
			dt.second,
			backupUserName,
			backupIdentifier)
	#

	#
	# TODO: maybe move this to ArchiveDataStore? or a separate class?
	#
	@staticmethod
	def tryParseDirName(dirName:str) -> dict:
		m = re.match(r"^([a-zA-Z_\.0-9-\+%]+)---(\d\d\d\d)(\d\d)(\d\d)(\d\d)(\d\d)(\d\d)---([a-zA-Z_\.0-9-\+%]+)---([a-zA-Z_\.0-9-\+%]+)$", dirName)
		if m:
			return {
				"systemName": m.group(1),
				"year": _toInt(m.group(2)),
				"month": _toInt(m.group(3)),
				"day": _toInt(m.group(4)),
				"hour": _toInt(m.group(5)),
				"minute": _toInt(m.group(6)),
				"second": _toInt(m.group(7)),
				"userName": m.group(8),
				"backupIdentifier": m.group(9),
			}
		return None
	#

	#
	# TODO: maybe move this to ArchiveDataStore? or a separate class?
	#
	@staticmethod
	def verifySystemName(systemName:str):
		assert re.match(r".*(---|\.\.|\s).*", systemName) is None
		assert re.match(r"^[a-zA-Z_\.0-9-\+%]+$", systemName) is not None
	#

	#
	# TODO: maybe move this to ArchiveDataStore? or a separate class?
	#
	@staticmethod
	def verifyBackupUserName(userName:str):
		assert re.match(r".*(---|\.\.|\s).*", userName) is None
		assert re.match(r"^[a-zA-Z_\.0-9-\+%]+$", userName) is not None
	#

	#
	# TODO: maybe move this to ArchiveDataStore? or a separate class?
	#
	@staticmethod
	def verifyBackupIdentifier(backupIdentifier:str):
		assert re.match(r".*(---|\.\.|\s).*", backupIdentifier) is None
		assert re.match(r"^[a-zA-Z_\.0-9-\+%]+$", backupIdentifier) is not None
	#

#


import os
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
from natsort import natsorted

first = True
# AdamW_10-40sec_Mel_Spectrogram / AdamW_10-40sec_MFCC/ RAdam_10-40sec_Mel_Spectrogram / RAdam_10-40sec_MFCC/history
PATH = 'AdamW_10-40sec_Mel_Spectrogram'
history_list = natsorted(os.listdir(PATH))
print(history_list)
file_name = history_list[0][:-6]

for idx, name in enumerate(history_list):
    df = pd.read_csv(os.path.join(PATH, name))

    globals()[f'loss_{idx+1}'] = df['loss'].to_list()
    globals()[f'accuracy_{idx+1}'] = df['accuracy'].to_list()
    globals()[f'val_loss_{idx+1}'] = df['val_loss'].to_list()
    globals()[f'val_accuracy_{idx+1}'] = df['val_accuracy'].to_list()

total_loss = list(zip(loss_1, loss_2, loss_3, loss_4, loss_5))
total_acc = list(zip(accuracy_1, accuracy_2, accuracy_3, accuracy_4, accuracy_5))
total_val_loss = list(zip(val_loss_1, val_loss_2, val_loss_3, val_loss_4, val_loss_5))
total_val_acc = list(zip(val_accuracy_1, val_accuracy_2, val_accuracy_3, val_accuracy_4, val_accuracy_5))

loss_lower_bound = [min(x) for x in total_loss]
loss_upper_bound = [max(x) for x in total_loss]
acc_lower_bound = [min(x) for x in total_acc]
acc_upper_bound = [max(x) for x in total_acc]

val_loss_lower_bound = [min(x) for x in total_val_loss]
val_loss_upper_bound = [max(x) for x in total_val_loss]
val_acc_lower_bound = [min(x) for x in total_val_acc]
val_acc_upper_bound = [max(x) for x in total_val_acc]

loss_avg = [sum(x)/len(x) for x in total_loss]
acc_avg = [sum(x)/len(x) for x in total_acc]
val_loss_avg = [sum(x)/len(x) for x in total_val_loss]
val_acc_avg = [sum(x)/len(x) for x in total_val_acc]

index = range(len(acc_avg))
df = pd.DataFrame(data=list(zip(loss_avg, acc_avg, val_loss_avg, val_acc_avg)),
                  columns=['loss', 'accuracy', 'val_loss', 'val_accuracy'], index=index)
df.to_csv(os.path.join(PATH, file_name+'_avg.csv'), mode='w')

x_axis = list(index)
plt.plot(x_axis, acc_avg)
plt.plot(x_axis, val_acc_avg)
plt.fill_between(x_axis, acc_lower_bound, acc_upper_bound, alpha=0.2)
plt.fill_between(x_axis, val_acc_lower_bound, val_acc_upper_bound, alpha=0.2)
plt.title('Model accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend(['Train', 'Validation'], loc='best')
plt.grid()
plt.savefig(os.path.join(PATH, file_name+'_avg.png'), dpi=300)
plt.show()

# ============================ Trying to plot LOSS for checking convergence ===================
x_axis = list(index)
plt.plot(x_axis, loss_avg)
plt.plot(x_axis, val_loss_avg)
plt.fill_between(x_axis, loss_lower_bound, loss_upper_bound, alpha=0.2)
plt.fill_between(x_axis, val_loss_lower_bound, val_loss_upper_bound, alpha=0.2)
plt.title('Model accuracy')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Train', 'Validation'], loc='best')
plt.grid()
plt.savefig(os.path.join(PATH, file_name+'_loss.png'), dpi=300)
plt.show()


# !/usr/local/bin/python3

import turtle

def draw_star(t, sz):
    for i in range(5):
        t.forward(sz)
        t.right(144)

def single_move_turtle(t, dis, ang):
    t.penup()
    t.right(ang)
    t.forward(dis)
    # t.left(ang)
    t.pendown()

wn = turtle.Screen()
wn.bgcolor("lightgreen")
wn.title("Hello, Francis")

FirstPolygons = turtle.Turtle()
FirstPolygons.color("blue")
FirstPolygons.pensize(3)

FirstPolygons.hideturtle()
size = int(input("input size: "))
for i in range(5):
    draw_star(FirstPolygons, size)
    single_move_turtle(FirstPolygons, 200, 144)

wn.mainloop()


from bitarray import bitarray
import sys
import os
def binStr(num, length):
    return bin(num)[2:].zfill(length)

def bytesToBin(bytes):
    if len(bytes) == 0:
        return ''
    else:
        return bin(int.from_bytes(bytes, byteorder='big'))[2:].zfill(8 * len(bytes))

def lz77_enc(buffer, wBits, lBits):
    maxWindow = (2**wBits) -1
    a = bitarray()
    w = i = 0
    while i < len(buffer):
        word = buffer[i:i + 1]
        l = 0
        back = 0
        first = i
        find = buffer.rfind(word, w, i)
        while find != -1:
            back = first - find
            l += 1
            i += 1
            # length can't exceed the end or our bit limit
            if i >= len(buffer) - 1 or i - find >= (2**lBits):
                break
            word += buffer[i:i + 1]
            wl = len(word)
            # only rfind again if next 2 chars not same
            if buffer[i:i + 1] != buffer[find + wl - 1:find + wl]:
                find = buffer.rfind(word, w, i)
        next = buffer[i:i + 1]
        i += 1
        if i - w > maxWindow:  # slide the window if exceeded max size
            w += i - w - maxWindow
        ls = binStr(l, lBits)
        a += binStr(back, wBits) + ls + bytesToBin(next)
    return a

#compresses a given file with the input number of wBits
def main():
    fn = sys.argv[1]
    wBits = int(sys.argv[2])
    lBits = int(sys.argv[3])
    plainFile = open(fn, 'rb')
    plainText = plainFile.read()
    encodedBitArr = lz77_enc(plainText, wBits, lBits)
    compressed_fn = os.path.splitext(fn)[0] + '.lz77'
    with open(compressed_fn, 'wb') as file:
        encodedBitArr.tofile(file)
    print('Your encoded data is stored in file', compressed_fn)
    return False

if __name__ == "__main__":
    main()


#!/usr/bin/python3

# Creates a TXF file for Form 1099-B to populate Form 8949.
# https://turbotax.intuit.com/txf/TXF042.jsp
# Look for codes 321, 711, 712, 323, 713, 714.

# python create-txf-2015.py 1099-b.csv > 1099-b.txf
# If you have multiple accounts, you may combine the .csv files by
# sorting them before creating the .txf file from the combined file.

# Import the .txf file into TurboTax via
# "File > Import > From TXF Files".
# You should see this:
# These Documents Are Now Ready for Import:
# - 1099-B (number of transactions)

# The result DOES NOT look correct in in EasyView.  You can verify it
# like this: "View > Go to Forms". Open "Form 8949". Review the
# transactions. Note that Line 1 in both Part I and II is scrollable
# if you have more than six transactions.  Also, there are multiple
# copies of the form if more than one of A, B, C or D, E, F is
# checked.

# If you don't like what you see, you can remove the imported data via
# "File > Remove Imported Data".

import sys
import csv
import datetime

box_dict = {'A': 321, 'B': 711, 'C': 712, 'D': 323, 'E': 713, 'F': 714}
# Only two taxrefs for wash sales are documented in Section IV.  One
# of those is used in Example 2C.  Example 2G for a long-term wash
# sale uses taxref 713.
wash_box_dict = {'A': 682, 'B': 718, 'C': 712, 'D': 323, 'E': 713, 'F': 714}

print('V042')
print('Aself')
print('D', datetime.date.today().strftime('%m/%d/%y'))
print('^')

with open(sys.argv[1], 'r') as csvfile:
    for row in csv.reader(csvfile):
        symbol = row[0]
        count = row[1]
        acquired = row[2]
        disposed = row[3]
        proceeds = row[4]
        try:
            # TXF042.jsp, example 2B; may be reported as "Missing" in 1099-B.
            float(row[5])
            base = row[5]
        except ValueError:
            base = ''
        adj = row[6]
        gain = row[7]
        adj_code = row[9]
        box = row[10]
        taxref = wash_box_dict[box] if adj_code == 'W' else box_dict[box]
        prefix = ' '
        if '/' in symbol:
            prefix = ' opt '
        else:
            prefix = ' sh '
        if float(count) < 0:
            prefix = ' short' + prefix
            count = count[1:]
            # https://www.irs.gov/pub/irs-pdf/i8949.pdf
            # Column (b)-Date Acquired
            # For a short sale, enter the date you acquired the property
            # delivered to the broker or lender to close the short sale.
            acquired = disposed
        descr = count + prefix + symbol
        print('TD')
        print('N' + str(taxref))
        print('P ' + descr)
        print('D ' + acquired.replace('VARIOUS', 'Various'))
        print('D ' + disposed)
        print('$' + base)
        print('$' + proceeds)
        # TurboTax 2016 ignores the sign of the adjustment and always puts "W"
        # in Box (f) if an adjustment is present.  For other types of
        # adjustments, e.g., "B", one has to override the values in Form 8949
        # in Forms View, including the new value in Box (h).
        if adj != '--' and adj != '0.00':
            print('$' + adj)
        print('^')


import threading
import time
from datetime import datetime
from itertools import cycle

import RPi.GPIO as GPIO

import display
import events
import ip
import messages
import threads
import update
from constants import *

def display_clock():
    now = datetime.now()
    device.write_text(1, " {:%H%M%S}".format(now.time()), dots=(3, 5))
    events.change_mode.wait(DISPLAY_RATE_CLOCK)

def display_date():
    now = datetime.now()
    device.write_text(1, "{:%d-%m-%y}".format(now.date()))
    events.change_mode.wait(DISPLAY_RATE_DATE)

def display_weather():
    device.write_text(1, "{:>6d}*C".format(update.temperature))
    events.change_mode.wait(DISPLAY_RATE_WEATHER)

def display_currency():
    device.write_text(1, " {:d} EUR".format(update.eur), dots=[6])
    events.change_mode.wait(DISPLAY_RATE_CURRENCY)
    if not events.change_mode.is_set():
        device.write_text(1, " {:d} USD".format(update.usd), dots=[6])
        events.change_mode.wait(DISPLAY_RATE_CURRENCY)

def display_instagram():
    device.write_text(1, "IG{:>6d}".format(update.followers))
    events.change_mode.wait(DISPLAY_RATE_IG)

def hide_display():
    events.change_mode.wait(5)

def wait_for_message_display():
    time.sleep(0.2)

def button_listener():
    global current_mode
    while True:
        if GPIO.event_detected(BUTTON_1):
            if messages.messages_to_read:
                message = messages.messages_to_read.popleft()
                threads.start_time_dependent()
                show_message(message)
                if messages.messages_to_read:
                    threads.start_flow()
                messages.send_read_id(message['id'])
            else:
                show_message()
            time.sleep(WAIT_TIME_AFTER_CLICK)
        elif GPIO.event_detected(BUTTON_2):
            current_mode = next(mode)
            events.change_mode.set()
            device.clear()
            time.sleep(WAIT_TIME_AFTER_CLICK)
        else:
            time.sleep(0.2)

def show_message(message=None):
    events.reading_message.set()
    device.clear()
    global current_mode
    remember_mode, current_mode = current_mode, 0
    if message:
        device.show_message(text=message['text'].upper(), mw=True)
    else:
        device.write_text(1, "NO MSGS", mw=True)
        time.sleep(2)
    current_mode = remember_mode
    events.reading_message.clear()

def setup_button_listener():
    threading.Thread(target=button_listener, daemon=True).start()

def init():
    GPIO.setmode(GPIO.BCM)
    GPIO.setup(BUTTON_1, GPIO.IN)
    GPIO.setup(BUTTON_2, GPIO.IN)
    GPIO.add_event_detect(BUTTON_1, GPIO.RISING, bouncetime=250)
    GPIO.add_event_detect(BUTTON_2, GPIO.RISING, bouncetime=250)

    global device
    device = display.device

    threads.thread_dependent_on_time.start()

    device.show_message(text=ip.ip)
    device.write_text(1, "LOADING", dots=[1])

    setup_button_listener()
    update.all_modes()
    messages.setup_messages_service()
    ip.send_ip()
    device.clear()

modes = {
    0: wait_for_message_display,
    1: display_clock,
    2: display_date,
    3: display_weather,
    4: display_currency,
    5: display_instagram,
    6: hide_display
}
current_mode = 1
mode = cycle(range(1, 7))
next(mode)

if __name__ == '__main__':
    init()
    while True:
        while not events.change_mode.is_set():
            modes[current_mode]()
        events.change_mode.clear()


import Adafruit_DHT
import RPi.GPIO as GPIO
import time
from datetime import datetime
import temp_sensor

class manage_temp():

    def __init__(self,pin_sensor):
        self.issummer=False
        self.iswinter=False
        GPIO.setmode(GPIO.BCM)
        self.pin_sensor = pin_sensor
        self.sensor_com = 6
        self.sensor_heat = 2
        self.sensor_cool = 3
        GPIO.setup(self.sensor_com, GPIO.OUT)
        GPIO.setup(self.sensor_heat, GPIO.OUT)
        GPIO.setup(self.sensor_cool, GPIO.OUT)
        if datetime.now().month>=5 and datetime.now().month<11:
            self.set_summer_mode()
        elif datetime.now().month>=11 and datetime.now().month<5:
            self.set_winter_mode()

    def set_winter_mode(self):
        self.issummer = False
        self.iswinter = True
        GPIO.output(self.sensor_heat, True)  ## enciendo el pin
        GPIO.output(self.sensor_cool, False)  ## apago el pin

    def set_summer_mode(self):
        self.issummer =True
        self.iswinter = False
        GPIO.output(self.sensor_cool, True)  ## enciendo el pin
        GPIO.output(self.sensor_heat, False)  ## apago el pin

    def set_temp(self, temp_setter):
        self.temp_target = temp_setter
        self.get_actual_temp()
        if self.temperature() < temp_setter and self.iswinter:
            return self.set_on_mode()
        elif self.temperature() >= temp_setter and self.iswinter:
            return self.set_off_mode()
        elif self.temperature() <= temp_setter and self.issummer:
            return self.set_off_mode()
        elif self.temperature() > temp_setter and self.issummer:
            return self.set_on_mode()

    def get_actual_temp(self):
        #temp_sensor = temp_sensor.get_(self.pin_sensor)
        self.humidity, self.temperature = temp_sensor.get_humidity_temp()
        self.temperature=5
        return (200, "La temperatura actual es de %s grados" %self.temperature)

    def set_on_mode(self):
        if GPIO.input(self.sensor_com):
            return (300, "La calefaccion ya esta encendida"  )
        else:
            GPIO.output(self.sensor_com, True)  ## enciendo el pin
            return (200, "encendiendo la caldera a %s grados en modo %s" % (self.temp_target, self.get_mode()))

    def set_off_mode(self):
        if not GPIO.input(self.sensor_com):
            return (300, "La calefaccion ya esta apagada"  )
        else:
            return (200, "Apagando la caldera a %s grados en modo %s" % (self.temp_target, self.get_mode()))
            GPIO.output(self.sensor_com, False)  ## enciendo el pin

    def get_mode(self):
        if self.issummer:
            return "verano"
        if self.iswinter:
            return "invierno"


import sys
import time

from PyQt5 import QtCore, QtWidgets
from PyQt5.QtGui import QColor

# https://evileg.com/ru/post/579/
# + something renamed...

class Ui_MainWindow(object):
    def setupUi(self, MainWindow):
        MainWindow.setObjectName("MainWindow")
        MainWindow.resize(453, 408)
        self.verticalLayout = QtWidgets.QVBoxLayout(MainWindow)
        self.verticalLayout.setObjectName("verticalLayout")
        self.verticalLayout_2 = QtWidgets.QVBoxLayout()
        self.verticalLayout_2.setObjectName("verticalLayout_2")
        self.textBrowser = QtWidgets.QTextBrowser(MainWindow)
        self.textBrowser.setObjectName("textBrowser")
        self.verticalLayout_2.addWidget(self.textBrowser)
        self.verticalLayout.addLayout(self.verticalLayout_2)
        self.horizontalLayout = QtWidgets.QHBoxLayout()
        self.horizontalLayout.setObjectName("horizontalLayout")
        spacerItem = QtWidgets.QSpacerItem(40, 20, QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Minimum)
        self.horizontalLayout.addItem(spacerItem)
        self.pushButton = QtWidgets.QPushButton(MainWindow)
        self.pushButton.setObjectName("pushButton")
        self.horizontalLayout.addWidget(self.pushButton)
        spacerItem1 = QtWidgets.QSpacerItem(40, 20, QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Minimum)
        self.horizontalLayout.addItem(spacerItem1)
        self.verticalLayout.addLayout(self.horizontalLayout)

        self.retranslateUi(MainWindow)
        QtCore.QMetaObject.connectSlotsByName(MainWindow)

    def retranslateUi(self, MainWindow):
        _translate = QtCore.QCoreApplication.translate
        MainWindow.setWindowTitle(_translate("MainWindow", "--- Пример ---"))
        self.pushButton.setText(_translate("MainWindow", "-- ВВОД --"))

# Объект, который будет перенесён в другой поток для выполнения кода
class BrowserHandler(QtCore.QObject):
    # running = False
    newTextAndColor = QtCore.pyqtSignal(str, object)

    # метод, который будет выполнять алгоритм в другом потоке
    def run(self):
        while True:
            # посылаем сигнал из второго потока в GUI поток
            self.newTextAndColor.emit(f'{time.strftime("%Y-%m-%d %H:%M:%S")} - thread 2 var 1.\n', QColor(0, 0, 150))
            QtCore.QThread.msleep(1000)

            # посылаем сигнал из второго потока в GUI поток
            self.newTextAndColor.emit(f'{time.strftime("%Y-%m-%d %H:%M:%S")} - thread 2 var 2.\n', QColor(150, 0, 0))
            QtCore.QThread.msleep(1000)

#class MyWindow():
class MyWindow(QtWidgets.QWidget):
#class MyWindow(QtWidgets.QMainWindow):

    def __init__(self, parent=None):
        super().__init__()
        self.ui = Ui_MainWindow()
        self.ui.setupUi(self)
        # используем кнопку для добавления текста с цветом в главном потоке
        self.ui.pushButton.clicked.connect(self.addAnotherTextAndColor)

        # создадим поток
        self.thread = QtCore.QThread()
        # создадим объект для выполнения кода в другом потоке
        self.browserHandler = BrowserHandler()
        # перенесём объект в другой поток
        self.browserHandler.moveToThread(self.thread)
        # после чего подключим все сигналы и слоты
        self.browserHandler.newTextAndColor.connect(self.addNewTextAndColor)
        # подключим сигнал старта потока к методу run у объекта, который должен выполнять код в другом потоке
        self.thread.started.connect(self.browserHandler.run)
        # запустим поток
        self.thread.start()

    @QtCore.pyqtSlot(str, object)
    def addNewTextAndColor(self, string, color):
        self.ui.textBrowser.setTextColor(color)
        self.ui.textBrowser.append(string)

    def addAnotherTextAndColor(self):
        self.ui.textBrowser.setTextColor(QColor(0, 200, 0))
        self.ui.textBrowser.append(f'Нажата кл.{time.strftime("%Y-%m-%d %H:%M:%S")} - thread 2 var 3.\n')

if __name__ == '__main__':
    app = QtWidgets.QApplication(sys.argv)
    window = MyWindow()
    window.show()
    sys.exit(app.exec())


import html
import re
import sys
from functools import lru_cache
from pathlib import Path
from typing import Dict

import pymorphy2

PATH = Path("../..")
sys.path.append(str(PATH))

cache: Dict = {}
morph = pymorphy2.MorphAnalyzer()

# read stopwords for RU
try:
    with open(PATH / "data/features/stopwords_ru.txt", "r") as file:
        stopwords = file.read().splitlines()
except FileNotFoundError:
    print("can't load /data/features/stopwords_ru.txt")
    stopwords = []

def clean_text(text: str = None) -> str:
    """
    clean text, leaving only tokens for clustering
    parameters
    ----------
        text : string
            input text
    returns
    -------
        cleaned string text without lower case
    """

    if not isinstance(text, str):
        text = str(text)

    text = html.unescape(text)

    text = re.sub(r"https?:\/\/.*[\r\n]*", "", text)  # remove urls
    text = re.sub(r"\S+@\S+", "", text)  # remove emails
    text = re.sub(r"ё", "е", text)
    text = re.sub(r"\!|\"|\:|\;|\.|\,|[<>]|\?|\@|\[|\]|\^|\_|\`|[{}]|\~|[—–-]|[«»]|[()]", " ", text)  # remove punctuation
    text = re.sub(r"\s+", " ", text)  # remove the long blanks

    text = text.strip()

    if len(text) < 3:
        return "TOREMOVE"
    else:
        return text

@lru_cache()
def get_morph4token(token: str = None) -> str:
    """
    get lemma for one tokens with decorator `@lru_cache`
    """

    return morph.parse(token)[0].normal_form

def lemmatize(text: str = None, char4split: str = " ") -> str:
    """
    lemmatizations text with cache
    parameters
    ----------
    input_text : string
        cleaned text
    char4split : string (default = " ")
        char-symbol how to split text
    returns
    -------
        lemmatized text
    """

    # get tokens from input text
    # in this case it's normal approach because we hard cleaned text
    if not isinstance(text, str):
        text = str(text)

    # get tokens from input text
    # in this case it's normal approach because we hard cleaned text
    list_tokens = text.split(char4split)

    words_lem = [
        get_morph4token(token) for token in list_tokens
        if token not in stopwords
    ]

    if len(words_lem) < 3:
        return "TOREMOVE"
    else:
        return " ".join(words_lem)