"""
Куйвалайнен Д.А.
kuyvalaynen@gmail.com
ЗЕБЗ-01-16
"""

import os.path
import sys
import shutil
import json
import datetime

class Backup:
    """Класс Backup выполняет резервное копирование файлов"""

    SETTFILE = "settings.json"

    def __init__(self, proj):
        """Инициализация класса
        Свойства:
            - self._proj (str) - название проекта
            - self._log_name (str) - имя файла для логирования
            - self._folders (list) - пути бекапа
            - self._tree (list) - дерево директорий
            - self._settings (dict) - настройки программы
            - self._log (dict) - данные для логирования:
                                 errors - ошибки при работе программы,
                                 begin_date - дата начала копирования,
                                 end_date - дата конца копирования,
                                 files - скопированные файлы
        """
        assert isinstance(proj, str), "Название должно быть типа 'str'!"
        self._proj = proj
        self._log_name = ""
        self._folders = []
        self._tree = []
        self._settings = {}
        self._log = {"errors": [],
                     "begin_date": None,
                     "end_date": None,
                     "files": []}
        self._run()

    def __str__(self):
        """Возвращает настройки программы"""
        text = "{0:12} {1}\n".format("Проект:", self._proj)
        text += "Список папок:\n"
        for i in self._folders:
            for key, value in i.items():
                text += "  {0:10} {1}\n".format(key+":", value)
        text += "Опции:"
        for key, value in self._settings.items():
            text += "\n  {0:10} {1}".format(key+":", value)
        return text

    def str_tree(self):
        """Возвращает строковое представление дерева директорий"""
        text = "Содержимое директорий:"
        sp = 0
        for folders in self._tree:
            for i in folders:
                text += "\n" + i
                for dr, prop in folders[i].items():
                    text += "\n  {}: {}".format(prop["path"],
                                                self._get_prop(prop["size"],
                                                               sp))
        return text

    def _get_prop(self, prop, sp):
        """Рекурсивная функция, фозвращает строковое представление
        ветки дерева
        Параметры:
            - prop - содержимое элемента ([] или (int, float))
            - sp (int) - отступ
        """
        try:
            if isinstance(prop, (int, float)):  # если файл
                return " - {0:.2f} Кб".format(prop)
            elif isinstance(prop, list):  # если папка
                text = ""
                for i in prop:
                    for dr, prop in i.items():
                        s = " " * sp + "|" + "_" * sp
                        text += "\n  {2}{0} {1}".\
                                format(dr, self._get_prop(prop["size"], sp+3),
                                       s)
                return text
        except Exception as err:
            mesg = "  Ошибка: {}".format(err)
            self._log['errors'].append(mesg)
            print(mesg)

    def _get_settings(self):
        """Считывает настройки из settings.json"""
        try:
            with open(Backup.SETTFILE, "r", encoding="utf-8") as fh:
                data = json.loads(fh.read())
            self._log_name = data["log_filename"]

            assert data["projects"].get(self._proj, None),\
                "В файле настроек отсутствует проект '{}'!".format(self._proj)

            self._folders = data["projects"][self._proj]["folders"]
            self._settings = data["projects"][self._proj]["options"]
            self._check_settings()
        except Exception as err:
            raise Exception(err)

    def _check_settings(self):
        """Проверяет поле options"""
        if self._settings.get("size_max", None):
            assert isinstance(self._settings["size_max"], int),\
                "В файле настроек значение секции 'size_max' не типа 'int'!"
        assert self._settings.get("confirm", 'None') != 'None',\
            "В файле настроек отсутствует секция 'confirm'!"
        assert isinstance(self._settings["confirm"], bool),\
            "В файле настроек значение секции 'confirm' не типа 'bool'!"

    def _run(self):
        """Работа программы"""
        try:
            self._get_settings()  # получение данных настроек
            print(self.__str__())  # вывод настроек

            for i in self._folders:  # создание дерева файлов
                data = dict()
                for key, value in i.items():
                    data[key] = self._get_tree(value)
                self._tree.append(data)

            print(self.str_tree())  # вывод дерева
            self._backup()  # бекап
            self._log['end_date'] = datetime.datetime.now()
        except Exception as err:
            mesg = "  Ошибка: {}".format(err)
            self._log['errors'].append(mesg)
            print(mesg)
        finally:
            if self._log:
                self._save_log()

    def _get_tree(self, value):
        """Рекурсивная функция, получает дерево файлов
        Параметры:
            - value (str) - путь
        Результат:
            {<Имя>:{path:<путь>,
                    size:<[содержимое папки] или
                         <размер файла(int,float)>>}}
        """
        path = os.path.normpath(value)
        data = dict()
        try:
            if self._check_dir(path):  # если папка
                dirname = os.path.split(path)
                data[dirname[1]] = {"path": path}
                files = os.listdir(path)  # получаем содержимое папки

                # заполнение данных о содержимом папки
                data[dirname[1]]["size"] =\
                    [self._get_tree(os.path.join(path, filename))
                     for filename in files]

            elif self._check_file(path):  #
                filename = os.path.split(path)
                # получение данных о файле
                data[filename[1]] = self._get_file(filename[1], path)
            else:
                # если не файл и не папка
                data[path] = {"path": path, "size": None}
                raise OSError("Дирректория {} не найдена".format(path))

        except OSError as err:
            mesg = "  OSError: {}".format(err)
            self._log['errors'].append(mesg)
            print(mesg)
        except Exception as err:
            mesg = "  Ошибка: {}".format(err)
            self._log['errors'].append(mesg)
            print(mesg)
        finally:
            return data

    def _get_file(self, filename, path):
        """Возвращает данные о файле"""
        size = os.path.getsize(path) / 1024
        return {"path": path, "size": size}

    def _check_dir(self, path):
        """Проверяет, является ли элемент в пути path папкой"""
        return os.path.isdir(path)

    def _check_file(self, path):
        """Проверяет, является ли элемент в пути path файлом"""
        return os.path.isfile(path)

    def _backup(self):
        """Выполняет бекап файлов"""
        self._log['begin_date'] = datetime.datetime.now()

        for i in range(len(self._folders)):
            try:
                f1 = self._check_dir(self._folders[i]["dest"])
                f2 = self._check_dir(self._folders[i]["src"])
                if f1 and f2:
                    # если пути существуют выполняется бекап
                    for dr, prop in self._tree[i]["src"].items():
                        self._copy(prop, self._folders[i]["dest"])
                else:
                    raise OSError("Не могу скопировать файлы из {} в {},"
                                  " т.к. какого-то из путей не существует".
                                  format(self._folders[i]["src"],
                                         self._folders[i]["dest"]))
            except OSError as err:
                mesg = "  OSError: {}".format(err)
                self._log['errors'].append(mesg)
                print(mesg)
            except Exception as err:
                mesg = "  Ошибка: {}".format(err)
                self._log['errors'].append(mesg)
                print(mesg)

    def _copy(self, prop, dest):
        """Рекурсивная функция, выполняет копирование содержимого директории
        Параметры:
            - prop (dict) - свойства копируемой директории
            - dest (str) - путь, куда нужно копировать
        """
        if isinstance(prop["size"], (int, float)):  # если файл
            self._copy_file(prop, dest)
        elif isinstance(prop["size"], list):  # если папка
            try:
                dir_name = os.path.split(prop["path"])
                #  если в dest отсутствует папка - создает ее
                if not self._check_dir(dest + "/" + dir_name[1]):
                    os.mkdir(dest + "/" + dir_name[1])
                    print("Папка {} была создана!".format(dir_name[1]))
                #  копирует содержимое папки в dest
                for i in prop["size"]:
                    for dr, pr in i.items():
                        self._copy(pr, dest + "/" + dir_name[1])

            except Exception as err:
                mesg = "  Ошибка: {}".format(err)
                self._log['errors'].append(mesg)
                print(mesg)

    def _copy_file(self, prop, dest):
        """Копирует файл
        Параметры:
            - prop (dict) - свойства файла
            - dest (str) - путь, куда нужно копировать
        """
        try:
            dir_name = os.path.split(prop["path"])
            assert self._check_file(prop["path"]),\
                "Файл {} не найден!".format(prop["path"])

            flag = True
            if self._settings.get("size_max"):
                if prop["size"] > self._settings["size_max"]:
                    mesg = "Файл {} превышает допустимый размер - {}кб!".\
                            format(prop["path"], self._settings["size_max"])

                    if self._settings["confirm"]:
                        answer = input("{}, скопировать его? (y/n) ".
                                       format(mesg)).upper()
                        if answer != "Y":
                            flag = False
                    else:
                        raise Exception(mesg)

            if self._check_file(dest + "/" + dir_name[1]):
                if self._settings["confirm"]:
                    answer = input("Файл {} уже существует, скопировать его?"
                                   " (y/n) ").upper()
                    if answer != "Y":
                        flag = False

            if flag:
                shutil.copy(prop["path"], dest)
                print("Файл {} скопирован!".format(prop["path"]))
                self._log['files'].append("{0} - {1:.2f}кб".
                                          format(prop["path"], prop["size"]))

        except Exception as err:
            mesg = "  Ошибка: {}".format(err)
            self._log['errors'].append(mesg)
            print(mesg)

    def _save_log(self):
        """Сохраняет результат выполнения программы"""
        try:
            with open(self._log_name, "w", encoding="utf-8") as fh:
                if self._tree:
                    fh.write(self.str_tree())
                fh.write("\n\nДата/время начала копирования: {}".
                         format(self._log['begin_date']))

                fh.write("\n\nДата/время конца копирования: {}".
                         format(self._log['end_date']))

                time = self._log['end_date'] - self._log['begin_date']
                fh.write("\n\nПрошло времени: {}".format(time))

                fh.write("\n\nСкопировано файлов: {}".
                         format(len(self._log['files'])))
                for line in self._log['files']:
                    fh.write("\n  " + line)

                fh.write("\n\nОшибки при выполнении программы {}:".
                         format(len(self._log['errors'])))
                for line in self._log['errors']:
                    fh.write("\n" + line)
        except Exception as err:
            print("При записи файла логов произошла ошибка: ", err)

if __name__ == "__main__":
    try:
        if len(sys.argv) == 2:
            proj = sys.argv[1]
        else:
            proj = input("Введите название проекта: ")
        bk = Backup(proj)
    except Exception as err:
        print("Во время работы произошла ошибка: ", err)


# vim: set fileencoding=utf-8 :
from flask import (
        Blueprint,
        current_app,
        request,
        )
from socketio import socketio_manage

from ..namespaces import (
        identity,
        campaign,
        )

socketio_blueprint = Blueprint("socketio", __name__)

@socketio_blueprint.route("/<path:path>")
def view_socketio(path):
    socketio_manage(request.environ, {
        "/identity": identity.IdentityNamespace,
        "/campaign": campaign.CampaignNamespace,
        },
        request=current_app._get_current_object(),
        )


import random
from user import *
import pygame
from settings import *

class Hud:
    def __init__(self, user):
        self.user = user
        self.display_surface = pygame.display.get_surface()
        try:
            self.font, self.bet_font = pygame.font.Font(FONT_STYLE, FONT_SIZE), pygame.font.Font(FONT_STYLE, FONT_SIZE)
            self.win_font = pygame.font.Font(FONT_STYLE, BIG_FONT_SIZE)
        except:
            print("Couldn't load font!")
            quit()
        self.win_text_angle = random.randint(-4, 4)

    def get_userdata(self):
        user_data=self.user.get_data()
        balance_surf = self.font.render("Balance:" + user_data['balance']+" HUF", True, "white", None)
        bet_surf=self.font.render("Bet:" + user_data['balance']+" HUF", True, "white", None)

        x, y = 20, self.display_surface.get_size()[1] - 30
        balance_rect = balance_surf.get_rect(bottomleft=(x, y))

        x = self.display_surface.get_size()[0] - 20
        bet_rect = bet_surf.get_rect(bottomright=(x, y))
        pygame.draw.rect(self.display_surface, False, balance_rect)
        pygame.draw.rect(self.display_surface, False, bet_rect)
        self.display_surface.blit(balance_surf, balance_rect)
        self.display_surface.blit(bet_surf, bet_rect)

    def update(self):
        pygame.draw.rect(self.display_surface, 'Black', pygame.Rect(0, 900, 1600, 100))
        self.display_info()


#!usr/bin/env python3

# Word segmentation
# John Goldsmith 2014-

# code refactoring/optimization in progress, Jackson Lee, 7/6/2015

import os
import math
import argparse
from pathlib import Path

from latexTable_py3 import MakeLatexTable
from lxa5lib import (get_language_corpus_datafolder, json_pdump,
                     changeFilenameSuffix, stdout_list,
                     load_config_for_command_line_help)

# Jan 6: added precision and recall.

def makeArgParser(configfilename="config.json"):

    language, \
    corpus, \
    datafolder, \
    configtext = load_config_for_command_line_help(configfilename)

    parser = argparse.ArgumentParser(
        description="Word segmentation program.\n\n{}"
                    .format(configtext),
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)

    parser.add_argument("--config", help="configuration filename",
                        type=str, default=configfilename)

    parser.add_argument("--language", help="Language name",
                        type=str, default=None)
    parser.add_argument("--corpus", help="Corpus file to use",
                        type=str, default=None)
    parser.add_argument("--datafolder", help="path of the data folder",
                        type=str, default=None)

    parser.add_argument("--cycles", help="number of cycles",
                        type=int, default=200)
    parser.add_argument("--candidates", help="number of candidates"
                        "per iteration",
                        type=int, default=25)
    parser.add_argument("--verbose", help="verbose output",
                        type=bool, default=False)

    return parser

class LexiconEntry:
    def __init__(self, key = "", count = 0):
        self.m_Key = key
        self.m_Count = count
        self.m_Frequency= 0.0
        self.m_CountRegister = list()

    def ResetCounts(self, current_iteration):
        if len(self.m_CountRegister) > 0:
            last_count = self.m_CountRegister[-1][1]
            if self.m_Count != last_count:
                self.m_CountRegister.append((current_iteration, self.m_Count))
        else:
            self.m_CountRegister.append((current_iteration, self.m_Count))
        self.m_Count = 0
    def Display(self, outfile):
        print("%-20s" % self.m_Key, file=outfile)
        for iteration_number, count in self.m_CountRegister:
            print("%6i %10s" % (iteration_number, "{:,}".format(count)), file=outfile)
# ---------------------------------------------------------#
class Lexicon:
    def __init__(self):
        self.m_LetterDict=dict()
        self.m_LetterPlog = dict()
        self.m_EntryDict = dict()
        self.m_TrueDictionary = dict()
        self.m_DictionaryLength = 0   #in bits! Check this is base 2, looks like default base in python
        self.m_Corpus     = list()
        self.m_SizeOfLongestEntry = 0
        self.m_CorpusCost = 0.0
        self.m_ParsedCorpus = list()
        self.m_NumberOfHypothesizedRunningWords = 0
        self.m_NumberOfTrueRunningWords = 0
        self.m_BreakPointList = list()
        self.m_DeletionList = list()  # these are the words that were nominated and then not used in any line-parses *at all*.
        self.m_DeletionDict = dict()  # They never stop getting nominated.
        self.m_Break_based_RecallPrecisionHistory = list()
        self.m_Token_based_RecallPrecisionHistory = list()
        self.m_Type_based_RecallPrecisionHistory = list()
        self.m_DictionaryLengthHistory = list()
        self.m_CorpusCostHistory = list()
    # ---------------------------------------------------------#
    def AddEntry(self,key,count):
        this_entry = LexiconEntry(key,count)
        self.m_EntryDict[key] = this_entry
        if len(key) > self.m_SizeOfLongestEntry:
            self.m_SizeOfLongestEntry = len(key)
    # ---------------------------------------------------------#
    # Found bug here July 5 2015: important, don't let it remove a singleton letter! John
    # don't del key-value pairs within still iterating through the pairs among the dict; fix by John Goldsmith July 6 2015
    def FilterZeroCountEntries(self, iteration_number):
        TempDeletionList = dict()
        for key, entry in self.m_EntryDict.items():
            if len(key) == 1 and entry.m_Count==0:
                entry.m_Count = 1
                continue
            if entry.m_Count == 0:
                self.m_DeletionList.append((key, iteration_number))
                self.m_DeletionDict[key] = 1
                TempDeletionList[key] = 1
                print("Excluding this bad candidate:", key)
        for key in TempDeletionList:
            del self.m_EntryDict[key]
    # ---------------------------------------------------------#
    def ReadCorpus(self, infilename):
        print("Name of data file:", infilename)
        if not os.path.isfile(infilename):
            print("Warning:", infilename, "does not exist.")
        infile = open(infilename)
        self.m_Corpus = infile.readlines() # bad code if the corpus is very large -- but then we won't use python.
        for line in self.m_Corpus:
            for letter in line:
                if letter not in self.m_EntryDict:
                    this_lexicon_entry = LexiconEntry()
                    this_lexicon_entry.m_Key = letter
                    this_lexicon_entry.m_Count = 1
                    self.m_EntryDict[letter] = this_lexicon_entry
                else:
                    self.m_EntryDict[letter].m_Count += 1
        self.m_SizeOfLongestEntry = 1
        self.ComputeDictFrequencies()
    # ---------------------------------------------------------#
    def ReadBrokenCorpus(self, infilename, numberoflines= 0):
        print("Name of data file:", infilename)
        if not os.path.isfile(infilename):
            print("Warning:", infilename, "does not exist.")
        infile = open(infilename)

        rawcorpus_list = infile.readlines() # bad code if the corpus is very large -- but then we won't use python.
        for line in rawcorpus_list:
            this_line = ""
            breakpoint_list = list()
            line = line.replace('.', ' .').replace('?', ' ?')
            line_list = line.split()
            if len(line_list) <=  1:
                continue
            for word in line_list:
                self.m_NumberOfTrueRunningWords += 1
                if word not in self.m_TrueDictionary:
                    self.m_TrueDictionary[word] = 1
                else:
                    self.m_TrueDictionary[word] += 1
                this_line += word
                breakpoint_list.append(len(this_line))
            self. m_Corpus.append(this_line)
            for letter in line:
                if letter not in self.m_EntryDict:
                    this_lexicon_entry = LexiconEntry()
                    this_lexicon_entry.m_Key = letter
                    this_lexicon_entry.m_Count = 1
                    self.m_EntryDict[letter] = this_lexicon_entry
                else:
                    self.m_EntryDict[letter].m_Count += 1
                if letter not in self.m_LetterDict:
                    self.m_LetterDict[letter] = 1
                else:
                    self.m_LetterDict[letter] += 1
            if numberoflines > 0 and len(self.m_Corpus) > numberoflines:
                break
            self.m_BreakPointList.append(breakpoint_list)
        self.m_SizeOfLongestEntry = 1
        self.ComputeDictFrequencies()

# ---------------------------------------------------------#
    def ComputeDictFrequencies(self):
        TotalCount = 0
        for (key, entry) in self.m_EntryDict.items():
            TotalCount += entry.m_Count
        for (key, entry) in self.m_EntryDict.items():
            entry.m_Frequency = entry.m_Count/float(TotalCount)
        TotalCount = 0
        for (letter, count) in self.m_LetterDict.items():
            TotalCount += count
        for (letter, count) in self.m_LetterDict.items():
            self.m_LetterDict[letter] = float(count)/float(TotalCount)
            self.m_LetterPlog[letter] = -1 * math.log(self.m_LetterDict[letter])
# ---------------------------------------------------------#
    # added july 2015 john
    def ComputeDictionaryLength(self):
        DictionaryLength = 0
        for word in self.m_EntryDict:
            wordlength = 0
            letters = list(word)
            for letter in letters:
                wordlength += self.m_LetterPlog[letter]
            DictionaryLength += wordlength
        self.m_DictionaryLength = DictionaryLength
        self.m_DictionaryLengthHistory.append(DictionaryLength)

# ---------------------------------------------------------#
    def ParseCorpus(self, outfile, current_iteration):
        self.m_ParsedCorpus = list()
        self.m_CorpusCost = 0.0
        self.m_NumberOfHypothesizedRunningWords = 0
        #total_word_count_in_parse = 0
        for word, lexicon_entry in self.m_EntryDict.items():
            lexicon_entry.ResetCounts(current_iteration)
        for line in self.m_Corpus:
            parsed_line,bit_cost =     self.ParseWord(line, outfile)
            self.m_ParsedCorpus.append(parsed_line)
            self.m_CorpusCost += bit_cost
            for word in parsed_line:
                self.m_EntryDict[word].m_Count +=1
                self.m_NumberOfHypothesizedRunningWords += 1
        self.FilterZeroCountEntries(current_iteration)
        self.ComputeDictFrequencies()
        self.ComputeDictionaryLength()
        print("\nCorpus     cost:", "{:,}".format(int(self.m_CorpusCost)))
        print("Dictionary cost:", "{:,}".format(int(self.m_DictionaryLength)))
        sum = int(self.m_CorpusCost + self.m_DictionaryLength)
        print("Total      cost:", "{:,}".format(sum))
        print("\nCorpus cost:", "{:,}".format(self.m_CorpusCost), file=outfile)
        print("Dictionary cost:", "{:,}".format(self.m_DictionaryLength), file=outfile)
        return
# ---------------------------------------------------------#
    def PrintParsedCorpus(self,outfile):
        for line in self.m_ParsedCorpus:
            PrintList(line, outfile)
# ---------------------------------------------------------#
    def ParseWord(self, word, outfile):
        wordlength = len(word)

        Parse=dict()
        Piece = ""
        LastChunk = ""
        BestCompressedLength = dict()
        BestCompressedLength[0] = 0
        CompressedSizeFromInnerScanToOuterScan = 0.0
        LastChunkStartingPoint = 0
        # <------------------ outerscan -----------><------------------> #
        #                  ^---starting point
        # <----prefix?----><----innerscan---------->
        #                  <----Piece-------------->
        if verboseflag: print("\nOuter\tInner", file=outfile)
        if verboseflag: print("scan:\tscan:\tPiece\tFound?", file=outfile)
        for outerscan in range(1,wordlength+1):
            Parse[outerscan] = list()
            MinimumCompressedSize= 0.0
            startingpoint = 0
            if outerscan > self.m_SizeOfLongestEntry:
                startingpoint = outerscan - self.m_SizeOfLongestEntry
            for innerscan in range(startingpoint, outerscan):
                if verboseflag: print("\n %3s\t%3s  " % (outerscan, innerscan), end=" ", file=outfile)
                Piece = word[innerscan: outerscan]
                if verboseflag: print(" %5s"% Piece, end=" ", file=outfile)
                if Piece in self.m_EntryDict:
                    if verboseflag: print("   %5s" % "Yes.", end=" ", file=outfile)
                    CompressedSizeFromInnerScanToOuterScan = -1 * math.log( self.m_EntryDict[Piece].m_Frequency )
                    newvalue =  BestCompressedLength[innerscan]  + CompressedSizeFromInnerScanToOuterScan
                    if verboseflag: print(" %7.3f bits" % (newvalue), end=" ", file=outfile)
                    if  MinimumCompressedSize == 0.0 or MinimumCompressedSize > newvalue:
                        MinimumCompressedSize = newvalue
                        LastChunk = Piece
                        LastChunkStartingPoint = innerscan
                        if verboseflag: print(" %7.3f bits" % (MinimumCompressedSize), end=" ", file=outfile)
                else:
                    if verboseflag: print("   %5s" % "No. ", end=" ", file=outfile)
            BestCompressedLength[outerscan] = MinimumCompressedSize
            if LastChunkStartingPoint > 0:
                Parse[outerscan] = list(Parse[LastChunkStartingPoint])
            else:
                Parse[outerscan] = list()
            if verboseflag: print("\n\t\t\t\t\t\t\t\tchosen:", LastChunk, end=" ", file=outfile)
            Parse[outerscan].append(LastChunk)
        if verboseflag:
            PrintList(Parse[wordlength], outfile)
        bitcost = BestCompressedLength[outerscan]
        return (Parse[wordlength],bitcost)
# ---------------------------------------------------------#
    def GenerateCandidates(self, howmany, outfile, iterationnumber):
        Nominees = dict()
        NomineeList = list()
        for parsed_line in self.m_ParsedCorpus:
            for wordno in range(len(parsed_line)-1):
                candidate = parsed_line[wordno] + parsed_line[wordno + 1]
                if candidate in self.m_EntryDict:
                    continue
                if candidate in Nominees:
                    Nominees[candidate] += 1
                else:
                    Nominees[candidate] = 1
        EntireNomineeList = sorted(Nominees.items(),key=lambda x:x[1],reverse=True)
        for nominee, count in EntireNomineeList:
            if nominee  in self.m_DeletionDict:
                continue
            else:
                NomineeList.append((nominee,count))
            if len(NomineeList) == howmany:
                break
        latex_data= list()
        latex_data.append("Iteration number " + str(iterationnumber))
        latex_data.append("piece   count   status")
        for nominee, count in NomineeList:
            self.AddEntry(nominee,count)
            print("%20s   %8i" %(nominee, count))
            latex_data.append(nominee +  "\t" + "{:,}".format(count) )
        MakeLatexTable(latex_data, outfile)
        self.ComputeDictFrequencies()
        return NomineeList

# ---------------------------------------------------------#
    def Expectation(self):
        self.m_NumberOfHypothesizedRunningWords = 0
        for this_line in self.m_Corpus:
            wordlength = len(this_line)
            ForwardProb = dict()
            BackwardProb = dict()
            Forward(this_line,ForwardProb)
            Backward(this_line,BackwardProb)
            this_word_prob = BackwardProb[0]

            if WordProb > 0:
                for nPos in range(wordlength):
                    for End in range(nPos, wordlength-1):
                        if End- nPos + 1 > self.m_SizeOfLongestEntry:
                            continue
                        if nPos == 0 and End == wordlength - 1:
                            continue
                        Piece = this_line[nPos, End+1]
                        if Piece in self.m_EntryDict:
                            this_entry = self.m_EntryDict[Piece]
                            CurrentIncrement = ((ForwardProb[nPos] * BackwardProb[End+1])* this_entry.m_Frequency ) / WordProb
                            this_entry.m_Count += CurrentIncrement
                            self.m_NumberOfHypothesizedRunningWords += CurrentIncrement

# ---------------------------------------------------------#
    def Maximization(self):
        for entry in self.m_EntryDict:
            entry.m_Frequency = entry.m_Count / self.m_NumberOfHypothesizedRunningWords

# ---------------------------------------------------------#
    def Forward (self, this_line,ForwardProb):
        ForwardProb[0]=1.0
        for Pos in range(1,Length+1):
            ForwardProb[Pos] = 0.0
            if (Pos - i > self.m_SizeOfLongestEntry):
                break
            Piece = this_line[i,Pos+1]
            if Piece in self.m_EntryDict:
                this_Entry = self.m_EntryDict[Piece]
                vlProduct = ForwardProb[i] * this_Entry.m_Frequency
                ForwardProb[Pos] = ForwardProb[Pos] + vlProduct
        return ForwardProb

# ---------------------------------------------------------#
    def Backward(self, this_line,BackwardProb):

        Last = len(this_line) -1
        BackwardProb[Last+1] = 1.0
        for Pos in range( Last, Pos >= 0,-1):
            BackwardProb[Pos] = 0
            for i in range(Pos, i <= Last,-1):
                if i-Pos +1 > m_SizeOfLongestEntry:
                    Piece = this_line[Pos, i+1]
                    if Piece in self.m_EntryDict[Piece]:
                        this_Entry = self.m_EntryDict[Piece]
                        if this_Entry.m_Frequency == 0.0:
                            continue
                        vlProduct = BackwardProb[i+1] * this_Entry.m_Frequency
                        BackwardProb[Pos] += vlProduct
        return BackwardProb

# ---------------------------------------------------------#
    def PrintLexicon(self, outfile):
        for key in sorted(self.m_EntryDict.keys()):
            self.m_EntryDict[key].Display(outfile)
        for iteration, key in self.m_DeletionList:
            print(iteration, key, file=outfile)

# ---------------------------------------------------------#
    def RecallPrecision(self, iteration_number, outfile,total_word_count_in_parse):

        total_true_positive_for_break = 0
        total_number_of_hypothesized_words = 0
        total_number_of_true_words = 0
        for linenumber in range(len(self.m_BreakPointList)):
            truth = list(self.m_BreakPointList[linenumber])
            if len(truth) < 2:
                print("Skipping this line:", self.m_Corpus[linenumber], file=outfile)
                continue
            number_of_true_words = len(truth) -1
            hypothesis = list()
            hypothesis_line_length = 0
            accurate_word_discovery = 0
            true_positive_for_break = 0
            word_too_big = 0
            word_too_small = 0
            real_word_lag = 0
            hypothesis_word_lag = 0

            for piece in self.m_ParsedCorpus[linenumber]:
                hypothesis_line_length += len(piece)
                hypothesis.append(hypothesis_line_length)
            number_of_hypothesized_words = len(hypothesis)

            # state 0: at the last test, the two parses were in agreement
            # state 1: at the last test, truth was # and hypothesis was not
            # state 2: at the last test, hypothesis was # and truth was not
            pointer = 0
            state = 0
            while (len(truth) > 0 and len(hypothesis) > 0):

                next_truth = truth[0]
                next_hypothesis  = hypothesis[0]
                if state == 0:
                    real_word_lag = 0
                    hypothesis_word_lag = 0

                    if next_truth == next_hypothesis:
                        pointer = truth.pop(0)
                        hypothesis.pop(0)
                        true_positive_for_break += 1
                        accurate_word_discovery += 1
                        state = 0
                    elif next_truth < next_hypothesis:
                        pointer = truth.pop(0)
                        real_word_lag += 1
                        state = 1
                    else: #next_hypothesis < next_truth:
                        pointer = hypothesis.pop(0)
                        hypothesis_word_lag = 1
                        state = 2
                elif state == 1:
                    if next_truth == next_hypothesis:
                        pointer = truth.pop(0)
                        hypothesis.pop(0)
                        true_positive_for_break += 1
                        word_too_big += 1
                        state = 0
                    elif next_truth < next_hypothesis:
                        pointer = truth.pop(0)
                        real_word_lag += 1
                        state = 1 #redundantly
                    else:
                        pointer = hypothesis.pop(0)
                        hypothesis_word_lag += 1
                        state = 2
                else: #state = 2
                    if next_truth == next_hypothesis:
                        pointer = truth.pop(0)
                        hypothesis.pop(0)
                        true_positive_for_break += 1
                        word_too_small +=1
                        state = 0
                    elif next_truth < next_hypothesis:
                        pointer = truth.pop(0)
                        real_word_lag += 1
                        state = 1
                    else:
                        pointer = hypothesis.pop(0)
                        hypothesis_word_lag += 1
                        state =2

            precision = float(true_positive_for_break) /  number_of_hypothesized_words
            recall    = float(true_positive_for_break) /  number_of_true_words

            total_true_positive_for_break += true_positive_for_break
            total_number_of_hypothesized_words += number_of_hypothesized_words
            total_number_of_true_words += number_of_true_words

        # the following calculations are precision and recall *for breaks* (not for morphemes)

        formatstring = "%30s %6.4f %12s %6.4f"
        total_break_precision = float(total_true_positive_for_break) /  total_number_of_hypothesized_words
        total_break_recall    = float(total_true_positive_for_break) /  total_number_of_true_words
        self.m_CorpusCostHistory.append( self.m_CorpusCost)
        self.m_Break_based_RecallPrecisionHistory.append((iteration_number,  total_break_precision,total_break_recall))
        print(formatstring %( "Break based Word Precision", total_break_precision, "recall", total_break_recall))
        print(formatstring %( "Break based Word Precision", total_break_precision, "recall", total_break_recall), file=outfile)

        # Token_based precision for word discovery:

        if (True):
            true_positives = 0
            for (word, this_words_entry) in self.m_EntryDict.items():
                if word in self.m_TrueDictionary:
                    true_count = self.m_TrueDictionary[word]
                    these_true_positives = min(true_count, this_words_entry.m_Count)
                else:
                    these_true_positives = 0
                true_positives += these_true_positives
            word_recall = float(true_positives) / self.m_NumberOfTrueRunningWords
            word_precision = float(true_positives) / self.m_NumberOfHypothesizedRunningWords
            self.m_Token_based_RecallPrecisionHistory.append((iteration_number,  word_precision,word_recall))

            print(formatstring %( "Token_based Word Precision", word_precision, "recall", word_recall), file=outfile)
            print(formatstring %( "Token_based Word Precision", word_precision, "recall", word_recall))

        # Type_based precision for word discovery:
        if (True):
            true_positives = 0
            for (word, this_words_entry) in self.m_EntryDict.items():
                if word in self.m_TrueDictionary:
                    true_positives +=1
            word_recall = float(true_positives) / len(self.m_TrueDictionary)
            word_precision = float(true_positives) / len(self.m_EntryDict)
            self.m_Type_based_RecallPrecisionHistory.append((iteration_number,  word_precision,word_recall))

            #print >>outfile, "\n\n***\n"
#            print "Type_based Word Precision  %6.4f; Word Recall  %6.4f" %(word_precision ,word_recall)
            print(formatstring %( "Type_based Word Precision", word_precision, "recall", word_recall), file=outfile)
            print(formatstring %( "Type_based Word Precision", word_precision, "recall", word_recall))

# ---------------------------------------------------------#
    def PrintRecallPrecision(self,outfile):
        print("\t\t\tBreak\t\tToken-based\t\tType-based", file=outfile)
        print("\t\t\tprecision\trecall\tprecision\trecall\tprecision\trecall", file=outfile)
        for iterno in (range(numberofcycles-1)):
            print("printing iterno", iterno)
            (iteration, p1,r1) = self.m_Break_based_RecallPrecisionHistory[iterno]
            (iteration, p2,r2) = self.m_Token_based_RecallPrecisionHistory[iterno]
            (iteration, p3,r3) = self.m_Type_based_RecallPrecisionHistory[iterno]
            cost1 = int(self.m_DictionaryLengthHistory[iterno])
            cost2 = int(self.m_CorpusCostHistory[iterno] )
            #print >>outfile,"%3i\t%8.3f\t%8.3f\t%8.3f\t%8.3f\t%8.3f\t%8.3f" %(iteration, r1,p1,r2,p2,r3,p3)
            print(iteration,"\t",cost1, "\t", cost2, "\t", p1,"\t",r1,"\t",p2,"\t",r2,"\t",p3,"\t",r3, file=outfile)

# ---------------------------------------------------------#
def PrintList(my_list, outfile):
    print(file=outfile)
    for item in my_list:
        print(item, end=" ", file=outfile)

def main(language, corpus, datafolder,
         numberofcycles, candidatesperiteration, verboseflag):

    corpusfile = corpus
    datadirectory = Path(datafolder, language)

    # to be taken away/replaced with something more generic
    shortoutname             = "wordbreaker-brownC-"

    total_word_count_in_parse     = 0
    numberoflines             =  0

    corpusfilename             = str(Path(datadirectory, corpusfile))
    outdirectory             = Path(datadirectory, "wordbreaking")

    if not outdirectory.exists():
        outdirectory.mkdir(parents=True)

    outfilename = str(Path(outdirectory, shortoutname + str(numberofcycles) + "i_py3.txt"))
    outfile_corpus_name = str(Path(outdirectory, shortoutname + str(numberofcycles) + "_brokencorpus_py3.txt"))
    outfile_lexicon_name = str(Path(outdirectory, shortoutname+ str(numberofcycles) + "_lexicon_py3.txt"))
    outfile_RecallPrecision_name = str(Path(outdirectory, shortoutname + str(numberofcycles) +  "_RecallPrecision_py3.tsv"))

    outfile = open(outfilename, "w")
    outfile_corpus = open(outfile_corpus_name, "w")
    outfile_lexicon = open(outfile_lexicon_name, "w")
    outfile_RecallPrecision = open(outfile_RecallPrecision_name, "w")

    print("#" + str(corpusfile), file=outfile)
    print("#" + str(numberofcycles) + " cycles.", file=outfile)
    print("#" + str(numberoflines) + " lines in the original corpus.", file=outfile)
    print("#" + str(candidatesperiteration) + " candidates on each cycle.", file=outfile)

    current_iteration = 0
    this_lexicon = Lexicon()
    this_lexicon.ReadBrokenCorpus (corpusfilename, numberoflines)
    print("#" + str(len(this_lexicon.m_TrueDictionary)) + " distinct words in the original corpus.", file=outfile)

    this_lexicon.ParseCorpus (outfile, current_iteration)

    for current_iteration in range(1, numberofcycles):
        print("\n Iteration number", current_iteration, "out of ", numberofcycles)
        print("\n\n Iteration number", current_iteration, file=outfile)
        this_lexicon.GenerateCandidates(candidatesperiteration, outfile,current_iteration)
        this_lexicon.ParseCorpus (outfile, current_iteration)
        this_lexicon.RecallPrecision(current_iteration, outfile,total_word_count_in_parse)

    this_lexicon.PrintParsedCorpus(outfile_corpus)
    this_lexicon.PrintLexicon(outfile_lexicon)
    this_lexicon.PrintRecallPrecision(outfile_RecallPrecision)
    outfile.close()
    outfile_corpus.close()
    outfile_lexicon.close()
    outfile_RecallPrecision.close()

if __name__ == "__main__":

    args = makeArgParser().parse_args()

    numberofcycles = args.cycles
    candidatesperiteration = args.candidates
    verboseflag = args.verbose

    description="You are running {}.\n".format(__file__) + \
                "This program works on word segmentation.\n" + \
                "numberofcycles = {}\n".format(numberofcycles) + \
                "candidatesperiteration = {}\n".format(candidatesperiteration) + \
                "verboseflag = {}\n".format(verboseflag)

    language, corpus, datafolder = get_language_corpus_datafolder(args.language,
                                      args.corpus, args.datafolder, args.config,
                                      description=description,
                                      scriptname=__file__)

    main(language, corpus, datafolder,
         numberofcycles, candidatesperiteration, verboseflag)


from collections import defaultdict
from functools import partial
from typing import List, Set, Tuple, Dict, Callable, Optional
import hashlib
import re
import string
import urllib

from datasets import Dataset

# ======== DEDUPLICATION FUNCTIONS ===================
from clean_helpers.utils import parse_meta

def build_dedup_template(min_template_line_size: int, min_template_line_occurence: int):
    def dedup_template(ds: Dataset, num_proc: int, batch_size: int) -> Dataset:
        """Computes and remove templates lines"""
        # Compute the hash of each lines
        split_into_lines_and_hashes = ds.map(
            split_text_to_lines_and_hash,
            num_proc=num_proc,
            batched=True,
            batch_size=batch_size,
            remove_columns=ds.column_names
        )
        lines_and_hashes = split_into_lines_and_hashes.remove_columns(
            set(split_into_lines_and_hashes.column_names) - {"lines", "hashes"}
        )

        # TODO: Batch read could help make it faster
        # Find template lines
        count_lines_occurence = defaultdict(lambda: 0)
        for row in lines_and_hashes:
            filtered_lines_and_hashes = [
                (line, hash_)
                for line, hash_ in zip(row["lines"], row["hashes"])
                if len(line) >= min_template_line_size
            ]
            for _, hash_ in filtered_lines_and_hashes:
                count_lines_occurence[hash_] += 1

        template_line_hashes = {k for k, v in count_lines_occurence.items() if v >= min_template_line_occurence}
        del count_lines_occurence

        # Clean dataset
        return split_into_lines_and_hashes.map(
            build_remove_template_lines(template_line_hashes),
            num_proc=num_proc,
            batched=True,
            batch_size=batch_size,
            remove_columns=split_into_lines_and_hashes.column_names
        )

    return dedup_template

def build_dedup_document(batch_normalizer: Callable[[Dict], List[str]]):
    def dedup_document(ds: Dataset, num_proc: int, batch_size: int) -> Dataset:
        hashed_documents = ds.map(
            lambda batch: {**batch, "hash": get_hash(batch_normalizer(batch))},
            num_proc=num_proc,
            batched=True,
            batch_size=batch_size,
        )

        hashes = set()

        return hashed_documents.map(
            partial(delete_text_from_duplicates, hashes=hashes),
            num_proc=1,  # VERY IMPORTANT: hashes will be updated, and is not thread safe.
            batched=True,
            batch_size=batch_size,
            remove_columns=hashed_documents.column_names
        )

        # # TODO: This version is memory efficient and runs faster, but we lose alignment
        # return hashed_documents.filter(
        #     lambda hashes_: [is_new_hash(hash_, hashes) for hash_ in hashes_],
        #     num_proc=1,  # VERY IMPORTANT: hashes will be updated, and is not thread safe.
        #     input_columns=["hash"],
        #     batched=True,
        #     batch_size=batch_size,
        # ).remove_columns("hash")
    return dedup_document

# =========== HELPERS ===============

def get_hash(texts: List[str]) -> List[str]:
    """Get hash of content field."""
    return [hashlib.md5(text.strip().encode("utf-8")).hexdigest() for text in texts]

def split_text_in_lines(text: str) -> List[str]:
    return [line.strip() for line in text.split("\n")]

def split_text_to_lines_and_hash(batch: Dict[str, List]):
    lines_per_texts = [split_text_in_lines(text) for text in batch["text"]]
    return {
        **{k: v for k, v in batch.items() if k != "text"},
        "lines": lines_per_texts,
        "hashes": [get_hash(lines) for lines in lines_per_texts]
    }

def clean_text(lines_and_hashes: List[Tuple[str, int]], template_line_hashes: Set[str]):
    return "\n".join([line for line, hash_ in lines_and_hashes if hash_ not in template_line_hashes])

def build_remove_template_lines(template_line_hashes: Set[str]):
    def remove_template_lines(batch: Dict[str, List]):
        cleaned_texts = [
            clean_text(
                list(zip(lines, hashes)),
                template_line_hashes
            )
            for lines, hashes in zip(batch["lines"], batch["hashes"])
        ]
        return {
            **{
                key: value
                for key, value in batch.items()
                if key not in ["lines", "hashes"]
            },
            "text": [cleaned_text for cleaned_text in cleaned_texts]
        }

    return remove_template_lines

def is_new_hash(hash_: str, hashes: Set[str]) -> bool:
    """Check if current hash is still in set of unique hashes and remove if true."""
    if hash_ in hashes:
        return False
    else:
        hashes.add(hash_)
        return True

def delete_text_from_duplicates(batch: Dict[str, List], hashes: Set[str]) -> Dict[str, List]:
    return {
        **{k: v for k, v in batch.items() if k != "hash"},
        "text": [text if is_new_hash(hash_, hashes) else "" for text, hash_ in zip(batch["text"], batch["hash"])]
    }

def url_with_only_some_query_param(url: str, query_param_map: Optional[dict] = None) -> str:
    url_parse = urllib.parse.urlparse(url)
    query = url_parse.query

    url_query_params = urllib.parse.parse_qsl(query)

    if query_param_map is None:
        url_query_params_new = {}
    else:
        url_query_params_new = [(query_param_map[old_key], old_value) for (old_key, old_value) in url_query_params if old_key in query_param_map]

    url_new_query = urllib.parse.urlencode(url_query_params_new, encoding="utf-8")
    url_parse = url_parse._replace(query=url_new_query)
    new_url = urllib.parse.urlunparse(url_parse)
    return new_url

# =========== BATCH NORMALISER ===============

# this only keeps letter characters
remove_non_character_regex = re.compile(f'\s+|\d+|[{re.escape(string.punctuation)}]')
def document_batch_normalizer(batch: Dict) -> List[str]:
    return [remove_non_character_regex.sub('', text) for text in batch["text"]]

def strict_url_batch_normalizer(batch: Dict) -> List[str]:
    return [parse_meta(meta)["url"] for meta in batch["meta"]]

url_host_and_path_regex = re.compile(r"^(.[^?]*)")
def url_host_and_path_batch_normalizer(batch: Dict) -> List[str]:
    return [url_host_and_path_regex.match(parse_meta(meta)["url"]).group(1) for meta in batch["meta"]]

lm_es_pseudocrawl_filtered_341_es_cointelegraph_com_regex = re.compile(r"^((?:(?!/amp)/?(?:[^?/]*))+)(?:/amp)?")
def url_lm_es_pseudocrawl_filtered_341_es_cointelegraph_com(batch: Dict) -> List[str]:
    return [lm_es_pseudocrawl_filtered_341_es_cointelegraph_com_regex.match(parse_meta(meta)["url"]).group(1) for meta in batch["meta"]]

def url_lm_en_pseudocrawl_filtered_619_www_qut_edu_au(batch: Dict) -> List[str]:
    return [url_with_only_some_query_param(parse_meta(meta)["url"], {"id": "id", "news-id": "id"}) for meta in batch["meta"]]


# -*- coding: utf-8 -*-
"""
Created on Tue Nov  2 20:29:42 2021

@author: jis
"""

import pandas as pd
import folium
from folium import plugins
import webbrowser
from collections import defaultdict, OrderedDict
import sys
import numpy as np
# from geopy.geocoders import Nominatim

# %%
## open cases_cleaned.csv and store it in an dataframe and then filter the date to the newest date
file = ".\\cases_cleaned_normal.csv"
cases_cleaned = pd.read_csv(file,index_col=0)

# %%
# cases_cleaned = cases_cleaned[cases_cleaned['Country_Region'].notna() &
#                               cases_cleaned['Province_State'].notna() &
#                               cases_cleaned['Admin2'].isna()]

#%%
cases_cleaned['date'] = pd.to_datetime(cases_cleaned['date'], format='%Y/%m/%d')
cases_cleaned.sort_values(by='date', inplace=True)
#cases_cleaned['date'] = cases_cleaned['date'].sort_values(ascending=True)

# %%
#removing nan or 0 lat long values
cases_cleaned = cases_cleaned[cases_cleaned['Lat'].notna() & cases_cleaned['Long_'].notna() &
                              (cases_cleaned['Lat'] != 0) & (cases_cleaned['Long_'] != 0)]

# %% Interpolating missing dates data
# min_date = cases_cleaned.date.min()
# max_date = cases_cleaned.date.max()
# from https://stackoverflow.com/questions/30056399/interpolate-and-fill-pandas-dataframe-with-datetime-index

# def fill_missing_dates(x):

#     # Setting date index
#     x.set_index('date',inplace=True)

#     #x = x.reindex(pd.date_range(start=min_date,end = max_date, freq='1D'))

#     # Rexampling by date and interpolating values
#     x = x.resample('D').interpolate().round()

#     #x = x.interpolate(method='linear')

#     # Returning the group with date as a column again
#     return x.reset_index()

# cases_cleaned = cases_cleaned.groupby(['Admin2', 'Province_State', 'Country_Region'],
#                       dropna=False).apply(fill_missing_dates).reset_index(drop=True)

# %% Formatting the date column to work appropriately
cases_cleaned['date'] = cases_cleaned['date'].dt.strftime('%Y/%m/%d')

# %%Getting only US data
filter_us = cases_cleaned['Country_Region'] == 'US'
cases_cleaned_us = cases_cleaned.loc[filter_us,:].copy()

# %% Getting only PR data
filter_pr = cases_cleaned['Province_State'] == 'Puerto Rico'
cases_cleaned_pr = cases_cleaned.loc[filter_pr,:].copy()

# %% creating weights
cases_cleaned['Weight'] = cases_cleaned['Confirmed'] / cases_cleaned['Confirmed'].max()

cases_cleaned_us['Weight'] = cases_cleaned_us['Confirmed']/cases_cleaned_us['Confirmed'].max()

cases_cleaned_pr['Weight'] = cases_cleaned_pr['Confirmed']/cases_cleaned_pr['Confirmed'].max()

# %% Replacing zeros and nan weights with smalles number

cases_cleaned['Weight'] = cases_cleaned['Weight'].replace(0, sys.float_info.min)
cases_cleaned['Weight'] = cases_cleaned['Weight'].replace(np.nan, sys.float_info.min)

cases_cleaned_us['Weight'] = cases_cleaned_us['Weight'].replace(0, sys.float_info.min)
cases_cleaned_us['Weight'] = cases_cleaned_us['Weight'].replace(np.nan, sys.float_info.min)

cases_cleaned_pr['Weight'] = cases_cleaned_pr['Weight'].replace(0, sys.float_info.min)
cases_cleaned_pr['Weight'] = cases_cleaned_pr['Weight'].replace(np.nan, sys.float_info.min)
#%% global data for heatmap
# Ref https://stackoverflow.com/questions/64325958/heatmapwithtime-plugin-in-folium

heat_data = defaultdict(list)

for row in cases_cleaned.itertuples():
    heat_data[row.date].append([row.Lat,row.Long_,row.Weight])

heat_data = OrderedDict(sorted(heat_data.items(),key=lambda t:t[0]))

# Export the heatmap
covid_country_map = folium.Map(location=[0,0],zoom_start=3, control_scale=True,attr="Stadia.AlidadeSmoothDark")

hm = plugins.HeatMapWithTime(data = list(heat_data.values()),index=list(heat_data.keys()),
                             auto_play=True,max_speed=60,max_opacity=1)
hm.add_to(covid_country_map)

covid_country_map.save("global_spread.html")

webbrowser.open("global_spread.html")
# %% US heatmap
heat_data = defaultdict(list)

for row in cases_cleaned_us.itertuples():
    heat_data[row.date].append([row.Lat,row.Long_,row.Weight])

heat_data = OrderedDict(sorted(heat_data.items(),key=lambda t:t[0]))

# Export the heatmap
covid_country_map = folium.Map(location=[cases_cleaned_us["Lat"].mean(),
                                        cases_cleaned_us["Long_"].mean()],
                                      zoom_start=4,control_scale=True,attr="Stadia.AlidadeSmoothDark")

hm = plugins.HeatMapWithTime(data = list(heat_data.values()),index=list(heat_data.keys()),
                             auto_play=True,max_speed=60,max_opacity=1)
hm.add_to(covid_country_map)

covid_country_map.save("us_spread.html")

webbrowser.open("us_spread.html")
# %% PR Heatmap
heat_data = defaultdict(list)

for row in cases_cleaned_pr.itertuples():
    heat_data[row.date].append([row.Lat,row.Long_,row.Weight])

heat_data = OrderedDict(sorted(heat_data.items(),key=lambda t:t[0]))

# Export the heatmap
covid_country_map = folium.Map(location=[cases_cleaned_pr["Lat"].mean(),
                                        cases_cleaned_pr["Long_"].mean()],
                                      zoom_start=9, control_scale=True,attr="Stadia.AlidadeSmoothDark")

hm = plugins.HeatMapWithTime(data = list(heat_data.values()),index=list(heat_data.keys()),
                             auto_play=True,max_speed=60,max_opacity=1)
hm.add_to(covid_country_map)

covid_country_map.save("pr_spread.html")

webbrowser.open("pr_spread.html")

# # %%
# california = cases_cleaned[(cases_cleaned['Country_Region'] == 'US') &
#                            (cases_cleaned['Province_State'] == 'California') &
#                            (cases_cleaned['date'] == '2020/02/21')]

# # %% plotting unique points to identify outlier
# unique_lat_long = cases_cleaned.groupby(['Lat','Long_']).size().reset_index(name='Freq')


class PruebaExcepcion():
    def Ejercicio(self):
        try:
            diction = {"Make": "Nissan", "Model":"Maxima", "Year": "2020"}
            print(diction['Make'])
            print(diction['Color'])
        except:
            print("Calling the wrong key.")
        finally:
            print("Remember to look for the correct key:value pair.")

Pr = PruebaExcepcion()
Pr.Ejercicio()


#	By: Tanushri Singh
#   CS 6350 - Big Data Management and Analytics
#  	Instructor: Latifur Khan

###################
#WHAT IT SHOULD DO:-
#The output should contain one line per user in the following format:
#<User_A>, <User_B><TAB><Mutual/Common Friend Number>
#where <User_A> & <User_B> are unique IDs corresponding to a user A and B (A and B are friend).
#< Mutual/Common Friend Number > is total number of common friends between user A and user B.
###################

#Import Lib
from pyspark.sql import SparkSession
from pyspark.sql.functions import udf, col
from pyspark.sql.types import ArrayType, StringType, IntegerType

spark = SparkSession.builder.getOrCreate()

#Read in soc-LiveJournal
inputDf = spark.read \
    .option("sep", "\t") \
    .csv("DataFiles/soc-LiveJournal1Adj.txt")
parseUdf = udf(lambda stringFriends:
               [] if stringFriends is None else
               [int(friend) for friend in stringFriends.split(",")],
               ArrayType(IntegerType(), False))
inputDf = inputDf \
    .withColumnRenamed("_c0", "userId") \
    .withColumnRenamed("_c1", "stringFriends")
inputDf = inputDf.withColumn("userId", inputDf["userId"].cast(IntegerType())) \
    .withColumn("friends", parseUdf(inputDf.stringFriends))

#Make pairs
def makePairs(userId, friends):
    pairs = []
    for friend in friends:
        if userId < friend:
            pairs.append([userId, friend])
    return pairs

makePairsUdf = udf(makePairs, ArrayType(
    ArrayType(IntegerType(), False), False))

#Make new Dataframe called pairedDf that stores just the pair of frieds
pairedDf = inputDf.sort("userId") \
    .withColumn("pairs", makePairsUdf(inputDf.userId, inputDf.friends)) \
    .select("pairs").rdd \
    .flatMap(lambda x: x) \
    .flatMap(lambda x: x) \
    .toDF()

#Rename columns of pairedDf
pairedDf = pairedDf \
    .withColumnRenamed("_1", "user1") \
    .withColumnRenamed("_2", "user2")
pairedDf = inputDf.select(col("userId"), col("friends").alias("user1friends")) \
    .join(pairedDf, inputDf.userId == pairedDf.user1, how="right")
pairedDf = inputDf.select(col("userId"), col("friends").alias("user2friends")) \
    .join(pairedDf, inputDf.userId == pairedDf.user2, how="right")
commonUdf = udf(lambda a, b: len(set(a).intersection(b)), IntegerType())

#Accumalate output
output = pairedDf.select("user1", "user2", "user1friends", "user2friends") \
    .withColumn("commonFriendsCount", commonUdf(pairedDf.user1friends, pairedDf.user2friends)) \
    .select("user1", "user2", "commonFriendsCount") \
    .orderBy("user1", "user2").collect()

#Output results to file
outputFile = open("PartOne_output.txt", "w")
for row in output:
    outputFile.write(str(row["user1"]) + ", " + str(row["user2"]) +
                     "\t" + str(row["commonFriendsCount"]) + "\n")
outputFile.close()


#!/usr/bin/env python

import os
import sys
import inspect
import pandas as pd
import geocoder
# import custom functions from tools folder
sys.path.append("tools")
import custom_geocode
import standardize
import unit_tests

def preprocess(geocode=False, standardizer=False, clean=False):
    '''All data wrangling steps prior to building the map.

    Args:
        geocode (bol): True, geocode the data and create a new data set ready to map
        standardizer (bol): True, create new standardized crime rate or change hover text
        clean (bol): True, load pre-cleaned dataset

    return:
        dataframe ready to map

    Lat long coordinates not included in department of Ed data set.
    Therefore, the processing of this data is a bit more involved than
    you may expect, as addresses need to be geocoded.

    Hover text for map is created in this function, as it is a
    major data wrangling step

    1. read data
    2. geocode
    3. standardize crime rates
    4. format hover text
    '''
    raw_file = 'oncampuscrime141516.csv'
    to_standardize_file = 'hand_input.csv'
    clean_file = 'final_geocoded_data.csv'

    in_folder = '/data/'
    out_folder = '/output/'

    geocode_out = 'geo_latest_attempts.csv'
    to_std_out = 'to_standardize_lastest_attempts.csv'
    standardized_out = 'final_standardized_data.csv'

    if geocode:
        ### read raw data into memory
        # locate the path of execution and find the
        # data in the data folder
        data_path = get_path() + in_folder + raw_file
        crime_df_raw = pd.read_csv(data_path, index_col=None)

        # UNIT TEST - check for dropped rows
        # making sure I didn't drop any rows on import
        # comparing to row count of raw csv in excel
        unit_tests.ut_row_count(test_df=crime_df_raw, target=11251)

        ### only keep the columns I need
        slim_crime_df = crime_df_raw.loc[:,('INSTNM', 'BRANCH', 'Address',
                                            'City', 'State', 'ZIP', 'sector_cd',
                                            'Sector_desc', 'men_total',
                                            'women_total', 'Total', 'RAPE16',
                                            'FONDL16', 'ROBBE16', 'AGG_A16')]

        # make complete address (housekeeping for Google geocoder) and
        # it doesn't like college, or business, names
        slim_crime_df["geo_address"] = slim_crime_df["Address"] \
                                    + ", " \
                                    + slim_crime_df["City"] \
                                    + ", " \
                                    + slim_crime_df["State"] \
                                    + ", " \
                                    + slim_crime_df["ZIP"]

        # UNIT TEST - check for dropped rows
        # I should still have the same number of rows as my raw data file
        unit_tests.ut_row_count(test_df=slim_crime_df, target=11251)

        ### only keep the schools that have a crime I am mapping
        # I also want to keep the number of rows under 2,500
        # which is the number of times your are allowed to hit the
        # google geocodeing server for free

        # fill na values with zeros, because I am subsetting the df by Crime > 0
        slim_crime_df.loc[:,('RAPE16','FONDL16','ROBBE16','AGG_A16')] = slim_crime_df.loc[:,('RAPE16','FONDL16',
                                                                                'ROBBE16','AGG_A16')].fillna(value=0)

        print("first")
        print(slim_crime_df.shape)

        # also dropping schools that can't fill out thier own address correctly
        slim_crime_df.loc[:,('City','State','ZIP')] = slim_crime_df.loc[:,('City','State','ZIP')].fillna(value='')

        print("sec")
        print(slim_crime_df.shape)

        # keep school that have a crime I am mapping
        crime_df_small = slim_crime_df[(slim_crime_df.RAPE16 > 0)
                                     | (slim_crime_df.FONDL16 > 0)
                                     | (slim_crime_df.ROBBE16 > 0)
                                     | (slim_crime_df.AGG_A16 > 0)]

        print("third")
        print(crime_df_small.shape)
        print(crime_df_small.dtypes)

        crime_df_smaller = crime_df_small[(crime_df_small.City != '')
                                     & (crime_df_small.State != '')
                                     & (crime_df_small.ZIP != '')]

        print("fourth")
        print(crime_df_smaller.shape)

        # UNIT TEST - check for poor logic and nulls
        # My exclusion set, plus my inclusion set, should equal the total
        # set prior to subsetting. Exclusion set: All crimes of interest == 0
        # ToDo: Need to include opp string sup logic is unit test
        # unit_tests.opposite_subset(df_total=slim_crime_df, df_keep=crime_df_smaller)

        # ----------------------------------------------------
        ### Geocode / get lat long for each school of interest
        # ----------------------------------------------------
        # via the google geocoder and a custom function
        # switch to True if you actually want to geocode stuff
        # unit test included in custom_geocoder
        # 'uncoded' addresses printed to data folder
        geo_crime_df = custom_geocode.custom_geocoder(crime_df_smaller, "geo_address")

        # write for stardardize parameter
        out_name = 'to_standardize_data.csv'
        out_path = get_path() + out_folder + out_name
        geo_crime_df.to_csv(out_path, sep=',')

        # standardize all crime rates for every crime and school
        # crimes getting standardized are in crime list
        crime_cols = ['AGG_A16', 'ROBBE16', 'FONDL16', 'RAPE16']
        standardized__df = standardize.standardize_crime_rates(geo_crime_df, crime_cols)

        # UNIT TEST - check for dropped rows
        unit_tests.compare_row_counts(df_new=standardized__df,
                                      df_test=geo_crime_df)

        # give df intuitive names
        renamed_df = rename_columns(standardized__df)
        # UNIT TEST - check for dropped rows
        unit_tests.compare_row_counts(df_new=renamed_df,
                                      df_test=standardized__df)

        # get text for mouse hover
        final_df = get_hover_text(renamed_df, crime_cols)
        # UNIT TEST - check for dropped rows
        unit_tests.compare_row_counts(df_new=final_df,
                                      df_test=renamed_df)

        # write for QA
        out_name = 'final_geocoded_data.csv'
        out_path = get_path() + out_folder + out_name
        final_df.to_csv(out_path, sep=',')

        return final_df

    if standardizer:
        # if you want to adjust how crime rates are calculated
        # but not re-geocode all the data
        data_path = get_path() + in_folder + to_standardize_file
        crime_df_clean = pd.read_csv(data_path, index_col=None)
        # unit_tests.ut_row_count(test_df=crime_df_clean, target=1929)

        # standardize all crime rates for every crime and school
        # crimes getting standardized are in crime list
        crime_cols = ['AGG_A16', 'ROBBE16', 'FONDL16', 'RAPE16']
        standardized__df = standardize.standardize_crime_rates(crime_df_clean, crime_cols)
        # UNIT TEST - check for dropped rows
        unit_tests.compare_row_counts(df_new=standardized__df,
                                      df_test=crime_df_clean)

        renamed_df = rename_columns(standardized__df)
        # UNIT TEST - check for dropped rows
        unit_tests.compare_row_counts(df_new=standardized__df,
                                      df_test=crime_df_clean)

        final_df = get_hover_text(renamed_df, crime_cols)
        # UNIT TEST - check for dropped rows
        unit_tests.compare_row_counts(df_new=standardized__df,
                                      df_test=crime_df_clean)

        # write to csv for QA
        out_name = 'hand_standardized_data_out.csv'
        out_path = get_path() + out_folder + out_name
        final_df.to_csv(out_path, sep=',')

        return final_df

    if clean:
        # if you just want to load the final data set from csv
        # find and return the clean data set located in the data folder
        data_path = get_path() + in_folder + clean_file

        # read csv into memory and check for dropped rows
        crime_df_clean = pd.read_csv(data_path, index_col=None)
        unit_tests.ut_row_count(test_df=crime_df_clean, target=2118)

        return crime_df_clean

def get_path():
    # simple dynamic file path
    pathname = os.path.dirname(sys.argv[0])
    abs_path = os.path.abspath(pathname)

    return abs_path

def rename_columns(df):
    # it is what it is
    renamed_df = df.rename(columns={
        'standardized_RAPE16_rate':'Rape',
        'standardized_FONDL16_rate': 'Fondling',
        'standardized_ROBBE16_rate':'Robbery',
        'standardized_AGG_A16_rate':'Assault'
        })

    return renamed_df

def get_hover_text(df, crime_list):
    # apply make_hover_text() across all the
    # rows of the df
    mod_df = df

    for crime in crime_list:
        crime_hover_text = '%s_hover_text' % crime
        # apply function across the rows of a df
        mod_df[crime_hover_text] = df.apply(make_hover_text,
                                            args=(crime,),
                                            axis=1)

    return mod_df

def make_hover_text(row, crime):
    # creating hover text for plotly map
    if crime == 'RAPE16':
        std_crime_rate = 'Rape'

    if crime == 'FONDL16':
        std_crime_rate = 'Fondling'

    if crime == 'ROBBE16':
        std_crime_rate = 'Robbery'

    if crime == 'AGG_A16':
        std_crime_rate = "Assault"

    # create hover text string
    if row['Total'] > 0:
        hover_text = row['INSTNM'] \
                       + '
' \
                       + str.title(row['City']) \
                       + ', ' \
                       + str(row['State']) \
                       + '
' \
                       + 'Crime Count: ' \
                       + str(row[crime]) \
                       + '
' \
                       + 'Enrollment: ' \
                       + str(int(row['Total'])) \
                       + '
' \
                       + 'Rate per 1000 students: ' \
                       + str(row[std_crime_rate])

        return hover_text

def geo_code_bad_addresses():

    # write for stardardize option
    in_name = 'failed_geo_after_5_times.csv'
    in_folder = '/output/'
    in_path = get_path() + in_folder + in_name
    to_geo_code = pd.read_csv(in_path, index_col=None)

    to_geo_code["geo_address"] = to_geo_code["INSTNM"] \
                               + ", " \
                               + to_geo_code["City"] \
                               + ", " \
                               + to_geo_code["State"]

    geo_crime_df = custom_geocode.custom_geocoder(to_geo_code,
                                                  "geo_address",
                                                  write=True)

    # ----------------------------------------------------------
    # ----------------------------------------------------------
    # ----------------------------------------------------------
    # ----------------------------------------------------------

    # standardize all crime rates for every crime and school
    # crimes getting standardized are in crime list
    crime_cols = ['AGG_A16', 'ROBBE16', 'FONDL16', 'RAPE16']
    standardized__df = standardize.standardize_crime_rates(geo_crime_df, crime_cols)

    # UNIT TEST - check for dropped rows
    unit_tests.compare_row_counts(df_new=standardized__df,
                                  df_test=geo_crime_df)

    # give df intuitive names
    renamed_df = rename_columns(standardized__df)
    # UNIT TEST - check for dropped rows
    unit_tests.compare_row_counts(df_new=renamed_df,
                                  df_test=standardized__df)

    # get text for mouse hover
    final_df = get_hover_text(renamed_df, crime_cols)
    # UNIT TEST - check for dropped rows
    unit_tests.compare_row_counts(df_new=final_df,
                                  df_test=renamed_df)

    # write for QA
    out_name = 'these_guys_are_back_from_the_dead_second.csv'
    out_folder = '/output/'
    out_path = get_path() + out_folder + out_name
    final_df.to_csv(out_path, sep=',')

    return final_df

def find_final_bad_addresses():
    # bad addresses df
    in_name = 'very_bad_addresses.csv'
    in_folder = '/output/'
    in_path = get_path() + in_folder + in_name
    df_bad = pd.read_csv(in_path, index_col=None)

    # good addresses df
    in_name = 'these_guys_are_back_from_the_dead_first.csv'
    in_folder = '/output/'
    in_path = get_path() + in_folder + in_name
    df_good = pd.read_csv(in_path, index_col=None)

    very_bad = df_bad[~df_bad['geo_address'].isin(df_good['geo_address'])]

    print("all addy")
    print(df_bad.shape)

    print("good addy")
    print(df_good.shape)

    print("need to find yet")
    print(very_bad.shape)

    # write out final bad addresses
    out_name = 'the_final_remaining_bad_schools.csv'
    out_folder = '/output/'
    out_path = get_path() + out_folder + out_name
    very_bad.to_csv(out_path, sep=',')

def concat_files():
    # df1
    in_name = 'final_geocoded_data_first.csv'
    in_folder = '/output/concat_files/'
    in_path = get_path() + in_folder + in_name
    df1 = pd.read_csv(in_path, index_col=None)

    # df2
    in_name = 'hand_standardized_data_out.csv'
    # in_folder = '/output/'
    in_path = get_path() + in_folder + in_name
    df2 = pd.read_csv(in_path, index_col=None)

    # df3
    in_name = 'passed_after_5_times.csv'
    # in_folder = '/output/'
    in_path = get_path() + in_folder + in_name
    df3 = pd.read_csv(in_path, index_col=None)

    # df4
    in_name = 'these_guys_are_back_from_the_dead_first.csv'
    # in_folder = '/output/'
    in_path = get_path() + in_folder + in_name
    df4 = pd.read_csv(in_path, index_col=None)

    # df5
    in_name = 'these_guys_are_back_from_the_dead_second.csv'
    # in_folder = '/output/'
    in_path = get_path() + in_folder + in_name
    df5 = pd.read_csv(in_path, index_col=None)

    # df6
    in_name = 'these_guys_are_good_i_guess.csv'
    # in_folder = '/output/'
    in_path = get_path() + in_folder + in_name
    df6 = pd.read_csv(in_path, index_col=None)

    print(df1.shape)
    print(df2.shape)
    print(df3.shape)
    print(df4.shape)
    print(df5.shape)
    print(df6.shape)

    final_dataset = pd.concat([df1,
                               df2,
                               df3,
                               df4,
                               df5,
                               df6]).drop_duplicates()

    # write out final bad addresses
    out_name = 'final_geocoded_data.csv'
    out_folder = '/output/'
    out_path = get_path() + out_folder + out_name
    final_dataset.to_csv(out_path, sep=',')

    return final_dataset

# if __name__ == "__main__":
# final = concat_files()
# print(final.shape)

# good_hand_shit = preprocess(standardizer=True)
# print(good_hand_shit)

# find_final_bad_addresses()

# bad_shit = geo_code_bad_addresses()
# print(bad_shit)


"""
@author: Javier Rodriguez (jrzaurin@gmail.com)
"""

import numpy as np
import pandas as pd
import os
import heapq
import argparse
import mxnet as mx

from mxnet import gluon, autograd, ndarray
from mxnet.gluon import Block, nn, HybridBlock
from GMF_gluon import train, evaluate, checkpoint

from Dataset import Dataset as ml1mDataset
from time import time
from utils import *

def parse_args():
    parser = argparse.ArgumentParser()

    parser.add_argument("--datadir", type=str, default="Data_Javier/",
        help="data directory.")
    parser.add_argument("--modeldir", type=str, default="models",
        help="models directory")
    parser.add_argument("--dataname", type=str, default="ml-1m",
        help="chose a dataset.")
    parser.add_argument("--epochs", type=int, default=20,
        help="number of epochs.")
    parser.add_argument("--batch_size", type=int, default=256,
        help="batch size.")
    parser.add_argument("--layers", type=str, default="[64,32,16,8]",
        help="layer architecture. The first elements is used for the embedding \
        layers and equals n_emb*2")
    parser.add_argument("--dropouts", type=str, default="[0,0,0]",
        help="dropout per dense layer. len(dropouts) = len(layers)-1")
    parser.add_argument("--l2reg", type=float, default=0.,
        help="l2 regularization")
    parser.add_argument("--lr", type=float, default=0.01,
        help="learning rate.")
    parser.add_argument("--learner", type=str, default="adam",
        help="Specify an optimizer: adagrad, adam, rmsprop, sgd")
    parser.add_argument("--validate_every", type=int, default=1,
        help="validate every n epochs")
    parser.add_argument("--save_model", type=int , default=1)
    parser.add_argument("--n_neg", type=int, default=4,
        help="number of negative instances to consider per positive instance.")
    parser.add_argument("--topK", type=int, default=10,
        help="number of items to retrieve for recommendation.")

    return parser.parse_args()

class MLP(HybridBlock):
    """
    Concatenate Embeddings that are then passed through a series of Dense
    layers
    """
    def __init__(self, n_user, n_item, layers, dropouts):
        super(MLP, self).__init__()

        self.layers = layers
        self.n_layers = len(layers)
        self.n_user = n_user
        self.n_item = n_item
        self.mlp = gluon.nn.HybridSequential()
        with self.name_scope():
            self.embeddings_user = nn.Embedding(n_user, int(layers[0]/2), weight_initializer='normal')
            self.embeddings_item = nn.Embedding(n_item, int(layers[0]/2), weight_initializer='normal')
            for i in range(1,self.n_layers):
                self.mlp.add(nn.Dense(in_units=layers[i-1], units=layers[i], activation="relu", prefix="linear{}".format(i)))
                self.mlp.add(nn.Dropout(rate=dropouts[i-1]))
            # self.out = nn.Dense(in_units=layers[-1], units=1, activation='sigmoid', weight_initializer='uniform')
            self.out = nn.Dense(in_units=layers[-1], units=1, weight_initializer='uniform')

    def forward(self, users, items):

        user_emb = self.embeddings_user(users)
        item_emb = self.embeddings_item(items)
        emb_vector = ndarray.concat(user_emb,item_emb,dim=1)
        emb_vector = self.mlp(emb_vector)
        preds = self.out(emb_vector)

        return preds

if __name__ == '__main__':
    args = parse_args()

    datadir = args.datadir
    dataname = args.dataname
    modeldir = args.modeldir
    layers = eval(args.layers)
    ll = str(layers[-1]) #last layer
    dropouts = eval(args.dropouts)
    dp = "wdp" if dropouts[0]!=0 else "wodp"
    l2reg = args.l2reg
    n_emb = int(layers[0]/2)
    batch_size = args.batch_size
    epochs = args.epochs
    learner = args.learner
    lr = args.lr
    validate_every = args.validate_every
    save_model = args.save_model
    topK = args.topK
    n_neg = args.n_neg

    modelfname = "gluon_MLP" + \
        "_".join(["_bs", str(batch_size)]) + \
        "_".join(["_reg", str(l2reg).replace(".", "")]) + \
        "_".join(["_lr", str(lr).replace(".", "")]) + \
        "_".join(["_n_emb", str(n_emb)]) + \
        "_".join(["_ll", ll]) + \
        "_".join(["_dp", dp]) + \
        ".params"
    modelpath = os.path.join(modeldir, modelfname)
    resultsdfpath = os.path.join(modeldir, 'results_df.p')

    dataset = ml1mDataset(os.path.join(datadir, dataname))
    trainRatings, testRatings, testNegatives = dataset.trainMatrix, dataset.testRatings, dataset.testNegatives
    n_users, n_items = trainRatings.shape

    test = get_test_instances(testRatings, testNegatives)
    test_dataset = mx.gluon.data.dataset.ArrayDataset(test)
    test_loader = mx.gluon.data.DataLoader(dataset=test_dataset,
        batch_size=100,
        shuffle=False)

    ctx =  mx.gpu() if mx.test_utils.list_gpus() else mx.cpu()
    model = MLP(n_users, n_items, layers, dropouts)
    model.hybridize()
    model.initialize(ctx=ctx)
    criterion = gluon.loss.SigmoidBCELoss(from_sigmoid=False)
    if learner.lower() == "adagrad":
        trainer = gluon.Trainer(model.collect_params(), 'AdaGrad', {'learning_rate': lr, 'wd': l2reg})
    elif learner.lower() == "rmsprop":
        trainer = gluon.Trainer(model.collect_params(), 'RMSProp', {'learning_rate': lr, 'wd': l2reg})
    elif learner.lower() == "adam":
        trainer = gluon.Trainer(model.collect_params(), 'Adam', {'learning_rate': lr, 'wd': l2reg})
    else:
        trainer = gluon.Trainer(model.collect_params(), 'SGD', {'learning_rate': lr, 'wd': l2reg})

    best_hr, best_ndcgm, best_iter=0,0,0
    for epoch in range(1,epochs+1):
        t1 = time()
        loss = train(model, criterion, trainer, epoch, batch_size, ctx,
            trainRatings,n_items,n_neg,testNegatives)
        t2 = time()
        if epoch % validate_every == 0:
            (hr, ndcg) = evaluate(model, test_loader, ctx, topK)
            print("Iteration {}: {:.2f}s, HR = {:.4f}, NDCG = {:.4f}, loss = {:.4f}, validated in {:.2f}s"
                .format(epoch, t2-t1, hr, ndcg, loss, time()-t2))
            if hr > best_hr:
                best_hr, best_ndcg, best_iter, train_time = hr, ndcg, epoch, t2-t1
                if save_model:
                    checkpoint(model, modelpath)

    print("End. Best Iteration {}:  HR = {:.4f}, NDCG = {:.4f}. ".format(best_iter, best_hr, best_ndcg))
    if save_model:
        print("The best MLP model is saved to {}".format(modelpath))

    if save_model:
        if not os.path.isfile(resultsdfpath):
            results_df = pd.DataFrame(columns = ["modelname", "best_hr", "best_ndcg", "best_iter",
                "train_time"])
            experiment_df = pd.DataFrame([[modelfname, best_hr, best_ndcg, best_iter, train_time]],
                columns = ["modelname", "best_hr", "best_ndcg", "best_iter","train_time"])
            results_df = results_df.append(experiment_df, ignore_index=True)
            results_df.to_pickle(resultsdfpath)
        else:
            results_df = pd.read_pickle(resultsdfpath)
            experiment_df = pd.DataFrame([[modelfname, best_hr, best_ndcg, best_iter, train_time]],
                columns = ["modelname", "best_hr", "best_ndcg", "best_iter","train_time"])
            results_df = results_df.append(experiment_df, ignore_index=True)
            results_df.to_pickle(resultsdfpath)


# A python script that performs fastqc on samples and then outputs the results as human and machine readable
import subprocess
import os
import logging
from pathlib import Path
import argparse
from Bio import SeqIO
import numpy
import operator
import json
import codecs
import re
from itertools import takewhile
import csv

logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
# Logging handler which catches EVERYTHING
file_logger = logging.FileHandler('fast_parser.log')
file_logger.setLevel(logging.DEBUG)
# Logging handler which logs less
console_logger = logging.StreamHandler()

#based on args flag,
def set_up_logger(quiet):
    if quiet:
        console_logger.setLevel(logging.WARNING)
    else:
        console_logger.setLevel(logging.INFO)

# Formats the logs so they are pretty
logFormatter = '%(asctime)s- %(name)s - %(lineno)s - %(levelname)s - %(message)s'
formatter = logging.Formatter(logFormatter)
file_logger.setFormatter(formatter)
console_logger.setFormatter(formatter)

# adds handlers to logger
logger.addHandler(file_logger)
logger.addHandler(console_logger)

#finds out how many files/dirs exist
def create_file_struct():
    q = Path.cwd()
    results_to_process = []
    for item in q.iterdir():
        if item.is_dir():
            logger.debug("dir found: %s" % item)
            results_to_process.append(item)
    return results_to_process

#goes into each samples subdirs and calls the parse fastq result function, then returns a result table
def create_result_table(results_to_process):
    z = Path.cwd()
    result_table = []
    for directory in results_to_process:
        os.chdir(directory)
        result = parse_fastq()
        #result = tuple([tuple([Kelly001,116603]),tuple([301, 36])])
        seq_name = result[0][0].strip()
        tot_seq = result[0][1]
        seq_len = result[1][0]
        mean_qual = result[1][1]
        logger.debug([seq_name, tot_seq, seq_len, mean_qual])
        result_table.append([seq_name, tot_seq, seq_len, mean_qual])
    return result_table

#goes intot the fastq folder pulls out the nessecary stats and returns it to create result table
def parse_fastq():
    temp_name = ''
    temp_seq = ''
    temp_len = ''
    temp_qual = ''
    parse = False
    base_table = []
    #only pulls the base table since it is more than one line long
    with open('fastqc_data.txt') as f:
        for line in f:
            if line.startswith('>>END_MODULE'):
                parse = False
            elif line.startswith('>>Per base sequence quality'):
                parse = True
            elif parse:
                base_table.append(str(line))
            else:
                continue
    with open('fastqc_data.txt') as p:
        data = p.readlines()
    #pulls only one line so must pass through the whole file twice
    for line in data:
        if re.match("(.*)Filename(.*)", line):
            temp_name = line
        elif re.match("(.*)Total Sequences(.*)", line):
            temp_seq = line
        elif re.match("(.*)Sequence length(.*)", line):
            temp_len = line

    logger.debug(temp_name)
    logger.debug(temp_seq)
    logger.debug(temp_len)

    # editing the base table to remove heading and then format in a manner to calc mean
    base = numpy.delete(base_table, 0, 0)
    quals = numpy.loadtxt(base, delimiter='\t', usecols=[1])
    fin_qual = round(numpy.mean(quals),2)

    #formats the text since it has naming next to numbers
    name = temp_name.split('\t')
    seq = temp_seq.split('\t')
    length = temp_len.split('\t')

    logger.debug(name)
    logger.debug(seq)
    logger.debug(length)

    #final formatted variables
    fin_name = name[1].strip()
    fin_seq = round(float(seq[1].strip()),2)
    fin_len = length[1].strip()

    logger.debug(fin_name)
    logger.debug(fin_seq)
    logger.debug(fin_len)

    logger.debug("processed file")

    result = tuple([tuple([fin_name, fin_seq]), tuple([fin_len, fin_qual])])
    return result

#function to print the result table to the console in a nice format
def print_table_to_console(result_table):
    for line in result_table:
        print(*line)

def main(args):
    set_up_logger(args.quiet)
    #Determines where the program started
    p = Path.cwd()
    logger.debug("I am here: %s" %p)
    files = create_file_struct()
    table = create_result_table(files)
    logger.info(table)
    if args.console:
        print_table_to_console(table)
    os.chdir(p)
    with open('fastq_output.csv','w') as csvfile:
        writer = csv.writer(csvfile)
        writer.writerows(table)
        #writer.writerows([['Filename','Number of Sequences','Lenght of Sequnences','Average Quality Score']]+[[a,b,c,d] for [a,b],[c,d] in table])

if __name__ == "__main__":
    # Build Argument Parser in order to facilitate ease of use for user
    parser = argparse.ArgumentParser(
        description="Perform Automated Formatting of Sequence Data")
    parser.add_argument('-n', action='store', required=True,
                        help="name for fastqc output dir", dest='dir_name')
    parser.add_argument('-c', action='store_true', default=True,
                        help="writes the parsed information to console in a way that is easier to read than the log file format", dest='console')
    parser.add_argument('-q', '--quiet', action='store_true', default=False,
                        help="Reduces the amount of text printed to terminal, check logfiles more often", dest='quiet')
    parser.add_argument('-v', '--version', action='version',
                        version='%(prog)s 1.0')

    args = parser.parse_args()
    main(args)