import re
import nltk
import html
import pymorphy2
import bs4
import pyphen
import math
import fnmatch
import os
import copy
import logging
import shutil
import django
django.setup()

from natsort import natsorted
from functools import reduce
from PIL import Image
from time import time
from nltk.tokenize import RegexpTokenizer
from . import settings
from . import models

"""
def read_files(source, request, only_these=None):
    res = []
    all_html = bs4.BeautifulSoup(
        '<!DOCTYPE html><meta charset="UTF-8"><title>Title</title>',
        'html.parser'
    )
    all_content = ''
    all_pages_number = 0
    for file_name in natsorted(os.listdir(source), key=lambda y: y.lower()):
        dir_path = os.path.join(source, file_name)
        if os.path.isdir(dir_path):
            dir_path = os.path.join(dir_path, 'html')
            if os.path.exists(dir_path):
                if file_name in only_these if only_these is not None else True:
                    mod_html = bs4.BeautifulSoup(
                        '<!DOCTYPE html><meta charset="UTF-8"><title>Title</title>',
                        'html.parser'
                    )
                    mod_content = ''
                    mod_pages_number = 0
                    for file_name in os.listdir(dir_path):
                        mod_pages_number += 1
                        file_path = os.path.join(dir_path, file_name)
                        if os.path.isfile(file_path) and re.match(fnmatch.translate('*.html'), file_path, re.IGNORECASE):
                            f = open(file_path, 'r', encoding='utf-8')
                            c = f.read()
                            soup = bs4.BeautifulSoup(c, 'html.parser')
                            imgs = soup.find_all('img')
                            for img in imgs:
                                img['src'] = os.path.join(dir_path, img['src'])
                            for element in soup.body.contents:
                                mod_html.body.append(copy.copy(element))
                            f.close()
                            tockens = re.split(r'<[^>]+>', c)
                            text = ' '.join(list(filter(lambda x: not re.fullmatch(r'\s*', x), tockens)))
                            mod_content += ' ' + html.unescape(text)
                    name, extension = os.path.splitext(os.path.basename(os.path.dirname(dir_path)))
                    db_modules = models.Module.objects.filter(
                        uid=request.session.session_key,
                        sdo=request.session['moodle'],
                        cid=request.session['course_id'],
                        mid=name,
                    )
                    if len(db_modules) > 0:
                        mod_name = db_modules[0].mod_name
                        sec_name = db_modules[0].sec_name
                    else:
                        mod_name = name
                        sec_name = name
                    indicators = request.POST.getlist('indicators-' + name)
                    res.append({
                        'name': name,
                        'content': mod_name,
                        'section': sec_name,
                        'indicators': indicators,
                        'dir_path': os.path.dirname(dir_path),
                        'html': str(mod_html),
                        'txt': mod_content,
                        'pgs_num': mod_pages_number
                    })
                    for element in mod_html.body.contents:
                        all_html.body.append(copy.copy(element))
                    all_content += mod_content
                    all_pages_number += mod_pages_number
        else:
            if re.match(fnmatch.translate('*.html'), os.path.basename(dir_path), re.IGNORECASE) and (file_name in only_these if only_these is not None else True):
                f = open(dir_path, 'r', encoding='utf-8')
                c = f.read()
                f.close()
                mod_html = bs4.BeautifulSoup(c, 'html.parser')
                tockens = re.split(r'<[^>]+>', c)
                text = ' '.join(list(filter(lambda x: not re.fullmatch(r'\s*', x), tockens)))
                mod_content = ' ' + html.unescape(text)
                name, extension = os.path.splitext(os.path.basename(os.path.dirname(dir_path)))
                db_modules = models.Module.objects.filter(
                    uid=request.session.session_key,
                    sdo=request.session['moodle'],
                    cid=request.session['course_id'],
                    mid=name,
                )
                if len(db_modules) > 0:
                    mod_name = db_modules[0].mod_name
                    sec_name = db_modules[0].sec_name
                else:
                    mod_name = name
                    sec_name = name
                indicators = request.POST.getlist('indicators-' + name)
                res.append({
                    'name': name,
                    'content': mod_name,
                    'section': sec_name,
                    'indicators': indicators,
                    'dir_path': os.path.dirname(dir_path),
                    'html': str(mod_html),
                    'txt': mod_content,
                    'pgs_num': 1
                })
                for element in mod_html.body.contents:
                    all_html.body.append(copy.copy(element))
                all_content += mod_content
                all_pages_number += 1
    res.insert(0, {
        'name': 'all',
        'content': 'Анализ всего курса',
        'section': '',
        'indicators': [],
        'dir_path': source,
        'html': str(all_html),
        'txt': all_content,
        'pgs_num': all_pages_number
    })
    clear_dir(os.path.join(settings.BASE_DIR, 'log', os.path.basename(source)))
    return res
"""

def read_files(source, request):
    res = []
    all_html = bs4.BeautifulSoup(
        '<!DOCTYPE html><meta charset="UTF-8"><title>Title</title>',
        'html.parser'
    )
    all_content = ''
    all_pages_number = 0
    for module_name in os.listdir(source):
        module_path = os.path.join(source, module_name)
        module_html = bs4.BeautifulSoup(
            '<!DOCTYPE html><meta charset="UTF-8"><title>Title</title>',
            'html.parser'
        )
        module_content = ''
        module_pages_number = 0
        for file_name in os.listdir(module_path):
            file_path = os.path.join(module_path, file_name)
            if re.fullmatch(r'.*\.html', file_name, re.I):
                module_pages_number += 1
                with open(file_path, 'r', encoding='utf-8') as f:
                    c = f.read()
                    bs = bs4.BeautifulSoup(c, 'html.parser')
                    imgs = bs.find_all('img')
                    for img in imgs:
                        img['src'] = os.path.join(module_path, os.path.basename(img['src']))
                    for content in bs.body.contents:
                        module_html.body.append(copy.copy(content))
                    tokens = re.split(r'<[^>]+>', c)
                    text = ' '.join(list(filter(lambda x: not re.fullmatch(r'\s*', x), tokens)))
                    module_content += ' ' + html.unescape(text)
        db_modules = models.Module.objects.filter(
            uid=request.session.session_key,
            sdo=request.session['moodle'],
            cid=request.session['course_id'],
            mid=module_name,
        )
        if len(db_modules) > 0:
            mod_name = db_modules[0].mod_name
            sec_name = db_modules[0].sec_name
        else:
            mod_name = module_name
            sec_name = ''
        indicators = request.POST.getlist('indicators-' + module_name)
        res.append({
            'name': module_name,
            'content': mod_name,
            'section': sec_name,
            'indicators': indicators,
            'dir_path': module_path,
            'html': str(module_html),
            'txt': module_content,
            'pgs_num': module_pages_number
        })
        for content in module_html.body.contents:
            all_html.body.append(copy.copy(content))
        all_content += module_content
        all_pages_number += module_pages_number
    if all_pages_number > 0:
        res.insert(0, {
            'name': 'all',
            'content': 'Анализ всего курса',
            'section': '',
            'indicators': [],
            'dir_path': source,
            'html': str(all_html),
            'txt': all_content,
            'pgs_num': all_pages_number
        })
    clear_dir(os.path.join(settings.BASE_DIR, 'log', request.session.session_key))
    return res

def get_words_number(logger, ru_norm_words, ru_uniq_norm_words, en_norm_words, en_uniq_norm_words):
    ru_words_number = len(ru_norm_words)
    ru_uniq_words_number = len(ru_uniq_norm_words)
    logger.info(f'Russian words number: {ru_words_number}')
    logger.info(f'Russian unique words number: {ru_uniq_words_number}')
    en_words_number = len(en_norm_words)
    en_uniq_words_number = len(en_uniq_norm_words)
    logger.info(f'English words number: {en_words_number}')
    logger.info(f'English unique words number: {en_uniq_words_number}')
    total_words_number = ru_words_number + en_words_number
    total_uniq_words_number = ru_uniq_words_number + en_uniq_words_number
    logger.info(f'Total words number: {total_words_number}')
    logger.info(f'Total unique words number: {en_uniq_words_number}')
    return ru_words_number, ru_uniq_words_number, en_words_number, en_uniq_words_number, total_words_number, total_uniq_words_number

def information_richness(logger, new_concepts_number, total_uniq_words_number, res):
    inf_saturation = (new_concepts_number / total_uniq_words_number) * 100
    res['inf_saturation'] = round(inf_saturation, 2)
    logger.info(f'Information saturation: {res["inf_saturation"]}')

def get_new_concepts_number(logger, soup, res):
    logger.info('The calculation of new concepts number...')
    start = time()

    italic_elements = soup.find_all('i')
    definitions = soup.find_all('div', 'definition')

    buf = []
    for italic_element in italic_elements:
        if re.fullmatch(r'\s*[a-zA-Zа-яА-Я]\s*', str(italic_element)):
            continue
        buf.append(italic_element)
        for definition in definitions:
            def_italic_elements = definition.find_all('i')
            for def_italic_element in def_italic_elements:
                if italic_element == def_italic_element:
                    buf.pop(len(buf) - 1)
                    break
            if italic_element not in buf:
                break

    new_concepts_number = reduce(
        lambda x, y: x + y,
        [len(definition.find_all('p')) for definition in definitions]
    ) if len(definitions) > 0 else 0

    new_concepts_number += len(buf)

    finish = time()
    logger.info(f'New concepts number: {new_concepts_number}')
    logger.info(f'Time spent: {finish - start}')
    res['new_concepts_number'] = new_concepts_number
    return new_concepts_number

def abstractness(logger, ru_norm_words, total_uniq_words_number, res):
    logger.info('Getting a list of abstract words...')
    start = time()
    abstract_words = list(
        filter(
            lambda x:
            re.fullmatch(
                r'.*(ость|есть|мость|ность|нность|емость|енность|ие|ье|ание|изм|ура|ано|ота|еств|ество|ет|ета|изн|изна|овизна|ин|ина|щин|щина|чин|чина|льщин|льщина|честв|чество|ств|ство|овств|овство|ничеств|ничество|тельств|тельство|енств|енство|шеств|шество|ур|ни|ние|нь|нье|ени|ение|тие|тье|овк|овка|еж|ёж|ежк|ежка|ёжк|ёжка|б|ба|об|оба|еб|еба|ёб|ёба|аци|ация|яци|яция|фукаци|фукация|инфикаци|инфикация|аж|от|н|ня|отн|отня|ын|ыня|ев|ева|знь|ьё|ьо|ец|ок|ёк|чик|ик|иц|ица|к|ка|инк|инка|ушк|ушка|очк|очка|юшк|юшка|ушко|юшко|ышк|ышко|ишк|ишка|ишко|оньк|онька|еньк|енька|онк|онка|ёнк|ёнка|ц|це|цо|ице|ецо|ца|ашк|ашка|ищ|ище|ища|очек|ечк|ечко|ушек|ышек)',
                x),
            ru_norm_words
        )
    )
    abstract_uniq_words = list(sorted(set(abstract_words)))
    res['abstract_uniq_words'] = abstract_uniq_words
    finish = time()
    logger.info(f'Abstract words: {abstract_words}')
    logger.info(f'Unique abstract words: {abstract_uniq_words}')
    logger.info(f'Time spent: {finish - start}')

    abstract_words_number = len(abstract_uniq_words)
    logger.info(f'Unique abstract words number: {abstract_words_number}')

    abstractness = (abstract_words_number / total_uniq_words_number) * 100
    res['abstractness'] = round(abstractness, 2)
    logger.info(f'Text abstractness: {res["abstractness"]}')

def readability(logger, ru_uniq_norm_words, ru_uniq_words_number, txt, res):
    logger.info('Splitting text into sentences...')
    start = time()
    sent_tockens = nltk.tokenize.sent_tokenize(txt)
    finish = time()
    logger.info(f'Sentences: {sent_tockens}')
    logger.info(f'Time spent: {finish - start}')
    sent_number = len(sent_tockens)
    logger.info(f'Sentences number: {sent_number}')

    res['sent_number'] = sent_number

    logger.info('Getting the number of compound words...')
    start = time()
    dic = pyphen.Pyphen(lang='ru')
    compound_words_number = 0
    for word in ru_uniq_norm_words:
        if len(dic.inserted(word).split('-')) > 4:
            compound_words_number += 1
    finish = time()
    logger.info(f'Compound words number: {compound_words_number}')
    logger.info(f'Time spent: {finish - start}')

    ganning_index = 0.4 * (0.78 * ru_uniq_words_number / sent_number +
                           100 * compound_words_number / ru_uniq_words_number)
    res['ganning_index'] = round(ganning_index, 2)
    logger.info(f'Text readability: {res["ganning_index"]}')

def wateriness(logger, ru_stop_words_number, en_stop_words_number, total_words_number, res):
    wateriness = ((ru_stop_words_number + en_stop_words_number) / total_words_number) * 100
    res['wateriness'] = round(wateriness, 2)
    logger.info(f'Text wateriness: {res["wateriness"]}')

def keyword_density(logger, ru_norm_words, en_norm_words, ru_uniq_norm_words, en_uniq_norm_words,  res):
    logger.info('Calculation of frequencies of Russian and English words...')
    start = time()
    ru_frequencies = [math.sqrt(ru_norm_words.count(word)) for word in ru_uniq_norm_words]
    en_frequencies = [math.sqrt(en_norm_words.count(word)) for word in en_uniq_norm_words]
    finish = time()
    logger.info(f'Russian frequencies: {ru_frequencies}')
    logger.info(f'English frequencies: {en_frequencies}')
    logger.info(f'Time spent: {finish - start}')

    density = max(ru_frequencies + en_frequencies)
    res['density'] = round(density, 2)
    logger.info(f'Text density: {res["density"]}')

def text_creolization(logger, indicators, soup, txt, res):
    logger.info('Calculation of textual characterization...')
    start = time()
    creolized_text_volume = 0
    creolization_vector = [0, 0, 0, 0, 0, 0, 0]
    if 'creolization_background_color' in indicators:
        bg_creolization = soup.find_all(['span', 'p', 'div'], {'style': re.compile(r'.*background-image: .+')})
        for element in bg_creolization:
            creolized_text_volume += len(''.join(re.split(r'<[^>]+>', str(element))))
        creolization_vector[0] = len(bg_creolization)
    if 'creolization_italics' in indicators:
        italics_creolization = soup.find_all('i')
        for element in italics_creolization:
            creolized_text_volume += len(''.join(re.split(r'<[^>]+>', str(element))))
        creolization_vector[1] = len(italics_creolization)
    if 'creolization_bold' in indicators:
        bold_creolization = soup.find_all('b')
        for element in bold_creolization:
            creolized_text_volume += len(''.join(re.split(r'<[^>]+>', str(element))))
        creolization_vector[2] = len(bold_creolization)
    if 'creolization_underline' in indicators:
        underline_creolization = soup.find_all('u')
        for element in underline_creolization:
            creolized_text_volume += len(''.join(re.split(r'<[^>]+>', str(element))))
        creolization_vector[3] = len(underline_creolization)
    if 'creolization_border' in indicators:
        border_creolization = soup.find_all(
            ['span', 'p', 'div'],
            {'style': re.compile(r'.*border(-left|-right|-top|-bottom)?: .+')}
        )
        for element in border_creolization:
            creolized_text_volume += len(''.join(re.split(r'<[^>]+>', str(element))))
        creolization_vector[4] = len(border_creolization)
    if 'creolization_pictogram' in indicators:
        pictogram_creolization = soup.find_all('div', {'class': ['definition', 'ex_head', 'cn_head']})
        for element in pictogram_creolization:
            creolized_text_volume += len(''.join(re.split(r'<[^>]+>', str(element))))
        creolization_vector[5] = len(pictogram_creolization)
    if 'creolization_link' in indicators:
        link_creolization = soup.find_all('a')
        for element in link_creolization:
            creolized_text_volume += len(''.join(re.split(r'<[^>]+>', str(element))))
        creolization_vector[6] = len(link_creolization)

    all_text_volume = len(txt)
    creolization_degree = round((creolized_text_volume / all_text_volume) * 100, 2)
    res['creolization_vector'] = creolization_vector
    res['all_text_volume'] = all_text_volume
    res['creolized_text_volume'] = creolized_text_volume
    res['creolization_degree'] = creolization_degree
    finish = time()
    logger.info(f'Background creolization: {creolization_vector[0]}')
    logger.info(f'Italics creolization: {creolization_vector[1]}')
    logger.info(f'Bold creolization: {creolization_vector[2]}')
    logger.info(f'Underline creolization: {creolization_vector[3]}')
    logger.info(f'Border creolization: {creolization_vector[4]}')
    logger.info(f'Pictogram creolization: {creolization_vector[5]}')
    logger.info(f'Link creolization: {creolization_vector[6]}')
    logger.info(f'All text volume: {all_text_volume}')
    logger.info(f'Creolized text volume: {creolized_text_volume}')
    logger.info(f'Creolization degree: {creolization_degree}')
    logger.info(f'Time spent: {finish - start}')

def get_tokens(logger, txt):
    logger.info('Partitioning into tokens...')
    start = time()
    tokenizer = RegexpTokenizer(r'[a-zA-Zа-яА-Я-]+')
    tokens = list(map(lambda x: x.lower(), tokenizer.tokenize(txt)))
    finish = time()
    logger.info(f'Tokens: {tokens}')
    logger.info(f'Time spent: {finish - start}')
    return tokens

def get_words(logger, tokens):
    morph = pymorphy2.MorphAnalyzer()
    logger.info('Parsing words...')
    start = time()
    words_objects = [morph.parse(tocken)[0] for tocken in tokens]
    finish = time()
    logger.info(f'Words objects: {words_objects}')
    logger.info(f'Time spent: {finish - start}')
    return words_objects

def get_filtered_words(logger, words):
    logger.info('Deletion of non-existent words...')
    start = time()
    filtered_words = list(filter(lambda x: x.tag.POS is not None, words))
    finish = time()
    logger.info(f'Filtered words objects: {filtered_words}')
    logger.info(f'Time spent: {finish - start}')
    return filtered_words

def get_norm_words(logger, filtered_words):
    logger.info('Normalization of words...')
    start = time()
    norm_words = list([word.normal_form for word in filtered_words])
    # unic_norm_words = list(sorted(set(norm_words)))
    finish = time()
    logger.info(f'Normalized russian words: {norm_words}')
    # logger.info(f'Unique normalized russian words: {unic_norm_words}')
    logger.info(f'Time spent: {finish - start}')
    return norm_words

def split_words(logger, norm_words, res):
    ru_norm_words = []
    en_norm_words = []

    logger.info('Separation into Russian and English normal words...')
    start = time()
    for word in norm_words:
        if re.fullmatch(r'[a-z0-9-/]+', word):
            en_norm_words.append(word)
        else:
            ru_norm_words.append(word)
    ru_uniq_norm_words = list(sorted(set(ru_norm_words)))
    en_uniq_norm_words = list(sorted(set(en_norm_words)))
    finish = time()
    logger.info(f'Russian normal words: {ru_norm_words}')
    logger.info(f'Russian unique normal words: {ru_uniq_norm_words}')
    logger.info(f'English normal words: {en_norm_words}')
    logger.info(f'English unique normal words: {en_uniq_norm_words}')
    logger.info(f'Time spent: {finish - start}')
    res['ru_norm_words'] = ru_norm_words
    res['en_norm_words'] = en_norm_words
    res['ru_uniq_norm_words'] = ru_uniq_norm_words
    res['en_uniq_norm_words'] = en_uniq_norm_words
    return ru_norm_words, en_norm_words, ru_uniq_norm_words, en_uniq_norm_words

def get_stop_words(logger):
    logger.info('Getting russian and english stop words...')
    start = time()
    ru_stop_words = set(nltk.corpus.stopwords.words('russian'))
    en_stop_words = set(nltk.corpus.stopwords.words('english'))
    finish = time()
    logger.info(f'Russian stop words: {ru_stop_words}')
    logger.info(f'English stop words: {en_stop_words}')
    logger.info(f'Time spent: {finish - start}')
    return ru_stop_words, en_stop_words

def get_words_without_stop_words(logger, ru_norm_words, ru_stop_words, en_norm_words, en_stop_words, res):
    logger.info('Removal of Russian and English stop words...')
    start = time()
    ru_filtered_words = [word for word in ru_norm_words if word not in ru_stop_words]
    en_filtered_words = [word for word in en_norm_words if word not in en_stop_words]
    finish = time()
    logger.info(f'Russian filtered words: {ru_filtered_words}')
    logger.info(f'English filtered words: {en_filtered_words}')
    logger.info(f'Time spent: {finish - start}')
    res['ru_filtered_words'] = ru_filtered_words
    res['en_filtered_words'] = en_filtered_words
    return ru_filtered_words, en_filtered_words

def get_stop_words_number(logger, ru_norm_words, ru_filtered_words, en_norm_words, en_filtered_words):
    logger.info('Calculation of the stop words number in the text...')
    start = time()
    ru_stop_words_number = len(list(set(ru_norm_words))) - len(list(set(ru_filtered_words)))
    en_stop_words_number = len(list(set(en_norm_words))) - len(list(set(en_filtered_words)))
    finish = time()
    logger.info(f'Russian stop words number: {ru_stop_words_number}')
    logger.info(f'English stop words number: {en_stop_words_number}')
    logger.info(f'Time spent: {finish - start}')
    return ru_stop_words_number, en_stop_words_number

def norm_en_words(logger, en_filtered_words, res):
    logger.info('Normalization of English words...')
    start = time()
    lemmatizer = nltk.stem.WordNetLemmatizer()
    en_norm_words = list([lemmatizer.lemmatize(word) for word in en_filtered_words])
    en_unic_norm_words = list(sorted(set(en_norm_words)))
    res['en_norm_words'] = en_norm_words
    res['en_unic_norm_words'] = en_unic_norm_words
    finish = time()
    logger.info(f'Normalized English words: {en_norm_words}')
    logger.info(f'Unique normalized English words: {en_unic_norm_words}')
    logger.info(f'Time spent: {finish - start}')
    return en_norm_words, en_unic_norm_words

def get_logger(file):
    log_path = os.path.join(settings.BASE_DIR, 'log')
    if not os.path.exists(log_path):
        os.mkdir(log_path)
    dir_name = os.path.basename(os.path.dirname(os.path.dirname(file["dir_path"])))
    log_path = os.path.join(log_path, dir_name)
    if not os.path.exists(log_path):
        os.mkdir(log_path)
    log_path = os.path.join(log_path, str(int(time())))
    if not os.path.exists(log_path):
        os.mkdir(log_path)
    log_name = os.path.basename(file["dir_path"])
    log_path = os.path.join(log_path, log_name + '.log')
    logger = logging.getLogger(log_name)
    logger.setLevel(logging.INFO)
    handler = logging.FileHandler(log_path, mode='w', encoding='utf-8')
    handler.setLevel(logging.INFO)
    logger.addHandler(handler)
    return logger

def text_characteristics(file):
    indicators = file['indicators']
    logger = get_logger(file)
    logger.info(f'Text characteristics for {file["dir_path"]}:')
    res = {}
    if 'information_richness' in indicators or \
            'abstractness' in indicators or \
            'readability' in indicators or \
            'water_content' in indicators or \
            'keyword_density' in indicators or \
            'creolization_background_color' in indicators or \
            'creolization_italics' in indicators or \
            'creolization_bold' in indicators or \
            'creolization_underline' in indicators or \
            'creolization_border' in indicators or \
            'creolization_pictogram' in indicators or \
            'creolization_link' in indicators:
        soup = bs4.BeautifulSoup(file['html'], 'html.parser')
        new_concepts_number = get_new_concepts_number(logger, soup, res)
        txt = file['txt']
        tokens = get_tokens(logger, txt)
        words = get_words(logger, tokens)
        filtered_words = get_filtered_words(logger, words)
        norm_words = get_norm_words(logger, filtered_words)
        ru_norm_words, en_norm_words, ru_uniq_norm_words, en_uniq_norm_words = split_words(logger, norm_words, res)
        ru_stop_words, en_stop_words = get_stop_words(logger)
        ru_filtered_words, en_filtered_words = get_words_without_stop_words(
            logger,
            ru_norm_words,
            ru_stop_words,
            en_norm_words,
            en_stop_words,
            res
        )
        ru_stop_words_number, en_stop_words_number = get_stop_words_number(
            logger,
            ru_norm_words,
            ru_filtered_words,
            en_norm_words,
            en_filtered_words
        )
        ru_words_number, \
        ru_uniq_words_number, \
        en_words_number, \
        en_uniq_words_number, \
        total_words_number, \
        total_uniq_words_number = get_words_number(
            logger, ru_norm_words,
            ru_uniq_norm_words,
            en_norm_words,
            en_uniq_norm_words
        )
        if 'information_richness' in indicators:
            information_richness(logger, new_concepts_number, total_uniq_words_number, res)
        if 'abstractness' in indicators:
            abstractness(logger, ru_norm_words, total_uniq_words_number, res)
        if 'readability' in indicators:
            readability(logger, ru_uniq_norm_words, ru_uniq_words_number, txt, res)
        if 'water_content' in indicators:
            wateriness(logger, ru_stop_words_number, en_stop_words_number, total_words_number, res)
        if 'keyword_density' in indicators:
            keyword_density(logger, ru_norm_words, en_norm_words, ru_uniq_norm_words, en_uniq_norm_words,  res)
        if 'creolization_background_color' in indicators or \
            'creolization_italics' in indicators or \
            'creolization_bold' in indicators or \
            'creolization_underline' in indicators or \
            'creolization_border' in indicators or \
            'creolization_pictogram' in indicators or \
            'creolization_link' in indicators:
            text_creolization(logger, indicators, soup, txt, res)
    logger.info('Text Characterization Completed.')
    return res

def text_characteristics_all_files(files, indicators):
    res = {
        'new_concepts_number': 0,
        'ru_filtered_words': [],
        'en_filtered_words': [],
        'ru_norm_words': [],
        'ru_uniq_norm_words': [],
        'en_norm_words': [],
        'en_uniq_norm_words': [],
        'abstract_uniq_words': [],
        'sent_number': 0,
        'creolization_vector': [0, 0, 0, 0, 0, 0, 0],
        'all_text_volume': 0,
        'creolized_text_volume': 0,
        'creolization_degree': 0
    }
    for file in files:
        if 'new_concepts_number' in file['txt_ch']:
            res['new_concepts_number'] += file['txt_ch']['new_concepts_number']
        if 'ru_filtered_words' in file['txt_ch']:
            res['ru_filtered_words'] += file['txt_ch']['ru_filtered_words']
        if 'en_filtered_words' in file['txt_ch']:
            res['en_filtered_words'] += file['txt_ch']['en_filtered_words']
        if 'ru_norm_words' in file['txt_ch']:
            res['ru_norm_words'] += file['txt_ch']['ru_norm_words']
        if 'ru_uniq_norm_words' in file['txt_ch']:
            res['ru_uniq_norm_words'] += file['txt_ch']['ru_uniq_norm_words']
        if 'en_norm_words' in file['txt_ch']:
            res['en_norm_words'] += file['txt_ch']['en_norm_words']
        if 'en_uniq_norm_words' in file['txt_ch']:
            res['en_uniq_norm_words'] += file['txt_ch']['en_uniq_norm_words']
        if 'abstract_uniq_words' in file['txt_ch']:
            res['abstract_uniq_words'] += file['txt_ch']['abstract_uniq_words']
        if 'sent_number' in file['txt_ch']:
            res['sent_number'] += file['txt_ch']['sent_number']
        if 'creolization_vector' in file['txt_ch']:
            for i in range(len(file['txt_ch']['creolization_vector'])):
                res['creolization_vector'][i] += file['txt_ch']['creolization_vector'][i]
        if 'all_text_volume' in file['txt_ch']:
            res['all_text_volume'] += file['txt_ch']['all_text_volume']
        if 'creolized_text_volume' in file['txt_ch']:
            res['creolized_text_volume'] += file['txt_ch']['creolized_text_volume']

    if 'creolization_background_color' in indicators or \
            'creolization_italics' in indicators or \
            'creolization_bold' in indicators or \
            'creolization_underline' in indicators or \
            'creolization_border' in indicators or \
            'creolization_pictogram' in indicators or \
            'creolization_link' in indicators:
        n = len(files)
        m = len(res['creolization_vector'])
        median = []
        for i in range(m):
            median.append(res['creolization_vector'][i] / n)
        res['creole_txt_uni_distr'] = [0 for _ in range(m)]
        for file in files:
            if 'creolization_vector' in file['txt_ch']:
                for i in range(len(file['txt_ch']['creolization_vector'])):
                    res['creole_txt_uni_distr'][i] += math.pow(file['txt_ch']['creolization_vector'][i] - median[i], 2)
        for i in range(m):
            res['creole_txt_uni_distr'][i] = round(res['creole_txt_uni_distr'][i] / n, 2)

    if 'information_richness' in indicators or \
            'abstractness' in indicators or \
            'readability' in indicators or \
            'water_content' in indicators:
        res['ru_uniq_norm_words'] = list(sorted(set(res['ru_uniq_norm_words'])))
        res['en_uniq_norm_words'] = list(sorted(set(res['en_uniq_norm_words'])))

        ru_words_number = len(res['ru_uniq_norm_words'])
        en_words_number = len(res['en_uniq_norm_words'])
        total_words_number = ru_words_number + en_words_number

        if 'information_richness' in indicators:
            inf_saturation = (res['new_concepts_number'] / total_words_number) * 100
            res['inf_saturation'] = round(inf_saturation, 2)

        if 'abstractness' in indicators:
            res['abstract_uniq_words'] = list(sorted(set(res['abstract_uniq_words'])))
            abstract_words_number = len(res['abstract_uniq_words'])
            abstractness = (abstract_words_number / total_words_number) * 100
            res['abstractness'] = round(abstractness, 2)

        if 'readability' in indicators:
            dic = pyphen.Pyphen(lang='ru')
            compound_words_number = 0
            for word in res['ru_uniq_norm_words']:
                if len(dic.inserted(word).split('-')) > 4:
                    compound_words_number += 1
            ganning_index = 0.4 * (0.78 * ru_words_number / res['sent_number'] + 100 * compound_words_number / ru_words_number)
            res['ganning_index'] = round(ganning_index, 2)

        if 'water_content' in indicators:
            ru_stop_words_number = len(list(set(res['ru_norm_words']))) - len(list(set(res['ru_filtered_words'])))
            en_stop_words_number = len(list(set(res['en_norm_words']))) - len(list(set(res['en_filtered_words'])))
            wateriness = ((ru_stop_words_number + en_stop_words_number) / total_words_number) * 100
            res['wateriness'] = round(wateriness, 2)

    if 'keyword_density' in indicators:
        ru_frequences = [math.sqrt(res['ru_norm_words'].count(word)) for word in res['ru_uniq_norm_words']]
        en_frequences = [math.sqrt(res['en_norm_words'].count(word)) for word in res['en_uniq_norm_words']]
        density = max(ru_frequences + en_frequences)
        res['density'] = round(density, 2)

    if res['all_text_volume'] != 0:
        res['creolization_degree'] = round(res['creolized_text_volume'] / res['all_text_volume'], 2) * 100

    return res

def get_points_brightness(img):
    width = img.size[0]
    height = img.size[1]
    px = img.load()

    res = []
    for i in range(width):
        for j in range(height):
            pixel = px[i, j]
            if type(pixel) == int:
                pixel = (pixel, pixel, pixel)
            R = pixel[0]
            G = pixel[1]
            B = pixel[2]
            res.append(0.299 * R + 0.587 * G + 0.114 * B)

    return res

def get_img_brightness(file_path):
    img = Image.open(file_path)
    width = img.size[0]
    height = img.size[1]

    psb = get_points_brightness(img)

    res = sum(psb) / (width * height)

    return res

def get_rel_img_brightness(file_path):
    img = Image.open(file_path)
    width = img.size[0]
    height = img.size[1]

    psb = get_points_brightness(img)

    res = sum(psb) / (width * height * max(psb))

    return res

def get_points_contrast(img):
    width = img.size[0]
    height = img.size[1]

    psb = get_points_brightness(img)

    imb = sum(psb) / (width * height)

    res = [(pb - imb) * (pb - imb) for pb in psb]

    return res

def get_img_contrast(file_path):
    img = Image.open(file_path)
    width = img.size[0]
    height = img.size[1]

    psc = get_points_contrast(img)

    res = sum(psc) / (width * height)

    return res

def get_rel_img_contrast(file_path):
    img = Image.open(file_path)
    width = img.size[0]
    height = img.size[1]

    psc = get_points_contrast(img)

    res = 2 * sum(psc) / (width * height * max(psc))

    return res

def img_characteristics(file):
    res = {}

    indicators = file['indicators']
    if 'illustrations_number' in indicators or \
            'average_illustrations_number_per_page' in indicators or \
            'image_brightness' in indicators or \
            'relative_image_brightness' in indicators or \
            'image_contrast' in indicators or \
            'relative_image_contrast' in indicators:
        soup = bs4.BeautifulSoup(file['html'], 'html.parser')
        img_list = soup.find_all('img')

        if 'illustrations_number' in indicators or \
                'average_illustrations_number_per_page' in indicators:
            img_number = len(img_list)
            if 'illustrations_number' in indicators:
                res['img_number'] = img_number
            if 'average_illustrations_number_per_page' in indicators:
                aver_img_in_pg_num = round(img_number / file['pgs_num'], 2)
                res['pgs_num'] = file['pgs_num']
                res['aver_img_in_pg_num'] = aver_img_in_pg_num

        if 'image_brightness' in indicators or \
                'relative_image_brightness' in indicators or \
                'image_contrast' in indicators or \
                'relative_image_contrast' in indicators:
            res['images'] = []
            for img in img_list:
                try:
                    img_char = {
                        'img_name': os.path.basename(img['src']),
                        'img_path': '/' + img['src'][img['src'].index('media'):]
                    }
                    if 'image_brightness' in indicators:
                        brightness = round(get_img_brightness(img['src']), 2)
                        img_char['brightness'] = brightness
                    if 'relative_image_brightness' in indicators:
                        rel_brightness = round(get_rel_img_brightness(img['src']), 2)
                        img_char['rel_brightness'] = rel_brightness
                    if 'image_contrast' in indicators:
                        contrast = round(get_img_contrast(img['src']), 2)
                        img_char['contrast'] = contrast
                    if 'relative_image_contrast' in indicators:
                        rel_contrast = round(get_rel_img_contrast(img['src']), 2)
                        img_char['rel_contrast'] = rel_contrast
                    res['images'].append(img_char)
                except Exception as e:
                    print(img['src'])
                    print(e)

    return res

def img_characteristics_all_files(files, indicators):
    res = {
        'img_number': 0,
        'pgs_num': 0,
        'images': []
    }

    for file in files:
        if 'img_number' in file['img_ch']:
            res['img_number'] += file['img_ch']['img_number']
        if 'pgs_num' in file['img_ch']:
            res['pgs_num'] += file['img_ch']['pgs_num']
        if 'images' in file['img_ch']:
            res['images'] += file['img_ch']['images']

    if res['pgs_num'] != 0:
        res['aver_img_in_pg_num'] = round(res['img_number'] / res['pgs_num'], 2)

    return res

def search_and_nav_characteristics(file):
    res = {
        'glossary': 0,
        'tables': 0,
        'images': 0,
        'literature': 0,
        'formula': 0,
        'litlist': 0
    }

    indicators = file['indicators']

    soup = bs4.BeautifulSoup(file['html'], 'html.parser')
    h1_list = soup.find_all('h1')

    if 'has_bibliography' in indicators or \
            'has_glossary' in indicators:
        for h1 in h1_list:
            if re.fullmatch(r'глоссарий', str(h1.string), re.I):
                res['glossary'] = 1
            if re.fullmatch(r'литература', str(h1.string), re.I):
                res['litlist'] = 1
    if 'has_tables_list' in indicators or \
            'has_illustrations_list' in indicators or \
            'has_links' in indicators or \
            'has_formula' in indicators:
        txt = file['txt']
        if 'has_tables_list' in indicators:
            tables = re.findall(
                r'(табл[а-я.]*\s*\d+(?:\.\d+)*(?:\s(?!\s*[-–](?!\s*(?:табл[а-я.]*\s*)?\d+(?:\.\d+)*))(?:[-–]\s*(?:табл[а-я.]*\s*)?\d+(?:\.\d+)*)?|[-–]\s*(?:табл[а-я.]*\s*)?\d+(?:\.\d+)*|(?=[.,:!?)](?!\d+))))',
                txt, re.I)
            if len(tables) != 0:
                res['tables'] += len(tables)
        if 'has_illustrations_list' in indicators:
            images = re.findall(
                r'(рис[а-я.]*\s*\d+(?:\.\d+)*(?:\s(?!\s*[-–](?!\s*(?:рис[а-я.]*\s*)?\d+(?:\.\d+)*))(?:[-–]\s*(?:рис[а-я.]*\s*)?\d+(?:\.\d+)*|\s*(?:(?:,\s*[а-яА-Яa-zA-Z](?![а-яА-Яa-zA-Z])\s*)(?:,\s*[а-яА-Яa-zA-Z](?![а-яА-Яa-zA-Z])\s*)*|(?:\(\s*[а-яА-Яa-zA-Z](?![а-яА-Яa-zA-Z])\s*\))(?:\s*,\s*\(\s*[а-яА-Яa-zA-Z](?![а-яА-Яa-zA-Z])\s*\))*))?|[-–]\s*(?:рис[а-я.]*\s*)?\d+(?:\.\d+)*|(?=[,])(?:,\s*[а-яА-Яa-zA-Z](?![а-яА-Яa-zA-Z])\s*)*|(?=[.:!?)](?!\d+))))',
                txt, re.I)
            if len(images) != 0:
                res['images'] += len(images)
                """
                for img in images:
                    if (re.fullmatch(r'рис[а-я.]*\s*\d+(\.\d+)*(\s*,\s*[а-яА-Яa-zA-Z](?![а-яА-Яa-zA-Z]))+\s*', img, re.I)):
                        res['images'] += len(img.split(',')) - 1
                    elif (re.fullmatch(r'рис[а-я.]*\s*\d+(\.\d+)*\s*(\(\s*[а-яА-Яa-zA-Z](?![а-яА-Яa-zA-Z])\s*\))(\s*,\s*\(\s*[а-яА-Яa-zA-Z](?![а-яА-Яa-zA-Z])\s*\))*\s*', img, re.I)):
                        res['images'] += len(img.split(','))
                    else:
                        matches = re.fullmatch(r'рис[а-я.]*\s*\d+(\.(?P<from>\d+))*\s*[-–]\s*(?:рис[а-я.]*\s*)?\d+(?:\.(?P<to>\d+))\s*', img, re.I)
                        if matches is not None:
                            res['images'] += int(matches.group('to')) - int(matches.group('from'))
                        else:
                            res['images'] += 1
                """
        if 'has_links' in indicators:
            literature = re.findall(r'(\[\d+(?:\s*[-–]\s*\d+)?(?:\s*,\s*\d+(?:\s*[-–]\s*\d+)?)*\])', txt, re.I)
            if len(literature) != 0:
                res['literature'] += len(literature)
        if 'has_formula' in indicators:
            formula = re.findall(r'((?<=[а-я]\s)\s*\(\s*\d+(?:.\d+)*(?:\s*[-–]\s*\d+(?:.\d+)*)?\s*\))', txt, re.I)
            if len(formula) != 0:
                res['formula'] += len(formula)

    return res

def search_and_nav_characteristics_all_files(files, indicators):
    res = {
        'glossary': 0,
        'tables': 0,
        'images': 0,
        'literature': 0,
        'formula': 0,
        'litlist': 0
    }

    for file in files:
        if 'glossary' in file['san_ch']:
            res['glossary'] = file['san_ch']['glossary'] if file['san_ch']['glossary'] == 1 else res['glossary']
        if 'tables' in file['san_ch']:
            res['tables'] += file['san_ch']['tables']
        if 'images' in file['san_ch']:
            res['images'] += file['san_ch']['images']
        if 'literature' in file['san_ch']:
            res['literature'] += file['san_ch']['literature']
        if 'formula' in file['san_ch']:
            res['formula'] += file['san_ch']['formula']
        if 'litlist' in file['san_ch']:
            res['litlist'] = file['san_ch']['litlist'] if file['san_ch']['litlist'] == 1 else res['litlist']

    return res

def clear_dir(dir_path):
    if os.path.exists(dir_path):
        for file_name in os.listdir(dir_path):
            file_path = os.path.join(dir_path, file_name)
            if os.path.isfile(file_path):
                os.unlink(file_path)
            elif os.path.isdir(file_path):
                shutil.rmtree(file_path)


#!/usr/bin/env python
""" Pipeline for .tex documents.
    The documents get converted from draft mode to
    compile ready documents and then compiled
    using the LaTeX engine

Created: 07.10.2021
@author: Max Weise
"""

import os
import shutil
import logging
import datetime
import argparse
import subprocess

# === DEFINITION OF CONSTANTS ===
WORKING_PATH = './'
FILE_PREFIX = '[piped]_'    # Use this to identify modified files
FILE_PREFIX_SPLIT = '[piped]'
DEPLOY_FOLER = 'DEPLOY'

LOGGING_LEVELS = {
    'DEBUG'     : logging.DEBUG,
    'INFO'      : logging.INFO,
    'WARNING'   : logging.WARNING,
    'ERROR'     : logging.ERROR,
    'CRITICAL'  : logging.CRITICAL,
}

def copy_file(__file_name: str) -> str:
    """ This function copies the wip-tex file to compile it without
        modifying the original file.
    """
    new_name = FILE_PREFIX + __file_name
    logging.info(f'Copying the file {__file_name}.tex -> {new_name}.tex')
    shutil.copy(__file_name + '.tex', new_name + '.tex')

    return new_name

def remove_draft_option(__file_name: str) -> None:  # sourcery skip: extract-method
    """ This function removes the draft option from a tex file."""

    try:
        lines_of_file = []
        logging.info(f'Now reading {__file_name}.tex')
        with open(__file_name + '.tex', 'r') as f:
            lines_of_file = f.readlines()

        parts_of_line = lines_of_file[0].split(',')
        parts_of_line.pop(parts_of_line.index(' draft'))

        lines_of_file[0] = ','.join(parts_of_line)

        logging.info(f'Now writing {__file_name}.tex')
        with open(__file_name + '.tex', 'w') as f:
            f.writelines(lines_of_file)
    except Exception as e:
        logging.error(e)

def compile_tex_file(__file_name: str, verbose: bool = False) -> None:
    """ Call LaTeX.exe and compile the tex document to a PDF file."""

    arg_list = ['pdflatex', __file_name + '.tex']

    if not verbose:
        arg_list.insert(1, '-quiet')

    try:
        logging.info(f'Calling <{arg_list}>')
        subprocess.check_call(arg_list)
    except Exception as e:
        logging.error(e)

def _get_date_prefix() -> str:
    """ Return a string representation of the current date in the format yyyy_mm_dd_HH_MM"""
    year = datetime.date.today().year
    month = datetime.date.today().month
    day = datetime.date.today().day
    hour = datetime.datetime.now().hour
    minute = datetime.datetime.now().minute

    return f'{year}_{month}_{day}_{hour}_{minute}_'

def _move_file_to_folder(__file_name: str) -> None:
    """ Move the compiled file to a specific folder"""
    deploy_path = os.path.join(WORKING_PATH, DEPLOY_FOLER)
    try:
        os.mkdir(deploy_path)
    except FileExistsError as e:
        logging.info(e)
        logging.warning(f'The directory {DEPLOY_FOLER} does allready exist.')
    except Exception as e:
        logging.error(e)

    today = _get_date_prefix()
    file_name = __file_name + '.pdf'

    # Remove the working prefix from the file
    string_list = __file_name.split('_')
    string_list.pop(string_list.index(FILE_PREFIX_SPLIT))
    new_name = '_'.join(string_list) + '.pdf'

    try:
        logging.info('Moving file to deploy directory')
        os.rename(os.path.join(WORKING_PATH, file_name), os.path.join(deploy_path, today + '_' + new_name))
    except Exception as e:
        logging.error(e)

def _clean_working_dir() -> None:
    """ Remove all auxiliary files which have been generated in the process"""
    logging.info('Cleaning up working directory')
    for f in os.listdir('.'):
        if FILE_PREFIX in f:
            os.remove(f)

def clean_up(__file_name: str = None) -> None:
    """ This function cleans the working directory"""
    _move_file_to_folder(__file_name)
    _clean_working_dir()

def _create_parser() -> argparse.ArgumentParser:
    """ Returns an argumentParser object which contains all valid CLI arguments."""
    # == Setup for system objects ==
    parser = argparse.ArgumentParser()      # Parse CLI Arguments

    # == Add Arguments for the parser to recognize ==
    # Positional Arguments
    parser.add_argument(
        'filename',
        help='Name of file which should be processed in the pipeline'
    )

    # Options
    parser.add_argument(
        '-v',
        help='Turn on the console output for LaTeX.exe',
        action='store_true'
    )

    parser.add_argument(
        '-log',
        help='Select the loglevel of this pipeline. Default: ERROR',
        choices=list(LOGGING_LEVELS.keys()),
        default=LOGGING_LEVELS['ERROR']
    )

    return parser

def main() -> None:
    """ Process of Pipeline"""
    parser = _create_parser()

    try:
        args = parser.parse_args()

        logging.basicConfig(format='[%(levelname)s] - %(message)s', level=LOGGING_LEVELS[args.log])

        verbose = bool(args.v)

        new_file_name = copy_file(args.filename)

        remove_draft_option(new_file_name)
        compile_tex_file(new_file_name, verbose=verbose)
        clean_up(new_file_name)
    except Exception as e:
        print(e)

if __name__ == '__main__':
    main()


from django.shortcuts import render
from rest_framework.decorators import api_view
from rest_framework.response import Response
from django.http import FileResponse
import wave
import struct

HEADER_SIZE = 44

@api_view(['POST'])
def volume(request):
        try:
            # value for the volume change
            factor = float(request.data["factor"])
            audio_file = request.data["audio_file"]

            try:
                input_wav = wave.open(audio_file, 'rb')
            except:
                return Response({"error": "Could not open audio file"})

            output_file = 'output.wav'
            output_wav = wave.open(output_file, 'wb')

            # get the params of the input WAV file
            params = input_wav.getparams()

            # write the header to the output file
            output_wav.setparams(params)
            for _ in range(HEADER_SIZE):
                output_wav.writeframes(input_wav.readframes(1))

            # read samples from input file, adjust volume, and write to the output file
            while True:
                frame = input_wav.readframes(1)
                if not frame:
                    break

                sample = struct.unpack("<i", frame)
                adjusted_sample = int(sample[0] * factor)
                output_wav.writeframes(struct.pack("<i", adjusted_sample))

            # close files
            input_wav.close()
            output_wav.close()

            # return the actual file
            with open(output_file, 'rb') as file:
                 response = FileResponse(file, as_attachment=True, filename='output.wav')

            return response
            # return Response({"message": "Volume changed", "output_audio_file": output_file})

        except Exception as error:
             return Response({"error": str(error)})


import json
import numpy as np
import csv

# functions for processing/analysing the json data

def openjson(path): # opens a json and reads the info into a dictionary
    with open(path, 'r') as j:
        info = json.loads(j.read())
    return info

def tempo(path): # reads the BPM information from json file
    info = openjson(path)
    bpm = info["tempo"]
    return int(bpm)

def beats(path): # reads the beats information (in seconds) from json
    info = openjson(path)
    beats = info["beats"]
    return beats

def beatseek(percentage, path, fs): # given a percentage (samples), outputs the index from beat array
    beats_array = np.asarray(beats(path))
    beats_array = beats_array * fs
    i = np.abs(beats_array - percentage).argmin()
    return i

def mouse_quantizetobeats(path, position, song_length, width): # given a position, outputs a position in line with beats info
    beats_array = np.asarray(beats(path))
    beats_array = (beats_array/song_length)*width
    index = (np.abs(beats_array - position)).argmin()
    return beats_array[index]

def segmentation(path): # outputs segmentation data array (time in sec) from json
    info = openjson(path)
    segmentation_array = info["segmentation"]
    extracted_seg = []
    for array in segmentation_array:
        extracted_seg = np.append(extracted_seg,array[0])
    return extracted_seg

def plot_segmentation(path, fs, kernel): # outputs an array with segmentation scaled to plot width
    extracted_seg = segmentation(path)
    plotting_segs = []
    for i in extracted_seg:
        i *= fs
        i = int(i/kernel)
        final = np.append(plotting_segs,i)
    return plotting_segs

def segmentation_beats(path): # uses segmentation data to output 2 arrays of closest beats and their indexes
    beats_array = np.asarray(beats(path))
    segmentation_array = np.asarray(segmentation(path))
    segmentation_closest_beats = []
    indexes = []
    for i in segmentation_array:
        index = np.abs(beats_array - i).argmin()
        indexes = np.append(indexes,index)
        segmentation_closest_beats = np.append(segmentation_closest_beats, beats_array[index])
    return segmentation_closest_beats,indexes


import requests
from bs4 import BeautifulSoup

def crawl():
    url = "https://suckhoedoisong.vn/Covid-19-cap-nhat-moi-nhat-lien-tuc-n168210.html"
    response = requests.get(url)
    if response.status_code == 200:
        text = response.text
        return text
    else:
        return 'No Response'

def analyzeFile(text):
    soup = BeautifulSoup(text, "html.parser")

    box_covid = soup.find(class_="box_covid")

    head_box_covid = box_covid.find(class_="head_box_covid")
    time = head_box_covid.find(class_="fs12").text
    number = head_box_covid.find(class_="fs30").text

    province_cases = []
    table_box_covid = box_covid.find(class_="box_covid_info_city")
    table_body_box_covid = table_box_covid.find("table")
    rows = table_body_box_covid.find_all('tr')
    for row in rows:
        cols = row.find_all('td')
        cols = [ele.text.strip() for ele in cols]
        province_cases.append([ele for ele in cols if ele])

    with open("box_covid.txt",'w', encoding="utf-8") as text_file:
        text_file.write("{} : {} \n".format(time, number))
        text_file.write("TỈNH | SỐ CA MỚI |TỔNG SỐ CA \n")
        for province in province_cases:
            for data in province:
                text_file.write('{} | '.format(data))
            text_file.write("\n")

text = crawl()
analyzeFile(text)
f = open("box_covid.txt", "r", encoding="utf-8")
print(f.read())


import os
import platform
import re
import shutil
import subprocess
import pytest

@pytest.mark.parametrize("tool", [
    "acyclic",
    "bcomps",
    "ccomps",
    "circo",
    "cluster",
    "diffimg",
    "dijkstra",
    "dot",
    "dot2gxl",
    "dot_builtins",
    "dotty",
    "edgepaint",
    "fdp",
    "gc",
    "gml2gv",
    "graphml2gv",
    "gv2gml",
    "gv2gxl",
    "gvcolor",
    "gvedit",
    "gvgen",
    "gvmap",
    "gvmap.sh",
    "gvpack",
    "gvpr",
    "gxl2dot",
    "gxl2gv",
    "lefty",
    "lneato",
    "mingle",
    "mm2gv",
    "neato",
    "nop",
    "osage",
    "patchwork",
    "prune",
    "sccmap",
    "sfdp",
    "smyrna",
    "tred",
    "twopi",
    "unflatten",
    "vimdot",
])
def test_tools(tool):

  if shutil.which(tool) is None:
    pytest.skip(f"{tool} not available")

  # FIXME: Remove skip when
  # https://gitlab.com/graphviz/graphviz/-/issues/1829 is fixed
  if tool == "smyrna" and os.getenv("build_system") == "msbuild":
    pytest.skip("smyrna fails to start because of missing DLLs in Windows MSBuild builds (#1829)")

  # FIXME: Remove skip when
  # https://gitlab.com/graphviz/graphviz/-/issues/1838 is fixed
  if tool == "gvpack" and platform.system() != "Windows":
    if os.getenv("build_system") == "cmake":
      pytest.skip("gvpack does not find libgvplugin_neato_layout.so.6"
                  "when built with CMake (#1838)")

  # Ensure that X fails to open display
  environ_copy = os.environ.copy()
  environ_copy.pop("DISPLAY", None)

  # Test usage
  p = subprocess.Popen([tool, "-?"], env=environ_copy, stdin=subprocess.DEVNULL,
                       stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
                       universal_newlines=True)
  output, _ = p.communicate()
  ret = p.returncode

  # FIXME: https://gitlab.com/graphviz/graphviz/-/issues/1934
  # cope with flaky failures, while also failing if this flakiness has been
  # fixed
  if tool == "lneato" and os.getenv("build_system") == "msbuild":
    has_pass = ret == 0
    has_fail = ret != 0
    for _ in range(100):
      if has_pass and has_fail:
        break
      p = subprocess.Popen([tool, "-?"], env=environ_copy,
                           stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
                           stderr=subprocess.STDOUT, universal_newlines=True)
      out, _ = p.communicate()
      if p.returncode == 0:
        has_pass = True
        ret = p.returncode
        output = out
      else:
        has_fail = True
    assert has_pass, "could not find passing execution"
    assert has_fail, "could not find failing execution (#1934 fixed?)"

  assert ret == 0, f"`{tool} -?` failed. Output was: {output}"

  assert re.match("usage", output, flags=re.IGNORECASE) is not None, \
    f"{tool} -? did not show usage. Output was: {output}"

  # Test unsupported option
  returncode = subprocess.call(
      [tool, "-$"],
      env=environ_copy,
  )

  assert returncode != 0, f"{tool} accepted unsupported option -$"


from collections import namedtuple
import config.training
import tensorflow as tf

Trainer = namedtuple("Trainer", [
    "d_train_op",
    "g_train_op",
    "d_loss",
    "g_loss",
    "accuracy",
])

def losses(real_logits, fake_logits):
    with tf.name_scope("d_loss"):
        with tf.name_scope("d_real_loss"):
            d_real_loss = tf.nn.sigmoid_cross_entropy_with_logits(
                labels = (
                    (1 - config.training.LABEL_SMOOTHING)
                    * tf.ones_like(real_logits)
                ),
                logits = real_logits
            )
        with tf.name_scope("d_fake_loss"):
            d_fake_loss = tf.nn.sigmoid_cross_entropy_with_logits(
                labels = tf.zeros_like(fake_logits),
                logits = fake_logits
            )
        d_loss = tf.reduce_mean(
            tf.concat(
                [d_real_loss, d_fake_loss],
                axis = 0
            )
        )

    with tf.name_scope("g_loss"):
        g_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(
            labels = tf.ones_like(fake_logits),
            logits = fake_logits
        ))

    return d_loss, g_loss

def accuracy(real_logits, fake_logits):
    with tf.name_scope("accuracy"):
        accuracy = tf.reduce_mean(tf.concat([
            tf.cast(tf.equal(
                tf.round(tf.sigmoid(real_logits)),
                tf.ones_like(real_logits)), tf.float32
            ),
            tf.cast(tf.equal(
                tf.round(tf.sigmoid(fake_logits)),
                tf.zeros_like(fake_logits)), tf.float32
            )
        ], axis = 0))

    return accuracy

def build(real_logits, fake_logits):
    d_loss, g_loss = losses(
        real_logits = real_logits,
        fake_logits = fake_logits
    )

    discriminator_vars = tf.get_collection(
        tf.GraphKeys.TRAINABLE_VARIABLES,
        scope = "discriminator"
    )
    generator_vars = tf.get_collection(
        tf.GraphKeys.TRAINABLE_VARIABLES,
        scope = "generator"
    )

    optimizer = tf.train.AdamOptimizer(
        learning_rate = config.training.LEARNING_RATE,
        beta1 = config.training.BETA1
    )

    update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
    with tf.control_dependencies(update_ops):
        d_train_op = optimizer.minimize(
            d_loss,
            var_list = discriminator_vars
        )

    with tf.control_dependencies(update_ops):
        g_train_op = optimizer.minimize(
            g_loss,
            var_list = generator_vars
        )

    return Trainer(
        d_train_op = d_train_op,
        g_train_op = g_train_op,
        d_loss = d_loss,
        g_loss = g_loss,
        accuracy = accuracy(
            real_logits = real_logits,
            fake_logits = fake_logits,
        ),
    )


import numpy as np
import cv2

def normalize(kernel_in):  # Map to the int(0, 255) range
    kernel = kernel_in.copy()
    kernel = kernel - np.amin(kernel)
    kernel = (kernel / np.amax(kernel)) * 255.0
    return np.uint8(kernel)

 #### preprocess  step ####

img_color = cv2.imread('Greek-ship.jpg')
img = cv2.cvtColor(img_color, cv2.COLOR_BGR2GRAY)
img = cv2.GaussianBlur(img, (13, 13), 3)

template_color = cv2.imread('patch.png')
template = cv2.cvtColor(template_color, cv2.COLOR_BGR2GRAY)

template = np.int64(cv2.resize(template,
                               (int(0.16 * template.shape[1]),
                                int(0.16 * template.shape[0]))))
img = np.int64(cv2.resize(
    img, (int(0.32 * img.shape[1]), int(0.32 * img.shape[0]))))

#### implement SSD , Zero-mean cross-corr from scratch

SSD = np.int64(np.zeros((img.shape[0], img.shape[1])))
ZM_ccor = np.int64(np.zeros((img.shape[0], img.shape[1])))

for m in range(0, img.shape[0] - template.shape[0]):
    for n in range(0, img.shape[1] - template.shape[1]):
        window = (img[m:m + template.shape[0], n:n + template.shape[1]])
        ZM_ccor[m, n] = np.sum(np.multiply(
            (window - np.mean(window)), template))
        SSD[m, n] = np.sum(np.power(window - template, 2))

##### apply process on SSD and ZM-cross-corr

res = (ZM_ccor) - 0.1 * (SSD)
res[res < 0] = 0
res = normalize(res)
_, res = cv2.threshold(res, 110, 255, cv2.THRESH_BINARY)
res = cv2.resize(
    res, (int(1 / 0.32 * res.shape[1]), int(1 / 0.32 * res.shape[0])))

#### find cordinate of best find results

pnt = []
for j in range(0, res.shape[1]):
    tmp = (np.where(res[:, j] > 0))
    if tmp[0].tolist() != []:
        pnt.append([j, int(np.average(tmp))])

pnt_final = [pnt[0]]
for i, row in enumerate(pnt):
    last = np.array(pnt_final[-1])
    dist = np.linalg.norm(last - np.array(row))
    if dist > 30:
        if np.abs(last[0] - row[0]) > 20:
            pnt_final.append(row)

### draw results on the original image

img_color = cv2.imread('Greek-ship.jpg')
template_color = cv2.imread('patch.png')
(h, w) = template_color.shape[:-1]

img_res = img_color.copy()
for y, x in pnt_final:
    img_res = cv2.rectangle(img_res, (y, x), (y + w, x + h), 255, 5)

cv2.imwrite('res15.jpg', img_res)


import cv2
import mediapipe as mp
import math
from typing import List, Mapping, Optional, Tuple, Union
import numpy as np

from mediapipe.framework.formats import landmark_pb2
from mediapipe.python.solutions.drawing_utils import DrawingSpec, _normalized_to_pixel_coordinates

WHITE_COLOR = (224, 224, 224)
BLACK_COLOR = (0, 0, 0)
RED_COLOR = (0, 0, 255)
GREEN_COLOR = (0, 255, 0)
YELLOW_COLOR = (0, 255, 255)
_PRESENCE_THRESHOLD = 0.5
_VISIBILITY_THRESHOLD = 0.5
_RGB_CHANNELS = 3

class UtilFunctions:
    """
    Estimates Pose points of a human body using the mediapipe library.
    """

    def __init__(self, view, mode=False, smooth=True,
                 detectionCon=0.6, trackCon=0.6):
        """
        :param mode: In static mode, detection is done on each image: slower
        :param upBody: Upper boy only flag
        :param smooth: Smoothness Flag
        :param detectionCon: Minimum Detection Confidence Threshold
        :param trackCon: Minimum Tracking Confidence Threshold
        """
        self.min_angle = 180
        self.max_distance = -1
        self.view = view
        self.mode = mode
        self.smooth = smooth
        self.detectionCon = detectionCon
        self.trackCon = trackCon
        self.visibility = [-1 for i in range(34)]
        self.stable_values = [[0, 0, 0] for i in range(34)]

        self.mpDraw = mp.solutions.drawing_utils
        self.mpPose = mp.solutions.pose
        self.pose = self.mpPose.Pose(static_image_mode=self.mode,
                                     smooth_landmarks=self.smooth,
                                     min_detection_confidence=self.detectionCon,
                                     min_tracking_confidence=self.trackCon, model_complexity=1)

    def findPose(self, img, world=False):
        """
        Find the pose landmarks in an Image of BGR color space.
        :param img: Image to find the pose in.
        :param draw: Flag to draw the output on the image.
        :return: Image with or without drawings
        """
        imgRGB = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
        results = self.pose.process(imgRGB)
        if world:
            return np.array(img), results.pose_world_landmarks
        else:
            return np.array(img), results.pose_landmarks

    def draw(self, img, landmarks_list):
        if landmarks_list:
            self.mpDraw.draw_landmarks(img, landmarks_list,
                                       self.mpPose.POSE_CONNECTIONS,
                                       landmark_drawing_spec=mp.solutions.drawing_styles.get_default_pose_landmarks_style()
                                       )

    def findPosition(self, img, landmarks):
        lmList = []
        if landmarks:
            h, w, c = img.shape
            for id, lm in enumerate(landmarks.landmark):
                lmList.append([id, int(lm.x * w), int(lm.y * h), lm.visibility])
        return lmList

    def check_visibility(self, landmarks_list, check):
        if landmarks_list:
            return landmarks_list.landmark[check].visibility

    def update_landmark(self, landmarks_list, joint):
        """
        Find the pose landmarks in an Image of BGR color space.
        :param landmarks_list: A normalized landmark list which needs to be updated
        :param joint: Flag to draw the output on the image.
        :ret : updated Landmarks_list
        """
        # This function updates the more stable value to the join under consideration
        if landmarks_list:
            if self.check_visibility(landmarks_list, joint) >= (
                    self.visibility[joint] - self.visibility[joint] * 0.008):
                self.stable_values[joint][0] = landmarks_list.landmark[joint].x
                self.stable_values[joint][1] = landmarks_list.landmark[joint].y
                self.stable_values[joint][2] = landmarks_list.landmark[joint].z

                self.visibility[joint] = self.check_visibility(landmarks_list, joint)

        landmarks_list.landmark[joint].x = self.stable_values[joint][0]
        landmarks_list.landmark[joint].y = self.stable_values[joint][1]
        landmarks_list.landmark[joint].z = self.stable_values[joint][2]

        return landmarks_list

    def drawLine(self, img, pt1, pt2, ptcolor, line_color, draw_points=True, r=8, t=2):
        cv2.line(img, pt1, pt2, line_color, t)
        if draw_points:
            cv2.circle(img, pt1, r, ptcolor, cv2.FILLED)
            cv2.circle(img, pt2, r, ptcolor, cv2.FILLED)

    def findAngle(self, img, p1, p2, p3, lmlist, draw=True):
        """
        Finds angle between three points. Inputs index values of landmarks
        instead of the actual points.
        :param img: Image to draw output on.
        :param p1: Point1 - Index of Landmark 1.
        :param p2: Point2 - Index of Landmark 2.
        :param p3: Point3 - Index of Landmark 3.
        :param draw:  Flag to draw the output on the image.
        :return:
        """

        # Get the landmarks
        x1, y1 = lmlist[p1][1:3]
        x2, y2 = lmlist[p2][1:3]
        x3, y3 = lmlist[p3][1:3]

        # Calculate the Angle
        angle = math.degrees(math.atan2((y3 - y2), (x3 - x2)) -
                             math.atan2((y1 - y2), (x1 - x2)))
        if angle < 0:
            angle+=360
        # Draw
        if draw:
            self.drawLine(img, (x1, y1), (x2, y2), RED_COLOR, WHITE_COLOR)
            self.drawLine(img, (x3, y3), (x2, y2), RED_COLOR, RED_COLOR)
            cv2.putText(img, str(int(angle)), (x2 - 50, y2 + 50),
                        cv2.FONT_HERSHEY_PLAIN, 2, (0, 0, 255), 2)
        return angle

    def findDistance(self, p1, p2, img, lmlist, draw=True, r=15, t=4):
        x1, y1 = lmlist[p1][1:3]
        x2, y2 = lmlist[p2][1:3]
        cx, cy = (x1 + x2) // 2, (y1 + y2) // 2

        if draw:
            self.drawLine(img, (x1, y1), (x2, y2), (255, 0, 255), (255, 0, 255))

        length = math.hypot(x2 - x1, y2 - y1)

        return length, img, [x1, y1, x2, y2, cx, cy]

    def checkCurvature(self, p1, p2, img, lmlist, draw=True):
        x1, y1 = lmlist[p1][1:3]
        x2, y2 = lmlist[p2][1:3]
        cx, cy = (x1 + x2) // 2, (y1 + y2) // 2

        length = math.hypot(x2 - x1, y2 - y1)
        self.max_distance = max(length, self.max_distance)
        #print(self.max_distance, length)
        if length + 25 >= self.max_distance:
            if draw:
                self.drawLine(img, (x1, y1), (cx, cy), GREEN_COLOR, GREEN_COLOR)
                self.drawLine(img, (x2, y2), (cx, cy), GREEN_COLOR, GREEN_COLOR)
            return GREEN_COLOR
        else:
            if draw:
                self.drawLine(img, (x1, y1), (cx, cy), RED_COLOR, RED_COLOR)
                self.drawLine(img, (x2, y2), (cx, cy), RED_COLOR, RED_COLOR)
            return RED_COLOR

    def angleCheck(self, img, p1, p2,p3, target, isleftView, lmlist, addOn=13, cutoff=45, draw=True):
        x1, y1 = lmlist[p1][1:3]
        x2, y2 = lmlist[p2][1:3]
        x3, y3 = x1, y2
        if isleftView:
            return self.angleCheckLeft(img, p1,p2,p3, target, lmlist,addOn, cutoff, draw)
        # Calculate the Angle
        angle = math.degrees(math.atan2((y3 - y2), (x3 - x2)) -
                             math.atan2((y1 - y2), (x1 - x2)))
        #print("here", angle)
        if -3 < angle < target + addOn and angle < cutoff:
            if draw:
                self.drawLine(img, (x3, y3), (x2, y2), GREEN_COLOR, GREEN_COLOR)
                cv2.putText(img, str(int(angle)), (x2 - 50, y2 + 50),
                            cv2.FONT_HERSHEY_PLAIN, 2, (0, 255, 0), 2)
            return GREEN_COLOR, angle
        elif angle < -3:
            if draw:
                self.drawLine(img, (x3, y3), (x2, y2), YELLOW_COLOR, YELLOW_COLOR)
                cv2.putText(img, str(int(angle)), (x2 - 50, y2 + 50),
                            cv2.FONT_HERSHEY_PLAIN, 2, (0, 0, 255), 2)
            return YELLOW_COLOR, angle
        elif angle < cutoff:
            if draw:
                self.drawLine(img, (x3, y3), (x2, y2), RED_COLOR, RED_COLOR)
                cv2.putText(img, str(int(angle)), (x2 - 50, y2 + 50),
                            cv2.FONT_HERSHEY_PLAIN, 2, (0, 0, 255), 2)

            return RED_COLOR, angle
        else:
            return BLACK_COLOR, angle

    def generate_angle_Feedback(self, back_angle_color):
        result = []
        if back_angle_color == RED_COLOR:
            result.append("Don't raise your back too much")
        elif back_angle_color == YELLOW_COLOR:
            result.append("Keep your torso parallel to the floor")
        else:
            result.append("Good Form! keep you back in this range.")

        return result[0], back_angle_color

    def generate_back_feedback(self, back_color):
        result = []
        if back_color == RED_COLOR:
            result.append("Don't round/curve your spine!")
        else:
            result.append("Good form! spine is natural and not rounded")
        return result[0], back_color

    def display_feedback(self, img, result_back, result_back_angle, color_back, color_angle):
        cv2.rectangle(img, (0, 0), (560, 80), (0, 0, 0), -1)
        cv2.putText(img, result_back, (0, 30), cv2.FONT_HERSHEY_SIMPLEX, 0.75, color_back, 2, cv2.LINE_AA)
        cv2.putText(img, result_back_angle, (0, 60), cv2.FONT_HERSHEY_SIMPLEX, 0.75, color_angle, 2, cv2.LINE_AA)

    def angleCheckLeft(self, img, p1, p2, p3, target, lmlist, addOn, cutoff, draw):
        # Calculate the Angle
        angle = self.findAngle(img,p1,p2,p3,lmlist, draw = False)

        x1, y1 = lmlist[p1][1:3]
        x2, y2 = lmlist[p2][1:3]
        x3, y3 = lmlist[p3][1:3]

        if angle > self.min_angle == 180:
            self.min_angle = angle
        print(self.min_angle-angle)
        if -3 < self.min_angle-angle < target + addOn and self.min_angle-angle < cutoff:
            if draw:
                self.drawLine(img, (x3, y3), (x2, y2), GREEN_COLOR, GREEN_COLOR)
                cv2.putText(img, str(int(angle)), (x2 - 50, y2 + 50),
                            cv2.FONT_HERSHEY_PLAIN, 2, (0, 255, 0), 2)
            return GREEN_COLOR, self.min_angle-angle
        elif self.min_angle - angle < -3:
            if draw:
                self.drawLine(img, (x3, y3), (x2, y2), YELLOW_COLOR, YELLOW_COLOR)
                cv2.putText(img, str(int(angle)), (x2 - 50, y2 + 50),
                            cv2.FONT_HERSHEY_PLAIN, 2, (0, 0, 255), 2)
            return YELLOW_COLOR, self.min_angle-angle
        elif self.min_angle - angle < cutoff:
            if draw:
                self.drawLine(img, (x3, y3), (x2, y2), RED_COLOR, RED_COLOR)
                cv2.putText(img, str(int(angle)), (x2 - 50, y2 + 50),
                            cv2.FONT_HERSHEY_PLAIN, 2, (0, 0, 255), 2)

            return RED_COLOR, self.min_angle-angle
        else:
            self.drawLine(img, (x3, y3), (x2, y2), (255,0,0), (255,0,0))
            return BLACK_COLOR, self.min_angle-angle

    def slopeCheckBarbell(self, img, landmarks, draw = True):
        img_h, img_w, _ = img.shape
        left_wrist = [int(landmarks.landmark[15].x*img_w), int(landmarks.landmark[15].y*img_h), int(landmarks.landmark[15].z*img_w)]
        right_wrist = [int(landmarks.landmark[16].x*img_w), int(landmarks.landmark[16].y*img_h), int(landmarks.landmark[16].z*img_w)]

        wrist_slope = pow((left_wrist[2] - right_wrist[2]), 2) / math.sqrt(
            pow((left_wrist[0] - right_wrist[0]), 2) + pow((left_wrist[1] - right_wrist[1]), 2))
        left_ankle = [landmarks.landmark[27].x, landmarks.landmark[27].y, landmarks.landmark[27].z]
        right_ankle = [landmarks.landmark[28].x, landmarks.landmark[28].y, landmarks.landmark[28].z]

        ankle_slope = pow((left_ankle[2] - right_ankle[2]), 2) / math.sqrt(
            pow((left_ankle[0] - right_ankle[0]), 2) + pow((left_ankle[1] - right_ankle[1]), 2))
        # print("slope of wrists:",wrist_slope)
        # print(wrist_slope - ankle_slope)
        if abs(wrist_slope - ankle_slope) < 2:
            if draw:
                self.drawLine(img, left_wrist[0:2], right_wrist[0:2], GREEN_COLOR, GREEN_COLOR, draw_points=False)
            return GREEN_COLOR
        else:
            if draw:
                self.drawLine(img, left_wrist[0:2], right_wrist[0:2], RED_COLOR, RED_COLOR, draw_points=False)
            return RED_COLOR

    def generate_barbell_feedback(self, barbell_color):
        result = []
        if barbell_color == RED_COLOR:
            result.append("The barbell is tilting keep it straight!")
        else:
            result.append("Good! Keep the barbell straight. ")
        return result[0], barbell_color


from threading import Timer
import asyncio

class InfiniteTimer:
    """
    Timer to update Threat Intelligence Files when Slips starts
    """

    def __init__(self, seconds, target):
        self.timer_running = False
        self.target_running = False
        self.seconds = seconds
        self.target = target
        self.thread = None

    def _handle_target(self):
        self.target_running = True
        asyncio.run(self.target())
        self.target_running = False
        self._start_timer()

    def _start_timer(self):
        if (
            self.timer_running
        ):
            self.thread = Timer(self.seconds, self._handle_target)
            self.thread.start()

    def start(self):
        if not self.timer_running and not self.target_running:
            self.timer_running = True
            self._start_timer()

    def cancel(self):
        if self.thread is not None:
            self.timer_running = (
                False
            )
            self.thread.cancel()


import FaceDetector as fd
import os
from options.test_options import TestOptions
from data import create_dataset
from models import create_model
import torch
import torchvision
import numpy as np

def cycle_gan_model():
    opt = TestOptions().parse()  # get test options
    # hard-code some parameters for test
    opt.num_threads = 0   # test code only supports num_threads = 0
    opt.batch_size = 1    # test code only supports batch_size = 1
    opt.serial_batches = True  # disable data shuffling; comment this line if results on randomly chosen images are needed.
    opt.no_flip = True    # no flip; comment this line if results on flipped images are needed.
    opt.display_id = -1   # no visdom display; the test code saves the results to a HTML file.
    model = create_model(opt)      # create a model given opt.model and other options
    model.setup(opt)               # regular setup: load and print networks; create schedulers
    if opt.eval:
        model.eval()            # regular setup: load and print networks; create schedulers

    return model

if __name__ == '__main__':

    cycle_gan = cycle_gan_model()
    face_detect = fd.FaceDetector('faceDetectionModel/deploy.prototxt.txt', 'faceDetectionModel/opencv_face_detector.caffemodel')

    face = face_detect.detect_face_from_image('me.jpg')
    original_size = face.shape

    data = {"A": None, "A_paths": None}
    face = np.array([face])
    face = face.transpose([0,3,1,2])
    data['A'] = torch.FloatTensor(face)

    cycle_gan.set_input(data)  # unpack data from data loader
    cycle_gan.test()

    result_image = cycle_gan.get_current_visuals()['fake']
    torchvision.utils.save_image(result_image, 'memonet.jpg')

    print('test')


from PIL import Image

from .cut import Cut
from ..utils.frange import frange
from ..utils.format_point import format_point
from ..utils.prelude import reset_position

class Raster(Cut):
    def make_cut(self, function, bounds):
        for depth in frange(-1, -self.depth, -self.depth_per_cut):
            self.add_commands(*reset_position())
            for x in frange(0, bounds[0]//10, 0.2):
                for y in frange(0, bounds[1]//10, 0.1):
                    actual_depth = max(depth, function(x, y)*self.depth-self.depth)
                    self.add_commands(format_point(x=x, y=y, z=actual_depth))
                x += 0.1
                for y in frange(bounds[1]//10, 0, -0.1):
                    actual_depth = max(depth, function(x, y)*self.depth-self.depth)
                    self.add_commands(format_point(x=x, y=y, z=actual_depth))


import requests
import json
from bs4 import BeautifulSoup
from requests.adapters import HTTPAdapter
import yaml

session = requests.Session()
adapter = HTTPAdapter(max_retries=5, pool_connections=100, pool_maxsize=100)
session.mount("http://www.allmusic.com", adapter)

session_cookies = {}
DATA = {}

def config():
    pass

def dispose():
    pass

def config_dump(data_file):
    GENRE_TREE = yaml.load(open(data_file))
    DATA["GENRE_TREE"] = GENRE_TREE
    GENRE_TREE_paths = styles(GENRE_TREE)
    DATA["GENRE_TREE_paths"] = GENRE_TREE_paths
    GENRE_TREE_styles = [p[-1] for p in GENRE_TREE_paths]
    DATA["GENRE_TREE_styles"] = set(GENRE_TREE_styles)

def styles(tree):
    styles = []
    for g, child in tree.items():
        if isinstance(child, list):
            for gs in child:
                if isinstance(gs, str):
                    styles.append((g, gs))
                if isinstance(gs, dict):
                    for gss, gsschild in gs.items():
                        for gsss in gsschild:
                            styles.append((g, gss, gsss))
        elif type(child) is dict:
            for gs, gschild in child.items():
                for gss in gschild:
                    styles.append((g, gs, gss))
        else:
            raise Exception("Unexpected genre tree format")
    return styles

def extract_style(styles, genres):
    results = []

    genre_set = set(genres)
    for s in styles:
        if s not in DATA["GENRE_TREE_styles"]:
            continue

        matched = False
        for style_path in DATA["GENRE_TREE_paths"]:
            if s == style_path[-1] and style_path[0] in genre_set:
                results.append(style_path)
                matched = True

    # We want to add all genres of a style, even if they aren't
    # explicitly listed
    all_genres = genre_set | set([r[0] for r in results])
    results += [(g,) for g in all_genres]
    return results

def parse_db_data(mbid, data, writer):

    album = data["album"]
    if album:
        genres = album["genres"]
        styles = album["styles"]

        if not genres:
            return

        # only use 1st and 2nd level genres
        annotations = []
        for a in extract_style(styles, genres):
            if len(a) == 0:
                continue
            elif len(a) == 2 or len(a) == 1:
                annotations.append(a)
            elif len(a) == 3:
                annotations.append((a[0], a[1]))
                annotations.append((a[0], a[2]))

        # remove duplicates that might occur after extracting only 2nd level subgenres
        annotations = list(set(annotations))
        annotations = ["---".join(list(a)).lower() for a in annotations]

        if annotations:
            if not isinstance(mbid, list):
                mbid = [mbid]

            for m in mbid:
                row = [m] + sorted(annotations)
                writer.writerow(row)

def _get_cookies():
    headers = query_data()
    r = session.get("http://www.allmusic.com/", headers=headers)
    session_cookies['allmusic_session'] = r.cookies.get('allmusic_session')

def scrape(query):
    if not session_cookies:
        _get_cookies()

    mbid = query['mbid']
    artist = query['artist']
    release = query['release']
    year = query['year']

    if not artist or not release or not year:
        raise Exception("Artist, release, and year information is required. Query: %s" % json.dumps(query))
    print("Scraping recording information for", mbid + ":", artist, "-", release, "-", year)

    if year:
        year = year[:4]

    res = search(artist, release)
    data = {}
    searchres = parse_search(res, artist, release)

    data["search_results"] = searchres
    data["album"] = {}
    if searchres:
        relres = get_first_release(searchres, year)
        if relres:
            release = parse_release(relres)
            data["album"] = release

        return data
    else:
        # No search results, return empty data
        return {}

def get_first_release(data, year):
    if data:
        if year:
            filtered = [d for d in data if d["year"] == year]
            if filtered:
                data = filtered
        rel = data[0]
        albumdata = rel["albumdata"]
        if albumdata:
            releaseurl = rel["albumdata"].get("href")
            if releaseurl:
                content = load_release(releaseurl)
                return content
    return {}

def parse_release(data):
    s = BeautifulSoup(data, "html5lib")

    genrenode = s.find("div", {"class": "genre"})
    stylenode = s.find("div", {"class": "styles"})
    moodnode = s.find("section", {"class": "moods"})
    themenode = s.find("section", {"class": "themes"})

    genres = []
    if genrenode:
        for a in genrenode.find_all("a"):
            genres.append(a.text.strip())
    styles = []
    if stylenode:
        for a in stylenode.find_all("a"):
            styles.append(a.text.strip())
    moods = []
    if moodnode:
        for a in moodnode.find_all("a"):
            moods.append(a.text.strip())
    themes = []
    if themenode:
        for a in themenode.find_all("a"):
            themes.append(a.text.strip())

    review = s.find("section", {"class": "review"})
    if review:
        review = review.text

    return {"genres": genres, "styles": styles, "moods": moods, "themes": themes, "review": review}

def load_release(url):
    headers = query_data()
    r = session.get(url, headers=headers, cookies=session_cookies)
    return r.content

def search(artist, release):
    headers = query_data()
    r = session.get('http://www.allmusic.com/search/albums/{} {}'.format(artist, release), headers=headers, cookies=session_cookies)
    return r.content

def parse_search(data, artist, release):
    s = BeautifulSoup(data, "html5lib")
    infos = s.find_all("div", {"class": "info"})
    data = []
    referrer = 'http://www.allmusic.com/search/albums/{} {}'.format(artist, release)
    for i in infos:
        title = i.find("div", {"class": "title"})
        if title:
            title = title.text.strip()
        artist = i.find("div", {"class": "artist"})
        if artist:
            artist = artist.text.strip()
        year = i.find("div", {"class": "year"})
        if year:
            year = year.text.strip()
        genres = i.find("div", {"class": "genre"})
        if genres:
            genres = genres.text.strip()
        artistnode = i.find("div", {"class": "artist"})
        artistdata = {}
        if artistnode:
            artistdata = artistnode.find("a")
            if artistdata:
                artistdata = artistdata.attrs
        albumnode = i.find("div", {"class": "title"})
        if albumnode:
            albumdata = albumnode.find("a")
            if albumdata:
                albumdata = albumdata.attrs
        data.append({"title": title, "artist": artist,
                     "year": year, "genres": genres,
                     "artistdata": artistdata, "albumdata": albumdata})
    return data

def query_data():
    headers = {
        'DNT': '1',
        'Accept-Encoding': 'gzip, deflate, sdch',
        'Accept-Language': 'en-NZ,en;q=0.8,es;q=0.6',
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.18 Safari/537.36',
        'Accept': '*/*',
        'Referer': 'http://www.allmusic.com/',
        'Connection': 'keep-alive',
        'Cache-Control': 'max-age=0',
    }
    return headers