"""Functions for creating new mixes from medleydb multitracks.
"""
import shutil
import sox

VOCALS = ["male singer", "female singer", "male speaker", "female speaker",
          "male rapper", "female rapper", "beatboxing", "vocalists"]

def mix_multitrack(mtrack, output_path, stem_indices=None,
                   alternate_weights=None, alternate_files=None,
                   additional_files=None):
    """Mix the stems of a multitrack to create a new mix.
    Can optionally adjust the volume of stems and replace, remove, or add
    stems.

    Parameters
    ----------
    mtrack : Multitrack
        Multitrack object
    output_path : str
        Path to save output file.
    stem_indices : list or None, default=None
        stem indices to include in mix.
        If None, mixes all stems
    alternate_weights : dict or None, default=None
        Dictionary with stem indices as keys and mixing coefficients as values.
        Stem indices present that are not in this dictionary will use the
        default estimated mixing coefficient.
    alternate_files : dict or None, default=None
        Dictionary with stem indices as keys and filepaths as values.
        Audio file to use in place of original stem. Stem indices present that
        are not in this dictionary will use the original stems.
    additional_files : list of tuple or None, default=None
        List of tuples of (filepath, mixing_coefficient) pairs to additionally
        add to final mix.

    Returns
    -------
    filepaths : list
        List of filepaths used in the mix
    weights : list
        List of weights used to mix filepaths

    """
    filepaths, weights = _build_mix_args(
        mtrack, stem_indices, alternate_weights, alternate_files,
        additional_files
    )

    if len(filepaths) == 1:
        shutil.copyfile(filepaths[0], output_path)
    else:
        cbn = sox.Combiner()
        cbn.build(
            filepaths, output_path, 'mix', input_volumes=weights
        )

    return filepaths, weights

def _build_mix_args(mtrack, stem_indices, alternate_weights, alternate_files,
                    additional_files):
    """Create lists of filepaths and weights to use in final mix.

    Parameters
    ----------
    mtrack : Multitrack
        Multitrack object
    stem_indices : list
        stem indices to include in mix.
        If None, mixes all stems
    alternate_weights : dict
        Dictionary with stem indices as keys and mixing coefficients as values.
        Stem indices present that are not in this dictionary will use the
        default estimated mixing coefficient.
    alternate_files : dict
        Dictionary with stem indices as keys and filepaths as values.
        Audio file to use in place of original stem. Stem indices present that
        are not in this dictionary will use the original stems.
    additional_files : list of tuples
        List of tuples of (filepath, mixing_coefficient) pairs to additionally
        add to final mix.

    Returns
    -------
    filepaths : list
        List of filepaths that were included in mix.
    weights : list
        List of weights that were used in mix

    """
    if stem_indices is None:
        stem_indices = list(mtrack.stems.keys())

    if alternate_files is None:
        alternate_files = {}

    if alternate_weights is None:
        alternate_weights = {}

    weights = []
    filepaths = []
    for index in stem_indices:
        if index in alternate_files.keys():
            filepaths.append(alternate_files[index])
        else:
            filepaths.append(mtrack.stems[index].audio_path)

        if index in alternate_weights.keys():
            weights.append(alternate_weights[index])
        else:
            if mtrack.stems[index].mixing_coefficient is not None:
                mix_coeff = mtrack.stems[index].mixing_coefficient
            else:
                print(
                    '[Warning] Multitrack does not have mixing coefficients. '
                    'Using uniform stem weights.'
                )
                mix_coeff = 1
            weights.append(mix_coeff)

    if additional_files is not None:
        for fpath, weight in additional_files:
            filepaths.append(fpath)
            weights.append(weight)

    return filepaths, weights

def mix_melody_stems(mtrack, output_path, max_melody_stems=None,
                     include_percussion=False, require_mono=False):
    """Creates a mix using only the stems labeled as melody.

    Parameters
    ----------
    mtrack : Multitrack
        Multitrack object
    output_path : str
        Path to save output wav file.
    max_melody_stems : int or None, default=None
        The maximum number of melody stems to mix. If None, uses the number of
        melody stems in the mix.
    include_percussion : bool, default=False
        If true, adds percussion stems to the mix.
    require_mono : bool, default=False
        If true, only includes melody stems that are monophonic instruments.

    Returns
    -------
    melody_indices : list
        List of selected melody indices.
    stem_indices : list
        List of stem indices used in mix.

    """
    if max_melody_stems is None:
        max_melody_stems = 100

    melody_rankings = mtrack.melody_rankings
    inverse_ranking = {v: k for k, v in melody_rankings.items()}
    n_melody_stems = len(list(melody_rankings.keys()))
    stem_indices = []
    melody_indices = []
    n_chosen = 0
    for i in range(1, n_melody_stems + 1):
        if n_chosen >= max_melody_stems:
            break

        this_stem_index = inverse_ranking[i]
        if require_mono:
            mono = all([
                f0_type == 'm' for f0_type
                in mtrack.stems[this_stem_index].f0_type
            ])
            if mono:
                stem_indices.append(this_stem_index)
                melody_indices.append(this_stem_index)
                n_chosen += 1
        else:
            stem_indices.append(this_stem_index)
            melody_indices.append(this_stem_index)
            n_chosen += 1

    if include_percussion:
        percussive_indices = [
            i for i, s in mtrack.stems.items() if s.f0_type == 'u'
        ]

        for i in percussive_indices:
            stem_indices.append(i)

    mix_multitrack(mtrack, output_path, stem_indices=stem_indices)
    return melody_indices, stem_indices

def mix_mono_stems(mtrack, output_path, include_percussion=False):
    """Creates a mix using only the stems that are monophonic. For example, in
    mix with piano, voice, and clarinet, the resulting mix would include
    only voice and clarinet.

    Parameters
    ----------
    mtrack : Multitrack
        Multitrack object
    output_path : str
        Path to save output wav file.
    include_percussion : bool, default=False
        If true, percussive instruments are included in the mix. If false, they
        are excluded.

    Returns
    -------
    mono_indices : list
        List of stem indices containing monophonic instruments.
    stem_indices : list
        List of stem indices used in mix.

    """
    stems = mtrack.stems
    stem_indices = []
    mono_indices = []
    for i in stems.keys():
        mono = all([
            f0_type == 'm' for f0_type in mtrack.stems[i].f0_type
        ])
        unvoiced = all([
            f0_type == 'u' for f0_type in mtrack.stems[i].f0_type
        ])
        if mono:
            stem_indices.append(i)
            mono_indices.append(i)
        elif include_percussion and unvoiced:
            stem_indices.append(i)

    mix_multitrack(mtrack, output_path, stem_indices=stem_indices)
    return mono_indices, stem_indices

def mix_no_vocals(mtrack, output_path):
    """Remixes a multitrack with anything type of vocals removed.
    If no vocals are present, the mix will be a simple weighted linear remix.

    Parameters
    ----------
    mtrack : Multitrack
        Multitrack object
    output_path : str
        Path to save output file.

    Returns
    -------
    stem_indices : list
        List of stem indices used in mix.

    """
    stems = mtrack.stems
    stem_indices = []
    for i in stems.keys():
        not_vocal = all([inst not in VOCALS for inst in stems[i].instrument])
        if not_vocal:
            stem_indices.append(i)

    mix_multitrack(mtrack, output_path, stem_indices=stem_indices)
    return stem_indices

def remix_vocals(mtrack, output_path, vocals_scale):
    """Remixes a multitrack, changing the volume of the vocals.

    Parameters
    ----------
    mtrack : Multitrack
        Multitrack object
    output_path : str
        Path to save output wav file.
    vocals_scale : float
        The target scale factor for vocals. A value of 1 keeps the volume the
        same. Values above 1 increase the volume and below 1 decrease it.

    Returns
    -------
    alternate_weights : dict
        Dictionary of vocal weights keyed by vocal stem index.

    """
    stems = mtrack.stems
    alternate_weights = {}
    for i in stems.keys():
        vocal = any([inst in VOCALS for inst in stems[i].instrument])
        if vocal:
            vocal_weight = stems[i].mixing_coefficient * vocals_scale
            alternate_weights[i] = vocal_weight

    mix_multitrack(
        mtrack, output_path, alternate_weights=alternate_weights
    )
    return alternate_weights


# This has to be imported before requests module to prevent warning
#   MonkeyPatchWarning: Monkey-patching ssl after ssl has already been imported may lead to errors,
#   including RecursionError on Python 3.6. It may also silently lead to incorrect behaviour on Python 3.7.
#   Please monkey-patch earlier. See https://github.com/gevent/gevent/issues/1016
from rocket_league.authentication.steam_authentication import steam_login, encode_steam_encrypted_app_ticket
from rocket_league.authentication.rocket_league_authentication import rl_auth
from rocket_league.requests import get_game_server_find_private_game_server_request
from rocket_league import rocket_league_constants
import getpass
import json
import logging
import websocket
import _thread as thread
import time

def rocket_league_connect_to_websocket(url, psy_token, session_id):
    def on_message(_, message):
        start = message.index('\r\n\r\n')
        start += 4
        message_length = len(message) - start
        if message_length == 0:
            print('[ROCKET LEAGUE] Message received with length 0')
        else:
            message = message[start:]
            message = json.loads(message)
            if len(message['Responses'][0]['Result']['Servers']) > 0:
                print('[MATCH FOUND] Match found!')

    def on_error(_, error):
        print(error)

    def on_close(_):
        print("[ROCKET LEAGUE] Websocket closed!")

    def on_open(_):
        print('[ROCKET LEAGUE] Connected to websocket!')

        def run(*_):
            brute_force(web_socket)

            time.sleep(1)
            web_socket.close()
            print("[ROCKET LEAGUE] Closing websocket...")

        thread.start_new_thread(run, ())

    websocket.enableTrace(True)
    web_socket = websocket.WebSocketApp(url,
                                        header={
                                            'PsyToken': psy_token,
                                            'PsySessionID': session_id,
                                            'PsyBuildID': rocket_league_constants.RLBuildId,
                                            'PsyEnvironment': rocket_league_constants.RLEnvironment,
                                            'User-Agent': rocket_league_constants.RLUserAgent
                                        },
                                        on_message=on_message,
                                        on_error=on_error,
                                        on_close=on_close,
                                        on_open=on_open)
    logging.getLogger('websocket').setLevel(logging.INFO)
    web_socket.run_forever()

def brute_force(web_socket):
    with open('rocket_league_private_server_wordlist') as file:
        for line in file:
            word = line.strip()
            print('[BRUTE FORCE] Testing {}:{}'.format(word, word))
            web_socket.send(get_game_server_find_private_game_server_request(word, word))
            rocket_league_constants.increase_id()
            time.sleep(1)

def main():
    print('[STEAM] Parsing config file...')
    with open('steam_login.config') as steam_login_config_file:
        steam_login_config = json.load(steam_login_config_file)

    print('[STEAM] Logging in...')
    two_factor_code = getpass.getpass('Enter 2FA> ')
    steam_client = steam_login(steam_login_config['username'], steam_login_config['password'], two_factor_code)
    print('[STEAM] Successfully logged in!')

    print('[STEAM] Getting id_64, encrypted_app_ticket and display_name...')
    steam_id_64 = steam_client.user.steam_id.as_64
    steam_encrypted_app_ticket = \
        steam_client.get_encrypted_app_ticket(rocket_league_constants.RLAppId, b'').encrypted_app_ticket
    steam_encoded_encrypted_app_ticket = encode_steam_encrypted_app_ticket(steam_encrypted_app_ticket)
    steam_display_name = steam_client.user.name

    print('[ROCKET LEAGUE] Authenticating with RL servers...')
    rl_auth_response = rl_auth(steam_id_64, steam_encoded_encrypted_app_ticket, steam_display_name)
    if rl_auth_response.get('Error', None) is not None:
        print('[ERROR][ROCKET LEAGUE] Failed to authenticate with the RL servers. Exiting.')
        exit(1)
    print('[ROCKET LEAGUE] Successfully authenticated with the RL servers!')

    print('[ROCKET LEAGUE] Grabbing per_con_url, psy_token and session_id...')
    per_con_url = rl_auth_response['Responses'][0]['Result']['PerConURL']
    psy_token = rl_auth_response['Responses'][0]['Result']['PsyToken']
    session_id = rl_auth_response['Responses'][0]['Result']['SessionID']

    print('[ROCKET LEAGUE] Connecting to websocket...')
    rocket_league_connect_to_websocket(per_con_url, psy_token, session_id)

if __name__ == '__main__':
    main()


from typing import Dict, Callable

from kallisticore.lib.action import Action
from kallisticore.lib.credential import Credential
from kallisticore.lib.expectation import Expectation

class PrometheusAction(Action):
    def __init__(self, module_func: Callable, arguments: Dict,
                 expectations: [Expectation] = None,
                 name: str = None, credential: Credential = None):
        super(PrometheusAction, self).__init__(
            module_func=module_func, arguments=arguments,
            expectations=expectations, name=name, credential=credential)

        self.arguments['configuration'] = {
            'prometheus_base_url': arguments.pop('base_url')}

__action_class__ = PrometheusAction

__actions_modules__ = ['chaosprometheus.probes']


from bs4 import BeautifulSoup
import time

class Seguidores():
    def __init__(self, dados, driver):
        self.dados = dados
        self.driver = driver

    def get_dados(self):
        self.scroll_final_page()
        self.salva_dados()

    def scroll_final_page(self):
        print('Realizando scroll da pagina.')
        SCROLL_PAUSE_TEMPO = 1.5
        ultima_altura_pagina = self.driver.browser.execute_script(
            "return document.body.scrollHeight")

        while True:
            self.driver.browser.execute_script(
                "window.scrollTo(0, document.body.scrollHeight);")
            time.sleep(SCROLL_PAUSE_TEMPO)
            nova_altura_pagina = self.driver.browser.execute_script(
                "return document.body.scrollHeight")
            if nova_altura_pagina == ultima_altura_pagina:
                break
            ultima_altura_pagina = nova_altura_pagina

    def get_html(self):
        print("Gerando HTML")
        time.sleep(2)
        element = self.driver.busca_css(self.dados["css_box_dados"])
        html_content = element.get_attribute('outerHTML')
        diretorio = self.dados["diretorio_html"]
        arquivo_html = open(diretorio, "w")
        arquivo_html.write(html_content)
        print('Pagina HTML salva com sucesso.')
        return html_content

    def salva_dados(self):
        html_content = self.get_html()
        bs = BeautifulSoup(html_content, 'html.parser')
        linhas = bs.findAll(
            'span', {'class': 'entity-result__title-text t-16'})

        lista = []
        diretorio = self.dados["diretorio_txt"]
        arquito_txt = open(diretorio, "w")

        for item in linhas:
            link = item.findChildren("a")
            url = link[0]['href']
            lista.append(url)
            arquito_txt.write(f'{url}\n')
        print('Dados salvos com sucesso.')


"""Implementations of different loss functions or loss utilities in PyTorch."""

import torch

def reinforce(loss, logits, baseline):
    """Return the simple REINFORCE loss with a baseline.

    NOTE: This does not add the loss tensor to the returned loss so make sure
          to add it separately!

    Arguments
    ---------
        loss: The tensor that contains the per-sample loss
        logits: The tensor that contains the per-sample logits
        baseline: The tensor that contains the baseline loss used to reduce the
                  variance of the gradient estimator
    """
    B = loss.shape[0]
    assert loss.shape == (B,) or loss.shape == (B, 1)
    assert baseline.numel() == 1
    assert logits.shape[0] == B

    return (
        (loss.view(-1, 1) - baseline.view(-1)).detach() * logits.view(B, -1)
    ).mean(1)

class Reinforce(torch.nn.Module):
    """Compute the simple REINFORCE loss with a exponential moving average
    baseline.

    Arguments
    ---------
        alpha: float, the parameter for the exponential moving
               average (default: 0.9)
    """
    def __init__(self, alpha=0.9):
        super().__init__()

        self.alpha = alpha
        self.baseline = None

    def forward(self, loss, logits):
        if self.baseline is None:
            self.baseline = torch.zeros(1, device=loss.device)
            with torch.no_grad():
                self.baseline[...] = loss.mean()

        rloss = reinforce(loss, logits, self.baseline)
        with torch.no_grad():
            self.baseline[...] = (
                self.alpha * self.baseline +
                (1-self.alpha)*loss.mean()
            )

        return rloss


"""Metrics base."""
import abc
from dataclasses import dataclass
from typing import Tuple

import tensorflow as tf

class PerformanceMetrics(tuple):
    """A set of performance metrics.

    NOTE:
    As python versions older than 3.9 does not support using
    tuple as a generic type hint, we still need to use typing.Tuple.
    Then the following does not work as an initializer:
        PerformanceMetrics = Tuple[tf.keras.metrics.Metric, ...]
    Therefore, this class is a wrapper around tuple, working both
    as a type hint and a type, until python version is updated.
    """

    def __new__(cls, *args: tf.keras.metrics.Metric):
        """Construct an instance."""
        assert all(isinstance(arg, tf.keras.metrics.Metric) for arg in args)
        return args

@dataclass(frozen=True)
class ObjectiveTerm:
    """A loss term in the loss formulation."""

    loss_func: tf.keras.losses.Loss
    loss_metric: tf.keras.metrics.Mean
    weight: float

    @classmethod
    def build(cls, name: str, loss_func: tf.keras.losses.Loss, weight: float = 1.0):
        """Create an objective term."""
        return cls(
            loss_func=loss_func, loss_metric=tf.keras.metrics.Mean(name), weight=weight
        )

@dataclass(frozen=True)
class ObjectiveFormulation:
    """A neural network loss formulation."""

    terms: Tuple[ObjectiveTerm]
    cumulative_loss_metric: tf.keras.metrics.Mean

    def calculate(self, labels, predictions):
        """Calculate the loss value given labels and predictions."""
        losses = []
        for objective_term in self.terms:
            loss = objective_term.loss_func(labels, predictions)
            objective_term.loss_metric(loss)
            losses.append(loss)

        cumulative_loss = tf.reduce_sum(
            tf.math.multiply(
                losses,
                [term.weight for term in self.terms],
            )
        )
        self.cumulative_loss_metric(cumulative_loss)
        return cumulative_loss

class Objective(abc.ABC):
    """Interface class for losses and performance metrics."""

    def __init__(self):
        """Create an objective."""
        self._objective_formulation = self._build_objective_formulation()
        self._performance_metrics = self._build_performance_metrics()

    def __call__(self, labels, predictions):
        """Calculate the cumulative loss value of labels and predictions."""
        cumulative_loss = self._objective_formulation.calculate(labels, predictions)

        for metric in self._performance_metrics:
            metric(labels, predictions)

        return cumulative_loss

    @property
    def formulation(self) -> ObjectiveFormulation:
        """Get the loss formulation."""
        return self._objective_formulation

    @property
    def performance_metrics(self) -> PerformanceMetrics:
        """Get the performance metric."""
        return self._performance_metrics

    @abc.abstractmethod
    def _build_objective_formulation(self) -> ObjectiveFormulation:
        pass

    @abc.abstractmethod
    def _build_performance_metrics(self) -> PerformanceMetrics:
        pass


import os
from xmind import XMindDocument
from xmind.document import SHAPE_RECTANGLE, SHAPE_ROUND_RECTANGLE, SHAPE_ELLIPSIS
from requirements import Attribute

import config

class AttributeNotRenderable(Exception):
    pass

class AttributeRenderer:
    markers = config.getAttributeMarkers()

    def __init__(self, attr):
        if not attr.isValid():
            raise AttributeNotRenderable("Attribute '%s' with value '%s' is not valid" % (attr.getName(), attr.getValue()))

        self.attribute = attr

    def isValid(self):
        return (self.attribute.getName() in AttributeRenderer.markers.keys()) and (self.attribute.getOrder() <= len(AttributeRenderer.markers[self.attribute.getName()]) )

    def render(self):
        order = self.attribute.getOrder()

        return AttributeRenderer.markers[self.attribute.getName()][order]

class Renderer:
    def __init__(self, chapters, requirements, doc=None, renderOrphans=True, renderFolded=True, maxDepth=999, verbose=False):
        self.requirements = requirements
        self.verbose = verbose
        self.chapters = chapters
        self.maxDepth = maxDepth
        self.xmindDoc = doc
        if self.xmindDoc is None:
            self.xmindDoc = XMindDocument.create(u"ReqTrees", u"text")

        self.levelsProgression = ['system', 'user', 'business']

        self._renderOrphans = renderOrphans
        self.renderFolded = renderFolded

        self.topicStyle = {}
        if doc is None:
            self.topicStyle['unknown'] = Renderer.setupStyle(self.xmindDoc, "unknown")
            self.topicStyle['business'] = Renderer.setupStyle(self.xmindDoc, "business")
            self.topicStyle['user'] = Renderer.setupStyle(self.xmindDoc, "user")
            self.topicStyle['system'] = Renderer.setupStyle(self.xmindDoc, "system")
        else:
            self.topicStyle = self.xmindDoc.get_styles()

        self.issues = []

    @staticmethod
    def setupStyle(xmindDoc, what):
        style = config.getMapStyle(what)
        return xmindDoc.create_topic_style( style['fill'], styleid=what, shape=style['shape'], line_color=style['lineColor'], line_width=style['lineWidth'])

    def setStyle(self, requirement, topic):
        style = 'unknown'
        if requirement.getCategory() in self.topicStyle.keys():
            style = requirement.getCategory()
        else:
            print("WARNING: Unknown category: %s" % requirement.getCategory())

        topic.set_style( self.topicStyle[ style ] )

    def setupSheet(self, rootSheet, rootTopic):
        pass

    def render(self, filename, sheet=None):
        rootSheet = sheet
        if rootSheet is None:
            rootSheet = self.xmindDoc.get_first_sheet()

        rootTopic = rootSheet.get_root_topic()

        cat = self.setupSheet(rootSheet, rootTopic)
        self.renderContent(rootSheet, cat )

        if self._renderOrphans:
            self.renderOrphans(rootTopic)
            self.renderNoLinks(rootTopic)

        if sheet is None:
            self.xmindDoc.save(filename)

    def renderContent(self, rootSheet, category ):
        self.attributeIssues = {}
        rootTopic = rootSheet.get_root_topic()

        for chapter in self.chapters:
            if chapter['category'] == category and len(chapter['reqs']) > 0:
                newTopic = rootTopic.add_subtopic(chapter['name'], folded=True)
                newTopic.set_style(self.topicStyle[category])

                children = self.sortLinks(chapter['reqs'])
                for reqID in children:
                    self.renderTopic(None, newTopic, reqID )

    def renderOrphans(self, rootTopic):
        for cat in self.levelsProgression[:-1]: # skip the business requirements
            orphans = []
            for key, req in self.requirements.items():
                if req.getCategory() == cat:
                    found = False
                    for key2, req2 in self.requirements.items():
                        for link in self.getNextLevel(req2):
                            if link == req.getID():
                                found = True
                                break

                        if found:
                            break

                    if not found and (key not in orphans):
                        orphans.append(key)

            if len(orphans) > 0:
                topic = rootTopic.add_subtopic("Orphaned %s Requirements" % cat )
                for key in orphans:
                    req = self.requirements[key]
                    self.createTopic(topic, req)

    def renderNoLinks(self, rootTopic):
        for cat in self.levelsProgression[1:]:   # skip the system requirements
            orphans = []
            for key, req in self.requirements.items():
                if req.getCategory() == cat and len(req.getLinks()) == 0:
                    orphans.append(key)

            if len(orphans) > 0:
                topic = rootTopic.add_subtopic("%s Requirements without links" % cat )
                for key in orphans:
                    req = self.requirements[key]
                    self.createTopic(topic, req)

    def createTopic(self, root, requirement):
        newTopic = root.add_subtopic(requirement.getText(), folded=self.renderFolded)
        self.setStyle(requirement, newTopic)

        note = ("%s\n%s" % (requirement.getID(), requirement.getFullText()))
        newTopic.set_note(note)

        for a in requirement.getAttributes():
            renderer = AttributeRenderer(a)
            if renderer.isValid():
                newTopic.add_marker(renderer.render())
            else:
                if a.getValue() is not None and len(a.getValue()) > 1 and requirement.getID() not in self.attributeIssues.keys():
                    msg = "WARNING: unknown attribute value ('%s') for '%s' in requirement %s" % (a.getValue(), a.getName(), requirement.getID())
                    self.attributeIssues[requirement.getID()] = msg
                    if self.verbose:
                        print(msg)

        return newTopic

    # sort the links attributed to a requirement.
    # use the 3 attributes priority, risk and difficulty
    def sortLinks(self, reqList):
        reqs = []
        for l in reqList:
          r = self.getLinkedRequirement(l)
          if r is not None:
              reqs.append(r)

        def reqOrder(req):
            order = []
            attrs = req.getAttributes()
            for attributeOrder in ['Priority', 'Risk', 'Difficulty']:
                for attribute in attrs:
                    if attribute.getName() == attributeOrder:
                       order.append( attribute.getOrder() )
                       break

            return order

        reqs.sort(key=reqOrder)
        return [x.getID() for x in reqs]

    def renderTopic(self, root, rootTopic, link ):
        linked = self.getLinkedRequirement(link)
        valid = (linked is not None )  # do we have valid links (consdiering the asymmetry between business and system)?

        if valid:
            newTopic = self.createTopic(rootTopic, linked)
            children = self.sortLinks(self.getNextLevel(linked))
            for l in children:
                self.renderTopic(linked, newTopic, l)

    def getNextLevel(self, requirement):
        pass

    def getLinkedRequirement(self, link):
        req = None
        if link in self.requirements.keys():
            req = self.requirements[link]

        return req

class BottomUpRenderer(Renderer):
    def setupSheet(self, rootSheet, rootTopic):
        rootSheet.set_title(u"BottomUp" )
        rootTopic.set_title(u"BottomUp Requirements tree" )

        return self.levelsProgression[0]

    def getNextLevel(self, requirement):
        links = []
        for i in range(0, len(self.levelsProgression)):
            if requirement.getCategory() == self.levelsProgression[i] and i< (len(self.levelsProgression) - 1):
                category = self.levelsProgression[i+1]
                for reqID, req in self.requirements.items():
                    if req.getCategory() == category and requirement.getID() in req.getLinks():
                        links.append(reqID)

        return links

class TopDownRenderer(Renderer):
    def setupSheet(self, rootSheet, rootTopic):
        rootSheet.set_title(u"TopDown" )
        rootTopic.set_title(u"TopDown Requirements tree" )

        return self.levelsProgression[-1]

    def getNextLevel(self, requirement):
        return requirement.getLinks()

class UnifiedRenderer:
    def __init__(self, chapters, requirements, renderFolded=True, maxDepth=999, verbose=False):
        self.xmindDoc = XMindDocument.create(u"ReqTrees", u"text")
        self.topicStyle = {}

        self.topicStyle['unknown'] = Renderer.setupStyle(self.xmindDoc, "unknown")
        self.topicStyle['business'] = Renderer.setupStyle(self.xmindDoc, "business")
        self.topicStyle['user'] = Renderer.setupStyle(self.xmindDoc, "user")
        self.topicStyle['system'] = Renderer.setupStyle(self.xmindDoc, "system")

        self.topdown = TopDownRenderer(chapters, requirements, doc=self.xmindDoc, renderOrphans=False, renderFolded=True, maxDepth=maxDepth, verbose=verbose)
        self.bottomup = BottomUpRenderer(chapters, requirements, doc=self.xmindDoc, renderOrphans=False, renderFolded=True, maxDepth=maxDepth, verbose=verbose)

    def render(self, filename ):

        # topdown is the first sheet
        sheet = self.xmindDoc.get_first_sheet()
        self.topdown.render(None, sheet=sheet )

        # bottomup is the first sheet
        newsheet = self.xmindDoc.create_sheet(u"ReqTrees", u"text")
        self.bottomup.render(None, sheet=newsheet)

        # now render orphans
        newsheet = self.xmindDoc.create_sheet(u"Issues", u"Issues")
        root = newsheet.get_root_topic()
        self.topdown.renderOrphans(root)
        self.topdown.renderNoLinks(root)

        # add a legend. the XMind legend creation does not seem to work properly anymore, so we create a dedicated sheet with all info
        newsheet = self.xmindDoc.create_sheet(u"Conventions", u"Conventions")
        root = newsheet.get_root_topic()
        self.renderLegend(root)

        # save our precious work!
        self.xmindDoc.save(filename)

    def renderLegend(self, root):
        # render topic styles legend
        rootTypes = root.add_subtopic("Requirement types", folded=False)

        for name, style in self.topicStyle.items():
            if name != "unknown":
                t = rootTypes.add_subtopic("%s Requirements" % name)
                t.set_style(style)

        # render markers legend
        rootMarkers = root.add_subtopic("Requirement attributes", folded=False)

        attributeMappings = config.getAttributeMappings()
        attributeTypes = attributeMappings.keys()

        for markerType in attributeTypes:
            generic = Attribute.getDescription(markerType)
            rootType = rootMarkers.add_subtopic(markerType, folded=True)
            rootType.set_note(generic)

            #levels = AttributeRenderer.getAttributeMapping(markerType)
            #levels = config.getAttributeMapping(markerType)
            for levelname, levelvalue in attributeMappings[markerType].items():
                descr = "indicates %s" % levelname
                m = rootType.add_subtopic(descr)
                m.add_marker(levelvalue)

        # render orphans + nolinks
        rootIssues = root.add_subtopic("Requirements Issues", folded=False)
        orphans = rootIssues.add_subtopic("Orphans: requirements that are not linked by any other requirement")
        orphans.set_style(self.topicStyle['unknown'])
        orphans.set_note("If a user requirement is not linked by any business requirement, this implies that this user requirement has no foundation in any business need and thus should not be implemented. If a system requirement is not linked by any user requirement, this implies that this system requirement is not related to any user feature, and thus may not be needed at all")

        nolinks = rootIssues.add_subtopic("NoLinks: requirements that do not link to any other requirement")
        nolinks.set_style(self.topicStyle['unknown'])
        nolinks.set_note("If a business requirement is not linked to any other requirement, the conclusion is that the requirement is NOT going to be implemented. In other words, any business requirement that is not linked to other (user) requirement results in a need that will not be addressed by the project. This situation should be avoided as much as possible: each business requirement is expected to have at least one link.\n If a user requirement has no link to any system requirement, this implies that, while there is a business need that was converted into a user requirement, this will not be implemented. This situation should be avoided as much as possible: each user requirement is expected to be linked to at least one system requirement.")

        # render traceability
        rootTrace = root.add_subtopic("Requirements traceability", folded=False)
        orphans = rootTrace.add_subtopic("TopDown: from business requirements down to system requirements traceability")
        orphans.set_style(self.topicStyle['unknown'])

        nolinks = rootTrace.add_subtopic("BottmUp: from system requirements up to business requirements traceability")
        nolinks.set_style(self.topicStyle['unknown'])


import copy
import unittest2

from savanna.conductor import resource as r
from savanna.utils import types

SAMPLE_DICT = {
    'first': [1, 2],
    'second': {'a': 1, 'b': 2}
}

SAMPLE_NESTED_LISTS_DICT = {
    'a': [[{'b': 123}]]
}

SAMPLE_CLUSTER_DICT = {
    'name': 'test-cluster',
    'cluster_configs': {
        'general': {
            'some_overridden_config': 'somevalue'
        }
    },
    'node_groups': [
        {
            'name': 'master',
        },
        {
            'name': 'worker',
            'node_processes': ['tasktracker', 'datanode'],
            'node_configs': {},
            'instances': [
                {
                    'name': 'test-cluster-001',
                    'ip': '1.1.1.1'
                }
            ]
        }
    ]
}

class TestResource(unittest2.TestCase):
    def test_resource_creation(self):
        res = r.Resource(SAMPLE_DICT)

        self.assertIsInstance(res.first, list)
        self.assertEqual(res.first, [1, 2])
        self.assertIsInstance(res.second, r.Resource)
        self.assertEqual(res.second.a, 1)
        self.assertEqual(res.second.b, 2)

    def test_resource_immutability(self):
        res = r.Resource(SAMPLE_DICT)

        with self.assertRaises(types.FrozenClassError):
            res.first.append(123)

        with self.assertRaises(types.FrozenClassError):
            res.first = 123

        with self.assertRaises(types.FrozenClassError):
            res.second.a = 123

    def test_nested_lists(self):
        res = r.Resource(SAMPLE_NESTED_LISTS_DICT)
        self.assertEqual(res.a[0][0].b, 123)

    def test_cluster_resource(self):
        cluster = r.ClusterResource(SAMPLE_CLUSTER_DICT)

        self.assertEqual(cluster.name, 'test-cluster')

        self.assertEqual(cluster.node_groups[0].name, 'master')
        self.assertIsInstance(cluster.node_groups[0], r.NodeGroupResource)
        self.assertEqual(cluster.node_groups[0].cluster.name, 'test-cluster')

        self.assertEqual(cluster.node_groups[1].instances[0].name,
                         'test-cluster-001')
        self.assertIsInstance(cluster.node_groups[1].instances[0],
                              r.InstanceResource)
        self.assertEqual(
            cluster.node_groups[1].instances[0].node_group.name,
            'worker')

    def test_to_dict(self):
        cluster = r.ClusterResource(SAMPLE_CLUSTER_DICT)
        self.assertEqual(cluster.to_dict(), SAMPLE_CLUSTER_DICT)

    def test_to_dict_filtering(self):
        cluster_dict = copy.deepcopy(SAMPLE_CLUSTER_DICT)
        cluster_dict['private_key'] = 'abacaba'
        cluster_dict['node_groups'][0]['id'] = 'some_id'

        cluster = r.ClusterResource(cluster_dict)
        self.assertEqual(cluster.to_dict(), SAMPLE_CLUSTER_DICT)

    def test_to_wrapped_dict(self):
        cluster = r.ClusterResource(SAMPLE_CLUSTER_DICT)
        wrapped_dict = cluster.to_wrapped_dict()
        self.assertEqual(len(wrapped_dict), 1)
        self.assertEqual(wrapped_dict['cluster'], SAMPLE_CLUSTER_DICT)

    def test_init_doesnt_mangle_input(self):
        initial_dict = copy.deepcopy(SAMPLE_CLUSTER_DICT)
        r.ClusterResource(SAMPLE_CLUSTER_DICT)
        self.assertEqual(initial_dict, SAMPLE_CLUSTER_DICT)

    def test_re_init_doesnt_mangle_input(self):
        res = r.ClusterResource(SAMPLE_CLUSTER_DICT)

        changed_dict = copy.deepcopy(SAMPLE_CLUSTER_DICT)
        changed_dict['name'] = 'abcdcwdw'
        changed_dict_copy = copy.deepcopy(changed_dict)

        res.re_init(changed_dict)

        self.assertEqual(changed_dict, changed_dict_copy)

    def test_re_init(self):
        cluster = r.ClusterResource(SAMPLE_CLUSTER_DICT)

        cluster.cluster_configs

        cluster_dict = copy.deepcopy(SAMPLE_CLUSTER_DICT)
        cluster_dict['name'] = 'somenewname'
        del cluster_dict['cluster_configs']

        cluster.re_init(cluster_dict)

        self.assertEqual('somenewname', cluster.name)
        with self.assertRaises(KeyError):
            cluster.cluster_configs
        self.assertEqual(cluster_dict, cluster.to_dict())

    def test_re_init_nested(self):
        cluster = r.ClusterResource(SAMPLE_CLUSTER_DICT)
        ng1_new_dict = copy.deepcopy(SAMPLE_CLUSTER_DICT['node_groups'][1])
        ng1_new_dict['instances'].append({
            'name': 'test_cluster_002',
            'ip': '2.2.2.2'})
        ng1_new_dict['name'] = 'new-ng-name'

        ng1 = cluster.node_groups[1]
        ng1.re_init(ng1_new_dict)

        self.assertEqual(ng1.cluster.name, cluster.name)
        self.assertEqual(cluster.node_groups[1].name, ng1.name)
        self.assertEqual(ng1.to_dict(), ng1_new_dict)


import tkinter as tk
from tkinter import messagebox

import subprocess
import threading

import macro
import utils

DATA = 0
MACRO = 1

modeNames = [
	"Data",
	"Macro"
]

class Tab:
	def __init__(self, gui, idx) :
		self.main = gui
		self.name = "Tab {0}".format(idx+1)
		self.mode = DATA

		self.frame = [
			self.init_frame(self.main.master, DATA),
			self.init_frame(self.main.master, MACRO)
		]

		self.data = bytearray([])
		self.filter = ""
		self.flt_output = None
		self.is_filtered = False
		self.binding = None
		self.macro = ""
		self.macro_thread = None

	def init_frame(self, root, mode) :
		head_label = ""
		recv_label = ""
		if mode == DATA:
			head_label = "  Filter  "
			recv_label = "Receiving"
		elif mode == MACRO:
			head_label = "Binding"
			recv_label = "Python Code"

		frame = tk.Frame(root)
		frame.grid_rowconfigure(3, weight=1)
		frame.grid_columnconfigure(2, weight=1)

		if mode == DATA:
			self.flt_switch = tk.IntVar();
			self.flt_switch.trace("w", self.toggle_filter)

			self.filter_cb = tk.Checkbutton(frame, text=head_label, variable=self.flt_switch)
			self.filter_cb.grid(row=0, column=0)

			self.filter_var = tk.StringVar()
			self.filter_var.trace("w", self.update_heading)

			self.filter_ent = tk.Entry(frame, width=200, textvariable=self.filter_var, name="filter")
			self.filter_ent.bind("<Return>", (lambda event: self.update()))
			self.filter_ent.grid(row=0, column=2, columnspan=5)

		frame.recv_lbl = tk.Label(frame, text=recv_label)
		frame.recv_lbl.grid(row=2, column=0, columnspan=7, sticky="w")

		frame.recv = tk.Text(frame, width=200, height=100)
		frame.recv.bind("<Control-Key>", self.main.key_down)
		frame.recv.bind("<FocusOut>", lambda event: self.update_model())
		frame.recv.grid(row=3, column=0, columnspan=7, sticky="w")

		if mode == DATA:
			self.clear_btn = tk.Button(frame, text="Clear", command=self.clear_data)
			self.clear_btn.grid(row=4, column=5, columnspan=2)
		elif mode == MACRO:
			self.binding_btn = tk.Button(frame, text="Binding", command=self.prompt_binding)
			self.action_btn = tk.Button(frame, text="Execute", command=self.run_macro)
			self.cancel_btn = tk.Button(frame, text="Cancel", state="disabled", command=self.cancel_macro)
			self.binding_btn.grid(row=4, column=0)
			self.action_btn.grid(row=4, column=4)
			self.cancel_btn.grid(row=4, column=5, columnspan=2)

		return frame

	def mode_name(self) :
		if self.mode >= DATA and self.mode <= MACRO:
			return modeNames[self.mode]

		# shouldn't get here
		return self.mode

	def append_byte(self, byte) :
		self.data.append(byte)
		if self.macro_running():
			self.macro_thread.queue.put(byte)

		if self.is_filtered and len(self.filter) == 0:
			text = self.frame[DATA].recv
			text.config(state="normal")
			text.insert("end", bytes([byte]).decode('cp437'))
			text.config(state="disabled")
		elif not self.is_filtered:
			self.frame[DATA].recv.insert("end", "{0:02x} ".format(byte))

	def update_model(self) :
		if self.is_filtered and self.mode == DATA:
			return

		content = self.frame[self.mode].recv.get("1.0", "end")
		if self.mode == DATA:
			self.data = utils.str2ba(content)
		elif self.mode == MACRO:
			if content[-1] == '\n':
				content = content[0:-1]

			self.macro = content

	def reply(self) :
		self.update_model()
		self.main.comms.send(self.data)

	def clear_data(self) :
		self.frame[DATA].recv.delete("1.0", "end")
		self.data = bytearray([])

	def set_data(self, data) :
		if not (isinstance(data, bytes) or isinstance(data, bytearray)) :
			return

		if not self.is_filtered:
			self.frame[DATA].recv.delete("1.0", "end")
			try:
				self.frame[DATA].recv.insert("end", utils.ba2str(self.data))
			except e:
				messagebox.showerror("Error", e)

		self.data = bytearray(data)

	def overwrite_data(self) :
		if self.mode != DATA:
			return

		if self.flt_output != None and len(self.flt_output) > 0:
			self.set_data(self.flt_output)

	def prompt_binding(self) :
		if self.main.binding_dlg is not None:
			return

		self.main.binding_dlg = macro.BindingDialog(self)

	def macro_running(self) :
		return self.macro_thread is not None and self.macro_thread.is_alive()

	def run_macro(self) :
		self.update_model()

		if self.macro_running() :
			messagebox.showinfo("Info", "Macro is already running")
			return

		if self.macro is not None and len(self.macro) > 0:
			self.macro_thread = macro.Macro(self.main, self)
			self.macro_thread.start()

	def cancel_macro(self) :
		if self.macro_running() :
			self.macro_thread.kill()

	def init_filter(self, flt, enabled) :
		value = 1 if enabled else 0
		self.filter_var.set(flt)
		self.flt_switch.set(value)

	def toggle_filter(self, *args) :
		self.update_model()
		self.is_filtered = self.flt_switch.get() != 0
		self.update()

	def update_heading(self, *args) :
		self.filter = self.filter_var.get()

	def run_filter(self) :
		text = ""
		cmd = self.filter
		# if a filter was given, use it as a shell command and provide our data to stdin
		if len(cmd) > 0:
			try:
				proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
				proc.stdin.write(self.data)
				proc.stdin.close()

				# To allow for overwriting our data later
				self.flt_output = proc.stdout.read()

				text = self.flt_output.decode('cp437')
				text += "\n" + proc.stderr.read().decode('cp437')

			except (FileNotFoundError, OSError) as e:
				text = "Filter Error: {0}".format(e)
		# otherwise treat our data as straight-up ASCII
		else:
			text = self.data.decode('cp437')

		return text

	def update(self, mode=None) :
		if mode is None:
			mode = self.mode

		frame = self.frame[mode]
		frame.recv.config(state="normal")
		frame.recv.delete("1.0", "end")

		text = ""
		editing = "normal"
		if mode == DATA:
			if self.is_filtered:
				self.filter_ent.grid()
				text = self.run_filter()
				editing = "disabled"
			else:
				self.filter_ent.grid_remove()
				text = utils.ba2str(self.data)

		elif mode == MACRO:
			text = self.macro

		frame.recv.insert("end", text)
		frame.recv.config(state=editing)


# ---
# jupyter:
#   jupytext:
#     formats: ipynb,py
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.3.3
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
import pickle
import random
import os
import torch
from tqdm import tqdm

import numpy as np
import torch.functional as F

from torch import nn
from torch.utils.data import Dataset, DataLoader
from torchvision import models

from hw_grapheme.dl_utils.radam import RAdam

# +
sz = 128
bs = 64
nfolds = 4 #keep the same split as the initial dataset
fold = 0
SEED = 2019
TRAIN = '../input/grapheme-imgs-128x128/'
LABELS = '../input/bengaliai-cv19/train.csv'

def seed_everything(seed):
    random.seed(seed)
    os.environ['PYTHONHASHSEED'] = str(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    torch.backends.cudnn.deterministic = True
    torch.backends.cudnn.benchmark = True

seed_everything(SEED)

# -

class Loss_combine(nn.Module):
    def __init__(self):
        super().__init__()

    def forward(self, input, target,reduction='mean'):
        x1,x2,x3 = input
        x1,x2,x3 = x1.float(),x2.float(),x3.float()
        y = target.long()
        return 0.7*F.cross_entropy(x1,y[:,0],reduction=reduction) + 0.1*F.cross_entropy(x2,y[:,1],reduction=reduction) + \
          0.2*F.cross_entropy(x3,y[:,2],reduction=reduction)

class GraphemeDataset(Dataset):
    def __init__(self, data_list, _type='train'):
        self.data_list = data_list

    def __len__(self):
        return len(self.data_list)

    def __getitem__(self, idx):
        data = self.data_list[idx]
        # name = data["name"]
        image = data["image"]
        grapheme_root = data["grapheme_root"]
        vowel_diacritic = data["vowel_diacritic"]
        consonant_diacritic = data["consonant_diacritic"]
        return image, vowel_diacritic, grapheme_root, consonant_diacritic

class ResNet50(nn.Module):
    def __init__(self):
        super(ResNet50,self).__init__()

        arch = models.resnet50(num_classes=1000, pretrained=True)
        arch = list(arch.children())
        w = arch[0].weight
        arch[0] = nn.Conv2d(1, 64, kernel_size=7, stride=2, padding=2, bias=False)
        arch[0].weight = nn.Parameter(torch.mean(w, dim=1, keepdim=True))
        self.backbone = nn.Sequential(*arch[:-1])

        # vowel_diacritic
        self.fc1 = nn.Linear(2048,11)
        # grapheme_root
        self.fc2 = nn.Linear(2048,168)
        # consonant_diacritic
        self.fc3 = nn.Linear(2048,7)

    def forward(self, x):
        x = self.backbone(x)
        x = x.view(-1, 2048)
        # print("x: ", x.shape)
        x1 = self.fc1(x)
        x2 = self.fc2(x)
        x3 = self.fc3(x)

        return x1,x2,x3

# +
batch_size = 32

with open("../data/processed_data/train_data_0.pickle", "rb") as f:
    data_list = pickle.load(f)

train_dataset = GraphemeDataset(data_list)
train_loader = torch.utils.data.DataLoader(train_dataset,batch_size=batch_size,shuffle=True)
# -

len(train_dataset)

iter(train_loader).next()

# +
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

model = ResNet50()
model = nn.DataParallel(model)
model.to(device);

# -

optimizer = RAdam(model.parameters(), lr=1e-3)
#scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer, base_lr=1e-4, max_lr=0.05)
criterion = nn.CrossEntropyLoss()

inputs, vowel, root, consonant = iter(train_loader).next()

inputs.shape

# +
epochs = 50 # original 50
model.train()
losses = []
accs = []
for epoch in range(epochs):
    running_loss = 0.0
    running_acc_vowel = 0.0
    running_acc_root = 0.0
    running_acc_consonant = 0.0

    for idx, (inputs, vowel, root, consonant) in tqdm(enumerate(train_loader),total=len(train_loader)):
        inputs = inputs.to(device)
        vowel = vowel.to(device)
        root = root.to(device)
        consonant = consonant.to(device)

        optimizer.zero_grad()
        outputs1, outputs2, outputs3 = model(inputs.unsqueeze(1).float())
        # print("outputs1: ", outputs1)
#         print("inputs: ", inputs.shape)
#         print("outputs1.shape: ", outputs1.shape)
#         print("vowel: ", vowel.shape)
        loss1 = criterion(outputs1, vowel.long())
        loss2 = criterion(outputs2, root.long())
        loss3 = criterion(outputs3, consonant.long())

        mini_batch_loss = loss1 + loss2 + loss3
        running_loss += mini_batch_loss * inputs.size(0)
        running_acc_vowel += (outputs1.argmax(1) == vowel.long()).sum()
        running_acc_root += (outputs2.argmax(1) == root.long()).sum()
        running_acc_consonant += (outputs3.argmax(1) == consonant.long()).sum()

        mini_batch_loss.backward()

        optimizer.step()
    #scheduler.step()
    losses.append(running_loss/len(train_dataset))
    print('running_acc_vowel : {:.2f}%'.format(running_acc_vowel.float()/len(train_dataset)))
    print('running_acc_root : {:.2f}%'.format(running_acc_root.float()/len(train_dataset)))
    print('running_acc_consonant : {:.2f}%'.format(running_acc_consonant.float()/len(train_dataset)))

    print('loss : {:.4f}'.format(running_loss/len(train_loader)))

torch.save(model.state_dict(), 'resnet34_50epochs_saved_weights.pth')
# -

model


import asyncio
import collections
import copy
import json
import random
import secrets

import socketio
from sanic import Sanic, response
from sanic.exceptions import abort

from diagnosis import Diagnosis
from nearest import getNearest
from sentiment import asyncAnalyze

app = Sanic(__name__)
sio = socketio.AsyncServer(async_mode="sanic", cors_allowed_origins=[])
sio.attach(app)

sessionSidStorage = {}

with open("./badwords.json", "r", encoding="utf-8") as fp:
    badwordsFilter = json.load(fp)

@app.middleware("response")
async def allowCors(_, response):
    response.headers["Access-Control-Allow-Origin"] = "*"
    response.headers["Access-Control-Allow-Methods"] = [
        "GET, POST, OPTIONS, PUT, PATCH, DELETE"
    ]

@app.options("/nearest")
async def __(request):
    return response.empty()

@app.post("/nearest")
async def nearestAPI(request):
    return response.json(
        getNearest(
            tuple(map(float, request.form["Pos"].pop().split(","))),
            request.form["type"].pop(),
        )
    )

def getQuestion(session):
    if not session.get("Sections"):
        session["Sections"] = [
            (copy.deepcopy(Diagnosis.Negative), "Negative"),
            (copy.deepcopy(Diagnosis.Positive), "Positive"),
            (copy.deepcopy(Diagnosis.Useless), "Useless"),
        ]

    if not session.get("Results"):
        session["Results"] = collections.defaultdict(list)

    if not session.get("Count"):
        session["Count"] = 0

    while True:
        Sections = list(filter(lambda x: bool(x[0]), session["Sections"]))

        if not Sections:
            break

        Section, SectionName = random.choice(Sections)
        random.shuffle(Section)

        if not Section:
            continue

        session["current"] = (Section.pop(), SectionName)
        return True

    return False

@sio.on("connect", namespace="/chatBot")
async def connect(sid, _):
    await sio.emit(
        "botMessage",
        {
            "content": f"""{random.choice(Diagnosis.Introduction)}

마음의 숲은 문맥속에 담긴 감정으로 우울증을 진단하는 서비스입니다.

정확한 분석을 위해서 예/아니오와 같은 단답보다는 글로 답변해주세요.""",
            "waitForNext": False,
        },
        to=sid,
        namespace="/chatBot",
    )

    async with sio.session(sid, namespace="/chatBot") as session:
        if getQuestion(session):
            await sio.emit(
                "botMessage",
                {"content": session["current"][0], "waitForNext": True},
                to=sid,
                namespace="/chatBot",
            )

@sio.on("clientMessage", namespace="/chatBot")
async def clientMessage(sid, data):
    async with sio.session(sid, namespace="/chatBot") as session:
        if any(filter(lambda x: x in data["content"].lower(), badwordsFilter)):
            return await sio.emit(
                "messageDenied",
                {"content": data["content"]},
                to=sid,
                namespace="/chatBot",
            )

        await sio.emit(
            "messageAccepted",
            {"content": data["content"], "isDelivered": True},
            to=sid,
            namespace="/chatBot",
        )

        Sentiment = await asyncAnalyze(data["content"], app.loop)

        if session.get("current"):
            session["Count"] += 1
            session["Results"][session["current"][1]].append(Sentiment)

        NewQuestion = getQuestion(session)

        if NewQuestion:
            return await sio.emit(
                "botMessage",
                {"content": session["current"][0], "waitForNext": True},
                to=sid,
                namespace="/chatBot",
            )

        Score = 0
        TotalScore = session["Count"] * 4

        for Result in session["Results"]["Positive"]:
            Operators = {
                "POS": lambda x: 4 - x,
                "NEUT": lambda _: 2,
                "NEG": lambda x: x,
            }

            Type, Value = max(Result.items(), key=lambda x: x[1])
            Value = Operators[Type](round(Value / 25))

            Score += Value

        for Result in session["Results"]["Negative"]:
            Operators = {
                "POS": lambda x: x,
                "NEUT": lambda _: 2,
                "NEG": lambda x: 4 - x,
            }

            Type, Value = max(Result.items(), key=lambda x: x[1])
            Value = Operators[Type](round(Value / 25))

            Score += Value

        return await sio.emit(
            "returnResult",
            {"score": Score, "totalScore": TotalScore},
            to=sid,
            namespace="/chatBot",
        )

Meetings = []

@app.options("/1")
async def __(request):
    return response.empty()

@app.post("/1")
async def _(request):
    print(request.form)
    Score, TotalScore = float(request.form["Score"].pop()), float(
        request.form["TotalScore"].pop()
    )

    Percentage = round(Score / TotalScore * 100)

    if Meetings:
        if Percentage >= 60:
            abort(403, message="Score is too high.")

        Matched = sorted(
            Meetings, keys=lambda x: abs(x[0] - Percentage), reverse=True
        ).pop()

        roomId = secrets.token_hex()

        Matched[1].put_nowait(roomId)
        return response.text(roomId)

    event = asyncio.Queue(maxsize=1)

    Meetings.append([Percentage, event])

    try:
        roomId = await asyncio.wait_for(
            event.get(), timeout=float(request.form.get("timeout", ["30.0"]))
        )
    except asyncio.TimeoutError:
        abort(408, message="Matching Timed out.")
    finally:
        Meetings.remove([Percentage, event])

    return response.text(roomId)

@sio.on("setRoom", namespace="/chat")
async def setRoom(sid, data):
    async with sio.session(sid, namespace="/chat") as session:
        session["roomId"] = data["roomId"]

        sio.enter_room(sid, session["roomId"], namespace="/chat")

        await sio.emit(
            "userJoined", room=session["roomId"], skip_sid=sid, namespace="/chat"
        )

@sio.on("sendMessage", namespace="/chat")
async def sendMessage(sid, data):
    async with sio.session(sid, namespace="/chat") as session:
        roomId = session.get("roomId")

        if not roomId or any(
            filter(lambda x: x in data["content"].lower(), badwordsFilter)
        ):
            return await sio.emit(
                "messageDenied",
                {"content": data["content"]},
                to=sid,
                namespace="/chat",
            )

        await sio.emit(
            "messageAccepted",
            {"content": data["content"], "isDelivered": True},
            to=sid,
            namespace="/chat",
        )

        await sio.emit(
            "userMessage",
            {"content": data["content"]},
            room=roomId,
            skip_sid=sid,
            namespace="/chat",
        )

@sio.on("disconnect", namespace="/chat")
async def disconnectRoom(sid):
    async with sio.session(sid, namespace="/chat") as session:
        roomId = session.get("roomId")

        if not roomId:
            return

        await sio.emit("userLeft", room=roomId, skip_sid=sid, namespace="/chat")

if __name__ == "__main__":
    app.run(host="0.0.0.0", port=5000)


from lxml import etree

html = etree.parse('./s20.html')

rsp = html.xpath("/bookstore/book")

print(rsp)

rst = html.xpath("//book[@category='sport']/year ")
#因为上面解析的rst是一个list，下面这一步将数据从list里面拿出来，重新放到rst里面
rst = rst[0]
print(rst.tag)
print(rst.text)


import torch
from transformers import pipeline, Pipeline, AutoTokenizer, AutoModelForSeq2SeqLM
from tqdm import tqdm
from srt import Subtitle

from engine.engine import Engine

class NllbEngine(Engine):
    translator: Pipeline

    def __init__(self, c) -> None:
        ec = c.engine.engine_configs
        tokenizer = AutoTokenizer.from_pretrained(ec["model"])
        model = AutoModelForSeq2SeqLM.from_pretrained(ec["model"])

        device = -1
        if (torch.cuda.is_available()):
            device = torch.cuda.current_device()

        self.translator = pipeline(
            "translation",
            device=device,
            model=model,
            tokenizer=tokenizer,
            src_lang=ec["srcLang"],
            tgt_lang=ec["tgtLang"],
            max_length=ec["maxLength"]
        )

    def Translate(self, query:str) -> str:
        return self.translator(query)[0]["translation_text"] # type: ignore

    def TranslateList(self, queries: list[Subtitle]) -> list[Subtitle]:
        st_idx = 0
        res = queries.copy()
        stream = (sub.content for sub in queries)

        for result in tqdm(self.translator(stream, batch_size=1), total=len(queries)):
            #if (type(result) is list):
            #    translated = result[0]["translation_text"]
            #    res[st_idx].content = translated
            #    st_idx += 1
            translated = result[0]["translation_text"] # type: ignore
            res[st_idx].content = translated
            st_idx += 1
        return res


import json
from django.views import View
from .models import User
from django.utils.decorators import method_decorator
from django.views.decorators.csrf import csrf_exempt
from django.http import JsonResponse
import json
import requests
# Create your views here.

def get_users():

        url = 'https://jsonplaceholder.typicode.com/users'
        response = requests.get(url)
        users = response.json()

        for i in users:
            User.objects.get_or_create(
                name = i['name'],
                user_name = i['username'],
                email = i['email'],
            )

class UserView(View):

    @method_decorator(csrf_exempt)
    def dispatch(self, request, *args, **kwargs):
        return super().dispatch(request, *args, **kwargs)

    def get(self, request, id=0):
        get_users()
        if id > 0:
            users = list(User.objects.filter(id=id).values())
            if len(users) > 0:
                user = users[0]
                datos={'message':"success", 'user': user}
            else:
                datos={'message':  'user not found'}
            return JsonResponse(datos)
        else:
            users = list(User.objects.values())
            if len(users) > 0:
                datos={'message':"success", 'users': users}
            else:
                datos={'message':  'users not found'}
            return JsonResponse(datos)

    def post(self, request):
        jd = json.loads(request.body)
        User.objects.create(name=jd['name'], user_name=jd['user_name'], email=jd['email'])
        datos={'message':"success"}
        return JsonResponse(datos)

    def put(self, request, id):
        jd = json.loads(request.body)
        users = list(User.objects.filter(id=id).values())
        if len(users) > 0:
            user = User.objects.get(id=id)
            user.name =jd["name"]
            user.user_name =jd["user_name"]
            user.email =jd["email"]
            user.save()
            datos={'message':"success"}
        else:
            datos={'message':  'user not found'}
        return JsonResponse(datos)

    def delete(self, request, id):
        users = list(User.objects.filter(id=id).values())
        if len(users) > 0:
            User.objects.filter(id=id).delete()
            datos={'message':"success"}
        else:
            datos={'message':  'user not found'}
        return JsonResponse(datos)

# def users(request):
#     #pull data from third party rest api
#     response = requests.get('https://jsonplaceholder.typicode.com/users')
#     #convert reponse data into json
#     users = response.json()
#     print(users)
#     return HttpResponse(response)


import requests
from lxml import html
import json
import time
import random
import hashlib
import sys

sys.path.insert(0, '/home/chaos/PycharmProjects/Crawler')
from page import user_agent

movie_index = 3878007
sortkey = "new_score"

def get_one_page(url, retry=0):
    while retry >= 0:
        headers = {
            'User-Agent':user_agent.UserAgents.get_agent()
        }

        response = requests.get(url, headers=headers)
        if response.status_code == 200:
            return response.text
        elif retry == 0:
            print("==> response code : " + str(response.status_code))
            return None

        time.sleep(3)
        retry -= 1

def parse_one_page(html):

    data = json.loads(html)['cmts']

    for item in data:
        yield{
            'comment': item['content'],
            'date': item['time'].split(' ')[0],
            'rate': item['score'],
            'city': item['cityName'],
            'nickname': item['nickName']
        }

def save_to_txt(filename):
    with open(filename, 'w+', encoding='utf-8') as f:
        uniq_list = []
        md5str = ''
        md5str.encode(encoding='utf-8')

        i = 0
        while i < 1001:
            i += 1
            url = 'http://m.maoyan.com/mmdb/comments/movie/1216365.json?_v_=yes&offset=' + str(i)
            html = get_one_page(url)
            print('saving %d page' % i)

            for item in parse_one_page(html):
                md5str = hashlib.md5(item['comment'].encode(encoding='utf-8')).hexdigest()

                if md5str not in uniq_list:
                    print(item['date'] + ',' + item['city'] + ',' + item['nickname'] + ',' + str(item['rate'])+','+item['comment'])
                    f.write(item['date'] + ',' + item['city'] + ',' + item['nickname'] + ',' + str(item['rate'])+','+item['comment']+'\n')
                    uniq_list.append(md5str)

            if len(uniq_list) > 10000:
                        uniq_list.clear()

            time.sleep(1 + float(random.randint(1, 100)) / 50)

def get_user_location(urls):
    locations = []
    for url in urls:
        page = get_one_page(url, 5)

        if page is None:
            locations.append("secrect")
            continue

        tree = html.fromstring(page)
        t = tree.xpath('//div[@class="user-info"]/a/text()')
        if len(t) > 0:
            locations += t
        else:
            locations.append("secrect")

        time.sleep(1 + float(random.randint(1, 50) / 50))
    return locations

def fetch_douban(filename):
    step = 20
    page_index = -1

    url_prefix = 'https://movie.douban.com/subject/' + str(movie_index) + '/comments?start='
    with open(filename, 'w+', encoding='utf-8') as f:
        while page_index < 1500:
            page_index += 1
            url = url_prefix + str(page_index*step) + '&limit= 20&sort='+ sortkey +'&status=P'
            url2 = 26985127
            page = get_one_page(url, 5)
            if page is None:
                continue
            tree = html.fromstring(page)

            names = tree.xpath('//div[@class="avatar"]/a/@title')
            comment_times = tree.xpath('//span[@class="comment-info"]/span[last()]/@title')
            locations = get_user_location(tree.xpath('//div[@class="avatar"]/a/@href'))
            comments = tree.xpath('//span[@class="short"]/text()')

            print("==> page " + str(page_index) + ", names: " + str(len(names)) + ", comment_times: " + str(len(comment_times)) + ", locations: " + str(len(locations)) + ", comments: " + str(len(comments)))
            for i in range(0, step - 1):
                print(comment_times[i] + ', ' + names[i] + ', ' + locations[i] + ', ' + comments[i] + '\n')
                f.write(comment_times[i] + ', ' + names[i] + ', ' + locations[i] + ', ' + comments[i] + '\n')

if __name__ == '__main__':
    movie_index = int(sys.argv[1])
    fetch_douban(sys.argv[2])


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from typing import List, Optional
from math import log2

from refinery.units.crypto.cipher import StandardBlockCipherUnit, Arg
from refinery.units.crypto.cipher.rc5 import rc5constants
from refinery.lib import chunks
from refinery.lib.crypto import (
    rotr,
    rotl,
    BlockCipher,
    BlockCipherFactory,
    CipherMode,
    CipherInterface,
    BufferType,
)

_W = 32
_R = 20

class RC6(BlockCipher):

    block_size: int
    key_size = range(256)
    _S: List[int]
    _w: int
    _r: int
    _u: int
    _m: int
    _g: int

    __slots__ = '_S', '_w', '_r', '_u', '_m', '_g'

    def __init__(self, key: BufferType, mode: Optional[CipherMode] = None, word_size: int = _W, rounds: int = _R):
        if word_size < 8 or word_size % 8:
            raise ValueError(F'Invalid word size: {word_size}')
        self._w = word_size
        self._u = word_size // 8
        self._r = rounds
        self._g = int(log2(word_size))
        self._m = (1 << word_size) - 1
        super().__init__(key, mode)

    @property
    def block_size(self):
        return self._u * 4

    def block_decrypt(self, block) -> BufferType:
        u = self._u
        w = self._w
        g = self._g
        r = self._r
        M = self._m
        S = self._S
        A, B, C, D = chunks.unpack(block, u)
        C = C - S[2 * r + 3] & M
        A = A - S[2 * r + 2] & M
        for i in range(r, 0, -1):
            A, B, C, D = D, A, B, C
            t = rotl(w, B * (2 * B + 1) & M, g)
            v = rotl(w, D * (2 * D + 1) & M, g)
            C = rotr(w, C - S[2 * i + 1] & M, t) ^ v
            A = rotr(w, A - S[2 * i + 0] & M, v) ^ t
        D = D - S[1] & M
        B = B - S[0] & M
        return chunks.pack((A, B, C, D), u)

    def block_encrypt(self, block) -> BufferType:
        u = self._u
        w = self._w
        g = self._g
        r = self._r
        M = self._m
        S = self._S
        A, B, C, D = chunks.unpack(block, u)
        B = B + S[0] & M
        D = D + S[1] & M
        for i in range(1, r + 1):
            t = rotl(w, B * (2 * B + 1) & M, g)
            v = rotl(w, D * (2 * D + 1) & M, g)
            A = rotl(w, A ^ t, v) + S[2 * i + 0] & M
            C = rotl(w, C ^ v, t) + S[2 * i + 1] & M
            A, B, C, D = B, C, D, A
        A = A + S[2 * r + 2] & M
        C = C + S[2 * r + 3] & M
        return chunks.pack((A, B, C, D), u)

    @property
    def key(self):
        return self._S

    @key.setter
    def key(self, key):
        w = self._w  # word size
        u = self._u  # length of a word in bytes
        r = self._r  # round count
        M = self._m  # bit mask
        L = list(chunks.unpack(key + (-len(key) % u) * B'\0', u))
        c = len(L)
        t = 2 * (r + 2)
        P, Q = rc5constants(w)
        S = [P]
        while len(S) < t:
            S.append(S[~0] + Q & M)
        i = j = 0
        A = B = 0
        for _ in range(3 * max(t, c)):
            A = S[i] = rotl(w, S[i] + A + B & M, 3)
            B = L[j] = rotl(w, L[j] + A + B & M, A + B)
            i = (i + 1) % t
            j = (j + 1) % c
        self._S = S

class rc6(StandardBlockCipherUnit, cipher=BlockCipherFactory(RC6)):
    """
    RC6 encryption and decryption. The parameter defaults are the RC6 parameters that were chosen
    for the AES candidacy. Only key sizes of 128, 192, and 256 bits are used for AES candidates, but
    the unit will allow any key size up to 256 bits.
    """
    def __init__(
        self, key, iv=b'', padding=None, mode=None, raw=False, segment_size=0,
        rounds    : Arg.Number('-k', help='Number of rounds to use, the default is {default}') = _R,
        word_size : Arg.Number('-w', help='The word size in bits, {default} by default.') = _W,
    ):
        super().__init__(
            key,
            iv,
            padding,
            mode,
            raw,
            segment_size,
            rounds=rounds,
            word_size=word_size
        )

    @property
    def block_size(self):
        return self.args.word_size // 2

    def _new_cipher(self, **optionals) -> CipherInterface:
        return super()._new_cipher(
            rounds=self.args.rounds,
            word_size=self.args.word_size,
            **optionals
        )


from datetime import datetime
from urllib.parse import urljoin
from uuid import UUID

from fastapi import APIRouter, Depends, Path
from fastapi.exceptions import HTTPException

from ... import models, schemas
from ...crud import CRUD
from .utils import get_textures_url

router = APIRouter(tags=["User information"])

async def resolve_user(user_id: UUID = Path(), crud: CRUD = Depends()):
    return await crud.get_user_by_uuid(user_id)

@router.get("/user/{user_id}", response_model=schemas.UserTextures)
async def get_user_textures_by_uuid(
    user: models.User | None = Depends(resolve_user),
    at: datetime | None = None,
    crud: CRUD = Depends(),
    textures_url: str = Depends(get_textures_url),
) -> schemas.UserTextures:
    """Get the currently logged in user information."""
    if user is None:
        raise HTTPException(404)
    return await get_user_textures(user, at, crud, textures_url)

async def get_user_textures(
    user: models.User,
    at: datetime | None,
    crud: CRUD,
    textures_url: str,
):
    textures = await crud.get_user_textures(user, at=at)
    return schemas.UserTextures(
        profile_id=user.uuid,  # type: ignore
        profile_name=user.name,  # type: ignore
        textures={
            k: schemas.Texture(
                url=urljoin(textures_url, v.upload.hash),
                metadata=v.meta,
            )
            for k, v in textures.items()
        },
    )


#!/usr/bin/python
#coding=utf-8

"""
auth: wuqichao、suguoxin
mail: wuqichao@playcrab.com、suguoxin@playcrab.com
create_time: 2015-9-17 10:00:00
usege: 用于hadoop下载文件任务操作

last_update: 2016-04-28 14:45:00
"""

from __future__ import absolute_import
import os

from download.celery import app
from custom.command import Custom_Command as cmd
from custom.db.mysql import Custom_MySQL
from custom.db.redis_tools import Custom_Redis
from celery.utils.log import get_task_logger

try:
    import xml.etree.cElementTree as ET
except ImportError:
    import xml.etree.ElementTree as ET

logger = get_task_logger(__name__)

'''
hadoop服务器集群中启动下载任务程池，由此脚本完成
'''
@app.task(bind=True, max_retries=5, default_retry_delay=6, soft_time_limit=270, time_limit=280)
def run_task(self, task_param):

    redis = Custom_Redis(using='etl_manage')
    mysql = Custom_MySQL(using='etl_manage')
    mysql.begin()
    datas = {'download_status': 0}
    where = {'id': int(task_param['id'])}

    local_log_dir = ""
    lzo_file_name = ""

    try:
        dir_param = {'game': task_param['game'], 'platform': task_param['platform'],
                     'log_date': task_param['log_date'], 'log_name': task_param['log_name']}
        filename_dict = {'log_name': task_param['log_name'], 'log_time': task_param['log_time'], 'source_ip': task_param['source_ip']}
        log_dir = "/%(game)s/%(platform)s/%(log_date)s/%(log_name)s/" % dir_param

        txt_file_name = "%(log_name)s_%(source_ip)s_%(log_time)s.txt" % filename_dict
        lzo_file_name = "%(log_name)s_%(source_ip)s_%(log_time)s.txt.lzo" % filename_dict
        md5_file_name = "%(log_name)s_%(source_ip)s_%(log_time)s.txt.lzo.md5" % filename_dict

        lzo_download_url = task_param['download_url'].rstrip('/') + log_dir + lzo_file_name
        md5_download_url = task_param['download_url'].rstrip('/') + log_dir + md5_file_name

        #从redis中，获取当前数据对应存储到哪块磁盘
        if redis.get("disk_xml") is None:
            disk_tmp = open('/data/etl_manage/conf/disk_game.xml', 'r')
            redis.set("disk_xml", str(disk_tmp.read()))

        disk_list = str(redis.get("disk_xml"))
        root = ET.fromstring(disk_list)
        disk = ""
        for gameinfo in root.findall('game'):
            if gameinfo.get('name') == task_param['game']:
                disk = gameinfo.get('disk')
                continue

        #local_log_dir = '/disk1/tmp_data' + log_dir
        local_log_dir = '/'+disk+'/data' + log_dir
        #创建本地目录
        if not os.path.exists(local_log_dir):
            os.makedirs(local_log_dir)

        #排除同名文件存在的可能，同时为修复执行提供方便
        if os.path.exists('%s%s' % (local_log_dir, txt_file_name)):
            cmd_remove = 'rm -f %s%s*' % (local_log_dir, txt_file_name)
            logger.info('remove history file: {0}'.format(cmd_remove))
            remove_result = cmd.run(cmd_remove)
            if remove_result['status'] != 0:
                logger.error('Error Code %s : %s  Cmd: %s' % (remove_result['status'], remove_result['output'], cmd_remove))

        '''
        下载md5文件,如果md5文件不存在则退出，不再继续执行程序，同时不向数据库写入任何标示
        '''
        md5_line = ' wget -o /tmp/log/wget_log -O %s%s %s' % (local_log_dir, md5_file_name, md5_download_url)
        logger.info('md5 info: {0}'.format(md5_line))
        md5_result = cmd.run(md5_line)

        if md5_result['status'] != 0:
            logger.error('Error Code %s : %s  Cmd: %s' % (md5_result['status'], md5_result['output'], md5_line))

        else:
            '''
            将任务标识为开始执行：1
            '''
            datas['download_status'] = 1
            mysql.update('etl_data_log', ' id = %(id)d' % where, **datas)
            mysql.commit()

            '''
            下载数据文件
            '''
            lzo_line = ' wget -o /tmp/log/wget_log -O %s%s %s' % (local_log_dir, lzo_file_name, lzo_download_url)
            logger.info('file info: {0}'.format(lzo_line))
            lzo_result = cmd.run(lzo_line)

            if lzo_result['status'] != 0:
                logger.error('Error Code %s : %s  Cmd: %s' % (lzo_result['status'], lzo_result['output'], lzo_line))
                datas['download_status'] = 0
            else:
                '''
                将任务标识为下载完成：2
                '''
                datas['download_status'] = 2
                mysql.update('etl_data_log', ' id = %(id)d' % where, **datas)
                mysql.commit()

                '''
                md5校验，如果未通过则不再继续执行程序
                '''
                check_line = "cat %s%s |grep `md5sum %s%s|cut -d ' ' -f 1`" % (local_log_dir, md5_file_name, local_log_dir, lzo_file_name)
                logger.info('md5 or md5 info: {0}'.format(check_line))
                check_result = cmd.run(check_line)

                if check_result['status'] != 0:
                    logger.error('Error Code %s : %s Cmd: %s' % (check_result['status'], check_result['output'], check_line))
                    datas['download_status'] = 0
                else:
                    '''
                    lzop解压缩
                    '''
                    cmd_line = ' lzop -dP %s%s' % (local_log_dir, lzo_file_name)
                    logger.info('file info: {0}'.format(cmd_line))
                    cmd_result = cmd.run(cmd_line)
                    if cmd_result['status'] != 0:
                        logger.error('Lzop Code %s : %s Cmd: %s' % (cmd_result['status'], cmd_result['output'], cmd_line))
                        datas['download_status'] = 0
                    else:
                        '''
                        将任务标识md5一致，完成下载任务：3
                        '''
                        datas['download_status'] = 3

        '''
        将任务标示为：(模拟) 已从任务队列中移除
        '''
        datas['in_download_queue'] = 0
        update_result = mysql.update('etl_data_log', ' id = %(id)d' % where, **datas)
        # 如果更新失败，则再调用一次，如果还是失败，则等待自动修复机制，但这种概率很小了
        if update_result != 1:
            mysql.update('etl_data_log', ' id = %(id)d' % where, **datas)

        mysql.commit()
        mysql.close()
        return True

    except Exception as exc:
        print (exc)
        logger.error('download error : %s' % exc)
        mysql.rollback()

        kill_proces = "kill -9 `ps -ef |grep wget |grep -v grep |grep '%s%s'|awk '{print $2}'`" % (local_log_dir, lzo_file_name)
        cmd.run(kill_proces)

        datas = {'in_download_queue': 0, 'download_status': 0}
        mysql.update('etl_data_log', ' id = %(id)d' % where, **datas)
        mysql.commit()

        mysql.close()
        raise self.retry(exc=exc, countdown=60)


import logging
import os
import pathlib
import subprocess
import time
import uuid

from django.db import models
from django.db.models import IntegerChoices
from django.db.models.signals import pre_delete
from django.dispatch import receiver

from base.base_model import BaseModel
from common.models.User import User
from website.applications.app_factory import AppFactory
from website.applications.core.application import Application
from website.applications.core.dataclass import (
    DataBaseDict,
    MariaDBConfig,
    MySQLDBConfig,
    SSLConfig,
    WebServerTypeEnum,
    WebSiteConfig,
)
from website.models.utils import (
    disable_section,
    enable_section,
    get_section,
    insert_section,
    update_nginx_server_name,
)

nginx_config_example = """
server {
    ########BASIC########

    listen 80;
    listen [::]:80;
    root {dir_path};
    server_name {domain};

    ########BASIC########

    ########SSL########

    #**#listen 443 ssl http2;
    #**#listen [::]:443 ssl http2;
    #**#ssl_certificate      /etc/letsencrypt/live/{domain}/fullchain.pem;
    #**#ssl_certificate_key     /etc/letsencrypt/live/{domain}/privkey.pem;
    #**#ssl_trusted_certificate /etc/letsencrypt/live/{domain}/chain.pem;

    #**#ssl_protocols TLSv1.1 TLSv1.2 TLSv1.3;
    #**#ssl_ciphers EECDH+CHACHA20:EECDH+CHACHA20-draft:EECDH+AES128:RSA+AES128:EECDH+AES256:RSA+AES256:EECDH+3DES:RSA+3DES:!MD5;
    #**#ssl_prefer_server_ciphers on;
    #**#ssl_session_cache shared:SSL:10m;
    #**#ssl_session_timeout 10m;
    #**#error_page 497  https://$host$request_uri;

    ########SSL########

    ########USER########

    ########USER########

}"""

class WebsiteManager(models.Manager):
    def get_queryset(self):
        return super().get_queryset().select_related("user")

class WebsiteMethod:
    @staticmethod
    def get_ssl_config_default() -> dict:
        return {
            "certbot": {
                "email": "",
                "provider": None,
            },
            "path": {
                "certificate": "",
                "key": "",
            },
            "method": "http-01",
        }

class WebsiteModel(BaseModel):
    """website model"""

    APP_FACTORY = AppFactory

    class StatusType(IntegerChoices):
        READY = 0, "准备中"
        VALID = 1, "有效"
        SUSPEND = 2, "暂停"
        EXPIRED = 3, "过期"
        DISABLED = 4, "禁用"
        ERROR = 5, "错误"

    class WebServerType(IntegerChoices):
        # WebServerTypeEnum
        # from website.applications.core.dataclass import WebServerTypeEnum
        Nginx = 1
        Apache = 2
        Lighttpd = 3
        IIS = 4
        Tomcat = 5
        Caddy = 6

    user = models.ForeignKey(
        User,
        related_name="user",
        blank=True,
        on_delete=models.CASCADE,
        verbose_name="用户",
    )
    name = models.CharField(verbose_name="名称", max_length=32)
    domain = models.CharField(max_length=1024, verbose_name="域名", unique=True)
    extra_domain = models.CharField(
        max_length=1024,
        null=True,
        blank=True,
        verbose_name="additional domain name",
        help_text='use "," split domain name',
    )
    # 配置说明
    # 证书签发主要依赖于 certbot ，需要定义
    # --preferred-challenges "验证方式"，其中验证方式为
    #   dns-01(自动处理DNS验证，请参考 User Guide https://eff-certbot.readthedocs.io/en/stable/using.html#dns-plugins,
    #   tls-sni-01 (443端口验证), http (80端口验证)
    # --email "邮件地址"
    # -----------------------------------------------------------------------------------------------------------------|
    # | --server "SERVER"                 The three parameters here mainly define the authentication information       |
    # |                                      of different issuing agencies.                                            |
    # | --eab-kid "EAB_KID"               To be abstracted as an optional field letsencrypt, ZeroSSL,                  |
    # |                                      or a custom issuing authority                                             |
    # | --eab-hmac-key "EAB_HMAC_KEY"     These three items should be in the system configuration, and the user should |
    # |                                      not be differentiated globally.                                           |
    # -----------------------------------------------------------------------------------------------------------------|
    # certbot certonly -n --nginx --agree-tos -m email@gmail.com -d "domain.com"
    # specific reference：https://www.moec.top/archives/2544.
    #
    # ssl_config configuration instructions
    # certbot
    #  - email: Email, required
    #  - server: optional
    # path: The certificate path, the subsequent nginx configuration files will be based on this path
    #  - certificate: fullchain.pem file absolute path
    #  - key: privkey.pem file absolute path
    ssl_config = models.JSONField(
        default=WebsiteMethod.get_ssl_config_default,
        blank=True,
        null=True,
        verbose_name="证书配置",
    )
    ssl_enable = models.BooleanField(default=False, verbose_name="enable SSL")
    index_root = models.CharField(
        default="/var/www/html", max_length=4096, verbose_name="站点目录"
    )
    status = models.IntegerField(
        choices=StatusType.choices,
        default=StatusType.READY,
        verbose_name="状态",
        help_text=StatusType.labels,
    )
    status_info = models.CharField(
        default="", blank=True, null=True, max_length=2048, verbose_name="status info"
    )

    web_server_type = models.IntegerField(
        default=WebServerType.Nginx, choices=WebServerType.choices
    )
    application = models.CharField(max_length=64, verbose_name="application")
    application_config = models.JSONField(
        default=dict, verbose_name="application config"
    )

    valid_web_server_config = models.TextField(
        max_length=102400,
        default=None,
        blank=True,
        null=True,
        verbose_name="valid configuration",
    )

    class Meta:
        unique_together = ["name"]
        verbose_name = "website"
        verbose_name_plural = "websites"

    def __str__(self):
        return self.name

    def get_application(self, app_config: dict = None) -> Application:
        if self.APP_FACTORY.MODULES == {}:
            self.APP_FACTORY.load()
        app = self.APP_FACTORY.get_application_module(
            self.application, self.get_app_website_config(), app_config
        )
        return app

    def or_create_ssl_config(self):
        if self.ssl_config is None or self.ssl_config["certbot"]["email"] == "":
            self.log("debug", f"{self.name} - {self.domain} ssl is first enable.")
            ssl_config = {
                "certbot": {
                    "email": self.user.email,
                    "provider": "default",
                },
                "path": {
                    "certificate": f"/etc/letsencrypt/live/{self.domain}/fullchain.pem",
                    "key": f"/etc/letsencrypt/live/{self.domain}/privkey.pem",
                },
                "method": "http-01",
            }

            ssl_folder = pathlib.Path(f"/etc/letsencrypt/live/{self.domain}")
            if not ssl_folder.exists():
                ssl_folder.mkdir(parents=True, exist_ok=True)

            self.ssl_config = ssl_config

    def get_app_website_config(self) -> WebSiteConfig:
        config = WebSiteConfig(
            domain=self.domain,
            root_dir=self.index_root,
            web_server_type=WebServerTypeEnum.Nginx,
        )

        # Get database config.
        logging.debug(f"get {self.name} - {self.domain} database config.")

        if hasattr(self, "databasemodel") and self.databasemodel:
            from database.models.database import DataBaseModel

            database: DataBaseModel = self.databasemodel
            if database.database_type == database.DBType.MariaDB:
                config.databases = DataBaseDict(
                    mariadb=MariaDBConfig(
                        self.databasemodel.name,
                        self.databasemodel.username,
                        self.databasemodel.password,
                    )
                )
            else:
                config.databases = DataBaseDict(
                    mariadb=MySQLDBConfig(
                        self.databasemodel.name,
                        self.databasemodel.username,
                        self.databasemodel.password,
                    )
                )
        # TODO:Get redis config.

        if self.ssl_enable:
            logging.debug(f"enable {self.name} - {self.domain} ssl toggle.")
            self.or_create_ssl_config()
            config.ssl_config = SSLConfig(
                ssl_certificate_path=self.ssl_config["path"]["certificate"],
                ssl_key_path=self.ssl_config["path"]["key"],
            )
            logging.debug(config.ssl_config.__str__())

        if self.valid_web_server_config is not None:
            config.web_server_config = self.valid_web_server_config

        return config

    def get_application_module(self, config: WebSiteConfig) -> Application:
        app_factory = AppFactory
        app_factory.load()
        return app_factory.get_application_module(self.application, config)

    def is_valid_configuration_001(self) -> subprocess.CompletedProcess:
        logging.info(
            f"001:verify {self.domain} configuration...\n\n{self.valid_web_server_config}"
        )

        minimum_config = f"""user www-data;
              worker_processes auto;
              pid /run/nginx.pid;
              include /etc/nginx/modules-enabled/*.conf;
              events {{
                      worker_connections 768;
              }}

              http {{
                      sendfile on;
                      tcp_nopush on;
                      types_hash_max_size 2048;
                      include /etc/nginx/mime.types;
                      default_type application/octet-stream;
                      ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3; # Dropping SSLv3, ref: POODLE
                      ssl_prefer_server_ciphers on;
                      gzip on;
                      {self.valid_web_server_config}

              }}
              """

        tmp_config = pathlib.Path(f"/tmp/{uuid.uuid4().hex}.conf")
        tmp_config.write_text(minimum_config)
        cmd = f"nginx -t -c {tmp_config.absolute()}"
        r = subprocess.run(cmd, shell=True, capture_output=True)

        return r

    def is_valid_configuration_002(self, web_server_config) -> bool:
        nginx_config_path = f"/etc/nginx/sites-available/{self.domain}.conf"
        enable_nginx_config_path = f"/etc/nginx/sites-enabled/{self.domain}.conf"

        old_web_server_config = ""
        if pathlib.Path(nginx_config_path).exists():
            with open(nginx_config_path, "r") as f:
                old_web_server_config = f.read()

        with open(nginx_config_path, "w") as f:
            f.write(web_server_config)

        if not pathlib.Path(enable_nginx_config_path).exists():
            os.system(f"ln -s {nginx_config_path} {enable_nginx_config_path}")
        res = os.system("nginx -t")
        if res == 0:
            os.system("systemctl reload nginx")
            return True
        else:
            if old_web_server_config:
                with open(nginx_config_path, "w") as f:
                    f.write(old_web_server_config)
            return False

    def get_nginx_config(self):
        # set domain and ssl path.
        data = nginx_config_example.replace("{domain}", self.domain)

        # set extra domain.
        if self.extra_domain:
            extra_domain = " ".join(self.extra_domain.split(","))
            server_name = f"{extra_domain}"
            data = update_nginx_server_name(data, self.domain, server_name)

        # set nginx root field.
        data = data.replace("{dir_path}", self.index_root)
        # set enter folder.
        app = self.get_application_module(self.get_app_website_config())
        app_config = app.read()
        logging.debug(app_config)
        if hasattr(app_config, "enter_folder_name") and app_config.enter_folder_name:
            old_field = f"root {self.index_root};"
            new_field = f"root {self.index_root}/{app_config.enter_folder_name};"
            if new_field not in data:
                data = data.replace(old_field, new_field)
        # set ssl
        if self.ssl_enable:
            data = enable_section(data, "ssl")
        else:
            data = disable_section(data, "ssl")
        return data

    def get_default_config(self):
        app = self.get_application_module(self.get_app_website_config())
        nginx_data = self.get_nginx_config()
        app_config = app.read()

        if not isinstance(app_config, str):
            app_config = app.read().nginx

        nginx_data = insert_section(nginx_data, app_config, "user")
        return nginx_data

    def sync_web_config(self, save=False):
        data = self.get_nginx_config()
        user_config = get_section(self.valid_web_server_config, "user")
        data = insert_section(data, user_config, "user")
        self.valid_web_server_config = data
        self.is_valid_configuration_002(self.valid_web_server_config)
        if save:
            self.save()


import os
import sys

import pandas as pd
import numpy as np
import datetime as dt

import sqlalchemy
from sqlalchemy.ext.automap import automap_base
from sqlalchemy.orm import Session
from sqlalchemy import create_engine, inspect

from flask import Flask, jsonify, render_template
from flask_sqlalchemy import SQLAlchemy

app = Flask(__name__)

#################################################
# Database Setup
#################################################

engine = create_engine("sqlite:///db/summaryTables.sqlite")

# reflect an existing database into a new model
Base = automap_base()
# reflect the tables
Base.prepare(engine, reflect=True)

# Save reference to the table
# Samples_Metadata = Base.classes.sample_metadata
print(Base.classes.keys(), file=sys.stderr)
statecount = Base.classes.statecount
wordcloud = Base.classes.wordcloud
yearName = Base.classes.yearName

# Create our session (link) from Python to the DB
session = Session(engine)

@app.route("/")
def index():
    """Return the homepage."""
    return render_template("index.html", statecount=statecount, wordcloud=wordcloud)

@app.route("/cloudchartdata")
def cloudchartdata():
    """Return the MetaData for a given sample."""
    sel = [
        wordcloud.Name,
        wordcloud.total_count,
    ]

    results = session.query(*sel).all()
    print(results)
    # Create a dictionary entry for each row of metadata information
    top50 = {}
    for result in results:
        top50["Name"] = result[0]
        top50["total_count"] = result[1]

    print(top50)
    return jsonify(results)

@app.route("/cloudchartdata2")
def cloudchartdata2():

    stmt = session.query(wordcloud).statement
    df = pd.read_sql_query(stmt, session.bind)

    data = df.to_json(orient='records')
    return jsonify(data)

@app.route("/linechart/<userSelection>")
def linechart(userSelection):

    stmt = session.query(yearName).statement
    df = pd.read_sql_query(stmt, session.bind)

    # Filter the data based on the sample number and
    # only keep rows with values above 1
    print(df)
    sample_data = df[df.Name.str.lower() == userSelection]
    print(sample_data)
    print(userSelection)
    data = sample_data.to_json(orient='records')
    return data

@app.route("/chlorodata")
def chlorodata():

    stmt = session.query(statecount).statement
    df = pd.read_sql_query(stmt, session.bind)

    data = df.to_json(orient='records')
    return jsonify(data)

if __name__ == "__main__":
    app.run()