import datetime
import json
import os
import random
import time
import xml.sax.saxutils as saxutils
from datetime import datetime

import azurelinuxagent.common.conf as conf
import azurelinuxagent.common.logger as logger
import azurelinuxagent.common.utils.textutil as textutil
from azurelinuxagent.common.datacontract import validate_param
from azurelinuxagent.common.event import add_periodic, WALAEventOperation, EVENTS_DIRECTORY
from azurelinuxagent.common.exception import ProtocolNotFoundError, \
    ResourceGoneError, ExtensionDownloadError, InvalidContainerError, ProtocolError, HttpError
from azurelinuxagent.common.future import httpclient, bytebuffer
from azurelinuxagent.common.protocol.goal_state import GoalState, TRANSPORT_CERT_FILE_NAME, TRANSPORT_PRV_FILE_NAME
from azurelinuxagent.common.protocol.hostplugin import HostPluginProtocol
from azurelinuxagent.common.protocol.imds import ComputeInfo
from azurelinuxagent.common.protocol.restapi import *
from azurelinuxagent.common.telemetryevent import TelemetryEventList
from azurelinuxagent.common.utils import fileutil, restutil
from azurelinuxagent.common.utils.archive import StateFlusher
from azurelinuxagent.common.utils.cryptutil import CryptUtil
from azurelinuxagent.common.utils.textutil import parse_doc, findall, find, \
    findtext, gettext, remove_bom, get_bytes_from_pem, parse_json
from azurelinuxagent.common.version import AGENT_NAME, CURRENT_VERSION

VERSION_INFO_URI = "http://{0}/?comp=versions"
HEALTH_REPORT_URI = "http://{0}/machine?comp=health"
ROLE_PROP_URI = "http://{0}/machine?comp=roleProperties"
TELEMETRY_URI = "http://{0}/machine?comp=telemetrydata"

WIRE_SERVER_ADDR_FILE_NAME = "WireServer"
INCARNATION_FILE_NAME = "Incarnation"
GOAL_STATE_FILE_NAME = "GoalState.{0}.xml"
HOSTING_ENV_FILE_NAME = "HostingEnvironmentConfig.xml"
SHARED_CONF_FILE_NAME = "SharedConfig.xml"
REMOTE_ACCESS_FILE_NAME = "RemoteAccess.{0}.xml"
EXT_CONF_FILE_NAME = "ExtensionsConfig.{0}.xml"
MANIFEST_FILE_NAME = "{0}.{1}.manifest.xml"

PROTOCOL_VERSION = "2012-11-30"
ENDPOINT_FINE_NAME = "WireServer"

SHORT_WAITING_INTERVAL = 1  # 1 second

MAX_EVENT_BUFFER_SIZE = 2 ** 16 - 2 ** 10

class UploadError(HttpError):
    pass

class WireProtocol(Protocol):
    """Slim layer to adapt wire protocol data to metadata protocol interface"""

    def __init__(self, endpoint):
        if endpoint is None:
            raise ProtocolError("WireProtocol endpoint is None")
        self.client = WireClient(endpoint)

    def detect(self):
        self.client.check_wire_protocol_version()

        trans_prv_file = os.path.join(conf.get_lib_dir(),
                                      TRANSPORT_PRV_FILE_NAME)
        trans_cert_file = os.path.join(conf.get_lib_dir(),
                                       TRANSPORT_CERT_FILE_NAME)
        cryptutil = CryptUtil(conf.get_openssl_cmd())
        cryptutil.gen_transport_cert(trans_prv_file, trans_cert_file)

        # Set the initial goal state
        logger.info('Initializing goal state during protocol detection')
        self.client.update_goal_state(forced=True)

    def update_goal_state(self):
        self.client.update_goal_state()

    def update_host_plugin_from_goal_state(self):
        self.client.update_host_plugin_from_goal_state()

    def get_endpoint(self):
        return self.client.get_endpoint()

    def get_vminfo(self):
        goal_state = self.client.get_goal_state()
        hosting_env = self.client.get_hosting_env()

        vminfo = VMInfo()
        vminfo.subscriptionId = None
        vminfo.vmName = hosting_env.vm_name
        vminfo.tenantName = hosting_env.deployment_name
        vminfo.roleName = hosting_env.role_name
        vminfo.roleInstanceName = goal_state.role_instance_id
        return vminfo

    def get_certs(self):
        certificates = self.client.get_certs()
        return certificates.cert_list

    def get_incarnation(self):
        return self.client.get_goal_state().incarnation

    def get_vmagent_manifests(self):
        goal_state = self.client.get_goal_state()
        ext_conf = self.client.get_ext_conf()
        return ext_conf.vmagent_manifests, goal_state.incarnation

    def get_vmagent_pkgs(self, vmagent_manifest):
        goal_state = self.client.get_goal_state()
        ga_manifest = self.client.get_gafamily_manifest(vmagent_manifest, goal_state)
        valid_pkg_list = ga_manifest.pkg_list
        return valid_pkg_list

    def get_ext_handlers(self):
        logger.verbose("Get extension handler config")
        goal_state = self.client.get_goal_state()
        ext_conf = self.client.get_ext_conf()
        # In wire protocol, incarnation is equivalent to ETag
        return ext_conf.ext_handlers, goal_state.incarnation

    def get_ext_handler_pkgs(self, ext_handler):
        logger.verbose("Get extension handler package")
        man = self.client.get_ext_manifest(ext_handler)
        return man.pkg_list

    def get_artifacts_profile(self):
        logger.verbose("Get In-VM Artifacts Profile")
        return self.client.get_artifacts_profile()

    def _download_ext_handler_pkg_through_host(self, uri, destination):
        host = self.client.get_host_plugin()
        uri, headers = host.get_artifact_request(uri, host.manifest_uri)
        success = self.client.stream(uri, destination, headers=headers, use_proxy=False)
        return success

    def download_ext_handler_pkg(self, uri, destination, headers=None, use_proxy=True):
        direct_func = lambda: self.client.stream(uri, destination, headers=None, use_proxy=True)
        # NOTE: the host_func may be called after refreshing the goal state, be careful about any goal state data
        # in the lambda.
        host_func = lambda: self._download_ext_handler_pkg_through_host(uri, destination)

        try:
            success = self.client.send_request_using_appropriate_channel(direct_func, host_func)
        except Exception:
            success = False

        return success

    def report_provision_status(self, provision_status):
        validate_param("provision_status", provision_status, ProvisionStatus)

        if provision_status.status is not None:
            self.client.report_health(provision_status.status,
                                      provision_status.subStatus,
                                      provision_status.description)
        if provision_status.properties.certificateThumbprint is not None:
            thumbprint = provision_status.properties.certificateThumbprint
            self.client.report_role_prop(thumbprint)

    def report_vm_status(self, vm_status):
        validate_param("vm_status", vm_status, VMStatus)
        self.client.status_blob.set_vm_status(vm_status)
        self.client.upload_status_blob()

    def report_ext_status(self, ext_handler_name, ext_name, ext_status):
        validate_param("ext_status", ext_status, ExtensionStatus)
        self.client.status_blob.set_ext_status(ext_handler_name, ext_status)

    def report_event(self, events):
        validate_param(EVENTS_DIRECTORY, events, TelemetryEventList)
        self.client.report_event(events)

def _build_role_properties(container_id, role_instance_id, thumbprint):
    xml = (u"<?xml version=\"1.0\" encoding=\"utf-8\"?>"
           u"<RoleProperties>"
           u"<Container>"
           u"<ContainerId>{0}</ContainerId>"
           u"<RoleInstances>"
           u"<RoleInstance>"
           u"<Id>{1}</Id>"
           u"<Properties>"
           u"<Property name=\"CertificateThumbprint\" value=\"{2}\" />"
           u"</Properties>"
           u"</RoleInstance>"
           u"</RoleInstances>"
           u"</Container>"
           u"</RoleProperties>"
           u"").format(container_id, role_instance_id, thumbprint)
    return xml

def _build_health_report(incarnation, container_id, role_instance_id,
                         status, substatus, description):
    # Escape '&', '<' and '>'
    description = saxutils.escape(ustr(description))
    detail = u''
    if substatus is not None:
        substatus = saxutils.escape(ustr(substatus))
        detail = (u"<Details>"
                  u"<SubStatus>{0}</SubStatus>"
                  u"<Description>{1}</Description>"
                  u"</Details>").format(substatus, description)
    xml = (u"<?xml version=\"1.0\" encoding=\"utf-8\"?>"
           u"<Health "
           u"xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\""
           u" xmlns:xsd=\"http://www.w3.org/2001/XMLSchema\">"
           u"<GoalStateIncarnation>{0}</GoalStateIncarnation>"
           u"<Container>"
           u"<ContainerId>{1}</ContainerId>"
           u"<RoleInstanceList>"
           u"<Role>"
           u"<InstanceId>{2}</InstanceId>"
           u"<Health>"
           u"<State>{3}</State>"
           u"{4}"
           u"</Health>"
           u"</Role>"
           u"</RoleInstanceList>"
           u"</Container>"
           u"</Health>"
           u"").format(incarnation,
                       container_id,
                       role_instance_id,
                       status,
                       detail)
    return xml

def ga_status_to_guest_info(ga_status):
    """
    Convert VMStatus object to status blob format
    """
    v1_ga_guest_info = {
        "computerName": ga_status.hostname,
        "osName": ga_status.osname,
        "osVersion": ga_status.osversion,
        "version": ga_status.version,
    }
    return v1_ga_guest_info

def ga_status_to_v1(ga_status):
    formatted_msg = {
        'lang': 'en-US',
        'message': ga_status.message
    }
    v1_ga_status = {
        "version": ga_status.version,
        "status": ga_status.status,
        "formattedMessage": formatted_msg
    }
    return v1_ga_status

def ext_substatus_to_v1(sub_status_list):
    status_list = []
    for substatus in sub_status_list:
        status = {
            "name": substatus.name,
            "status": substatus.status,
            "code": substatus.code,
            "formattedMessage": {
                "lang": "en-US",
                "message": substatus.message
            }
        }
        status_list.append(status)
    return status_list

def ext_status_to_v1(ext_name, ext_status):
    if ext_status is None:
        return None
    timestamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
    v1_sub_status = ext_substatus_to_v1(ext_status.substatusList)
    v1_ext_status = {
        "status": {
            "name": ext_name,
            "configurationAppliedTime": ext_status.configurationAppliedTime,
            "operation": ext_status.operation,
            "status": ext_status.status,
            "code": ext_status.code,
            "formattedMessage": {
                "lang": "en-US",
                "message": ext_status.message
            }
        },
        "version": 1.0,
        "timestampUTC": timestamp
    }
    if len(v1_sub_status) != 0:
        v1_ext_status['status']['substatus'] = v1_sub_status
    return v1_ext_status

def ext_handler_status_to_v1(handler_status, ext_statuses, timestamp):
    v1_handler_status = {
        'handlerVersion': handler_status.version,
        'handlerName': handler_status.name,
        'status': handler_status.status,
        'code': handler_status.code,
        'useExactVersion': True
    }
    if handler_status.message is not None:
        v1_handler_status["formattedMessage"] = {
            "lang": "en-US",
            "message": handler_status.message
        }

    if len(handler_status.extensions) > 0:
        # Currently, no more than one extension per handler
        ext_name = handler_status.extensions[0]
        ext_status = ext_statuses.get(ext_name)
        v1_ext_status = ext_status_to_v1(ext_name, ext_status)
        if ext_status is not None and v1_ext_status is not None:
            v1_handler_status["runtimeSettingsStatus"] = {
                'settingsStatus': v1_ext_status,
                'sequenceNumber': ext_status.sequenceNumber
            }
    return v1_handler_status

def vm_status_to_v1(vm_status, ext_statuses):
    timestamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())

    v1_ga_guest_info = ga_status_to_guest_info(vm_status.vmAgent)
    v1_ga_status = ga_status_to_v1(vm_status.vmAgent)
    v1_handler_status_list = []
    for handler_status in vm_status.vmAgent.extensionHandlers:
        v1_handler_status = ext_handler_status_to_v1(handler_status,
                                                     ext_statuses, timestamp)
        if v1_handler_status is not None:
            v1_handler_status_list.append(v1_handler_status)

    v1_agg_status = {
        'guestAgentStatus': v1_ga_status,
        'handlerAggregateStatus': v1_handler_status_list
    }
    v1_vm_status = {
        'version': '1.1',
        'timestampUTC': timestamp,
        'aggregateStatus': v1_agg_status,
        'guestOSInfo': v1_ga_guest_info
    }
    return v1_vm_status

class StatusBlob(object):
    def __init__(self, client):
        self.vm_status = None
        self.ext_statuses = {}
        self.client = client
        self.type = None
        self.data = None

    def set_vm_status(self, vm_status):
        validate_param("vmAgent", vm_status, VMStatus)
        self.vm_status = vm_status

    def set_ext_status(self, ext_handler_name, ext_status):
        validate_param("extensionStatus", ext_status, ExtensionStatus)
        self.ext_statuses[ext_handler_name] = ext_status

    def to_json(self):
        report = vm_status_to_v1(self.vm_status, self.ext_statuses)
        return json.dumps(report)

    __storage_version__ = "2014-02-14"

    def prepare(self, blob_type):
        logger.verbose("Prepare status blob")
        self.data = self.to_json()
        self.type = blob_type

    def upload(self, url):
        try:
            if not self.type in ["BlockBlob", "PageBlob"]:
                raise ProtocolError("Illegal blob type: {0}".format(self.type))

            if self.type == "BlockBlob":
                self.put_block_blob(url, self.data)
            else:
                self.put_page_blob(url, self.data)
            return True

        except Exception as e:
            logger.verbose("Initial status upload failed: {0}", e)

        return False

    def get_block_blob_headers(self, blob_size):
        return {
            "Content-Length": ustr(blob_size),
            "x-ms-blob-type": "BlockBlob",
            "x-ms-date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
            "x-ms-version": self.__class__.__storage_version__
        }

    def put_block_blob(self, url, data):
        logger.verbose("Put block blob")
        headers = self.get_block_blob_headers(len(data))
        resp = self.client.call_storage_service(restutil.http_put, url, data, headers)
        if resp.status != httpclient.CREATED:
            raise UploadError(
                "Failed to upload block blob: {0}".format(resp.status))

    def get_page_blob_create_headers(self, blob_size):
        return {
            "Content-Length": "0",
            "x-ms-blob-content-length": ustr(blob_size),
            "x-ms-blob-type": "PageBlob",
            "x-ms-date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
            "x-ms-version": self.__class__.__storage_version__
        }

    def get_page_blob_page_headers(self, start, end):
        return {
            "Content-Length": ustr(end - start),
            "x-ms-date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
            "x-ms-range": "bytes={0}-{1}".format(start, end - 1),
            "x-ms-page-write": "update",
            "x-ms-version": self.__class__.__storage_version__
        }

    def put_page_blob(self, url, data):
        logger.verbose("Put page blob")

        # Convert string into bytes and align to 512 bytes
        data = bytearray(data, encoding='utf-8')
        page_blob_size = int((len(data) + 511) / 512) * 512

        headers = self.get_page_blob_create_headers(page_blob_size)
        resp = self.client.call_storage_service(restutil.http_put, url, "", headers)
        if resp.status != httpclient.CREATED:
            raise UploadError(
                "Failed to clean up page blob: {0}".format(resp.status))

        if url.count("?") <= 0:
            url = "{0}?comp=page".format(url)
        else:
            url = "{0}&comp=page".format(url)

        logger.verbose("Upload page blob")
        page_max = 4 * 1024 * 1024  # Max page size: 4MB
        start = 0
        end = 0
        while end < len(data):
            end = min(len(data), start + page_max)
            content_size = end - start
            # Align to 512 bytes
            page_end = int((end + 511) / 512) * 512
            buf_size = page_end - start
            buf = bytearray(buf_size)
            buf[0: content_size] = data[start: end]
            headers = self.get_page_blob_page_headers(start, page_end)
            resp = self.client.call_storage_service(
                restutil.http_put,
                url,
                bytebuffer(buf),
                headers)
            if resp is None or resp.status != httpclient.CREATED:
                raise UploadError(
                    "Failed to upload page blob: {0}".format(resp.status))
            start = end

def event_param_to_v1(param):
    param_format = '<Param Name="{0}" Value={1} T="{2}" />'
    param_type = type(param.value)
    attr_type = ""
    if param_type is int:
        attr_type = 'mt:uint64'
    elif param_type is str:
        attr_type = 'mt:wstr'
    elif ustr(param_type).count("'unicode'") > 0:
        attr_type = 'mt:wstr'
    elif param_type is bool:
        attr_type = 'mt:bool'
    elif param_type is float:
        attr_type = 'mt:float64'
    return param_format.format(param.name,
                               saxutils.quoteattr(ustr(param.value)),
                               attr_type)

def event_to_v1(event):
    params = ""
    for param in event.parameters:
        params += event_param_to_v1(param)
    event_str = ('<Event id="{0}">'
                 '<![CDATA[{1}]]>'
                 '</Event>').format(event.eventId, params)
    return event_str

class WireClient(object):

    def __init__(self, endpoint):
        logger.info("Wire server endpoint:{0}", endpoint)
        self._endpoint = endpoint
        self._goal_state = None
        self._host_plugin = None
        self.status_blob = StatusBlob(self)
        self.goal_state_flusher = StateFlusher(conf.get_lib_dir())

    def get_endpoint(self):
        return self._endpoint

    def call_wireserver(self, http_req, *args, **kwargs):
        try:
            # Never use the HTTP proxy for wireserver
            kwargs['use_proxy'] = False
            resp = http_req(*args, **kwargs)

            if restutil.request_failed(resp):
                msg = "[Wireserver Failed] URI {0} ".format(args[0])
                if resp is not None:
                    msg += " [HTTP Failed] Status Code {0}".format(resp.status)
                raise ProtocolError(msg)

        # If the GoalState is stale, pass along the exception to the caller
        except ResourceGoneError:
            raise

        except Exception as e:
            raise ProtocolError("[Wireserver Exception] {0}".format(
                ustr(e)))

        return resp

    def decode_config(self, data):
        if data is None:
            return None
        data = remove_bom(data)
        xml_text = ustr(data, encoding='utf-8')
        return xml_text

    def fetch_config(self, uri, headers):
        resp = self.call_wireserver(restutil.http_get,
                                    uri,
                                    headers=headers)
        return self.decode_config(resp.read())

    def fetch_cache(self, local_file):
        if not os.path.isfile(local_file):
            raise ProtocolError("{0} is missing.".format(local_file))
        try:
            return fileutil.read_file(local_file)
        except IOError as e:
            raise ProtocolError("Failed to read cache: {0}".format(e))

    def save_cache(self, local_file, data):
        try:
            fileutil.write_file(local_file, data)
        except IOError as e:
            fileutil.clean_ioerror(e, paths=[local_file])
            raise ProtocolError("Failed to write cache: {0}".format(e))

    @staticmethod
    def call_storage_service(http_req, *args, **kwargs):
        # Default to use the configured HTTP proxy
        if not 'use_proxy' in kwargs or kwargs['use_proxy'] is None:
            kwargs['use_proxy'] = True

        return http_req(*args, **kwargs)

    def fetch_manifest_through_host(self, uri):
        host = self.get_host_plugin()
        uri, headers = host.get_artifact_request(uri)
        response = self.fetch(uri, headers, use_proxy=False)
        return response

    def fetch_manifest(self, version_uris):
        logger.verbose("Fetch manifest")
        version_uris_shuffled = version_uris
        random.shuffle(version_uris_shuffled)

        for version in version_uris_shuffled:
            # GA expects a location and failoverLocation in ExtensionsConfig, but
            # this is not always the case. See #1147.
            if version.uri is None:
                logger.verbose('The specified manifest URL is empty, ignored.')
                continue

            direct_func = lambda: self.fetch(version.uri)
            # NOTE: the host_func may be called after refreshing the goal state, be careful about any goal state data
            # in the lambda.
            host_func = lambda: self.fetch_manifest_through_host(version.uri)

            try:
                response = self.send_request_using_appropriate_channel(direct_func, host_func)

                if response:
                    host = self.get_host_plugin()
                    host.update_manifest_uri(version.uri)
                    return response
            except Exception as e:
                logger.warn("Exception when fetching manifest. Error: {0}".format(ustr(e)))

        raise ExtensionDownloadError("Failed to fetch manifest from all sources")

    def stream(self, uri, destination, headers=None, use_proxy=None):
        success = False
        logger.verbose("Fetch [{0}] with headers [{1}] to file [{2}]", uri, headers, destination)

        response = self._fetch_response(uri, headers, use_proxy)
        if response is not None:
            chunk_size = 1024 * 1024  # 1MB buffer
            try:
                with open(destination, 'wb', chunk_size) as destination_fh:
                    complete = False
                    while not complete:
                        chunk = response.read(chunk_size)
                        destination_fh.write(chunk)
                        complete = len(chunk) < chunk_size
                success = True
            except Exception as e:
                logger.error('Error streaming {0} to {1}: {2}'.format(uri, destination, ustr(e)))

        return success

    def fetch(self, uri, headers=None, use_proxy=None, decode=True):
        logger.verbose("Fetch [{0}] with headers [{1}]", uri, headers)
        content = None
        response = self._fetch_response(uri, headers, use_proxy)
        if response is not None:
            response_content = response.read()
            content = self.decode_config(response_content) if decode else response_content
        return content

    def _fetch_response(self, uri, headers=None, use_proxy=None):
        resp = None
        try:
            resp = self.call_storage_service(
                restutil.http_get,
                uri,
                headers=headers,
                use_proxy=use_proxy)

            host_plugin = self.get_host_plugin()

            if restutil.request_failed(resp):
                error_response = restutil.read_response_error(resp)
                msg = "Fetch failed from [{0}]: {1}".format(uri, error_response)
                logger.warn(msg)

                if host_plugin is not None:
                    host_plugin.report_fetch_health(uri,
                                                    is_healthy=not restutil.request_failed_at_hostplugin(resp),
                                                    source='WireClient',
                                                    response=error_response)
                raise ProtocolError(msg)
            else:
                if host_plugin is not None:
                    host_plugin.report_fetch_health(uri, source='WireClient')

        except (HttpError, ProtocolError, IOError) as e:
            logger.verbose("Fetch failed from [{0}]: {1}", uri, e)
            if isinstance(e, ResourceGoneError) or isinstance(e, InvalidContainerError):
                raise
        return resp

    # Type of update performed by _update_from_goal_state()
    class _UpdateType(object):
        # Update the Host GA Plugin client (Container ID and RoleConfigName)
        HostPlugin = 0
        # Update the full goal state only if the incarnation has changed
        GoalState = 1
        # Update the full goal state unconditionally
        GoalStateForced = 2

    def update_host_plugin_from_goal_state(self):
        """
        Fetches a new goal state and updates the Container ID and Role Config Name of the host plugin client
        """
        self._update_from_goal_state(WireClient._UpdateType.HostPlugin)

    def update_goal_state(self, forced=False):
        """
        Updates the goal state if the incarnation changed or if 'forced' is True
        """
        self._update_from_goal_state(
            WireClient._UpdateType.GoalStateForced if forced else WireClient._UpdateType.GoalState)

    def _update_from_goal_state(self, refresh_type):
        """
        Fetches a new goal state and updates the internal state of the WireClient according to the requested 'refresh_type'
        """
        max_retry = 3

        for retry in range(1, max_retry + 1):
            try:
                if refresh_type == WireClient._UpdateType.HostPlugin:
                    goal_state = GoalState.fetch_goal_state(self)
                    self._update_host_plugin(goal_state.container_id, goal_state.role_config_name)
                    return

                if self._goal_state is None or refresh_type == WireClient._UpdateType.GoalStateForced:
                    new_goal_state = GoalState.fetch_full_goal_state(self)
                else:
                    new_goal_state = GoalState.fetch_full_goal_state_if_incarnation_different_than(self, self._goal_state.incarnation)

                if new_goal_state is not None:
                    self._goal_state = new_goal_state
                    self._save_goal_state()
                    self._update_host_plugin(new_goal_state.container_id, new_goal_state.role_config_name)

                return

            except IOError as e:
                logger.warn("IOError processing goal state (attempt {0}/{1}) [{2}]", retry, max_retry, ustr(e))

            except ResourceGoneError:
                logger.info("Goal state is stale, re-fetching (attempt {0}/{1})", retry, max_retry)

            except ProtocolError as e:
                logger.verbose("ProtocolError processing goal state (attempt {0}/{1}) [{2}]", retry, max_retry, ustr(e))

            except Exception as e:
                logger.verbose("Exception processing goal state (attempt {0}/{1}) [{2}]", retry, max_retry, ustr(e))

        raise ProtocolError("Exceeded max retry updating goal state")

    def _update_host_plugin(self, container_id, role_config_name):
        if self._host_plugin is not None:
            self._host_plugin.update_container_id(container_id)
            self._host_plugin.update_role_config_name(role_config_name)

    def _save_goal_state(self):
        try:
            self.goal_state_flusher.flush(datetime.utcnow())

        except Exception as e:
            logger.warn("Failed to save the previous goal state to the history folder: {0}", ustr(e))

        try:
            local_file = os.path.join(conf.get_lib_dir(), INCARNATION_FILE_NAME)
            self.save_cache(local_file, self._goal_state.incarnation)

            def save_if_not_none(goal_state_property, file_name):
                file_path = os.path.join(conf.get_lib_dir(), file_name)

                if goal_state_property is not None and goal_state_property.xml_text is not None:
                    self.save_cache(file_path, goal_state_property.xml_text)

            # NOTE: Certificates are saved in Certificate.__init__
            save_if_not_none(self._goal_state, GOAL_STATE_FILE_NAME.format(self._goal_state.incarnation))
            save_if_not_none(self._goal_state.hosting_env, HOSTING_ENV_FILE_NAME)
            save_if_not_none(self._goal_state.shared_conf, SHARED_CONF_FILE_NAME)
            save_if_not_none(self._goal_state.ext_conf, EXT_CONF_FILE_NAME.format(self._goal_state.incarnation))
            save_if_not_none(self._goal_state.remote_access, REMOTE_ACCESS_FILE_NAME.format(self._goal_state.incarnation))

        except Exception as e:
            logger.warn("Failed to save the goal state to disk: {0}", ustr(e))

    def _set_host_plugin(self, new_host_plugin):
        if new_host_plugin is None:
            logger.warn("Setting empty Host Plugin object!")
        self._host_plugin = new_host_plugin

    def get_goal_state(self):
        if self._goal_state is None:
            raise ProtocolError("Trying to fetch goal state before initialization!")
        return self._goal_state

    def get_hosting_env(self):
        if self._goal_state is None:
            raise ProtocolError("Trying to fetch Hosting Environment before initialization!")
        return self._goal_state.hosting_env

    def get_shared_conf(self):
        if self._goal_state is None:
            raise ProtocolError("Trying to fetch Shared Conf before initialization!")
        return self._goal_state.shared_conf

    def get_certs(self):
        if self._goal_state is None:
            raise ProtocolError("Trying to fetch Certificates before initialization!")
        return self._goal_state.certs

    def get_ext_conf(self):
        if self._goal_state is None:
            raise ProtocolError("Trying to fetch Extension Conf before initialization!")
        return self._goal_state.ext_conf

    def get_ext_manifest(self, ext_handler):
        if self._goal_state is None:
            raise ProtocolError("Trying to fetch Extension Manifest before initialization!")

        local_file = MANIFEST_FILE_NAME.format(ext_handler.name, self.get_goal_state().incarnation)
        local_file = os.path.join(conf.get_lib_dir(), local_file)

        try:
            xml_text = self.fetch_manifest(ext_handler.versionUris)
            self.save_cache(local_file, xml_text)
            return ExtensionManifest(xml_text)
        except Exception as e:
            raise ExtensionDownloadError("Failed to retrieve extension manifest. Error: {0}".format(ustr(e)))

    def get_remote_access(self):
        if self._goal_state is None:
            raise ProtocolError("Trying to fetch Remote Access before initialization!")
        return self._goal_state.remote_access

    def get_gafamily_manifest(self, vmagent_manifest, goal_state):
        local_file = MANIFEST_FILE_NAME.format(vmagent_manifest.family, goal_state.incarnation)
        local_file = os.path.join(conf.get_lib_dir(), local_file)

        try:
            xml_text = self.fetch_manifest(vmagent_manifest.versionsManifestUris)
            fileutil.write_file(local_file, xml_text)
            return ExtensionManifest(xml_text)
        except Exception as e:
            raise ProtocolError("Failed to retrieve GAFamily manifest. Error: {0}".format(ustr(e)))

    def check_wire_protocol_version(self):
        uri = VERSION_INFO_URI.format(self.get_endpoint())
        version_info_xml = self.fetch_config(uri, None)
        version_info = VersionInfo(version_info_xml)

        preferred = version_info.get_preferred()
        if PROTOCOL_VERSION == preferred:
            logger.info("Wire protocol version:{0}", PROTOCOL_VERSION)
        elif PROTOCOL_VERSION in version_info.get_supported():
            logger.info("Wire protocol version:{0}", PROTOCOL_VERSION)
            logger.info("Server preferred version:{0}", preferred)
        else:
            error = ("Agent supported wire protocol version: {0} was not "
                     "advised by Fabric.").format(PROTOCOL_VERSION)
            raise ProtocolNotFoundError(error)

    def send_request_using_appropriate_channel(self, direct_func, host_func):
        # A wrapper method for all function calls that send HTTP requests. The purpose of the method is to
        # define which channel to use, direct or through the host plugin. For the host plugin channel,
        # also implement a retry mechanism.

        # By default, the direct channel is the default channel. If that is the case, try getting a response
        # through that channel. On failure, fall back to the host plugin channel.

        # When using the host plugin channel, regardless if it's set as default or not, try sending the request first.
        # On specific failures that indicate a stale goal state (such as resource gone or invalid container parameter),
        # refresh the goal state and try again. If successful, set the host plugin channel as default. If failed,
        # raise the exception.

        # NOTE: direct_func and host_func are passed as lambdas. Be careful about capturing goal state data in them as
        # they will not be refreshed even if a goal state refresh is called before retrying the host_func.

        if not HostPluginProtocol.is_default_channel():
            ret = None
            try:
                ret = direct_func()

                # Different direct channel functions report failure in different ways: by returning None, False,
                # or raising ResourceGone or InvalidContainer exceptions.
                if not ret:
                    logger.periodic_info(logger.EVERY_HOUR, "[PERIODIC] Request failed using the direct channel, "
                                                            "switching to host plugin.")
            except (ResourceGoneError, InvalidContainerError) as e:
                logger.periodic_info(logger.EVERY_HOUR, "[PERIODIC] Request failed using the direct channel, "
                                                        "switching to host plugin. Error: {0}".format(ustr(e)))

            if ret:
                return ret
        else:
            logger.periodic_info(logger.EVERY_HALF_DAY, "[PERIODIC] Using host plugin as default channel.")

        try:
            ret = host_func()
        except (ResourceGoneError, InvalidContainerError) as e:
            host_plugin = self.get_host_plugin()
            old_container_id = host_plugin.container_id
            old_role_config_name = host_plugin.role_config_name

            msg = "[PERIODIC] Request failed with the current host plugin configuration. " \
                  "ContainerId: {0}, role config file: {1}. Fetching new goal state and retrying the call." \
                  "Error: {2}".format(old_container_id, old_role_config_name, ustr(e))
            logger.periodic_info(logger.EVERY_SIX_HOURS, msg)

            self.update_host_plugin_from_goal_state()

            new_container_id = host_plugin.container_id
            new_role_config_name = host_plugin.role_config_name
            msg = "[PERIODIC] Host plugin reconfigured with new parameters. " \
                  "ContainerId: {0}, role config file: {1}.".format(new_container_id, new_role_config_name)
            logger.periodic_info(logger.EVERY_SIX_HOURS, msg)

            try:
                ret = host_func()
                if ret:
                    msg = "[PERIODIC] Request succeeded using the host plugin channel after goal state refresh. " \
                          "ContainerId changed from {0} to {1}, " \
                          "role config file changed from {2} to {3}.".format(old_container_id, new_container_id,
                                                                             old_role_config_name, new_role_config_name)
                    add_periodic(delta=logger.EVERY_SIX_HOURS,
                                 name=AGENT_NAME,
                                 version=CURRENT_VERSION,
                                 op=WALAEventOperation.HostPlugin,
                                 is_success=True,
                                 message=msg,
                                 log_event=True)

            except (ResourceGoneError, InvalidContainerError) as e:
                msg = "[PERIODIC] Request failed using the host plugin channel after goal state refresh. " \
                      "ContainerId changed from {0} to {1}, role config file changed from {2} to {3}. " \
                      "Exception type: {4}.".format(old_container_id, new_container_id, old_role_config_name,
                                                    new_role_config_name, type(e).__name__)
                add_periodic(delta=logger.EVERY_SIX_HOURS,
                             name=AGENT_NAME,
                             version=CURRENT_VERSION,
                             op=WALAEventOperation.HostPlugin,
                             is_success=False,
                             message=msg,
                             log_event=True)
                raise

        if not HostPluginProtocol.is_default_channel():
            logger.info("Setting host plugin as default channel from now on. "
                        "Restart the agent to reset the default channel.")
            HostPluginProtocol.set_default_channel(True)

        return ret

    def upload_status_blob(self):
        ext_conf = self.get_ext_conf()

        if ext_conf.status_upload_blob is None:
            # the status upload blob is in ExtensionsConfig so force a full goal state refresh
            self.update_goal_state(forced=True)
            ext_conf = self.get_ext_conf()

        if ext_conf.status_upload_blob is None:
            raise ProtocolNotFoundError("Status upload uri is missing")

        blob_type = ext_conf.status_upload_blob_type
        if blob_type not in ["BlockBlob", "PageBlob"]:
            blob_type = "BlockBlob"
            logger.verbose("Status Blob type is unspecified, assuming BlockBlob")

        try:
            self.status_blob.prepare(blob_type)
        except Exception as e:
            raise ProtocolError("Exception creating status blob: {0}", ustr(e))

        # Swap the order of use for the HostPlugin vs. the "direct" route.
        # Prefer the use of HostPlugin. If HostPlugin fails fall back to the
        # direct route.
        #
        # The code previously preferred the "direct" route always, and only fell back
        # to the HostPlugin *if* there was an error. We would like to move to
        # the HostPlugin for all traffic, but this is a big change.  We would like
        # to see how this behaves at scale, and have a fallback should things go
        # wrong. This is why we try HostPlugin then direct.
        try:
            host = self.get_host_plugin()
            host.put_vm_status(self.status_blob, ext_conf.status_upload_blob, ext_conf.status_upload_blob_type)
            return
        except ResourceGoneError:
            # refresh the host plugin client and try again on the next iteration of the main loop
            self.update_host_plugin_from_goal_state()
            return
        except Exception as e:
            # for all other errors, fall back to direct
            msg = "Falling back to direct upload: {0}".format(ustr(e))
            self.report_status_event(msg, is_success=True)

        try:
            if self.status_blob.upload(ext_conf.status_upload_blob):
                return
        except Exception as e:
            msg = "Exception uploading status blob: {0}".format(ustr(e))
            self.report_status_event(msg, is_success=False)

        raise ProtocolError("Failed to upload status blob via either channel")

    def report_role_prop(self, thumbprint):
        goal_state = self.get_goal_state()
        role_prop = _build_role_properties(goal_state.container_id,
                                           goal_state.role_instance_id,
                                           thumbprint)
        role_prop = role_prop.encode("utf-8")
        role_prop_uri = ROLE_PROP_URI.format(self.get_endpoint())
        headers = self.get_header_for_xml_content()
        try:
            resp = self.call_wireserver(restutil.http_post,
                                        role_prop_uri,
                                        role_prop,
                                        headers=headers)
        except HttpError as e:
            raise ProtocolError((u"Failed to send role properties: "
                                 u"{0}").format(e))
        if resp.status != httpclient.ACCEPTED:
            raise ProtocolError((u"Failed to send role properties: "
                                 u",{0}: {1}").format(resp.status,
                                                      resp.read()))

    def report_health(self, status, substatus, description):
        goal_state = self.get_goal_state()
        health_report = _build_health_report(goal_state.incarnation,
                                             goal_state.container_id,
                                             goal_state.role_instance_id,
                                             status,
                                             substatus,
                                             description)
        health_report = health_report.encode("utf-8")
        health_report_uri = HEALTH_REPORT_URI.format(self.get_endpoint())
        headers = self.get_header_for_xml_content()
        try:
            # 30 retries with 10s sleep gives ~5min for wireserver updates;
            # this is retried 3 times with 15s sleep before throwing a
            # ProtocolError, for a total of ~15min.
            resp = self.call_wireserver(restutil.http_post,
                                        health_report_uri,
                                        health_report,
                                        headers=headers,
                                        max_retry=30,
                                        retry_delay=15)
        except HttpError as e:
            raise ProtocolError((u"Failed to send provision status: "
                                 u"{0}").format(e))
        if restutil.request_failed(resp):
            raise ProtocolError((u"Failed to send provision status: "
                                 u",{0}: {1}").format(resp.status,
                                                      resp.read()))

    def send_event(self, provider_id, event_str):
        uri = TELEMETRY_URI.format(self.get_endpoint())
        data_format = ('<?xml version="1.0"?>'
                       '<TelemetryData version="1.0">'
                       '<Provider id="{0}">{1}'
                       '</Provider>'
                       '</TelemetryData>')
        data = data_format.format(provider_id, event_str)
        try:
            header = self.get_header_for_xml_content()
            # NOTE: The call to wireserver requests utf-8 encoding in the headers, but the body should not
            #       be encoded: some nodes in the telemetry pipeline do not support utf-8 encoding.
            resp = self.call_wireserver(restutil.http_post, uri, data, header)
        except HttpError as e:
            raise ProtocolError("Failed to send events:{0}".format(e))

        if restutil.request_failed(resp):
            logger.verbose(resp.read())
            raise ProtocolError(
                "Failed to send events:{0}".format(resp.status))

    def report_event(self, event_list):
        buf = {}
        # Group events by providerId
        for event in event_list.events:
            if event.providerId not in buf:
                buf[event.providerId] = ""
            event_str = event_to_v1(event)
            if len(event_str) >= MAX_EVENT_BUFFER_SIZE:
                details_of_event = [ustr(x.name) + ":" + ustr(x.value) for x in event.parameters if x.name in
                                    ["Name", "Version", "Operation", "OperationSuccess"]]
                logger.periodic_warn(logger.EVERY_HALF_HOUR,
                                     "Single event too large: {0}, with the length: {1} more than the limit({2})"
                                     .format(str(details_of_event), len(event_str), MAX_EVENT_BUFFER_SIZE))
                continue
            if len(buf[event.providerId] + event_str) >= MAX_EVENT_BUFFER_SIZE:
                self.send_event(event.providerId, buf[event.providerId])
                buf[event.providerId] = ""
            buf[event.providerId] = buf[event.providerId] + event_str

        # Send out all events left in buffer.
        for provider_id in list(buf.keys()):
            if len(buf[provider_id]) > 0:
                self.send_event(provider_id, buf[provider_id])

    def report_status_event(self, message, is_success):
        from azurelinuxagent.common.event import report_event, \
            WALAEventOperation

        report_event(op=WALAEventOperation.ReportStatus,
                     is_success=is_success,
                     message=message,
                     log_event=not is_success)

    def get_header(self):
        return {
            "x-ms-agent-name": "WALinuxAgent",
            "x-ms-version": PROTOCOL_VERSION
        }

    def get_header_for_xml_content(self):
        return {
            "x-ms-agent-name": "WALinuxAgent",
            "x-ms-version": PROTOCOL_VERSION,
            "Content-Type": "text/xml;charset=utf-8"
        }

    def get_header_for_cert(self):
        trans_cert_file = os.path.join(conf.get_lib_dir(),
                                       TRANSPORT_CERT_FILE_NAME)
        content = self.fetch_cache(trans_cert_file)
        cert = get_bytes_from_pem(content)
        return {
            "x-ms-agent-name": "WALinuxAgent",
            "x-ms-version": PROTOCOL_VERSION,
            "x-ms-cipher-name": "DES_EDE3_CBC",
            "x-ms-guest-agent-public-x509-cert": cert
        }

    def get_host_plugin(self):
        if self._host_plugin is None:
            goal_state = GoalState.fetch_goal_state(self)
            self._set_host_plugin(HostPluginProtocol(self.get_endpoint(),
                                                     goal_state.container_id,
                                                     goal_state.role_config_name))
        return self._host_plugin

    def has_artifacts_profile_blob(self):
        ext_conf = self.get_ext_conf()
        return ext_conf and not \
            textutil.is_str_none_or_whitespace(ext_conf.artifacts_profile_blob)

    def get_artifacts_profile_through_host(self, blob):
        host = self.get_host_plugin()
        uri, headers = host.get_artifact_request(blob)
        profile = self.fetch(uri, headers, use_proxy=False)
        return profile

    def get_artifacts_profile(self):
        artifacts_profile = None

        if self.has_artifacts_profile_blob():
            blob = self.get_ext_conf().artifacts_profile_blob
            direct_func = lambda: self.fetch(blob)
            # NOTE: the host_func may be called after refreshing the goal state, be careful about any goal state data
            # in the lambda.
            host_func = lambda: self.get_artifacts_profile_through_host(blob)

            logger.verbose("Retrieving the artifacts profile")

            try:
                profile = self.send_request_using_appropriate_channel(direct_func, host_func)
            except Exception as e:
                logger.warn("Exception retrieving artifacts profile: {0}".format(ustr(e)))
                return None

            if not textutil.is_str_empty(profile):
                logger.verbose("Artifacts profile downloaded")
                try:
                    artifacts_profile = InVMArtifactsProfile(profile)
                except Exception:
                    logger.warn("Could not parse artifacts profile blob")
                    msg = "Content: [{0}]".format(profile)
                    logger.verbose(msg)

                    from azurelinuxagent.common.event import report_event, WALAEventOperation
                    report_event(op=WALAEventOperation.ArtifactsProfileBlob,
                                 is_success=False,
                                 message=msg,
                                 log_event=False)

        return artifacts_profile

class VersionInfo(object):
    def __init__(self, xml_text):
        """
        Query endpoint server for wire protocol version.
        Fail if our desired protocol version is not seen.
        """
        logger.verbose("Load Version.xml")
        self.parse(xml_text)

    def parse(self, xml_text):
        xml_doc = parse_doc(xml_text)
        preferred = find(xml_doc, "Preferred")
        self.preferred = findtext(preferred, "Version")
        logger.info("Fabric preferred wire protocol version:{0}",
                    self.preferred)

        self.supported = []
        supported = find(xml_doc, "Supported")
        supported_version = findall(supported, "Version")
        for node in supported_version:
            version = gettext(node)
            logger.verbose("Fabric supported wire protocol version:{0}",
                           version)
            self.supported.append(version)

    def get_preferred(self):
        return self.preferred

    def get_supported(self):
        return self.supported

class ExtensionManifest(object):
    def __init__(self, xml_text):
        if xml_text is None:
            raise ValueError("ExtensionManifest is None")
        logger.verbose("Load ExtensionManifest.xml")
        self.pkg_list = ExtHandlerPackageList()
        self._parse(xml_text)

    def _parse(self, xml_text):
        xml_doc = parse_doc(xml_text)
        self._handle_packages(findall(find(xml_doc,
                                           "Plugins"),
                                      "Plugin"),
                              False)
        self._handle_packages(findall(find(xml_doc,
                                           "InternalPlugins"),
                                      "Plugin"),
                              True)

    def _handle_packages(self, packages, isinternal):
        for package in packages:
            version = findtext(package, "Version")

            disallow_major_upgrade = findtext(package,
                                              "DisallowMajorVersionUpgrade")
            if disallow_major_upgrade is None:
                disallow_major_upgrade = ''
            disallow_major_upgrade = disallow_major_upgrade.lower() == "true"

            uris = find(package, "Uris")
            uri_list = findall(uris, "Uri")
            uri_list = [gettext(x) for x in uri_list]
            pkg = ExtHandlerPackage()
            pkg.version = version
            pkg.disallow_major_upgrade = disallow_major_upgrade
            for uri in uri_list:
                pkg_uri = ExtHandlerVersionUri()
                pkg_uri.uri = uri
                pkg.uris.append(pkg_uri)

            pkg.isinternal = isinternal
            self.pkg_list.versions.append(pkg)

# Do not extend this class
class InVMArtifactsProfile(object):
    """
    deserialized json string of InVMArtifactsProfile.
    It is expected to contain the following fields:
    * inVMArtifactsProfileBlobSeqNo
    * profileId (optional)
    * onHold (optional)
    * certificateThumbprint (optional)
    * encryptedHealthChecks (optional)
    * encryptedApplicationProfile (optional)
    """

    def __init__(self, artifacts_profile):
        if not textutil.is_str_empty(artifacts_profile):
            self.__dict__.update(parse_json(artifacts_profile))

    def is_on_hold(self):
        # hasattr() is not available in Python 2.6
        if 'onHold' in self.__dict__:
            return str(self.onHold).lower() == 'true'
        return False


#!/usr/bin/env python
# _*_ coding: utf-8 _*_

import serial
import time
from functools import reduce

MAX_ANGLE = 270
MIN_ANGLE = 0

def ListToStr(data):
    strings = "".join(map(str, data))
    return strings

def DataToStr(x, y):
    if isinstance(x, str):
        return x + chr(y)
    else:
        return chr(x) + chr(y)

con = None

def SetSerial(port="/dev/ttyUSB0"):
    global con
    con = serial.Serial(port,
                        115200,
                        parity=serial.PARITY_EVEN,
                        stopbits=serial.STOPBITS_ONE,
                        timeout=1)

class Servo:
    def __init__(self, con, addr, angle=None):
        self.con = con
        self.addr = int(addr)
        self.angle = angle
        self.bef_angle = 0
        if angle != None:
            self.Pos(angle)

    def GetId(self):
        return self.addr

    def Pos(self, angle):
        if angle > 270: angle = 270
        if angle < 0:   angle = 0
        # self.bef_angle = self.angle
        self.angle = angle
        angle = int(3500 + int((8000 * angle) / 270))
        data = reduce(DataToStr, [(0x80 | int(self.addr)), angle >> 7, angle & 0x7f])
        self.con.write(data)

        time.sleep(0.0001)
        rcv = [ord(i) for i in self.con.read(6)]
        bef_angle = rcv[-2:]
        self.bef_angle = 270.0*((bef_angle[0] << 7 | bef_angle[1] & 0x7f) - 3500)/8000
        self.con.flushInput()
        return self.bef_angle

    def GetPos(self):
        return self.angle

    def Speed(self, speed):
        self.con.flushInput()
        self.con.write(ListToStr([(0xc0 | self.addr), 0x03, speed]))
        rcv = [ord(i) for i in self.con.read(6)]
        return rcv[-1]

    def GetSpeed(self):
        self.con.flushInput()
        self.con.write(ListToStr([(0xa0 | self.addr), 0x03]))
        rcv = [ord(i) for i in self.con.read(5)]
        return rcv[-1]

    def Stretch(self, stretch):
        self.con.flushInput()
        self.con.write(ListToStr([(0xc0 | self.addr), 0x02, stretch]))
        rcv = [ord(i) for i in self.con.read(6)]
        return rcv[-1]

    def GetStretch(self):
        self.con.flushInput()
        self.con.write(ListToStr([(0xc0 | self.addr), 0x02]))
        rcv = [ord(i) for i in self.con.read(5)]
        return rcv[-1]


from django.shortcuts import render
from rest_framework.views import APIView
from django.http import JsonResponse, HttpResponse
from .models import TopicHub, Hub, PlaylistOfHub, HubOfTopic
from .serializers import TopicHubSerializers, HubSerializers
from random import choice
from playlists.models import Playlists, ArtistOfPlaylist
from playlists.serializers import ArtistSerializers, PlaylistSerializers
from rest_framework.permissions import AllowAny

# Create your views here.
class TopicHubAPIView(APIView):
    permission_classes = [AllowAny]

    def post(self, reuqest):
        data = reuqest.data
        if not data:
            return HttpResponse(status=404)
        topics = TopicHub.objects.create(
            title=data["title"]
        )

        topics.save()
        serializers = TopicHubSerializers(topics).data

        return JsonResponse(serializers, safe=False)

    def get(self, request):
        topics = TopicHub.objects.all()
        if not topics.exists():
            return HttpResponse(status=404)

        serializer = TopicHubSerializers(topics, many=True)
        return JsonResponse(serializer.data, safe=False)

class HubAPIView(APIView):
    permission_classes = [AllowAny]

    def post(self, request):
        data = request.data
        if not data:
            return HttpResponse(status=404)

        hubs = Hub.objects.create(
            cover=data["cover"],
            thumbnail=data["thumbnail"],
            thumbnail_has_text=data["thumbnailHasText"],
            thumbnail_r=data["thumbnailR"],
            title=data["title"],
            description=data["description"]
        )
        hubs.save()

        serializers = HubSerializers(hubs).data

        return JsonResponse(serializers, safe=False)

    def get(self, request):
        hubs = Hub.objects.all()

        if not hubs.exists():
            return HttpResponse(status=404)

        serializer = HubSerializers(hubs, many=True)
        return JsonResponse(serializer.data, safe=False)

class HubPageAPIView(APIView):
    permission_classes = [AllowAny]

    def get(self, request):
        # hub highlight
        topic_highlight = TopicHub.objects.filter(title="Nổi bật")
        hub_topicHighlight = HubOfTopic.objects.filter(topic=topic_highlight[0])
        data_hub_highlight = []
        for hub_topic in hub_topicHighlight:
            data_hub_highlight.append(HubSerializers(hub_topic.hub).data)

        # banner
        hub = Hub.objects.all()
        banner = []
        for hub in hub:
            banner.append(HubSerializers(hub).data)
        banner_data = choice(banner)

        # hub nations
        topic_nation = TopicHub.objects.filter(title="Quốc gia")
        hub_topicNation = HubOfTopic.objects.filter(topic=topic_nation[0])
        data_hub_nations = []
        for hub_topic in hub_topicNation:
            data_hub_nations.append(HubSerializers(hub_topic.hub).data)

        # Top topic
        topic_top = TopicHub.objects.filter(title="Top topic")
        hub_topTopic = HubOfTopic.objects.filter(topic=topic_top[0])
        data_hub_top = []
        for hub_topic in hub_topTopic:
            hub_playlist = PlaylistOfHub.objects.filter(hub=hub_topic.hub)
            playlist_data = []
            for art_pl in hub_playlist:
                artist_playlist = ArtistOfPlaylist.objects.filter(playlist=art_pl.playlist)
                data_artist = []
                for artist in artist_playlist:
                    data_artist.append(ArtistSerializers(artist.artist).data)
                playlist_dict = dict(PlaylistSerializers(art_pl.playlist).data, **{"artists": data_artist})
                playlist_data.append(playlist_dict)
            data_hut = dict(HubSerializers(hub_topic.hub).data, **{"playlists": playlist_data})
            data_hub_top.append(data_hut)

        # topic thịnh hành
        topic_popular = TopicHub.objects.filter(title="Thịnh hành")
        hub_topicPopular = HubOfTopic.objects.filter(topic=topic_popular[0])
        data_hub_popular = []
        for hub_topic in hub_topicPopular:
            hub_playlist = PlaylistOfHub.objects.filter(hub=hub_topic.hub)
            playlist_data = []
            for art_pl in hub_playlist:
                artist_playlist = ArtistOfPlaylist.objects.filter(playlist=art_pl.playlist)
                data_artist = []
                for artist in artist_playlist:
                    data_artist.append(ArtistSerializers(artist.artist).data)
                playlist_dict = dict(PlaylistSerializers(art_pl.playlist).data, **{"artists": data_artist})
                playlist_data.append(playlist_dict)
            data_hut = dict(HubSerializers(hub_topic.hub).data, **{"playlists": playlist_data})
            data_hub_popular.append(data_hut)

        # Response trả về
        res = {
            "err": 0,
            "msg": "Success",
            "data": {
                "sectionId": "hub",
                "banners": banner_data,
                "featured": {
                    "title": "Nổi bật",
                    "items": data_hub_highlight
                },
                "nations": {
                    "title": "Quốc gia",
                    "items": data_hub_nations
                },
                "topTopic": {
                    "title": "Top topic",
                    "items": data_hub_top,
                },
                "topic": {
                    "title": "Top topic",
                    "items": data_hub_top[0:8],
                },
                "popular": {
                    "title": "Thịnh hành",
                    "items": data_hub_popular,
                }
            }
        }
        return JsonResponse(res, safe=False)


import random
import math
import collections
import datetime
import math

def get_user_brands(reco_file_path, end_date):
    '''用户商品倒插表'''
    beta = 0.5
    reco_file=open(reco_file_path,"r")
    reco_file.readline()
    brand_pref = collections.defaultdict(lambda :0)
    for line in reco_file:
        line = line.strip()
        user_name, brand_id, behavior_id, date_str = tuple(line.split(","))

        cur_date = datetime.date(2013,
            int(date_str[0:date_str.find('月')]),
            int(date_str[date_str.find('月')+1:-1]))
        # 流行度
        popularity = 1 / (1 + beta * (end_date - cur_date).days)
        brand_pref[brand_id] += popularity
        # print(brand_pref)

    pref_thres = 1.5
    # print(brand_pref)
    # brand_pref = {brand_id:pref for brand_id, pref in brand_pref.items() if pref > pref_thres }
    item_pool = [brand_id for brand_id, pref in brand_pref.items() if pref > pref_thres]

    reco_file=open(reco_file_path,"r")
    reco_file.readline()

    user_brands = collections.defaultdict(lambda :
        collections.defaultdict(lambda :0))

    for line in reco_file:
        line = line.strip()
        user_name, brand_id, behavior_id, date_str = tuple(line.split(","))

        # if behavior_id == '1' or behavior_id == '0':
        if behavior_id == '1':
            user_brands[user_name][brand_id] += 1
        else:
            user_brands[user_name][brand_id] += 0

    # for user_name, brand_ids in user_brands.items():
    #     print(user_name, brand_ids)

    return (user_brands, item_pool)

def init_model(user_brands, K):
    '''初始化LFM'''

    P = {key:[random.random()/math.sqrt(K) for x in range(K)]
        for key in user_brands.keys()}

    brand_keys = set()
    for user_name, brand_id_dict in user_brands.items():
        for brand_id in brand_id_dict:
            brand_keys.add(brand_id)

    Q = {key:[random.random()/math.sqrt(K) for x in range(K)]
        for key in brand_keys}

    return (P, Q)

def preference(user, brand, p, q):
    '''用户对品牌的兴趣'''
    return sum(p[user][f]*q[brand][f] for f in range(0,len(p[user])))

def select_samples(items, item_pool):
    '''随机选择负样本'''

    ret = {key:1 for key, buy_time in items.items() if buy_time >= 1}

    samples_size = 10 * len(items)
    for i in range(0, len(ret) * 3):
        item = item_pool[random.randint(0,len(item_pool)-1)]
        if item in ret:
            continue
        ret[item] = 0
        if len(ret) > samples_size:
            break
    return ret

def lfm(user_brands, item_pool, K, step_time, alpha, lamb):
    '''隐语义模型'''
    P, Q = init_model(user_brands, K)

    for step in range(0, step_time):
        for user, brands in user_brands.items():
            samples = select_samples(brands, item_pool)
            for item, rui in samples.items():
                eui = rui - preference(user, item, P, Q)
                for f in range(0, K):
                    P[user][f] += alpha * (eui * Q[item][f] - lamb * P[user][f])
                    Q[item][f] += alpha * (eui * P[user][f] - lamb * Q[item][f])
        alpha *= 0.9
        # print(Q['7868'])
    # 归一化
    # for key, values in P.items():
    #     m = math.sqrt(sum(value*value for value in values))
    #     P[key] = [value/m for value in values]

    # for key, values in Q.items():
    #     m = math.sqrt(sum(value*value for value in values))
    #     Q[key] = [value/m for value in values]

    # print(Q['7868'])

    return (P, Q)

#推荐
def recommend(user, P, Q):

    rank = {brand:preference(user, brand, P, Q) for brand, q in Q.items()}

    return rank

if __name__ == '__main__':

    ali_file_path = "Z:\CodeSpace\Python\天池\data\\t_alibaba_data.csv"
    data_file_path = "Z:\CodeSpace\Python\天池\data\data_4_7_15.txt"
    result_file_path = "Z:\CodeSpace\Python\天池\\result\lfm_7_15.txt"
    result_file = open(result_file_path, 'w')

    # data_file_path = "Z:\CodeSpace\Python\天池\data\\test.txt"

    K, step_time, alpha, lamb = 15, 200, 0.04, 0.02

    user_brands, item_pool = get_user_brands(data_file_path, datetime.date(2013, 7, 15))

    P, Q = lfm(user_brands, item_pool, K, step_time, alpha, lamb)

    rank_thres = 0.6

    print("recommending...")

    for user_name in user_brands:
        rank = recommend(user_name, P, Q)

        rank = {brand:pref for brand, pref in rank.items() if pref > rank_thres}
        # print(rank)

        commit_brand = sorted(rank.items(), key=lambda d:d[1], reverse = True)[:11]
        # if len(commit_brand) > 1 and commit_brand[0][0] == '27791' and commit_brand[0][1] < 0.7:
        #     commit_brand = commit_brand[1:11]
        # else:
        #     commit_brand = commit_brand[:10]
        commit_brand = [brand for brand, pref in commit_brand]

        if len(commit_brand) < 1:
            continue

        result_file.write(user_name + '\t' + ','.join(commit_brand) + '\n')
        # print(sorted_rank[:10])


#-----------------------------------------------
# Import
#-----------------------------------------------

from __future__ import print_function
import pickle
import os.path
from googleapiclient.discovery import build
from google_auth_oauthlib.flow import InstalledAppFlow
from google.auth.transport.requests import Request

import pandas as pd
import os, csv
import re
import enchant

from operator import itemgetter

#-----------------------------------------------
# Global variables
#-----------------------------------------------

BAU_folder = "G:/My Drive/Learning and Development/L&D Systems Training/BAU and Aris Processes and Activities" # master folder
d = enchant.Dict("en_US")
funct_excel_files = 	[]
rasci_excel_files = 	[]
manual_excel_files = 	[]

#-----------------------------------------------
# Definitions
#-----------------------------------------------

def documents_setup():

	# If modifying these scopes, delete the file token.pickle.
	print(" - Getting Scopes")
	SCOPES = 	[
					'https://www.googleapis.com/auth/presentations',
					'https://www.googleapis.com/auth/spreadsheets'
				]

	# document ID's
	print(" - Getting document ID's")
	BAU_SHEET_ID		= '19R5DH0wZWuntZ81RR1Pd6eUxqDbkCutKp5uW9lipcT0'

	# sheet ranges
	print(" - Getting sheet ranges")
	BAU_SHEET_RANGE 	= '*DC!A1:BL'

	creds = None

	print(" - Checking for token (pickle) ")
	if os.path.exists('token.pickle'):
		with open('token.pickle', 'rb') as token:
			creds = pickle.load(token)
	# If there are no (valid) credentials available, let the user log in.
	if not creds or not creds.valid:
		if creds and creds.expired and creds.refresh_token:
			creds.refresh(Request())
		else:
			flow = InstalledAppFlow.from_client_secrets_file(
				'client_secret.json', SCOPES)
			creds = flow.run_local_server(port=0)
		# Save the credentials for the next run
		with open('token.pickle', 'wb') as token:
			pickle.dump(creds, token)

	service 		= build('slides', 'v1', credentials=creds)
	service_sheet 	= build('sheets', 'v4', credentials=creds)

	# Call the Slides API
	print(" - Calling document API's")
	global bau_sheet_result

	bau_sheet_result 	= service_sheet.spreadsheets().values().get(spreadsheetId=BAU_SHEET_ID, range=BAU_SHEET_RANGE).execute()

def get_bau_values():

	bau_rows 	= bau_sheet_result.get('values', [])

	# get the number for each column by using their name
	L4Name_col_num 				= bau_rows[0].index("L4Name")
	ArtifactNum_col_num			= bau_rows[0].index("Artifact Number (Auto)")
	System_col_num				= bau_rows[0].index("System")
	TransactionCode_col_num		= bau_rows[0].index("Txs Code")
	DC_Returns_col_num 			= bau_rows[0].index("RETURNS_DC")
	RC_Returns_CEN_col_num		= bau_rows[0].index("RETURNS_RC (CEN)")
	RC_Returns_BRK_col_num		= bau_rows[0].index("BRK RC")

	global BAU_L4Name
	global BAU_ArtifactNum
	global BAU_System
	global BAU_TransactionCode
	global BAU_DC_Returns
	global BAU_RC_Returns_CEN
	global BAU_RC_Returns_BRK

	BAU_L4Name 			= [row[L4Name_col_num] for row in bau_rows]
	BAU_ArtifactNum 	= [row[ArtifactNum_col_num] for row in bau_rows]
	BAU_System			= [row[System_col_num] for row in bau_rows]
	BAU_TransactionCode = [row[TransactionCode_col_num] for row in bau_rows]
	BAU_DC_Returns		= [row[DC_Returns_col_num] for row in bau_rows]
	BAU_RC_Returns_CEN	= [row[RC_Returns_CEN_col_num] for row in bau_rows]
	BAU_RC_Returns_BRK	= [row[RC_Returns_BRK_col_num] for row in bau_rows]

#..................................................................
# Get files
def getFiles():

	xlsx_files_count = 0
	gsheet_files_count = 0

	# go through all the files in the folder
	for path, subdirs, files in os.walk(BAU_folder):
		for name in files:

			# find all excel files
			if name.endswith(".xlsx"):

				xlsx_files_count += 1

				# get all files that are in Functions folder
				if(path.split("\\")[-1] == "Functions"):

					# get exported files
					if name.endswith("Manual Copy Paste.xlsx") == False:
						funct_excel_files.append(os.path.join(path, name))

					# get manual copy-paste files
					else:
						manual_excel_files.append(os.path.join(path, name))

				# get all files that are in RASCI folder
				if(path.split("\\")[-1] == "RASCI"):
					rasci_excel_files.append(os.path.join(path, name))

			# get all google sheet files
			if name.endswith(".gsheet"):
				gsheet_files_count += 1

	print("Function files found: " + str(len(funct_excel_files)))
	print("Manual copy-paste files fount: " + str(len(manual_excel_files)))
	print("RASCI files found: " + str(len(rasci_excel_files)))
	print(xlsx_files_count)
	print(gsheet_files_count)

L3_Number = []
L3_function_name = []
L3_description = []

L4_function = []
L4_description = []
row_counter = 0

# ====================================
# MANUAL COPY PASTE
# ====================================

manual_list = []
not_transaction_code = "HEADERDETAIL,MARKOUT,EQUIPUNITID,FLOWTHRU,PICKLIST,WRICEF"

def readL4Manual_Excel_Sheet1(manual_excel_files):
	print("Reading Manual Copy-Paste Excel")

	for manual_path in manual_excel_files:

		split_path = manual_path.split("\\")[-1].split(" ")
		names = ["L3Name", "L3Num", "L3Desc", "L4Name", "L4Desc", "L4Roles", "L4System"]
		df = pd.read_excel(manual_path, sheet_name="Sheet1", engine='openpyxl', usecols="A:D", header=None, names=names)

		row_activities = 2
		row_name = 1
		for row in df.iloc[:,0].tolist():
			if row=="Activities" or row=="[-]":
				break
			else:
				row_activities += 1

		for row in df.iloc[:,0].tolist():
			if row=="Name":
				break
			else:
				row_name += 1

		# L3 Information
		L3Name = " ".join(split_path[2:]).split("-")[0]
		L3Num = split_path[1].replace("0", "").replace("-", ".")
		L3Desc = df.iloc[row_activities-3][0]

		for i in range(row_name,len(df)):#list((df.iloc[row_counter:len(df)])):
			row = list(df.iloc[i])

			RC = False
			DC = False
			Overview = ""
			ProcessSteps = ""
			Note = ""
			Other = ""
			Transaction_code = ""
			switch = "Overview"

			# print(row)
			if (row[0]!=""):
				# print(i)

				try:
					L4Name = row[0]

					if "RC" in L4Name:
						RC = True
					if "DC" in L4Name:
						DC = True
				except:
					L4Name = ""

				try:
					L4Desc = row[1]
					print("-----------------------------------------")

					# process_list = ["process steps:", "process steps", "Process steps:", "Process steps"]
					# note_list = ["Note:", "Note", "note"]
					# other_list =

					for line in L4Desc.splitlines():#.split(char(10)):
						line_clean = line.strip().replace(":", "").lower()

						for wrong_char in "#.,/_+()'':-*+&[ ];":
							line_clean = line_clean.replace(wrong_char, "").replace('"', "").replace("CTRL+", "").strip().lower()
						# print(line_clean)

						if line_clean == "overview":
							switch == "Overview"

						if line_clean[:12] == "processsteps":
							print(line_clean)
							# print("Process Step")
							switch = "Process Steps"

						if line_clean == "note":
							switch = "Note"

						if line_clean == "other":
							switch = "Other"

						# get transaction codes
						for word in line.split(" "):
							clean_word = word
							for wrong_char in " #.,/_+()'':-*+&[];":
								clean_word = clean_word.replace(wrong_char, "").replace('"', "").replace("CTRL+", "").strip()

							# remove words that are not transaction codes
							for bad_word in not_transaction_code.split(","):
								if bad_word in clean_word or clean_word in bad_word:

									clean_word = ""

									# print(bad_word)

							if("WRICEF58" == clean_word):
								print(clean_word)

							if clean_word == clean_word.upper() and clean_word != "" and clean_word.isdigit() == False and len(clean_word) >= 5 and d.check(clean_word) == False and clean_word not in Transaction_code and clean_word not in not_transaction_code:
								Transaction_code += clean_word + ", "

						if switch == "Overview":
							Overview += line + chr(10)
						if switch == "Process Steps":
							ProcessSteps += line + chr(10)
						if switch == "Note":
							Note += line + chr(10)
						if switch == "Other":
							Other += line + chr(10)

					if "RC" in L4Desc:
						RC = True
					if "DC" in L4Desc:
						DC = True

							# process_line.append(line)

					# for PS in process_list:
					# 	if PS in L4Desc:
					# 		ProcessSteps = re.split(PS,L4Desc)[-1].lstrip("\n").lstrip(" ").lstrip("\n").lstrip(" ")
					# 		Overview = re.split(PS,L4Desc)[0]
					# 		break

					# for note in note_list:
					# 	if note in L4Desc:
					# 		Note = re.split(grp,L4Desc)[-1].lstrip("\n").lstrip(" ").lstrip("\n").lstrip(" ")
					# 		Overview = re.split(grp,L4Desc)[0]
					# 		break

					# if ProcessSteps == "":
					# 	Overview = L4Desc

					# # seperate and get Process Steps and Overview
					# if "Process Steps:" in L4Desc:
					# 	ProcessSteps = re.split(r'Process Steps:',L4Desc)[-1].lstrip("\n").lstrip(" ").lstrip("\n").lstrip(" ")
					# 	Overview = re.split(r'Process Steps:|Process Steps',L4Desc)[0]
					# elif "Process Steps" in L4Desc:
					# 	ProcessSteps = re.split(r'Process Steps',L4Desc)[-1].lstrip("\n").lstrip(" ").lstrip("\n").lstrip(" ")
					# 	Overview = re.split(r'Process Steps:|Process Steps',L4Desc)[0]
					# elif "Process steps:" in L4Desc:
					# 	ProcessSteps = re.split(r'Process steps:',L4Desc)[-1].lstrip("\n").lstrip(" ").lstrip("\n").lstrip(" ")
					# 	Overview = re.split(r'Process Steps:|Process Steps|Process steps:',L4Desc)[0]
					# elif "Process steps" in L4Desc:
					# 	ProcessSteps = re.split(r'Process steps',L4Desc)[-1].lstrip("\n").lstrip(" ").lstrip("\n").lstrip(" ")
					# 	Overview = re.split(r'Process Steps:|Process Steps|Process steps',L4Desc)[0]
					# else:
					# 	print("There is NO PS")
					# 	Overview = L4Desc

					# check for RC and DC

				except:
					L4Desc = ""

				try:
					L4Roles = row[2]
				except:
					L4Roles = ""

				try:
					L4System = row[3]
				except:
					L4System = ""

				bau_num = ""
				num_counter = 0
				for num in BAU_L4Name:
					if num.lower().strip().replace(' ', '') == L4Name.lower().strip().replace(' ', ''):
						bau_num = 		BAU_ArtifactNum[num_counter]
						BAU_L4Name_value 			= BAU_L4Name[num_counter]
						BAU_ArtifactNum_value		= BAU_ArtifactNum[num_counter]
						BAU_System_value			= BAU_System[num_counter]
						BAU_TransactionCode_value 	= BAU_TransactionCode[num_counter]
						BAU_DC_Returns_value		= BAU_DC_Returns[num_counter]
						BAU_RC_Returns_CEN_value	= BAU_RC_Returns_CEN[num_counter]
						BAU_RC_Returns_BRK_value	= BAU_RC_Returns_BRK[num_counter]
						break

					num_counter += 1
				if bau_num == "":
					# print(L4Name.lower().strip().replace(' ', ''))
					pass

				# leadingZero_bau_num = []
				# for single_num in bau_num:
				if "-" in bau_num:
					leadingZero_bau_num = (bau_num.split("-")[0] + "-" + bau_num.split("-")[1].zfill(2))
				elif bau_num == "":
					leadingZero_bau_num = "NA"
				else:
					leadingZero_bau_num = bau_num

				# print(leadingZero_bau_num)

				# 	print
				# try:
				# 	bau_num = BAU_ArtifactNum[BAU_L4Name.index(L4Name.strip())]
				# except:
				# 	bau_num = "~ " + str(L4Name)

				manual_list.append([
					L3Name,
					L3Num,
					L3Desc,
					L4Name,
					L4Desc,
					Overview,
					ProcessSteps,
					Note,
					Other,
					Transaction_code[:-2],
					RC,
					DC,
					L4Roles,
					L4System,
					leadingZero_bau_num,
					BAU_L4Name_value,
					BAU_ArtifactNum_value,
					BAU_System_value,
					BAU_TransactionCode_value,
					BAU_DC_Returns_value,
					BAU_RC_Returns_CEN_value,
					BAU_RC_Returns_BRK_value
					])

	global manual_list_sorted
	manual_list_sorted = sorted(manual_list, key=itemgetter(14))

# ====================================
# Start
# ====================================

documents_setup()
get_bau_values()
getFiles()
# readL3Function_Excel_SheetA(funct_excel_files)
readL4Manual_Excel_Sheet1(manual_excel_files)

# ====================================
# Write to CSV
# ====================================

# write to csv
with open('ARIS_manual_list.csv', 'w', newline='', encoding='utf-8') as f:
	print("Exporting to CSV")

	fieldnames =[
	"L3Name",
	"L3Num",
	"L3Desc",
	"L4Name",
	"L4Desc",
	"Overview",
	"Process Steps",
	"Note",
	"Other",
	"Transaction Code",
	"RC",
	"DC",
	"L4Roles",
	"L4System",
	"BAU Numbers",
	"BAU_L4Name_value",
	"BAU_ArtifactNum_value",
	"BAU_System_value",
	"BAU_TransactionCode_value",
	"BAU_DC_Returns_value",
	"BAU_RC_Returns_CEN_value",
	"BAU_RC_Returns_BRK_value"
	]

	thewriter = csv.DictWriter(f, fieldnames=fieldnames)

	counter = 0

	# write headers
	thewriter.writerow({
		"L3Name":"L3Name",
		"L3Num":"L3Num",
		"L3Desc":"L3Desc",
		"L4Name":"L4Name",
		"L4Desc":"L4Desc",
		"Overview":"Overview",
		"Process Steps":"Process Steps",
		"Note":"Note", "Other":"Other",
		"Transaction Code":"Transaction Code",
		"RC":"RC",
		"DC":"DC",
		"L4Roles":"L4Roles",
		"L4System":"L4System",
		"BAU Numbers":"BAU Numbers",
		"BAU_L4Name_value":"BAU_L4Name_value",
		"BAU_ArtifactNum_value":"BAU_ArtifactNum_value",
		"BAU_System_value":"BAU_System_value",
		"BAU_TransactionCode_value":"BAU_TransactionCode_value",
		"BAU_DC_Returns_value":"BAU_DC_Returns_value",
		"BAU_RC_Returns_CEN_value":"BAU_RC_Returns_CEN_value",
		"BAU_RC_Returns_BRK_value":"BAU_RC_Returns_BRK_value"
		})

	for i in manual_list_sorted:
		# for x in i:
		thewriter.writerow({
			"L3Name":i[0],
			"L3Num":i[1],
			"L3Desc":i[2],
			"L4Name":i[3],
			"L4Desc":i[4],
			"Overview":i[5],
			"Process Steps":i[6],
			"Note":i[7],
			"Other":i[8],
			"Transaction Code":i[9],
			"RC":i[10],
			"DC":i[11],
			"L4Roles":i[12],
			"L4System":i[13],
			"BAU Numbers":i[14],
			"BAU_L4Name_value":i[15],
			"BAU_ArtifactNum_value":i[16],
			"BAU_System_value":i[17],
			"BAU_TransactionCode_value":i[18],
			"BAU_DC_Returns_value":i[19],
			"BAU_RC_Returns_CEN_value":i[20],
			"BAU_RC_Returns_BRK_value":i[21]
			})


M = 998244353

from collections import Counter

N = 1 + 10 ** 6
v = [False] * N
spf = [1] * N

for i in range(2, N, 2):
    spf[i] = 2

for i in range(3, N, 2):
    if not v[i]:
        spf[i] = i
        j = i
        while j * i < N:
            if not v[j * i]:
                v[j * i] = True
                spf[j * i] = i
            j += 2

def facts(n, mod):
    res = [1] * (n + 1)
    ctr = Counter()
    for i in range(1, n + 1):
        a = n - i + 1
        b = i
        while a > 1:
            ctr[spf[a]] += 1
            a //= spf[a]
        while b > 1:
            ctr[spf[b]] -= 1
            b //= spf[b]
        for p in ctr:
            res[i] *= pow(p, ctr[p], mod)
            res[i] %= mod
    return res

print(*facts(1000, 18))


from PyQt5.QtWidgets import QMainWindow, QTableWidget, QTableWidgetItem, QApplication, QHeaderView,\
    QMessageBox, QInputDialog, QDialog

from UI import Ui_MainWindow
from binarytree import BinaryTree

class MainWindow(QMainWindow):
    def __init__(self, parent=None):
        super(MainWindow, self).__init__(parent)
        self.ui = Ui_MainWindow()
        self.ui.setupUi(self)
        self.currentTree_ = BinaryTree()
        self.currentTree_.load('BinaryTree.db')
        self.ui.editorTranslate.returnPressed.connect(self.translate)
        self.ui.editorTranslate.textChanged.connect(self.distanceSearch)
        self.ui.listDistance.itemClicked.connect(self.cclicked)
        self.ui.add.clicked.connect(self.newNode)
        self.ui.editorRu.returnPressed.connect(self.newNode)

    def newNode(self):
        in_tree = self.currentTree_.getAll()
        if self.ui.editorEn.text().lower() not in in_tree:
            if self.ui.editorEn.text() and self.ui.editorRu.text().lower():
                self.currentTree_.put(self.ui.editorEn.text().lower(), self.ui.editorRu.text().lower())
                self.ui.editorEn.clear()
                self.ui.editorRu.clear()
            else:
                if not self.ui.editorEn.text():
                    QMessageBox.about(self,
                    'Неполные данные', 'Вы не ввели слово на английском')
                if not self.ui.editorRu.text():
                    QMessageBox.about(self,
                    'Неполные данные', f'Вы не ввели слово для перевода слова {self.ui.editorEn.text()}')
        else:
            QMessageBox.about(
            self, 'Уже есть', 'В базе данных был найден перевод данного слова!!!')
            self.ui.editorEn.clear()
            self.ui.editorRu.clear()

    def translate(self):
        self.ui.labelRu.setText(self.currentTree_.translate(self.ui.editorTranslate.text()))

    def distanceSearch(self, text):
        self.ui.listDistance.clear()
        for i in self.currentTree_.search(text.lower()):
            self.ui.listDistance.addItem(i)

    def cclicked(self, item):
        self.ui.editorTranslate.setText(item.text())
        self.translate()

    def __del__(self):
        self.currentTree_.save('BinaryTree.db')


class Events:
    def __init__(self, config, template, timesheet, text_color):
        self.config = config
        self.template = template
        self.timesheet = timesheet
        self.text_color = text_color

    def read_config(self):
        import configparser
        import logging
        logger = logging.getLogger(__name__)

        config = configparser.ConfigParser()
        config.read(self.config)
        config_dump = {}
        for each_section in config.sections():
            for (each_key, each_val) in config.items(each_section):
                config_dump[each_key] = each_val
        logger.debug({"config": config_dump})
        return config

    def get_dimensions(*, image, font, item, config, key):
        import logging
        logger = logging.getLogger(__name__)

        image_width, image_height = image.size
        logger.info({
            "image": {
                "width": image_width,
                "height": image_height
            }
        })
        canvas_width, canvas_height = font.getsize(item)
        try:
            item_width = config.getint(key, 'width', fallback=0)
        except KeyError:
            item_canvas_center = int((image_width - canvas_width) / 2)
            item_left_offset = config.getint(key, 'width_offset_left', fallback=0)
            item_right_offset = config.getint(key, 'width_offset_right', fallback=0)
            item_width = item_left_offset + item_canvas_center - item_right_offset
        try:
            item_height = int(config.get(key, 'height'))
        except KeyError:
            raise
        dimensions = (item_width, item_height)
        return dimensions

    def fill(self, data, template, uuid, cert_dir):
        from datetime import datetime
        from PIL import Image
        from PIL import ImageFont
        from PIL import ImageDraw
        import logging
        logger = logging.getLogger(__name__)

        config = read_config(self)
        logger.info({
            "data": data,
        })
        img = Image.open(template)
        draw = ImageDraw.Draw(img)
        # Credits:
        # https://stackoverflow.com/questions/1970807/center-middle-align-text-with-pil

        for key, value in data.items():
            logger.info({
                key: value
            })
            item = data[key].strip()
            font_size = config.getint(key, 'font_size')
            font = ImageFont.truetype(r'./news-serif.ttf', font_size)

            item_dimensions = get_dimensions(image=img, font=font, item=item, config=config, key=key)
            item_colour = self.text_color
            draw.text(item_dimensions, item, item_colour, font=font)

        cert_name = data['name'].lower().replace(' ', '_')
        cert_path = f'{cert_dir}/{cert_name}.jpg'
        try:
            img.save(cert_path)
            img.close()
            logger.info({
                "status": "saved",
                "certificate": cert_path,
                "identifier": uuid,
                "time": str(datetime.now())
            })
        except FileNotFoundError:
            logger.error({
                "status": "error",
                "exception": "Unable to save file. Please check the given directory exists.",
                "identifier": uuid,
                "time": datetime.now()
            })
            exit(1)


import roslibpy
import numpy as np
import quaternion
import time

import ros_mobile_client as rmc
from grounded_action import GroundedAction, GroundedActionPool, GroundedStep
import area_globalization as ag
from occupancygrid_tools import AreaChecker

from typing import List, Dict, Sequence

import matplotlib.pyplot as plt
# %matplotlib inline

COLOR_OBSTACLE = 100
COLOR_UNKNOWN = -1
COLOR_PASSABLE = 0

obs_thresh = 3
unk_thresh = 3
view_options = 10
view_r = 6

sel_count = 3

def random_select(l: Sequence, c: int=sel_count) -> List:
    return [l.pop(np.random.randint(len(l))) for i in range(min(len(l), c))]

class AreaSelection(GroundedAction):
    def _gen_candidates(self) -> GroundedStep:
        obs = ag.get_obstacles(self.mc.map_data, obs_thresh, COLOR_OBSTACLE)
        filterd = list(filter(lambda o: o.convex_area>0, obs))
        sel = random_select(filterd)

        gs = [s.convex_gravity for s in sel]
        gsxy = [self.mc.get_coordinates_from_index(g) for g in gs]

        pos = [np.quaternion(0,g[0],g[1],0) for g in gsxy]
        ori = [self.mc.orientation for n in range(len(pos))]
        are = [s.convex_area for s in sel]

        return GroundedStep(self, pos=pos, ori=ori, are=are)

    def _evaluate(self, candidates: GroundedStep) -> GroundedStep:
        if len(candidates.candidates)==0: return candidates
        are = candidates.candidates['are']
        areasum = sum(are)

        for i in range(len(are)):
            ev = are[i]/areasum*np.random.random()
            candidates.evaluations[i] = ev

        candidates.selected_cand_id = np.argmax(candidates.evaluations)
        return candidates

class Observation(GroundedAction):
    def _gen_candidates(self):
        obs = ag.get_obstacles(self.mc.map_data, obs_thresh, COLOR_OBSTACLE)
        print(f'found obstacles: {len(obs)}')
        selob = random_select(obs)
        selob = selob[np.argmin([o.convex_area for o in selob])]
        print(f'selected: {selob.convex_gravity}')
        del obs

        selob_r = selob.diameter/2
        selob_vec = selob_r+view_r
        selob_g_qua = np.quaternion(0,selob.convex_gravity[0],selob.convex_gravity[1],0)

        yaws = np.random.random([view_options])*(np.pi*2)
        qs = np.array([quaternion.from_euler_angles(0,0,y) for y in yaws])
        vs = np.array([np.quaternion(0,selob_vec,0,0) for _ in range(view_options)])

        vecs = list(filter(lambda q: 0<=q.x and q.x<self.mc.map_data.shape[0] and 0<=q.y and q.y<self.mc.map_data.shape[1], selob_g_qua+qs*vs*qs.conj()))
        print(f'1 filterd vecs: {vecs}')
        vecs = list(filter(lambda q: self.mc.map_data[int(q.x),int(q.y)]==COLOR_PASSABLE, vecs))
        print(f'2 filterd vecs: {vecs}')
        sel = random_select(vecs)

        pos = [self.mc.get_coordinates_from_index((q.x, q.y)) for q in sel]
        posq = [np.quaternion(0,p[0],p[1],0) for p in pos]
        ori = [self.mc.get_relative_orientation(selob_g_qua-q) for q in sel]
#         ori = [self.mc.orientation for _ in sel]

        return GroundedStep(self, pos=posq, ori=ori)

    def _evaluate(self, candidates: GroundedStep):
        if len(candidates.candidates)==0: return candidates
        ## all random evaluation
        evas = np.random.random([len(candidates.candidates['pos'])])
        evsum = np.sum(evas)
        candidates.evaluations = evas/evsum
        candidates.selected_cand_id = np.argmax(candidates.evaluations)
        return candidates

class MapExploration(GroundedAction):
    def _gen_candidates(self):
        ac = AreaChecker(self.mc.map_data, self.mc.get_index_from_coordinates(self.mc.position))
        print(np.unique(ac.color_map), ac.color_map.shape)
        unks = ag.get_obstacles(ac.color_map, unk_thresh, COLOR_UNKNOWN)
        print(f'found unknown area: {len(unks)}')
        sel = random_select(unks)

        gs = [s.convex_gravity+ac.start for s in sel]
        print(f'gs: {gs}')
        gsxy = [self.mc.get_coordinates_from_index(g) for g in gs]

        pos = [np.quaternion(0,g[0],g[1],0) for g in gsxy]
        ori = [self.mc.get_orientation_from_body(g) for g in pos]
        are = [s.convex_area for s in sel]

        return GroundedStep(self, pos=pos, ori=ori, are=are)

    def _evaluate(self, candidates):
        if len(candidates.candidates)==0: return candidates
        are = candidates.candidates['are']
        areasum = sum(are)

        for i in range(len(are)):
            ev = 0 if areasum == 0 else are[i]/areasum
            candidates.evaluations[i] = ev

        candidates.selected_cand_id = np.argmax(candidates.evaluations)
        return candidates

class GAScheduler(GroundedAction):
    def __init__(self, name, client, pool):
        super().__init__(name, client)
        self.pool = pool

    def _gen_candidates(self) -> GroundedStep:
        stps = [ga.step() for ga in self.pool.pool.values()]
        return GroundedStep(self, stp=stps)

    def _evaluate(self, candidates) -> GroundedStep:
        stps = candidates.candidates['stp']
        e = np.array([0 if s.selected_cand_id==-1 else 1 for s in stps])

        ## all random evaluation
        evas = np.random.random([len(candidates.candidates['stp'])])*e
        evsum = np.sum(evas)
        candidates.evaluations = evas/evsum
        candidates.selected_cand_id = np.argmax(candidates.evaluations)
        return stps[candidates.selected_cand_id]

def main():
    rosclient = roslibpy.Ros('10.244.1.176', port=9090)
    rosclient.on_ready(lambda: print('is ROS connected: ', rosclient.is_connected))
    rosclient.run()

    mc = rmc.MobileClient(rosclient, lambda *s: print(f'【reached】 result: {s}'))
    mc.wait_for_ready()
    print(f'pos: {mc.position}, map_shape: {mc.map_data.shape}')
    ga_map = MapExploration('map_exploration', mc)
    ga_are = AreaSelection('area_selection', mc)
    ga_obs = Observation('observation', mc)

    pool_grounded = GroundedActionPool([ga_map, ga_are, ga_obs])
    ga_sched = GAScheduler('top_scheduler', mc, pool_grounded)

    mc.start()
    # ------------------------------------------------

    for i in range(10):
        s = ga_sched.step()
        pos = s.candidates['pos'][s.selected_cand_id]
        ori = s.candidates['ori'][s.selected_cand_id]

        mc.set_goal(pos, ori)
        mc.wait_for_goal_accepted()

    # ------------------------------------------------
    mc.stop()
    rosclient.terminate()
    print(f'finished {s.candidates}')

if __name__ == '__main__':
    main()


import unittest
from bitcoin import LecturaInputs

class Test_Lecturas(unittest.TestCase):

    def test_dict(self):
        # Comprobamos si las funciones nos retornan un diccionario
        dict_test = {}  # Creo uno vacio que nos sirva de referencia
        self.assertEqual(type(LecturaInputs.lectura_blocks()), type(dict_test))
        self.assertEqual(type(LecturaInputs.lectura_transacciones()), type(dict_test))

    def test_cant_bloques(self):
        # Compruebo que la longitud del diccionario es igual al numero de
        # bloques que se proponen en la PEC
        self.assertEqual( len(LecturaInputs.lectura_blocks().keys()) , 144)

if __name__ == '__main__':
    unittest.main()


#coding:utf-8
'''
定义一个graph Runner

'''

from PySide6.QtCore import QObject,Signal
from time import sleep

class VGRunner(QObject):

    progress = Signal(int)
    finished = Signal()

    def __init__(self,func,*args,**kwds):
        super().__init__()
        self.func = func
        self.args = args
        self.kwds = kwds

    def run(self):
        try:
            self.func(*self.args,**self.kwds)
        except ValueError as e:
            print(e)
        finally:
            self.finished.emit()


import torch
import os
import wandb
import json
import torch.optim as optim
from PIL import Image
from utils import parser, misc
from utils.config import *
from tools import builder
from tqdm import tqdm
from my_datasets.my_PCN import get_dataloader
from utils.logger import *
from utils.AverageMeter import AverageMeter
from utils.metrics import Metrics
from extensions.chamfer_dist import ChamferDistanceL1

# os.environ["CUDA_VISIBLE_DEVICES"] = '1'
device = 'cuda' if torch.cuda.is_available() else 'cpu'

def main(args, config):

    wandb.login()
    run = wandb.init(project='my_foldingnet')
    start_epoch = 0

    model = builder.model_builder(config.model).to(device)
    (train_sampler, train_dataloader), (_, test_dataloader) = builder.dataset_builder(args, config.dataset.train), \
                                                            builder.dataset_builder(args, config.dataset.val)

    optimizer, scheduler = builder.build_opti_sche(model, config)

    for epoch in range(start_epoch, config.max_epoch + 1):
        losses = AverageMeter()
        model.train()
        for (taxonomy_ids, model_ids, data) in tqdm(train_dataloader):
            model.zero_grad()
            gt = data[1].to(device)
            _, recon = model(gt)
            loss_func = ChamferDistanceL1().to(device)
            loss = loss_func(recon, gt)
            loss.backward()
            optimizer.step()
            losses.update(loss.item())
        wandb.log({"train_loss_epoch": losses.avg()})

        #test metric
        if epoch % 20 == 0:
            validate(model, test_dataloader, epoch, args)

        # #visulize result
        # if epoch % 50 == 0:
        #     with torch.no_grad():
        #         (taxonomy_ids, model_ids, data) = next(iter(test_dataloader))
        #         model.eval()
        #         gt = data[1].to(device)
        #         input_pc = gt.squeeze().detach().cpu().numpy()
        #         input_pc = misc.get_ptcloud_img(input_pc)
        #         _, recon = model(gt)
        #         recon = recon.squeeze().detach().cpu().numpy()
        #         recon = misc.get_ptcloud_img(recon)
        #         wandb.log({"epoch": epoch, "input": wandb.Image(input_pc), "recon": wandb.Image(recon)})

def validate(model, test_dataloader, epoch, args):
    with torch.no_grad():
        print("Validate the test dataset and visulize points")
        model.eval()
        losses = AverageMeter()
        category_metrics = dict()
        test_metrics = AverageMeter(Metrics.names())

        #创建保存图片的文件夹
        save_pth = os.path.join(args.saveimg_path, 'epoch%d' % epoch)
        os.makedirs(save_pth, exist_ok=True)
        gt_pth = os.path.join(save_pth, 'gt')
        os.makedirs(gt_pth, exist_ok=True)
        recon_pth = os.path.join(save_pth, 'recon')
        os.makedirs(recon_pth, exist_ok=True)

        for idx, (taxonomy_ids, model_ids, data) in enumerate(tqdm(test_dataloader)):
            gt = data[1].to(device)
            _, recon = model(gt)
            _metrics = Metrics.get(recon, gt)

            #记录每个种类的指标
            for _taxonomy_id in taxonomy_ids:
                if _taxonomy_id not in category_metrics:
                    category_metrics[_taxonomy_id] = AverageMeter(Metrics.names())
                category_metrics[_taxonomy_id].update(_metrics)

            #保存图片
            if idx % 500 == 0:
                input_gt = gt.squeeze().detach().cpu().numpy()
                input_gt = misc.get_ptcloud_img(input_gt)
                im = Image.fromarray(input_gt)
                gt_savepth = os.path.join(gt_pth, '%s.jpg' % model_ids)
                im.save(gt_savepth)

                recon = recon.squeeze().detach().cpu().numpy()
                recon = misc.get_ptcloud_img(recon)
                im = Image.fromarray(recon)
                recon_savepth = os.path.join(recon_pth, '%s.jpg' % model_ids)
                im.save(recon_savepth)

        #计算平均指标
        for _,v in category_metrics.items():
            test_metrics.update(v.avg())
        wandb.log({'test_loss_epoch': test_metrics.avg()[1]})

        #打印结果
        shapenet_dict = json.load(open('./data/shapenet_synset_dict.json', 'r'))
        print('============================ TEST RESULTS ============================')
        print('[Validation] EPOCH: %d  Metrics = %s' % (epoch, ['%.4f' % m for m in test_metrics.avg()]))
        msg = ''
        msg += 'Taxonomy' + '\t'
        msg += 'ClassName' + '\t'
        msg += 'nsamples' + '\t'
        for metric in test_metrics.items:
            msg += metric + '\t'
        print(msg)

        for taxonomy_id in category_metrics:
            msg = ''
            msg += (taxonomy_id + '\t')
            msg += shapenet_dict[taxonomy_id] + '\t'
            msg += (str(category_metrics[taxonomy_id].count(0)) + '\t')
            for value in category_metrics[taxonomy_id].avg():
                msg += '%.3f' % value + '\t'
            print(msg)

        msg = ''
        msg += 'Overall\t\t'
        for value in test_metrics.avg():
            msg += '%.3f \t' % value
        print(msg)

        #使用wandb track
        columns = ['Taxonomy', 'ClassName', 'nsamples', 'F1_Score', 'CDL1', 'CDL2', 'EMD']
        data = []
        for taxonomy_id in category_metrics:
            data_id = [taxonomy_id, shapenet_dict[taxonomy_id], str(category_metrics[taxonomy_id].count(0))]
            for value in category_metrics[taxonomy_id].avg():
                data_id.append(value)
            data.append(data_id)
        data_all = ['overall', '', '']
        for value in test_metrics.avg():
            data_all.append(value)
        data.append(data_all)

        table = wandb.Table(data=data, columns=columns)
        wandb.log({'test_table': table})

if __name__ == '__main__':
    args = parser.get_args()
    args.distributed = False
    config = get_config(args)
    config.dataset.train.others.bs = config.total_bs
    main(args, config)