#! /usr/bin/env python
# -*- coding: utf-8 -*-

"""
Module that contains ArtellaDrive Python client implementation
"""

from __future__ import print_function, division, absolute_import

import os
import json
import time
import codecs
import socket
import base64
import random
import hashlib
import logging
import threading
import traceback
from collections import OrderedDict
try:
    from urllib.parse import urlparse, urlencode, urlunparse
    from urllib.request import urlopen, Request
    from urllib.error import HTTPError, URLError
except ImportError:
    from urlparse import urlparse, urlunparse
    from urllib import urlencode
    from urllib2 import urlopen, Request, HTTPError, URLError

from artella import dcc
from artella.core import consts, utils, exceptions, dccplugin

logger = logging.getLogger('artella')

class ArtellaDriveClient(object):
    """
    Class used to interact with ArtellaDrive application on client side
    """

    _challenge_path = None

    def __init__(self, host=consts.DEFAULT_HOST, port=consts.DEFAULT_PORT, extensions=None):
        self._host = host               # Contains default IP host used by the client.
        self._port = port               # Contains default port used by the client.
        self._auth_header = ''          # Contains authentication header read from challenge file.
        self._remote_sessions = list()  # Contains list of available remote sessions.
        self._local_projects = dict()   # Contains dictionary of all available user local Artella projects.
        self._remote_projects = dict()  # Contains a dictionary of all available user remote Artella projects.
        self._batch_ids = set()         # Contains a list that tracks all calls made to the client during a session.
        self._socket_buffer = None      # Contains instance of the socket buffer used to communicate with Artella App.
        self._running = False           # Flag that is True while Artella Drive thread is running.
        self._available = True          # Flag that is True while Artella Drive is available.
        self._exception = None          # Contains exception caused while Artella Drive is running.
        self._extensions = extensions   # Contains a list with all extensions managed by the client.
        self._client_thread = None      # Contains Python client thread

    # ==============================================================================================================
    # PROPERTIES
    # ==============================================================================================================

    @property
    def is_running(self):
        """
        Returns whether or not current Artella Drive thread is running.
        While Artella Drive thread is running, it listens responses from Artella Drive

        :return: True if the Artella Drive thread is running; False otherwise.
        :rtype: bool
        """

        return self._running

    @property
    def is_available(self):
        """
        Returns whether or not current Artella Drive client is available.
        Artella Drive Client is not available, for example, is the user closes Artella Drive App manually.

        :return: True if Artella Drive Client is available; False otherwise.
        :rtype: bool
        """

        return self._available

    # ==============================================================================================================
    # CLASS FUNCTIONS
    # ==============================================================================================================

    @classmethod
    def get(cls, extensions=None):
        """
        Returns a new instance of ArtellaDriveClient making sure that authorization is valid

        :param list(str) extensions: List of extensions
        :return: New instance of Artella Drive object
        :rtype: ArtellaDriveClient
        """

        artella_client = cls(extensions=extensions)

        # Challenge value gets updated when the Artella drive restarts.
        # We need to check this each time in case the local server got restarted
        auth_header = artella_client.update_auth_challenge()
        if not auth_header:
            logger.warning('Local Artella Drive is not available. Please launch Artella Drive App.')
            return None

        # Updates the list of available remotes
        artella_client.update_remotes_sessions(show_dialogs=False)

        return artella_client

    # ==============================================================================================================
    # CHALLENGE FILE
    # ==============================================================================================================

    def check(self, update=False):
        """
        Checks whether or not current client is available and running

        :param update: bool, Whether or not remote sessions should be updated
        :return: True if the client is available and running; False otherwise.
        :rtype: bool
        """

        return self._available and self._running and self.get_remote_sessions(update=update)

    def get_challenge_file_path(self):
        """
        Returns path where challenge file is located in local desktop machine

        :return: dictionary containing challenge file path
        :rtype: dict
        :example:
        >>> self.get_challenge_file_path()
        {
            "challenge_file_path": "C:\\users\\artella\\Application Data\\artella\\artella-challenge"
        }
        """

        req = Request('http://{}:{}/v2/localserve/auth/challenge-file-path'.format(self._host, self._port))
        rsp = self._communicate(req, skip_auth=True)

        return rsp

    def update_auth_challenge(self):
        """
        Collects some data from a local file to confirm we are running on the same that Artella drive is running.

        :return: string containing authentication header for current session or None if authentication was invalid
        :rtype: str or None
        """

        if not ArtellaDriveClient._challenge_path:
            rsp = self.get_challenge_file_path()
            if not rsp or 'error' in rsp:
                logger.error('Unable to get challenge file path "{}"'.format(rsp))
                self._auth_header = None
                return self._auth_header
            ArtellaDriveClient._challenge_path = rsp.get('challenge_file_path')

        if not os.path.exists(ArtellaDriveClient._challenge_path):
            logger.error('Challenge file not found: "{}"'.format(ArtellaDriveClient._challenge_path))
            self._auth_header = None
            return self._auth_header

        with open(os.path.expanduser(ArtellaDriveClient._challenge_path), 'rb') as fp:
            base_auth_header = fp.read()

        self._auth_header = consts.AUTH_HEADER.format(codecs.encode(base_auth_header[:64], 'hex').decode('ascii'))

        return self._auth_header

    def has_remote_sessions(self):
        """
        Returns whether or not current Artella Drive Client has remote sessions available or not
        :return: bool
        """

        return self._remote_sessions and len(self._remote_sessions) > 0

    def get_remote_sessions(self, update=False):
        """
        Returns a list with all available remote sessions

        :return: List of cached remote sessions in current Artella Drive Client
        :rtype: list(str)
        """

        if not self._remote_sessions and update:
            self.update_remotes_sessions()

        return self._remote_sessions

    def update_remotes_sessions(self, show_dialogs=True):
        """
        Collects all available remote servers available.

        :return: List of available remote sessions received from Artella Drive App.
        :rtype: list(str)
        """

        def _ping_remote_sessions():
            rsp = self.ping()
            if 'error' in rsp:
                msg = 'Attempting to retrieve remote sessions: "{}" '.format(rsp.get('error'))
                logger.error(msg)
            self._remote_sessions = rsp.get('remote_sessions', list())

        utils.clear_list(self._remote_sessions)

        if not self.update_auth_challenge():
            msg = 'Unable to authenticate to Artella Drive App.'
            logger.error(msg)
            if show_dialogs:
                dcc.show_error('Artella - Authentication error', msg)
            return

        _ping_remote_sessions()

        # If not remote sessions found, we recheck after 1 second to make sure Artella Drive App is running.
        if not self._remote_sessions:
            time.sleep(1)
            _ping_remote_sessions()
            if not self._remote_sessions:
                msg = 'No remote sessions available. Please visit your Project Drive in Artella Web App and try again!'
                logger.error(msg)

        return self._remote_sessions

    # ==============================================================================================================
    # SESSION
    # ==============================================================================================================

    def get_storage_id(self):
        """
        Returns storage ID of the machine this client is running on

        :return: ID indicating this desktop instance
        :rtype: str
        :example:
        >>> self.get_storage_id()
        "d7apalzl2rdnphe5wuccytqq3i"
        """

        req = Request('http://{}:{}/v2/localserve/kv/settings/machine-id'.format(self._host, self._port))
        storage_id = self._communicate(req)

        return storage_id

    def get_metadata(self):
        """
        Returns general data related with current session by asking remote server

        :return: Returns a dictionary containing all keys and values in kv namespace
        :rtype: dict
        :example:
        >>> self.get_metadata()
        {
            "machine-id": "d7apalzl2rdnphe5wuccytqq3i",
            "workspace": "C:/Users/artella/artella-files",
            "openers.log": "C:/Users/artella/AppData/Roaming/artella/openers.log"
        }
        """

        params = urlencode({'dump': 'true'})
        req = Request('http://{}:{}/v2/localserve/kv/settings?{}'.format(self._host, self._port, params))
        rsp = self._communicate(req)

        return rsp

    # ==============================================================================================================
    # PROJECTS
    # ==============================================================================================================

    def get_local_projects(self, force_update=False):
        """
        Returns all available project files found in the local user machine

        :param bool force_update: Whether to force the cache of the current available local projects
        :return: List of dictionaries containing all the information of the current available
        Artella on local user machine projects
        :rtype: dict(str, dict(str, str))
        """

        if self._local_projects and not force_update:
            return self._local_projects

        local_root = self.get_local_root()
        if not local_root or not os.path.isdir(local_root):
            logger.warning(
                'Impossible to retrieve current available projects because local root is not valid: {}'.format(
                    local_root))
            return list()

        project_files = utils.get_files(local_root, pattern=consts.PROJECT_METADATA_FILE)

        for project_file in project_files:
            project_dir = os.path.dirname(project_file)
            project_name = os.path.basename(project_dir)
            self._local_projects.setdefault(project_name, None)
            with open(project_file, 'r') as fh:
                lines = fh.readlines()
                project_data = {
                    'directory': project_dir
                }
                for line in lines:
                    line_split = line.split(': ')
                    if not line_split:
                        continue
                    project_data[line_split[0]] = line_split[1].strip()
                if project_data:
                    self._local_projects[project_name] = project_data

        return self._local_projects

    def get_remote_projects(self, force_update=False):
        """
        Returns all available projects in Artella

        :param bool force_update: Whether to force the cache of the current available remote projects
        :return: Dictionary that contains all projects for each one of the available remote sessions
        :rtype: dict(str, list(str))
        """

        if self._remote_projects and not force_update:
            return self._remote_projects

        if not self._remote_sessions:
            return dict()

        for remote_session in self._remote_sessions:
            remote_session_api = remote_session.get('api', None)
            if not remote_session_api:
                continue
            remote_session_projects = remote_session.get('projects', dict())
            self._remote_projects[remote_session_api] = remote_session_projects

        return self._remote_projects

    def get_project_name(self, project_id, remote_session=None):
        """
        Returns the project name giving its remote Artella ID

        :param str project_id: ID of the project in remote Artella server
        :param str remote_session: Remote session we want to search project on. If not given, the project will be
            search in all available remote sessions
        :return: Name of the project
        :rtype: str
        """

        remote_projects = self.get_remote_projects()
        if not remote_projects:
            logger.warning('No remote projects available to search project name for: "{}"'.format(project_id))
            return ''

        project_name = ''
        if remote_session:
            if remote_session not in remote_projects:
                logger.warning(
                    'Given remote session: "{}" is not available. Impossible to retrieve name for project: "{}"'.format(
                        remote_session, project_id))
                return project_name

            if project_id not in remote_projects[remote_session]:
                logger.warning('Project ID "{}" not found in remote session: "{}"'.format(
                    project_id, remote_session))
                return project_name

            project_name = remote_projects[remote_session][project_id].get('name', '')
        else:
            for remote_id, remote_data in remote_projects.items():
                for remote_project_id, project_data in remote_data.items():
                    if project_id != remote_project_id:
                        continue
                    project_name = project_data.get('name', '')
                    break

        if not project_name:
            logger.warning('No project name found for project: "{}"'.format(project_id))

        return project_name

    # ==============================================================================================================
    # PATHS
    # ==============================================================================================================

    def is_artella_path(self, file_path):
        """
        Returns whether or not given file path is an Artella file path or not
        A path is considered to be an Artella path if the path is located inside the Artella project folder
        in the user machine

        :param str file_path: path to check. If not given, current DCC scene file path will be used
        :return: True if the given file path is an Artella path; False otherwise.
        :rtype: bool
        """

        if not file_path or not os.path.exists(file_path):
            return False

        local_root = os.path.normpath(os.path.abspath(self.get_local_root()))
        file_path = os.path.normpath(os.path.abspath(file_path))

        return file_path.startswith(local_root)

    def get_local_root(self):
        """
        Returns the local storage root path for this machine by asking to remote server.
        .. note::
            If local root cannot be retrieved from server (this can happen because server is down or because user
            doesn't have a working internet connection) then this function will try to retrieve the local root path
            from ARTELLA_FOLDER_ROOT environment variable. This environment variable must be defined by users manually
            to point to local root path.

        :return: Absolute path where files are stored locally
        :rtype: str or None
        :example:
        >>> self.get_local_root()
        "C:/Users/artella/artella-files"
        """

        req = Request('http://{}:{}/v2/localserve/kv/settings/workspace'.format(self._host, self._port))
        local_root = self._communicate(req)
        if not local_root or (isinstance(local_root, dict)):
            alr = os.environ.get(consts.ALR, None)
            if alr:
                logger.warning('Unable to get local storage root. Using env var instead: "{}"'.format(consts.ALR))
                local_root = alr
            else:
                local_root = ''
                logger.error('Unable to get local storage root.')
                logger.info(
                    'Check that the local Artella Drive service is running and you have a working internet connection.')
                logger.info(
                    'To work offline set the "{}" environment variable to the proper local project directory'.format(
                        consts.ALR))
        else:
            local_root = utils.clean_path(local_root)
            os.environ[consts.ALR] = local_root

        return local_root

    def resolve_path(self, path):
        """
        Converts a local path to a remote handle path representation and vice versa

        :param str path: local path or remote server path
        :return: Returns a dictionary containing resolved path or error message is the path cannot be resolve
        :rtype: dict
        :example:
        >>> os.environ['ART_LOCAL_ROOT'] = "C:/Users/artella/artella-files"
        >>> self.resolve_path("ART_LOCAL_ROOT/project/refs/ref.png")
        {
            'handle': 'project__fo0pohsa2sea4wyr5zmzcwnzse/refs/ref.png',
            'file_path': u'C:\\Users\\artella\\artella-files\\project\\refs\\ref.png'
        }
        >>> self.resolve_path("artella:project__fo0pohsa2sea4wyr5zmzcwnzse/refs/ref.png")
        {
            'handle': 'project__fo0pohsa2sea4wyr5zmzcwnzse/refs/ref.png',
            'file_path': u'C:\\Users\\artella\\artella-files\\project\\refs\\ref.png'
        }
        >>> self.resolve_path("C:/Users/artella/artella-files/project/refs/ref.png")
        {
            'handle': 'project__fo0pohsa2sea4wyr5zmzcwnzse/refs/ref.png',
            'file_path': 'C:\\Users\\artella\\artella-files\\project\\refs\\ref.png'
        }
        >>> self.resolve_path("C:/Invalid/Path/ref.png")
        {
            'url': 'http://127.0.0.1:29282/v2/localserve/resolve',
            'error': 'Failed to reach the local ArtellaDrive: "Bad Request"'
        }
        """

        # We set request object depending whether given path is an URI one or not
        if is_uri_path(path):
            payload = {'handle': path}
        else:
            payload = {'file_path': path}

        req = Request('http://{}:{}/v2/localserve/resolve'.format(self._host, self._port))
        req.add_header('Content-Type', 'application/json')
        rsp = self._communicate(req, json.dumps(payload).encode())
        if 'error' in rsp:
            logger.error('Attempting to resolve "{}" "{}"'.format(path, rsp.get('error')))

        return rsp

    def translate_path(self, file_path):
        """
        Converts a file path to a local file path taking into account the available projects.

        :param str file_path: File path we want to to translate to its user local version
        :return: User local version of the given path
        :rtype: str
        :example:
        >>> self.translate_path("C:/Users/Bobby/artella-files/ProjectA/Assets/Characters/A/Model/a.ma")
        "C:/Users/Tomi/artella/data/ProjectA/Assets/Characters/A/Model/a.ma"
        >>> self.translate_path("/ProjectA/Assets/Characters/A/Model/a.ma")
        "C:/Users/Tomi/artella/data/ProjectA/Assets/Characters/A/Model/a.ma"
        >>> self.translate_path("$ART_LOCAL_ROOT/ProjectA/refs/ref.png")
        "C:/Users/Tomi/artella/data/ProjectA/refs/ref.png"
        """

        if is_uri_path(file_path):
            return self.resolve_path(file_path)

        path = file_path
        if not os.path.isabs(file_path):
            path = self.relative_path_to_absolute_path(file_path)
        path = utils.clean_path(os.path.expanduser(os.path.expandvars(path)))

        local_root = self.get_local_root()
        local_project_names = list((self.get_local_projects() or dict()).keys())

        for old_alr in consts.OLD_LOCAL_ROOTS:
            old_alr_str = '${}'.format(old_alr)
            if path.startswith(old_alr_str):
                path = utils.clean_path(path.replace(old_alr, local_root))
                path = path[1:] if path.startswith('$') else path

        path_split = path.split('/')
        total_chars = 0
        for split in path_split:
            if split in local_project_names:
                break
            total_chars += len(split) + 1

        common_path = path[total_chars:]
        translated_path = '{}/{}'.format(local_root, common_path)

        rsp = self.resolve_path(translated_path)
        if 'error' in rsp:
            logger.warning('Error while resolving translated file path: "{}"!'.format(path))
            return path

        if not translated_path.startswith(local_root):
            logger.warning('Was not possible to translate file path: "{}"!'.format(path))
            return path

        return translated_path

    def is_path_translated(self, file_path):
        """
        Returns whether or not given path is already translated to a valid Artella path

        :param file_path: str, path you want to check translation validation
        :return: True if the given path is an already translated Artella path; False otherwise
        :rtype: bool
        :example:
        >>> self.is_path_translated("C:/Users/Bobby/artella-files/ProjectA/Assets/Characters/A/Model/a.ma")
        False
        >>> self.is_path_translated("$ART_LOCAL_ROOT/ProjectA/refs/ref.png")
        True
        """

        if not file_path:
            return False

        return file_path.startswith('${}'.format(consts.ALR))

    def convert_path(self, file_path):
        """
        Converts given path to a path that Artella can understand

        :param str file_path: File path we want to convert
        :return: str
        :rtype: str
        :example:
        >>> self.translate_path("C:/Users/Bobby/artella-files/ProjectA/Assets/Characters/A/Model/a.ma")
        "$$ART_LOCAL_ROOT/ProjectA/Assets/Characters/A/Model/a.ma"
        >>> self.translate_path("/ProjectA/Assets/Characters/A/Model/a.ma")
        "$ART_LOCAL_ROOT/ProjectA/Assets/Characters/A/Model/a.ma"
        """

        local_root = self.get_local_root()

        path_to_convert = file_path
        if not os.path.isabs(file_path):
            path_to_convert = self.relative_path_to_absolute_path(file_path)
        translated_path = utils.clean_path(self.translate_path(path_to_convert))

        converted_path = translated_path
        if translated_path.startswith(local_root):
            converted_path = translated_path.replace(local_root, '${}'.format(consts.ALR))

        return converted_path

    def relative_path_to_absolute_path(self, relative_path, project_id=None):
        """
        Converts a relative path to an absolute local file path taking into account the given project id

        :param str relative_path: Relative path to a file
        :param str or None project_id: project id the relative path file belongs to
        :return: Absolute local file path of the relative path in the given project
        :rtype: str
        """

        if os.path.isabs(relative_path):
            return relative_path

        local_root = self.get_local_root()

        if not project_id:
            project_id = utils.split_path(relative_path)[0]

        local_project_names = self.get_local_projects() or dict()
        for project_name, project_data in local_project_names.items():
            if project_id == project_name:
                return utils.clean_path(os.path.join(local_root, relative_path))
            if project_id == project_data.get('remote', None):
                return utils.clean_path(os.path.join(local_root, project_name, relative_path))

        return relative_path

    def project_name_from_path(self, file_path):
        """
        Returns project name given file path belongs to. None if the path does not belongs to nay user local project.

        :param str file_path: path we want to retrieve project name of.
        :return: project name given path belongs to.
        :rtype: str
        """

        local_project_names = list((self.get_local_projects() or dict()).keys())

        project_name = None

        path_split = file_path.split('/')
        for split in path_split:
            if split in local_project_names:
                project_name = split
                break

        return project_name

    def project_id_from_path(self, file_path):
        """
        Returns project ID given file path belongs to. None if the path does not belongs to any user local project.

        :param str file_path: path we want to retrieve project ID of.
        :return: Project ID given path belongs to.
        :rtype: str
        """

        local_projects = self.get_local_projects() or dict()
        if not local_projects:
            logger.warning('No local projects available to retrieve project ID from path!')
            return None

        project_name = self.project_name_from_path(file_path)
        if not project_name:
            logger.warning('Was not possible to retrieve Project ID from path: {}!'.format(file_path))
            return None

        if project_name not in list(local_projects.keys()):
            logger.warning(
                'Was not possible to retrieve Project ID from path because project "{}" is not recognized!'.format(
                    project_name))
            return None

        project_data = local_projects[project_name]
        project_id = project_data.get('remote', None)

        return project_id

    # ==============================================================================================================
    # FILES/FOLDERS
    # ==============================================================================================================

    def exists_in_server(self, file_path):
        """
        Returns whether or not given file path exists in Artella remote server.

        :param str file_path: File path we want to check. It can be an absolute local file path or an URI path
        :return: True if the file exists in server; False otherwise.
        :rtype: bool
        """

        rsp = self.resolve_path(file_path)

        return False if 'error ' in rsp else True

    def status(self, file_paths, include_remote=False):
        """
        Returns the status of a file from the remote server

        :param str or list(str) file_paths: Local path(s) or Resolved URI path(s) of a folder/file or list of
            folders/files
        :param bool include_remote: Whether to include remote server information into the response. Take into account
            that this call can slow down the server response.
        :return:
        :rtype: dict
        :example:
        >>> self.status('artella:project__fo0pohsa2sea4wyr5zmzcwnzse/refs/ref.png')
        >>> self.status('C:/Users/artella/artella-files/projects/refs/ref.png')
        {
            'project__fo0pohsa2sea4wyr5zmzcwnzse/refs/ref.png':
            {
                'local_info':
                {
                    'content_length': 390223,
                    'remote_version': 1,
                    'name': u'Area.png',
                    'modified_time': '2020-03-11T23:38:01.3985577+01:00',
                    'mode': u'0666',
                    'signature':
                    'sha1:f67817bdfa6a828ab3a80110d2e52ca8a430afb7',
                    'path': u'C:\\Users\\artella\\artella-files\\project\\refs\\ref.png'
                }
            }
        }
        """

        if not self.get_remote_sessions(update=True):
            logger.warning(
                'No remote sessions available. Artella App Drive status call aborted.')
            return list()

        file_paths = utils.force_list(file_paths, remove_duplicates=True)
        result = list()

        # Make sure that paths are clean
        file_paths = [utils.clean_path(file_path) for file_path in file_paths]

        for file_path in file_paths:
            uri_path = path_to_uri(file_path) if not is_uri_path(file_path) else file_path
            uri_parts = urlparse(uri_path)
            uri_handle = utils.clean_path(uri_parts.path)
            params = urlencode({'handle': uri_handle, 'include-remote': str(bool(include_remote)).lower()})
            req = Request('http://{}:{}/v2/localserve/fileinfo?{}'.format(
                self._host, self._port, params))
            rsp = self._communicate(req)
            if 'error' in rsp:
                logger.error(
                    'Attempting to retrieve status "{}" "{}"'.format(
                        uri_path.encode(), rsp.get('error').encode('utf-8')))
                continue

            result.append(rsp)

        return result

    def download(self, paths, version=None, recursive=False, overwrite=True, folders_only=False):
        """
        Downloads files from remote server

        :param list(str) paths: List of file/folder paths (full or URI paths) to download from remote server
        :param int version: If given, specified version of the file/folder will be downloaded
        :param bool recursive: If True and we download a folder, all folder contents will be downloaded also
        :param bool overwrite: If True, already downloaded files will be overwritten with the contents in remote server
        :param bool folders_only: If True, only folder structure will be downloaded
        :return: Dictionary containing batch ID of the operation
        :rtype: dict
        :example:
        >>> self.download('artella:project__fo0pohsa2sea4wyr5zmzcwnzse/refs/ref.png')
        >>> self.download('C:/Users/artella/artella-files/project/refs/', recursive=True)
        >>> self.download('C:/Users/artella/artella-files/project/refs/', recursive=True, version=1)
        >>> self.download('C:/Users/artella/artella-files/project/refs/ref.png')
        {
            'batch_id': '1584199877-4'
        }
        >>> self.download('Non/Valid/Path')
        {}
        """

        path_handle_map = paths_to_handles(paths, as_dict=True)
        handles = list(path_handle_map.values())
        if not path_handle_map or not handles:
            return dict()
        logger.debug('Handles: "{}"'.format(handles))

        payload = {
            'handles': handles,
            'recursive': recursive,
            'replace_locals': overwrite,
            'folders_only': folders_only
        }
        if version is not None:
            payload['version'] = int(version)
        try:
            req = Request('http://{}:{}/v2/localserve/transfer/download'.format(self._host, self._port))
            rsp = self._communicate(req, json.dumps(payload).encode())
        except Exception:
            rsp = {'error': traceback.format_exc()}
        if 'error' in rsp:
            logger.warning(
                'Unable to download file paths "{}" "{}"'.format(list(path_handle_map.keys()), rsp.get('error')))
            return rsp

        return self._track_response(rsp)

    def pause_downloads(self):
        """
        Pauses current active downloads from Artella server

        :return: Response from Artella server after downloads are stopped
        :rtype: dict
        """

        req = Request('http://{}:{}/v2/localserve/pause/pause/downloads'.format(self._host, self._port))
        rsp = self._communicate(req)

        return rsp

    def pause_uploads(self):
        """
        Pauses current active uploads to Artella server

        :return: Response from Artella server after downloads are stopped
        :rtype: dict
        """

        req = Request('http://{}:{}/v2/localserve/pause/pause/uploads'.format(self._host, self._port))
        rsp = self._communicate(req)

        return rsp

    def resume_downloads(self):
        """
        Resume current paused downloads from Artella server

        :return: Response from Artella server after downloads are resumed
        :rtype: dict
        """

        req = Request('http://{}:{}/v2/localserve/pause/resume/downloads'.format(self._host, self._port))
        rsp = self._communicate(req)

        return rsp

    def resume_uploads(self):
        """
        Resume current paused uploads to Artella server

        :return: Response from Artella server after uploads are resumed
        :rtype: dict
        """

        req = Request('http://{}:{}/v2/localserve/pause/resume/uploads'.format(self._host, self._port))
        rsp = self._communicate(req)

        return rsp

    def upload(self, paths, folders_only=False, comment=''):
        """
        Uploads a new file/folder or a new version of an existing file/folder to the remote server

        :param list(str) paths: List of file/folder paths (full or URI paths) to upload to the remote server
        :param bool folders_only: If True, only folder structure will be uploaded
        :param str comment: Comment that will be linked to the uploaded files/folders
        :return: Response from Artella server after the upload operation is done.
        :rtype: dict
        >>> self.upload('artella:project__fo0pohsa2sea4wyr5zmzcwnzse/refs/ref.png')
        >>> self.upload('C:/Users/artella/artella-files/project/refs/', folders_only=True, comment='new folder')
        >>> self.upload('C:/Users/artella/artella-files/project/refs/ref.png', comment='new file version')
        {
            'batch_id': '1584199877-4'
        }
        >>> self.upload('Non/Valid/Path')
        {}
        """

        paths = utils.force_list(paths, remove_duplicates=True)

        path_handle_map = paths_to_handles(paths, as_dict=True)
        handles = list(path_handle_map.values())
        if not path_handle_map or not handles:
            return dict()
        logger.debug('Handles: "{}"'.format(handles))

        payload = {
            'handles': handles,
            'recursive': False,
            'commit_message': comment or '',
            'folders_only': folders_only
        }
        req = Request('http://{}:{}/v2/localserve/transfer/upload'.format(self._host, self._port))

        rsp = self._communicate(req, json.dumps(payload).encode())
        if 'error' in rsp:
            logger.warning(
                'Unable to upload file paths "{}" "{}"'.format(list(path_handle_map.keys()), rsp.get('error')))
            return rsp

        return self._track_response(rsp)

    def lock_file(self, file_paths, note=consts.DEFAULT_LOCK_NOTE):
        """
        Lock given file path in the web platform so that other users know it is in use

        :param str or list(str) file_paths: Absolute local file path(s) to lock
        :param str or None note: Optional note to add to the lock operation
        :return: Returns True if the lock file was successfully locked or if the if the file was already locked
            by the same user; False otherwise
        :rtype: bool
        :example:
        >>> self.lock_file('C:/Users/artella/artella-files/project/refs/ref.png')
        True
        """

        file_paths = utils.force_list(file_paths, remove_duplicates=True)
        result = OrderedDict()

        for file_path in file_paths:
            payload = {
                'handle': file_path,
                'note': note or consts.DEFAULT_LOCK_NOTE
            }
            req = Request('http://{}:{}/v2/localserve/lock'.format(self._host, self._port))
            req.add_header('Content-Type', 'application/json')
            rsp = self._communicate(req, json.dumps(payload), method='POST')
            if isinstance(rsp, dict):
                if 'error' in rsp:
                    logger.error('Unable to lock file path "{}" "{}"'.format(file_path, rsp.get('error')))
                    return False
            else:
                logger.error('Unable to lock file path "{}" "{}"'.format(file_path, rsp))
                return False

            result[file_path] = rsp.get('response', False) and rsp.get('status_code', 0) == 200

        return result

    def unlock_file(self, file_paths):
        """
        Unlocks given file paths in the web platform so that other users can use it

        :param str or list(str) file_paths: Absolute local file path(s) to unlock
        :return: Returns True if the file was successfully unlocked; False otherwise
        :rtype: bool
        """

        file_paths = utils.force_list(file_paths, remove_duplicates=True)
        result = OrderedDict()

        for file_path in file_paths:
            payload = {
                'handle': file_path
            }
            req = Request('http://{}:{}/v2/localserve/lock'.format(self._host, self._port))
            req.add_header('Content-Type', 'application/json')
            rsp = self._communicate(req, json.dumps(payload).encode(), method='DELETE')
            if 'error' in rsp:
                logger.error('Unable to unlock file path "{}" "{}"'.format(file_path, rsp.get('error')))
                return result

            if isinstance(rsp, dict):
                result[file_path] = rsp.get('response', False) and rsp.get('status_code', 0) == 200
            else:
                # If we try to unlock files that are not remote yet the response will be an string telling us that
                # information. We consider that as a valid lock.
                result[file_path] = True

        return result

    def file_current_version(self, file_path, _status=None):
        """
        Returns current version of the given file

        :param str file_path: Absolute local file path to retrieve current local version of
        :param str _status: new file status
        :return: Current local version of the given file path
        :rtype: int
        """

        status = _status or self.status(file_path, include_remote=True)
        if not status:
            return None

        current_version = -1
        for file_status in status:
            for file_uri_path, file_status_data in file_status.items():
                if 'local_info' not in file_status_data or not file_status_data['local_info'] or \
                        'remote_info' not in file_status_data:
                    continue
                else:
                    current_version = file_status_data['remote_info'].get('version', 0)
                    break

        return current_version

    def file_latest_version(self, file_path, _status=None):
        """
        Returns latest version of the given file
        :param str file_path: Absolute local file path to retrieve latest server version of
        :return: int
        """

        status = _status or self.status(file_path, include_remote=True)

        latest_version = 0
        for file_status in status:
            for file_uri_path, file_status_data in file_status.items():
                if 'local_info' not in file_status_data or not file_status_data['local_info'] or \
                        'remote_info' not in file_status_data:
                    continue
                else:
                    latest_version = file_status_data['remote_info'].get('raw', dict()).get('highest_version', 0)
                    break

        return latest_version

    def file_is_latest_version(self, file_path):
        """
        Returns whether or not given local file path is updated to the latest version available in Artella server.

        :param str file_path: Absolute local file path or URI path we want to check version of
        :return: True if the file is updated to the latest version available; False otherwise.
        :rtype: bool
        """

        status = self.status(file_path, include_remote=True)

        local_version = self.file_current_version(file_path, _status=status)
        latest_version = self.file_latest_version(file_path, _status=status)

        return local_version == latest_version

    def can_lock_file(self, file_path):
        """
        Returns whether or not current opened DCC file (or given file path) can be locked or not
        A file only can be locked if it is not already locked by other user.

        :param str or None file_path: Absolute local file path we want to check if can be locked or not.
        :return: True if the file can be locked by current user; False otherwise.
        :rtype: bool
        """

        is_locked, is_locked_by_me, is_locked_by_name, remote_record_found = self.check_lock(file_path)
        if not remote_record_found:
            logger.info('File "{}" is not versioned yet. No need to lock'.format(file_path))
            return True

        if not is_locked or (is_locked and is_locked_by_me):
            return True

        return False

    def check_lock(self, file_path):
        """
        Returns whether or not the given file is locked and whether or not current user is the one that has the file
        locked.

        :param str file_path: Absolute local file path to check lock status of
        :return: Returns a tuple with the following fields:
            - is_locked: True if the given file is locked; False otherwise
            - is_locked_by_me: True if the given file is locked by current user; False otherwise
            - locked_by_name: Name of the user that currently has the file locked
            - remote_record_found: Indicates whether the request relates to an existing remote file record or not
        :rtype: tuple(bool, bool, str, bool)
        """

        payload = {
            'handle': file_path
        }
        req = Request('http://{}:{}/v2/localserve/lock'.format(self._host, self._port))
        req.add_header('Content-Type', 'application/json')
        rsp = self._communicate(req, json.dumps(payload).encode(), method='GET')
        if 'error' in rsp:
            logger.error('Unable to check lock status for file path "{}" "{}"'.format(file_path, rsp.get('error')))
            return False, False, '', False

        is_locked = rsp.get('is_locked', False)
        is_locked_by_me = rsp.get('is_locked_by_me', False)
        locked_by_name = rsp.get('locked_by_name', '')
        remote_record_found = rsp.get('remote_record_found', False)

        return is_locked, is_locked_by_me, locked_by_name, remote_record_found

    def get_progress(self):
        """
        Returns the download progress information of the current download operation from Artella server

        :return: A tuple containing the following information:
            - amount of done download operations
            - amount of total download operations in progress
            - amount of total download operations that are going to be done
            - amount of total bytes downloaded
            - amount of total bytes to download
        :rtype: int, int, int, int, int
        """

        tx_count_done = 0
        tx_count_total = 0
        tx_bytes_done = 0
        tx_bytes_total = 0
        batches_complete = set()

        batch_ids = list(self._batch_ids)[:]

        for batch_id in batch_ids:
            params = urlencode({'batch-id': batch_id, 'details': True})
            req = Request('http://{}:{}/v2/localserve/progress/summary?{}'.format(self._host, self._port, params))
            rsp = self._communicate(req)
            tcd = int(rsp.get('transfer_count_done', 0))
            tct = int(rsp.get('transfer_count_total', 0))
            tbd = int(rsp.get('transfer_bytes_download_done', 0))
            tbt = int(rsp.get('transfer_bytes_download_total', 0))
            if tcd == tct or tbd == tbt:
                batches_complete.add(batch_id)
                continue
            tx_count_done += tcd
            tx_count_total += tct
            tx_bytes_done += tbd
            tx_bytes_total += tbt

        self._batch_ids -= batches_complete

        progress_value = 1
        if tx_bytes_total > 0:
            progress_value = float(tx_bytes_done) / float(tx_bytes_total)
        elif tx_count_total > 0:
            progress_value = float(tx_count_done) / float(tx_count_total)

        return int(progress_value * 100), tx_count_done, tx_count_total, tx_bytes_done, tx_bytes_total

    # ==============================================================================================================
    # TEST
    # ==============================================================================================================

    def ping(self):
        """
        Test call that returns whether the Artella Drive is valid or not

        :return: Returns a success response or auth failure message
        :rtype: dict
        """

        req = Request('http://{}:{}/v2/localserve/ping'.format(self._host, self._port))
        rsp = self._communicate(req, skip_auth=True)

        return rsp

    # ==============================================================================================================
    # ARTELLA DRIVE APP
    # ==============================================================================================================

    def artella_drive_connect(self):
        """
        Connects to the local user Artella Drive App via web socket so we can listen for realtime events
        :return:
        """

        server_address = (self._host, self._port)
        logger.info('Connecting to Artella Drive App web socket: {}'.format(server_address))
        try:
            sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            sock.connect(server_address)
        except Exception as exc:
            logger.error('Artella Plugin failed to connect to the Artella Drive App web socket: {}'.format(exc))
            return

        self.artella_drive_send_request(sock)
        self._socket_buffer = SocketBuffer(sock)
        rsp = self.artella_drive_read_response()
        self._running = True

        return rsp

    def artella_drive_disconnect(self):
        logger.debug('Disconnecting Artella Drive Client ...')
        self._running = False

        self._disconnect()

        return True

    def artella_drive_listen(self):
        self.artella_drive_connect()
        if not self._socket_buffer:
            logger.error('Socket to Artella Drive not connected.')
            return
        self._client_thread = threading.Thread(target=self._pull_messages,).start()

    def artella_drive_send_request(self, sock):

        if not self.update_auth_challenge():
            logger.error('Unable to authenticate to Artella Drive App.')
            return

        extensions_str = ''
        for extension in self._extensions:
            if extension != '':
                extension = '{}&'.format(extension)
            if extension.startswith('*'):
                extension = extension[1:]
            if extension.startswith('.'):
                extension = '%2A{}'.format(extension)
            extensions_str += 'file-handler={}'.format(extension)
        if extensions_str:
            extensions_str = extensions_str[:-1]

        key, expected_key_response = make_ws_key()
        path = '/v2/localserve/ws' if not extensions_str else '/v2/localserve/ws?{}'.format(extensions_str)
        r = [
            'GET {} HTTP/1.1'.format(path),
            'Upgrade: websocket',
            'Connection: Upgrade',
            'Host: {}:{}'.format(self._host, self._port),
            'Sec-WebSocket-Key: {}'.format(key),
            'Sec-WebSocket-Version: 13',
            'Authorization: {}'.format(self._auth_header),
        ]
        rr = consts.CRLF.join(r) + consts.CRLF + consts.CRLF
        logger.debug(
            'Artella Plugin sending websocket access request to the local Artella Drive App: {}'.format(r))

        try:
            sock.sendall(rr.encode())
        except Exception as exc:
            logger.warning('Was not possible to send request to Artella Drive App: {}'.format(exc))
            return False

        return True

    def artella_drive_read_response(self):
        logger.debug('Reading socket response ...')
        line = ''
        while line != consts.CRLF:
            line = self._socket_buffer.read_line()

        return line

    # ==============================================================================================================
    # INTERNAL
    # ==============================================================================================================

    def _communicate(self, req, data=None, skip_auth=False, method=None):
        """
        Internal function that sends a request to ArtellaDriver server

        :param Request req: URL request object
        :param str data: additional encoded data to send to the server
        :param bool skip_auth: whether authorization check should be skip or not
        :param str or None method: Method that should be used bu urrllib to send the request
        :return: dict, dictionary containing the answer from the server.
            If the call is not valid, a dictionary containing the url and error message is returned.
        """

        logger.debug('Making request to Artella Drive "{}"'.format(req.get_full_url()))
        if data:
            logger.debug('Request payload dump: "{}" | {}'.format(json.loads(data), type(data)))

        if not self._auth_header and not skip_auth:
            rsp = self.update_auth_challenge()
            if not rsp:
                msg = 'Unable to authenticate'
                logger.error(msg)
                return {'error': msg, 'url': req.get_full_url()}

        req.add_header('Authorization', self._auth_header)
        if method is not None:
            req.get_method = lambda: method
        try:
            rsp = urlopen(req, data)
        except URLError as exc:
            if hasattr(exc, 'reason'):
                msg = 'Failed to reach the local ArtellaDrive: "{}"'.format(exc.reason)
            elif hasattr(exc, 'code'):
                msg = 'ArtellaDrive is unable to fulfill the request "{}"'.format(exc.code)
            else:
                msg = exc
            logger.debug(exc)
            logger.error(msg)
            return {'error': msg, 'url': req.get_full_url()}
        else:
            raw_data = rsp.read()
            if not raw_data:
                return {'error': 'No Artella data response.', 'url': req.get_full_url()}
            else:
                try:
                    raw_data = "".join(chr(x) for x in bytearray(raw_data))
                    json_data = json.loads(raw_data)
                except ValueError:
                    logger.debug('ArtellaDrive data response: "{}"'.format(raw_data))
                    return raw_data
                except Exception as exc:
                    logger.error(exc)
                    return raw_data
                else:
                    logger.debug('ArtellaDriver JSON response: "{}"'.format(json_data))
                    return json_data

    def _disconnect(self):
        """
        Internal function that closes current Artella Drive client web socket

        :return: True if the web socket was successfully disconnected; False otherwise.
        :rtype: bool
        """

        try:
            if self._socket_buffer:
                self._socket_buffer.close()
                self._socket_buffer = None
                self._running = False
        except Exception as exc:
            logger.error(
                'Error while disconnect Artella Plugin from Artella Drive App web socket ... {}'.format(exc))
            return False

        return True

    def _track_response(self, rsp):
        """
        Internal function that extracts batch_id and transfer_data from response so that we can continue the
        tracking process

        :param dict rsp: Dictionary returned by a call to remote server
        :return: String containing ID of the operation
        :rtype: str
        """

        batch_id = rsp.get('batch_id')
        self._batch_ids.add(batch_id)

        return rsp

    def _pull_messages(self):
        logger.debug('Listening for commands on websocket')
        while self._running:
            try:
                msg = self._get_message()
            except Exception as exc:
                self._available = False
                try:
                    if exc.errno == 10054:
                        self._disconnect()
                except Exception as exc:
                    try:
                        logger.debug(str(exc))
                    except Exception as exc:
                        pass
                continue

            self._available = True
            logger.debug('Message received: {}'.format(msg))
            dccplugin.DccPlugin().pass_message(msg)

        # If Artella Drive Client is nor running we force disconnection
        # self._disconnect()

    def _get_message(self):
        op_code = ord(self._socket_buffer.get_char())
        v = ord(self._socket_buffer.get_char())
        if op_code != 129:
            raise exceptions.ArtellaException('Not a final text frame: {}'.format(op_code))
        if v < 126:
            length = v
        elif v == 126:
            a = ord(self._socket_buffer.get_char()) << 8
            b = ord(self._socket_buffer.get_char())
            length = a + b
        elif v == 127:
            # 8 byte payload length - we do not have any of these
            raise exceptions.ArtellaException('Unsupported payload length')
        else:
            raise exceptions.ArtellaException('Bad payload length: {}'.format(v))

        payload = self._socket_buffer.get_chars(length)

        return json.loads(payload)

class SocketBuffer(object):
    def __init__(self, sock):
        super(SocketBuffer, self).__init__()

        self._sock = sock
        self._buffer = ''

    def read_line(self):
        line = ''
        while True:
            c = self.get_char()
            line += c
            if c == '\n':
                return line

    def get_char(self):
        if len(self._buffer) == 0:
            self._buffer = self._sock.recv(consts.DEFAULT_BUFFER_SIZE)
        r = self._buffer[0]
        self._buffer = self._buffer[1:]

        return chr(r) if type(r) == int else r

    def get_chars(self, count):
        cc = ''
        for x in range(count):
            cc += self.get_char()

        return cc

    def close(self):
        self._sock.shutdown(socket.SHUT_RDWR)
        self._sock.close()

def make_ws_key():
    """
     https://tools.ietf.org/html/rfc6455
    :return:
    """

    key = base64.b64encode(str(int(random.random() * 9999999)).encode()).decode()
    h = hashlib.sha1()
    h.update((key + "258EAFA5-E914-47DA-95CA-C5AB0DC85B11").encode())
    expected_key_response = base64.b64encode(h.digest()).decode()

    return key, expected_key_response

def is_uri_path(file_path):
    """
    Returns whether or not given file path is using Artella URI schema or not

    :param str file_path: File path to check.
    :return: True if given path is using Artella URI schema; False otherwise.
    :rtype: bool
    :example:
    >>> is_uri_path("artella:project__fo0pohsa2sea4wyr5zmzcwnzse/refs/ref.png")
    True
    >>> is_uri_path("C:/Users/artella/artella-files/project/refs/ref.png")
    False
    """

    if not file_path:
        return False

    uri = urlparse(file_path)

    return uri.scheme == consts.ARTELLA_URI_SCHEME

def path_to_uri(path):
    """
    Converts a path to a path that uses Artella API scheme

    :param str path: File to convert to Artella URI scheme format
    :return: Returns a path converted into Artella URI scheme format. If the conversion cannot be done, the path
        without any change will be returned.
    :rtype: str
    :example:
    >>> path_to_uri("artella:project__fo0pohsa2sea4wyr5zmzcwnzse/refs/ref.png")
    "artella:project__fo0pohsa2sea4wyr5zmzcwnzse/refs/ref.png"
    >>> path_to_uri("C:/Users/artella/artella-files/project/refs/ref.png")
    "artella:project__fo0pohsa2sea4wyr5zmzcwnzse/refs/ref.png"
    >>> path_to_uri("Invalid/Path")
    "Invalid/Path"
    """

    path_uri = paths_to_uri(path)

    return path_uri[0]

def paths_to_uri(paths):
    """
    Converts a list of paths to paths that use Artella URI scheme

    :param str or list(str) paths: List of paths to convert to Artella URI scheme
    :return: List of paths converted into Artella URI scheme. If a path cannot be converted, it will be returned
        without any changes.
    :rtype: list(str)
    """

    paths = utils.force_list(paths)

    fixed_paths = list()

    for pth in paths:
        full_path = os.path.abspath(os.path.expandvars(os.path.expanduser(pth)))
        rsp = ArtellaDriveClient.get().resolve_path(full_path)
        if 'error' in rsp:
            logger.warning('Unable to translate path "{}" "{}"'.format(pth, rsp.get('error')))
            fixed_paths.append(pth)
            continue

        url_parts = (consts.ARTELLA_URI_SCHEME, '', rsp.get('handle'), '', '', '')
        fixed_path = urlunparse(url_parts)
        if not is_uri_path(fixed_path):
            logger.error('Failed to translate "{}" to URI: "{}"'.format(pth, fixed_path))
            fixed_paths.append(pth)
            continue

        fixed_paths.append(fixed_path)

    return fixed_paths

def path_to_handle(path):
    """
    Converts a path to an Artella handle path

    :param path: str, Path to convert to handle path, it can be an URI path or an absolute local file path
    :return: Returns an Artella handle path
    :rtype: str
    :example:
    >>> path_to_handle("artella:project__fo0pohsa2sea4wyr5zmzcwnzse/refs/ref.png")
    "file__12345"
    >>> path_to_handle("C:/Users/artella/artella-files/project/refs/ref.png")
    "file__12345"
    """

    handles = paths_to_handles(path, as_dict=False)

    return handles[0] if handles else path

def paths_to_handles(paths, as_dict=False):
    """
    Converts a list of paths (full paths or URI paths) to Artella handle paths

    :param str or list(str) paths:
    :param bool as_dict: If True, a dictionary mapping file paths with their handle will be returned.
    :return: List of handles associated to given file paths
    :rtype: list(str) or dict(str, str)
    """

    paths = utils.force_list(paths)

    handles = set()
    path_handle_map = dict()

    for pth in paths:
        if not is_uri_path(pth):
            uri_path = path_to_uri(pth)
            # If converted URI path is the same as the original path means that path is not valid, so we skip it
            if uri_path == pth:
                continue
        else:
            uri_path = pth
        uri_parts = urlparse(uri_path)
        handle = uri_parts.path
        handles.add(handle)
        path_handle_map[pth] = handle

    if as_dict:
        return path_handle_map

    return list(handles)

if __name__ == '__main__':

    from artella import loader
    loader.init()

    artella_cli = ArtellaDriveClient.get()
    print(artella_cli.ping())


"""
l'opérateur // est vachement utile dans cet exemple car il prend en compte uniquement la partie entière de la division ,
l'opérateur % lui prend le reste

exemple 624 :

624 // 100 - > 6
reste = 624 % 100 = 624 - (6.xxx * 100) = 70
etc.

"""

def calculer_billets(montant):
    billets_100 = montant // 100
    reste = montant % 100
    billets_50 = reste // 50
    reste = reste % 50
    billets_10 = reste // 10
    reste = reste % 10
    billets_2 = reste // 2
    billets_1 = reste % 2
    return billets_100, billets_50, billets_10, billets_2, billets_1

montant = int(input("Entrez une somme d'argent: "))
billets_100, billets_50, billets_10, billets_2, billets_1 = calculer_billets(montant)

print(f'{billets_100} billets de 100')
print(f'{billets_50} billets de 50')
print(f'{billets_10} billets de 10')
print(f'{billets_2} billets de 2')
print(f'{billets_1} billets de 1')


#%%from http.client import TOO_MANY_REQUESTS
import json
from tracemalloc import start
import pvlib as pv
import numpy as np
import pandas as pd
import requests

#%%
# c = cdsapi.Client()
# c.retrieve("reanalysis-era5-pressure-levels",
# {
# "variable": "temperature",
# "pressure_level": "1000",
# "product_type": "reanalysis",
# "year": "2008",
# "month": "01",
# "day": "01",
# "time": "12:00",
# "format": "grib"
# }, "download.grib")

#%% TMY Data (PVGIS)
def tmy_data(lat, lon):
    tmy_data_api = pv.iotools.get_pvgis_tmy(
        latitude= lat,
        longitude= lon,
        map_variables= True
    )
    tmy = tmy_data_api[0]

    return tmy

def wind_weather_openfred(lat, lon, year, hub_height):
    start_date = str(year) + '-1-1'
    end_date = str(year+1) + '-1-1'
    location = Point(lon, lat)
    raw_data_wind = Weather(
        start=start_date,
        stop=end_date,
        locations=[location],
        heights= [hub_height],
        variables = "windpowerlib",
        **defaultdb())
    return raw_data_wind

def sky_data(lat, lon, year):
    query = {
        'date' : str(year) + "-01-01",
        'last_date': str(year+1) + "-01-01",
        'lat': lat,
        'lon': lon
    }
    response_API = requests.get("https://api.brightsky.dev/weather", params=query)
    data = response_API.text
    sky_data_api =  json.loads(data)
    sky = pd.DataFrame.from_dict(sky_data_api["weather"][0:8760])

    return sky

def era5_weather(lat, lon, year, variable):
    target_file = f'weather_era5_{variable}.nc'
    start_date, end_date = f'{year}-01-01', f'{year+1}-01-01'
    ds = era5.get_era5_data_from_datespan_and_position(
        variable=variable,
        start_date=start_date, end_date=end_date,
        latitude=lat, longitude=lon,
        target_file=target_file)

    return ds


from django.shortcuts import render, get_object_or_404, redirect
from django.views.decorators.http import require_POST

from goods.models import Goods
from shopcart.cart import Cart
from shopcart.forms import AddProductForm

# Create your views here.
def detail(request):
    cart = Cart(request)

    for product in cart:
        product['quantity_form'] = AddProductForm(initial={'quantity':product['quantity'],'id_update':True})

    context = {'cart':cart}
    return render(request, 'mypage/bag_detail.html', context)

@require_POST
# def add(request, product_id):
#     cart = Cart(request)
#     product = get_object_or_404(Goods, id=product_id)
#
#     form = AddProductForm(request.POST) # 입력된 수량
#     if form.is_valid():
#         cd = form.cleaned_data  # 유효성검사가 끝난 데이터
#         cart.add(product=product,quantity=cd['quantity'], is_update=cd['is_update'])
#         return redirect('shopcart:detail')

def add(request, product_id):
    cart = Cart(request)
    product = get_object_or_404(Goods, id=product_id)

    form = AddProductForm(request.POST)  # 입력된 수량
    if form.is_valid():
        cd = form.cleaned_data  # 유효성검사가 끝난 데이터
        cart.add(product=product, quantity=cd['quantity'], is_update=cd.get('is_update', False))
        return redirect('shopcart:detail')

    # 유효성 검사 실패 시에도 장바구니 페이지로 리다이렉트
    return redirect('shopcart:detail')

def remove(request, product_id):
    cart = Cart(request)
    product = get_object_or_404(Goods, id=product_id)
    cart.remove(product)
    return redirect('shopcart:detail')


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import os, sys, time
from datetime import datetime
from pytz import timezone

# set email to send notification
mailto = ['email@email.com']
mail_message = []

def log(logmessage):
    agora = datetime.now(timezone('America/Sao_Paulo'))
    data = agora.strftime('%Y-%m-%d %H:%M:%S %Z%z')
    mail_message.append('{} - {}'.format(data, logmessage))
    print('{} - {}'.format(data, logmessage))

dir1 = '' # diretory with control file

lista1 = []
lista2 = []

with open(arq1, 'r') as f:
    lista1[:] = f.read().splitlines()

print(50 * '-')
log('Starting cleanup ...')

for root, directories, filenames in os.walk(os.path.join(dir1, 'site_data')):
    for filename in filenames:
        if filename.endswith('.php'):
            lista2.append(os.path.join(root,filename))

dif = list(set(lista2) - set(lista1))

if len(dif) > 0:
    log('Found {} file(s) to remove !!!'.format(len(dif)))
else:
    log('Nothing to remove !!!')
    sys.exit(0)

for i in dif:
    log('Removing {}'.format(i))
    os.remove(i)

log('Files removed !!!')

from send_email import send_simple_message
for i in mailto:
    send_simple_message(i, 'Check File Just Removed File(s)', '\n'.join(mail_message))


#
#  L9-5: sum_multiples_raw.py
#

# each of the sum_multiple_N.py examples, in succession...
#    (no submission required)

#
# the imperative state-changing way
#

s = 0 # this accumulator object changes state...
for n in range(1, 10):
    if n % 3 == 0 or n % 5 == 0:
        s += n
print(s)

#
# the object-oriented way with a list
#

m = list() # this list object changes state...
for n in range(1, 10):
    if n % 3 == 0 or n % 5 == 0:
        m.append(n)
print(sum(m))

# #
# # subclass of list which has its own sum
# # (sum() as function is equivalent to sum() as method
# #
#
# class Summable_List(list):
#     def sum(self):
#         s = 0
#         for v in self:
#             s += v
#         return s

# defining sum recursively for lists: look, no state!
#   (but no filtering - yet)

def sumr(seq):
    if len(seq) == 0:
        return 0
    return seq[0] + sumr(seq[1:])

print (sumr([k for k in range(1,10)]))

# defining sequence of values recursively:
# note use of passed function filter_func
# functional programming makes functions first-class data

def until(n, filter_func, v):
    if v == n:
        return []
    if filter_func(v):
        return [v] + until(n, filter_func, v+1)
    else:
        return until(n, filter_func, v+1)

mult_3_5 = lambda n: n%3==0 or n%5==0

print(until (10, mult_3_5, 0))
print(until (10, lambda n: n%3==0 or n%5==0, 0))
print(sumr(until (10, lambda n: n%3==0 or n%5==0, 0)))

# mostly functional version

print(sum(n*n for n in range(1, 10) if n%3==0 or n%5==0))
#  range(n) is a generator expression: it yields a sequence
#   of values 0,1,...,n-1 and is iterable

# now to iteratively compute math.sqrt(2)

def next_(n, x):
    return (x+n/x)/2

f = lambda x: next_(2,x)

print ([round(x,4) for x in (1.0, f(1.0), f(f(1.0)), f(f(f(1.0))))])

def repeat(f, a):
    yield a
    for v in repeat(f, f(a)):
        yield v

print (repeat,2)

# treating function as object
# first-class objects are:
# Created at runtime
# • Assigned to a variable or element in a data structure
# • Passed as an argument to a function
# • Returned as the result of a function

def factorial(n):
    '''returns n!'''
    return 1 if n < 2 else n * factorial(n-1)

print (factorial(47))
print (factorial.__doc__)
print (type(factorial))

# use function through different name, passing as argument

fact = factorial

print (list(map(fact,range(11)))) # applies fact to each element of range(11)

# higher-order functions take function as arg and/or return function as result

# list comprehensions and generator expressions

# A quick way to build a sequence is using a list comprehension (if the target is a list)
# or a generator expression (for all other kinds of sequences).
#

symbols = '0123456789'
codes = []
for symbol in symbols:
    if ord(symbol)%2==0:
        codes.append(ord(symbol))
print (codes)

# list comprehension (lc) of above: all names are local
symbols = '0123456789'
codes = [ord(symbol) for symbol in symbols if ord(symbol)%2 == 0]
print (codes)
print (list(ord(symbol) for symbol in symbols if ord(symbol)%2 == 0))

# lc's build lists from sequences or other iterable types by filtering and transforming

# most functional languages have functions filter() and map() which can be composed to
#   achieve the same effect

codes = list(map(ord,filter(lambda c: ord(c)%2==0, symbols))) # , map(ord,symbols)))
print (codes)

# finally: do original problem using map and filter

codes = sum(filter(lambda x: x%3==0 or x%5==0, map(lambda x: x,range(1,10))))
print (codes)

import functools
import operator

# can also do this using functools.reduce and operator.add: sum is shortcut for reduce

codes = functools.reduce(operator.add,filter(lambda x: x%3==0 or x%5==0, map(lambda x: x,range(1,10))))
print (codes)


"""Put a direct text annotation at the beginning of all lines in a docx file.
"""

from pathlib import Path
import re

import docx

from ..utils.languagesusser import determine_language_from_text as dlt
from ..utils.textlineannotator import TextlineAnnotation
from ..resources.constants import FileFormat

class DocxParagraphTextAnnotations(TextlineAnnotation):
    """Some of the plaintext annotations to put at the beginning of a line,
    placed in a class so it doesn't need to be substantiated for each par.
    """

    @classmethod
    def _annotate_docx_par(cls, paragraph_text):
        """Roughly annotate the type of the line according to the information in the line.

        Arguments:
            paragraph -- a line of text from a docx Paragaph
        """
        new_text = TextlineAnnotation.annotate_line(paragraph_text)
        return new_text

    @classmethod
    def _annotate_all_docx_pars(cls, corpus_docx_file: docx.document.Document) -> docx.document.Document:
        """Annotate all the lines in a corpus docx file with some qualifications
        stipulated in annotate_docx_line.

        Arguments:
            corpus_docx_file -- the path to a docx file to put in the corpus
        """

        for paragraph in corpus_docx_file.paragraphs:
            # split lines and discard extra space

            paragraph_text = [line for line in paragraph.text.splitlines() if len(line) > 0]

            # do nothing if nothing is in the line
            if len(paragraph_text) == 0:
                continue

            # annotate each line of text, more than one if there is a linebreak
            elif len(paragraph_text) >= 1:
                new_text = [cls._annotate_docx_par(paragraph_line) for paragraph_line in paragraph_text]
                paragraph.text = '\n'.join(new_text)
                continue

        return corpus_docx_file

    @classmethod
    def annotate_docx_file(cls, docx_filepath: Path, **kwargs):
        """ask a docx file to be annotated and save a new version of it

        Arguments:
            docx_filepath -- path to a docx file
        Kwargs:
            output_path: desired output Path
        """

        output_path = kwargs.get('output_path') # type: Path
        verbose = kwargs.get('verbose')
        new_docx_name = None # initialize docx name
        docx_document = docx.Document(docx_filepath)

        # construct output filepath
        if output_path is not None: # use desired path if provided
            output_path = Path(output_path)

        else: # make default folder
            output_dir = docx_filepath.parent / 'annotated'
            if output_dir.exists() is False:
                output_dir.mkdir()

        # if no new name is provided: come up with default "annotated" name
        if new_docx_name is None:
            new_docx_name = Path(docx_filepath.stem + ' annotated').with_suffix(docx_filepath.suffix)
            output_path = output_path / new_docx_name # make new name

        # resolve in case of relative path
        output_path = output_path.resolve()

        if verbose is not None:
            print(f'#####\ndocx_filename = {docx_filepath}\nnew_docx_name = {new_docx_name}\noutput_path = {output_path}\n')

        annotated_docx = cls._annotate_all_docx_pars(docx_document)
        annotated_docx.save(output_path)


#RandomForestTreeClassifier

from sklearn.datasets import load_iris
from sklearn import tree
from sklearn.metrics import accuracy_score
import os
from sklearn.model_selection import GridSearchCV

from sklearn.model_selection import ShuffleSplit

from sklearn.ensemble import RandomForestClassifier
from sklearn.datasets import make_classification

import pandas as pd
import numpy as np
from sklearn import cluster, datasets

import graphviz

from sklearn.tree import export_graphviz
import pydot

from sklearn import preprocessing
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_curve, auc
from matplotlib.legend_handler import HandlerLine2D
import matplotlib.pyplot as plt
from sklearn import metrics
from subprocess import call
from IPython.display import Image

efile_normal_datac = os.path.join("Elephant_miceflows/bbr/ele_nm.csv")
edfnormaldatac=pd.read_csv(efile_normal_datac)#nrows=250
mfile_normal_datac = os.path.join("Elephant_miceflows/bbr/mice_nm.csv")
mdfnormaldatac=pd.read_csv(mfile_normal_datac)#nrows=1000

efile_loss01_data = os.path.join("Elephant_miceflows/bbr/ele_loss01.csv")
edfloss01data=pd.read_csv(efile_loss01_data)
mfile_loss01_data = os.path.join("Elephant_miceflows/bbr/mice_loss01.csv")
mdfloss01data=pd.read_csv(mfile_loss01_data)

efile_loss05_data = os.path.join("Elephant_miceflows/bbr/ele_loss05.csv")
edfloss05data=pd.read_csv(efile_loss05_data)
mfile_loss05_data = os.path.join("Elephant_miceflows/bbr/mice_loss05.csv")
mdfloss05data=pd.read_csv(mfile_loss05_data)

efile_loss10_data = os.path.join("Elephant_miceflows/bbr/ele_loss10.csv")
edfloss10data=pd.read_csv(efile_loss10_data)
mfile_loss10_data = os.path.join("Elephant_miceflows/bbr/mice_loss10.csv")
mdfloss10data=pd.read_csv(mfile_loss10_data)

efile_dup1_data = os.path.join("Elephant_miceflows/bbr/ele_dup1.csv")
edfdup1data=pd.read_csv(efile_dup1_data)
mfile_dup1_data = os.path.join("Elephant_miceflows/bbr/mice_dup1.csv")
mdfdup1data=pd.read_csv(mfile_dup1_data)

efile_dup5_data = os.path.join("Elephant_miceflows/bbr/ele_dup5.csv")
edfdup5data=pd.read_csv(efile_dup5_data)
mfile_dup5_data = os.path.join("Elephant_miceflows/bbr/mice_dup5.csv")
mdfdup5data=pd.read_csv(mfile_dup5_data)

efile_reord25_data = os.path.join("Elephant_miceflows/bbr/ele_reord25.csv")
edfreord25data=pd.read_csv(efile_reord25_data)
mfile_reord25_data = os.path.join("Elephant_miceflows/bbr/mice_reord25.csv")
mdfreord25data=pd.read_csv(mfile_reord25_data)

efile_reord50_data = os.path.join("Elephant_miceflows/bbr/ele_reord50.csv")
edfreord50data=pd.read_csv(efile_reord50_data)
mfile_reord50_data = os.path.join("Elephant_miceflows/bbr/mice_reord50.csv")
mdfreord50data=pd.read_csv(mfile_reord50_data)
#print(mdfreord50data)

# test data

#Genomeworkflows/bbr/ham_pord5050dn skip 250 for elrphant, skip 800 for mice, workflows skip 200
cfile_test_data = os.path.join("Genomeworkflows/bbr/bbr_reor50dn.csv")
cdftestdata=pd.read_csv(cfile_test_data,skiprows=200)

cdftestdata.columns=['#15#c_ip:1','c_port:2','c_pkts_all:3','c_rst_cnt:4','c_ack_cnt:5',
'c_ack_cnt_p:6','c_bytes_uniq:7','c_pkts_data:8','c_bytes_all:9','c_pkts_retx:10',
'c_bytes_retx:11','c_pkts_ooo:12','c_syn_cnt:13','c_fin_cnt:14','s_ip:15',
's_port:16','s_pkts_all:17','s_rst_cnt:18','s_ack_cnt:19','s_ack_cnt_p:20',
's_bytes_uniq:21','s_pkts_data:22','s_bytes_all:23','s_pkts_retx:24',
's_bytes_retx:25','s_pkts_ooo:26','s_syn_cnt:27','s_fin_cnt:28','first:29',
'last:30','durat:31','c_first:32','s_first:33','c_last:34','s_last:35',
'c_first_ack:36','s_first_ack:37','c_isint:38','s_isint:39','c_iscrypto:40',
's_iscrypto:41','con_t:42','p2p_t:43','http_t:44','c_rtt_avg:45','c_rtt_min:46',
'c_rtt_max:47','c_rtt_std:48','c_rtt_cnt:49','c_ttl_min:50','c_ttl_max:51',
's_rtt_avg:52','s_rtt_min:53','s_rtt_max:54','s_rtt_std:55','s_rtt_cnt:56',
's_ttl_min:57','s_ttl_max:58','p2p_st:59','ed2k_data:60','ed2k_sig:61',
'ed2k_c2s:62','ed2k_c2c:63','ed2k_chat:64','c_f1323_opt:65','c_tm_opt:66',
'c_win_scl:67','c_sack_opt:68','c_sack_cnt:69','c_mss:70','c_mss_max:71',
'c_mss_min:72','c_win_max:73','c_win_min:74','c_win_0:75','c_cwin_max:76',
'c_cwin_min:77','c_cwin_ini:78','c_pkts_rto:79','c_pkts_fs:80','c_pkts_reor:81',
'c_pkts_dup:82','c_pkts_unk:83','c_pkts_fc:84','c_pkts_unrto:85','c_pkts_unfs:86',
'c_syn_retx:87','s_f1323_opt:88','s_tm_opt:89','s_win_scl:90','s_sack_opt:91',
's_sack_cnt:92','s_mss:93','s_mss_max:94','s_mss_min:95','s_win_max:96',
's_win_min:97','s_win_0:98','s_cwin_max:99','s_cwin_min:100','s_cwin_ini:101',
's_pkts_rto:102','s_pkts_fs:103','s_pkts_reor:104','s_pkts_dup:105',
's_pkts_unk:106','s_pkts_fc:107','s_pkts_unrto:108','s_pkts_unfs:109',
's_syn_retx:110','http_req_cnt:111','http_res_cnt:112','http_res:113',
'c_pkts_push:114','s_pkts_push:115','c_tls_SNI:116','s_tls_SCN:117',
'c_npnalpn:118','s_npnalpn:119','c_tls_sesid:120','c_last_handshakeT:121',
's_last_handshakeT:122','c_appdataT:123','s_appdataT:124','c_appdataB:125',
's_appdataB:126','fqdn:127','dns_rslv:128','req_tm:129','res_tm:130',
'DeclaredThroughput','Actualthroughput']
#print(cdftestdata)

edfnormaldatac['Target'] = 0
mdfnormaldatac['Target'] = 1

#edfnormaldata['Target'] = 4
#mdfnormaldata['Target'] = 5

#edfnormaldatab['Target'] = 6
#mdfnormaldatab['Target'] = 7

edfloss01data['Target'] = 2
mdfloss01data['Target'] = 3
edfloss05data['Target'] = 4
mdfloss05data['Target'] = 5
edfloss10data['Target'] = 6
mdfloss10data['Target'] = 7
edfdup1data['Target'] = 8
mdfdup1data['Target'] = 9
edfdup5data['Target'] = 10
mdfdup5data['Target'] = 11
edfreord25data['Target'] = 12
mdfreord25data['Target'] = 13
edfreord50data['Target'] = 14
mdfreord50data['Target'] = 15

all_data_frames = [edfnormaldatac, mdfnormaldatac,edfloss01data, mdfloss01data, edfloss05data, mdfloss05data,
edfloss10data, mdfloss10data, edfdup1data, mdfdup1data, edfdup5data, mdfdup5data,
edfreord25data, mdfreord25data, edfreord50data, mdfreord50data]

all_data_result = pd.concat(all_data_frames)

test_data_frames = cdftestdata

#convert all strings and time to zero- to remove them from rules
all_data_result['#15#c_ip:1']=0
all_data_result['s_ip:15']=0
all_data_result['first:29']=0
all_data_result['last:30']=0
all_data_result['http_res:113']=0
all_data_result['c_tls_SNI:116']=0
all_data_result['s_tls_SCN:117']=0
all_data_result['fqdn:127']=0
all_data_result['dns_rslv:128']=0
all_data_result['http_hostname:131']=0
all_data_result['c_port:2']=0
all_data_result['s_port:16']=0
all_data_result['first:29']=0
all_data_result['last:30']=0
all_data_result['http_t:44']=0

features=list(all_data_result.columns[:133])

test_data_frames['#15#c_ip:1']=0
test_data_frames['s_ip:15']=0
test_data_frames['first:29']=0
test_data_frames['last:30']=0
test_data_frames['http_res:113']=0
test_data_frames['c_tls_SNI:116']=0
test_data_frames['s_tls_SCN:117']=0
test_data_frames['fqdn:127']=0
test_data_frames['dns_rslv:128']=0
test_data_frames['http_hostname:131']=0
test_data_frames['c_port:2']=0
test_data_frames['s_port:16']=0
test_data_frames['first:29']=0
test_data_frames['last:30']=0
test_data_frames['http_t:44']=0

testfeatures=list(test_data_frames.columns[:133])

#print(all_data_result['http_res:113'])

X=all_data_result[features]
y=all_data_result['Target']
target_names=['0','1', '2', '3','4','5', '6','7','8','9','10','11','12','13','14','15']
print(X)
testX=test_data_frames[testfeatures]
print(testX)

#Original tree
#X,y= make_classification(n_samples=1000, n_features=4,
#	n_informative=2, n_redundant=0,
#	random_state=0, shuffle=False)
clf = RandomForestClassifier(n_estimators=10, max_depth=6, random_state=0)

clf = clf.fit(X, y)

print("train score =")
print(clf.score(X,y))
#pruning for optimum parameters

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.3, random_state = 100)

y_pred_1 = clf.predict(X_test)
print(y_pred_1)

# first fit the model to get baseline
#clf=clf.fit(X_test, y_test)
print("test tree score =")
print(clf.score(X_test, y_test))

#evaluating optimum parameters
y_pred = clf.predict(X_test)
# Model Accuracy, how often is the classifier correct?
print("Accuracy of test tree:",metrics.accuracy_score(y_test, y_pred))
#printing AUC
#false_positive_rate, true_positive_rate, thresholds = roc_curve(y_test, y_pred)
#roc_auc = auc(false_positive_rate, true_positive_rate)

#print("roc_auc =")
#print(roc_auc)

#tuning max depth

max_depths = np.linspace(1, 32, 32, endpoint=True)
train_results = []
test_results = []
for max_depth in max_depths:
	dt = RandomForestClassifier(max_depth=max_depth, max_features=15)
	dt.fit(X_train, y_train)
	train_pred = dt.predict(X_train)

	tscore=metrics.accuracy_score(y_train, train_pred)
	train_results.append(tscore)

	y_pred=dt.predict(X_test)
	###

	nscore=metrics.accuracy_score(y_test, y_pred)
	test_results.append(nscore)

#print(train_results)
line1, = plt.plot(max_depths, train_results, 'b', label="Training Data")
line2, = plt.plot(max_depths, test_results, 'r', label="Test Data")
plt.legend(handler_map={line1: HandlerLine2D(numpoints=2)})
plt.ylabel('Score')
plt.xlabel('Tree depth')
plt.show()

print("Printing leaf nodes?")
train_results2 = []
test_results2 = []
#tuning the leaf nodes
for j in [10,30,50, 60,70, 80, 90, 100]:
	clf_stump=RandomForestClassifier(max_depth=5,n_estimators=j,max_features=15)
	clf_stump.fit(X_train, y_train)
	train_pred2 = clf_stump.predict(X_train)
	tscore2=metrics.accuracy_score(y_train, train_pred2)
	train_results2.append(tscore2)

	newy_pred=clf_stump.predict(X_test)

	nscore2=metrics.accuracy_score(y_test, newy_pred)
	test_results2.append(nscore2)
	print(j,tscore2, nscore2)

#predicting values
#after parameter tuning

#clf = tree.DecisionTreeClassifier(criterion = "gini", splitter = 'random', max_leaf_nodes = 20, min_samples_leaf = 10, max_depth= 10)
#clf = clf.fit(X, y)
clf = RandomForestClassifier(n_estimators=10, max_depth=5, random_state=0,max_features=15)
clf = clf.fit(X, y)
print("tree generated")
importances=clf.feature_importances_

std = np.std([tree.feature_importances_ for tree in clf.estimators_],
             axis=0)
indices = np.argsort(importances)[::-1]

# Print the feature ranking
print("Feature ranking:")

for f in range(X.shape[1]):
    print("%d. feature %d (%f)" % (f + 1, indices[f], importances[indices[f]]))

estimator=clf.estimators_[5]

clf = RandomForestClassifier(n_estimators=10, max_depth=5, random_state=0,max_features=15)
clf = clf.fit(X, y)

y_pred_1 = clf.predict(testX)
print("Test size %s" %len(testX))
for n in y_pred_1:
	print(n)

dot_data = export_graphviz(estimator, out_file='osg-test.dot',
	feature_names=features,
	class_names=['0','1','2', '3','4','5','6','7','8','9','10','11','12','13','14','15'],
	rounded=True, proportion=False,
	precision=2, filled=True)

#graph = graphviz.Source(dot_data)
#graph

call(['dot', '-Tpng', 'osg-test.dot', '-o', 'osg-test.png'])

#feat_importance = clf.tree_.compute_feature_importances(normalize=False)
#print("feat importance = " + str(feat_importance))

Image(filename = 'osg-test.png')


#Q1.circle class

class Circle:
    def __init__(self,r):                           #Initialising value
        self.radius = r
        print("Radius: ", self.radius)
    def getArea(self):                              #Function to calculate & print area of circle
        area = 3.14*self.radius*self.radius
        print("Area: ", area)
    def getCircumference(self):                     #Function to calculate & print perimeter of circle
        per = 2*3.14*self.radius
        print("Circumference: %.2f" % per)

#Driving Code
obj = Circle(5)             #Object creation of class Circle
obj.getArea()
obj.getCircumference()

#Q2.Student class

class Student:
    def __init__(self,name,roll):                  #Initialising values
        self.n = name
        self.r = roll
    def Display(self):                             #Function to display Details of students
        print("Name: ", self.n)
        print("Roll_Number: ", self.r)

#Driving Code
S1 = Student("X", 1)         #Object creation of class Student
S1.Display()
S2 = Student("Y", 2)         #Object Creation of class Student
S2.Display()

#Q3.Temperature class

class Temperature:
    def __init__(self,temp):                      #Initialising value
        self.t = temp
    def convertFahrenheit(self):                  #Function to convert Celsius to Fahrenheit
        fah = (9/5)*self.t + 32
        print("Fahrenheit Equivalent: ", fah)
    def convertCelsius(self):                     #Function to convert Fahrenheit to Celsius
        cel = (self.t-32)*(5/9)
        print("Celsius Equivalent: ", cel)
t1 = Temperature(53)          #Object Creation of class Temperature
t2 = Temperature(96.8)        #Object Creation of class Temperature
t1.convertFahrenheit()
t2.convertCelsius()

#Q4.MovieDetails

class MovieDetails:
    def __init__(self, mn, an, yr, r):            #Initialising values
        self.movie_name = mn
        self.artist_name = an
        self.release_year = yr
        self.ratings = r
    def Display(self):             				               #Function to display details of Movie
        print("Movie: ",self.movie_name)
        print("Artist_Name: ",self.artist_name)
        print("Release_Year: ",self.release_year)
        print("Ratings_(Out of 10): ", self.ratings)
    def Update(self, new_moviename, new_artistname, new_release, new_ratings):  #Function to update details of Movie as provided
        print("Updating Movie Details.....")
        self.movie_name = new_moviename
        self.artist_name = new_artistname
        self.release_year = new_release
        self.ratings = new_ratings
Movie1 = MovieDetails("xyz","abc",2000,5.5)    				      #Object creation of class MovieDetails
Movie2 = MovieDetails("jkl","iop",2001,7.3)              	              #Object creation of class MovieDetails
Movie1.Display()
Movie2.Display()
Movie2.Update("jkl","ytu",2001,7.5)
Movie2.Display()

#Q5.Expenditure

class Expenditure:
    def __init__(self,expenditure, savings):     #Initialising values
        self.e = expenditure
        self.s = savings
    def Display(self):                           #Function to Display Expenditure & Savings
        print("Expenditure: ", self.e)
        print("Savings: ", self.s)
    def Calculate_salary(self):                  #Function to Calculate salary
        self.salary = self.s - self.e
    def Display_salary(self):
        print("Salary: ", self.salary)           #Function to Display salary
E = Expenditure(2240, 4000)                      #Object creation of class Expenditure
E.Display()
E.Calculate_salary()
E.Display_salary()


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from absl.testing import parameterized

import numpy as np
from tensor2tensor.data_generators import problem_hparams
from tensor2tensor.models.research import transformer_aux
import tensorflow.compat.v1 as tf
from tensorflow.compat.v1 import estimator as tf_estimator

class TransformerAuxTest(parameterized.TestCase, tf.test.TestCase):

  @parameterized.parameters(
      dict(
          tensor=np.array(
              [1, 2, 3, 4]
          ),
          shift=0,
          axis=0,
          target=np.array(
              [1, 2, 3, 4]
          ),
      ),
      dict(
          tensor=np.array(
              [1, 2, 3, 4]
          ),
          shift=2,
          axis=0,
          target=np.array(
              [0, 0, 1, 2]
          ),
      ),
      dict(
          tensor=np.array(
              [1, 2, 3, 4]
          ),
          shift=-2,
          axis=0,
          target=np.array(
              [3, 4, 0, 0]
          ),
      ),
      dict(
          tensor=np.array(
              [[1, 2, 3, 4],
               [5, 6, 7, 8]]
          ),
          shift=2,
          axis=1,
          target=np.array(
              [[0, 0, 1, 2],
               [0, 0, 5, 6]]
          ),
      ),
  )
  def test_shift_and_pad(self, tensor, shift, axis, target):
    with self.test_session() as session:
      output = transformer_aux.shift_and_pad(tensor, shift, axis)
      output_val = session.run(output)
      self.assertAllEqual(output_val, target)

  def test_transformer_aux_body(self):
    batch_size = 3
    input_length = 5
    target_length = 16
    vocab_size = 9
    hparams = transformer_aux.transformer_aux_tiny()
    hparams.shift_values = "-5,1,2,3"
    p_hparams = problem_hparams.test_problem_hparams(vocab_size,
                                                     vocab_size,
                                                     hparams)
    hparams.problem_hparams = p_hparams
    inputs = np.random.randint(
        vocab_size, size=(batch_size, input_length, 1, 1))
    targets = np.random.randint(
        vocab_size, size=(batch_size, target_length, 1, 1))
    features = {
        "inputs": tf.constant(inputs, dtype=tf.int32),
        "targets": tf.constant(targets, dtype=tf.int32),
        "target_space_id": tf.constant(1, dtype=tf.int32),
    }
    tf.train.create_global_step()
    model = transformer_aux.TransformerAux(hparams, tf_estimator.ModeKeys.TRAIN,
                                           p_hparams)
    logits, losses = model(features)

    self.assertIn("training", losses)
    self.assertIn("auxiliary", losses)

    with self.test_session() as session:
      session.run(tf.global_variables_initializer())
      logits_val = session.run(logits)
      self.assertEqual(logits_val.shape,
                       (batch_size, target_length, 1, 1, vocab_size))

if __name__ == "__main__":
  tf.test.main()


import tensorflow as tf
import numpy as np
import scipy.misc as misc
import os, sys
import scipy.io
import matplotlib.cm as cm
from functools import partial

def save_image(image, save_dir, name):
    """
    Save image by unprocessing if mean given else just save
    :param mean:
    :param image:
    :param save_dir:
    :param name:
    :return:
    """
    misc.imsave(os.path.join(save_dir, name + ".png"), image)

def add_to_regularization_and_summary(var):
    if var is not None:
        tf.summary.histogram(var.op.name, var)
        tf.add_to_collection("reg_loss", tf.nn.l2_loss(var))

def add_activation_summary(var):
    if var is not None:
        tf.summary.histogram(var.op.name + "/activation", var)
        tf.summary.scalar(var.op.name + "/sparsity", tf.nn.zero_fraction(var))

def add_gradient_summary(grad, var):
    if grad is not None:
        tf.summary.histogram(var.op.name + "/gradient", grad)

def add_image_summary(var,**kwargs):
    if var is None:
        return
    num_ch = int(var.get_shape()[-1])
    var_ch_lst = tf.split(var, num_or_size_splits=num_ch, axis=3)
    for ch, var_ch in enumerate(var_ch_lst):
        tf.summary.image(var.op.name + "/image_ch%d"%ch, var_ch, **kwargs)

def colorize(value, vmin=None, vmax=None, cmap=None):
    """
    src="https://gist.github.com/jimfleming/c1adfdb0f526465c99409cc143dea97b.js"

    A utility function for TensorFlow that maps a grayscale image to a matplotlib
    colormap for use with TensorBoard image summaries.
    By default it will normalize the input value to the range 0..1 before mapping
    to a grayscale colormap.
    Arguments:
      - value: 2D Tensor of shape [height, width] or 3D Tensor of shape
        [height, width, 1].
      - vmin: the minimum value of the range used for normalization.
        (Default: value minimum)
      - vmax: the maximum value of the range used for normalization.
        (Default: value maximum)
      - cmap: a valid cmap named for use with matplotlib's `get_cmap`.
        (Default: 'gray')
    Example usage:
    ```
    output = tf.random_uniform(shape=[256, 256, 1])
    output_color = colorize(output, vmin=0.0, vmax=1.0, cmap='viridis')
    tf.summary.image('output', output_color)
    ```

    Returns a 3D tensor of shape [height, width, 3].
    """

    # normalize
    vmin = tf.reduce_min(value) if vmin is None else vmin
    vmax = tf.reduce_max(value) if vmax is None else vmax
    value = (value - vmin) / (vmax - vmin) # vmin..vmax

    # squeeze last dim if it exists
    value = tf.squeeze(value)

    # quantize
    indices = tf.to_int32(tf.round(value * 255))

    # gather
    c_map = cm.get_cmap(cmap if cmap is not None else 'gray')
    colors = c_map(np.arange(256))[:, :3]
    colors = tf.constant(colors, dtype=tf.float32)
    value = tf.gather(colors, indices)

    return value

def batch_colorize(tensor, vmin=None, vmax=None, cmap=None):
    tensor = tf.cast(tensor, tf.float32)
    colorize_tensor = partial(colorize, vmin=vmin, vmax=vmax, cmap=cmap)
    colorized_tensor = tf.map_fn(colorize_tensor, tensor)
    return colorized_tensor

def batch_colorize_ndarray(value, vmin=None, vmax=None, cmap=None):
    # normalize
    vmin = np.min(value[:]) if vmin is None else vmin
    vmax = np.max(value[:]) if vmax is None else vmax
    value = 255 * (value - vmin) / (vmax - vmin) # vmin..vmax

    # squeeze last dim if it exists
    value = np.squeeze(value)

    # gather
    c_map = cm.get_cmap(cmap if cmap is not None else 'gray')
    value = 255 * c_map(value.astype(np.int32))
    value = value.astype(np.uint8)
    return value


from ml.ap.ap_models import Best_CNN_categorical, Best_LSTM_categorical, Best_CNN_LSTM_categorical, \
    Best_BiLSTM_categorical, Best_aCNN_categorical
import numpy as np
from ml.training_utils import plot_by_fold, get_distribution_of_labels_categorical_from_caselabels, \
    class_weights_from_onehot_categorical
from ml.dataseries import DataSeries
import pandas as pd

dataseries = DataSeries(path="../../exports/ap_qrs_all",
                        labelfunction=DataSeries.get_labels_by_case_ap_left_right,
                        #caselabels="C:\\Users\\J\\Box Sync\\CLAIM-Data\\CLAIM-AP\\test.xlsx",
                        caselabels="C:\\Users\\James\\Box\\CLAIM-Data\\CLAIM-AP\\test.xlsx",
                        classes=['L','R'],
                        include_only=("NN2",1))

valaccuracy_by_fold = []
valloss_by_fold = []
prop_by_fold = []
verbose = 2

modeltype = Best_aCNN_categorical
downsample_ratio = 0.4

results_dict = {}

print(get_distribution_of_labels_categorical_from_caselabels(dataseries.caselabels))

for fold in range(4):
    print(f"FOLD {fold}")
    (train_x, train_y, train_n, train_caseids),\
    (test_x, test_y, test_n, test_caseids) = dataseries.get_train_test_data(reverse=True, fold_num=fold,
                                                                            downsample_ratio=downsample_ratio)

    seq_length = train_x.shape[1]
    input_channels = train_x.shape[2]

    if fold == 0:
        print(f"Train: {train_x.shape} from {train_n} cases")
        print(f"Test: {test_x.shape} from {test_n} cases")

    model = modeltype(sequence_length=seq_length, input_channels=input_channels, n_classes=3)

    results = model.train(train_x, train_y, epochs=10, batch_size=32, tensorboard_modelname=f"cnn_lstm_newcat_{fold}",
                          validation_data=(test_x, test_y), verbose=verbose)

    print(f"Categories are {list(enumerate(dataseries.classes))}")
    predictions_by_case = {caseid:[] for caseid in set(test_caseids)}
    answers_by_case = {}
    for case, x, y in zip(test_caseids, test_x, test_y):
        x = x.reshape(1, x.shape[0], x.shape[1])
        predictions_by_case[case].append(model.predict(x)[0])
        answers_by_case[case] = y
    for case in predictions_by_case.keys():
        result = np.mean(np.stack(predictions_by_case[case]),axis=0)
        if np.argmax(result) == np.argmax(answers_by_case[case]):
            correct = "CORRECT"
        else:
            correct = "INCORRECT"
        print(f"CASE {case}: {result} - {correct}")
        results_dict[case] = [result[0], result[1], result[2], np.argmax(result), np.argmax(answers_by_case[case]), correct]

    # print(f"Categories are {list(enumerate(dataseries.classes))}")
    # for case, x, y in zip(train_caseids, train_x, train_y):
    #     x = x.reshape(1, x.shape[0], x.shape[1])
    #     result = model.predict(x)
    #     if np.argmax(result) == np.argmax(y):
    #         correct = "CORRECT"
    #     else:
    #         correct = "INCORRECT"
    #     print(f"TRAIN: I think case {case} is {result} - should be {y} - {correct}")
    # for case, x, y in zip(test_caseids, test_x, test_y):
    #     x = x.reshape(1, x.shape[0], x.shape[1])
    #     result = model.predict(x)
    #     if np.argmax(result) == np.argmax(y):
    #         correct = "CORRECT"
    #     else:
    #         correct = "INCORRECT"
    #     print(f"TEST: I think case {case} is {result} - should be {y} - {correct}")

    valaccuracy_by_fold.append(results.history['val_acc'])
    valloss_by_fold.append(results.history['val_loss'])

valaccuracy_by_fold = np.stack(valaccuracy_by_fold)
valloss_by_fold = np.stack(valloss_by_fold)

pd.DataFrame.from_dict(results_dict,orient='index').to_csv("ap_results.csv")

print(get_distribution_of_labels_categorical_from_caselabels(dataseries.caselabels))

plot_by_fold(valaccuracy_by_fold, valloss_by_fold)