#!/usr/bin/env python3

CLOUD = False
try:
    from polyinterface import Controller,LOGGER
except ImportError:
    from pgc_interface import Controller,LOGGER

try:
    import polyinterface
except ImportError:
    import pgc_interface as polyinterface
    CLOUD = True

import sys
import json
import time
import http.client
import urllib.parse
from datetime import datetime
import os
import os.path
import re
import logging
from copy import deepcopy

from pgSession import pgSession
from nodes import Thermostat
from node_funcs import *

LOGGER = polyinterface.LOGGER

ECOBEE_API_URL = 'api.ecobee.com'

class Controller(Controller):
    def __init__(self, polyglot):
        super().__init__(polyglot)
        self.name = 'Ecobee Controller'
        self.tokenData = {}
        self.msgi = {}
        self.in_discover = False
        self.discover_st = False
        self.refreshingTokens = False
        self.pinRun = False
        self._last_dtns = False
        self.hb = 0
        self.ready = False
        self.waiting_on_tokens = False
        self._cloud = CLOUD

    def start(self):
        LOGGER.info('Started Ecobee v2 NodeServer')
        self.removeNoticesAll()
        self.heartbeat()
        self.serverdata = get_server_data(LOGGER)
        LOGGER.info('Ecobee NodeServer Version {}'.format(self.serverdata['version']))
        nsv = 'nodeserver_version'
        ud  = False
        cust_data = self.polyConfig['customData']
        if not nsv in cust_data:
            LOGGER.info("Adding {}={} to customData".format(nsv,self.serverdata['version']))
            cust_data[nsv] = self.serverdata['version']
            ud = True
        elif cust_data[nsv] != self.serverdata['version']:
            LOGGER.info("Update {} from {} to {} in customData".format(nsv,cust_data[nsv],self.serverdata['version']))
            cust_data[nsv] = self.serverdata['version']
            ud = True
        # Delete the saved token data we were doing for a while
        # For PG3 we can start doing it again, but not in PG2 since it costs $$
        # Can't iterate over keys, since the keys get removed, causes python crash
        keys = list(cust_data.keys())
        for key in keys:
            if re.match('tokenData2020.*',key):
                LOGGER.info("Deleting old customData key {}".format(key))
                del cust_data[key]
        if ud:
            self.saveCustomDataWait(cust_data)
        #LOGGER.debug("init=\n"+json.dumps(self.poly.init,sort_keys=True,indent=2))
        LOGGER.debug("customData=\n"+json.dumps(cust_data,sort_keys=True,indent=2))
        self.set_debug_mode()
        self.get_session()
        # Anything special to do in pgtest development mode?
        #if self.poly.init['development']:
        #
        # Cloud uses OAuth, local users PIN
        #
        self.pg_test = False
        if self._cloud:
            self.grant_type = 'authorization_code'
            self.api_key    = self.serverdata['api_key']
            # TODO: Need a better way to tell if we are on pgtest!
            #       "logBucket": "pgc-test-logbucket-19y0vctj4zlk5",
            if self.poly.stage == 'test':
                self.pg_test = True
                LOGGER.warning("Looks like we are running on to pgtest")
                self.redirect_url = 'https://pgtest.isy.io/api/oauth/callback'
            else:
                LOGGER.warning("Looks like we are running on to pgc")
                self.redirect_url = 'https://polyglot.isy.io/api/oauth/callback'
        else:
            self.grant_type = 'ecobeePin'
            self.api_key    = self.serverdata['api_key_pin']
            self.redirect_url = None
        # Force to false, and successful communication will fix it
        #self.set_ecobee_st(False) Causes it to always stay false.
        # Make sure they are using the latest API
        if self.check_api():
            if 'tokenData' in self.polyConfig['customData']:
                self.tokenData = self.polyConfig['customData']['tokenData']
                if self._checkTokens():
                    LOGGER.info("start: Calling discover...")
                    self.discover()
            else:
                LOGGER.info('No tokenData, will need to authorize...')
                self.authorize()
                self.reportDrivers()
        else:
            LOGGER.info('No tokenData, will need to authorize...')
            self._reAuth("Using old api key.")
        self.ready = True
        LOGGER.debug('done')

    def shortPoll(self):
        if not self.ready:
            LOGGER.debug("{}:shortPoll: not run, not ready...".format(self.address))
            return False
        if self.in_discover:
            LOGGER.debug("{}:shortPoll: Skipping since discover is still running".format(self.address))
            return
        if self.waiting_on_tokens is False:
            return
        elif self.waiting_on_tokens == "OAuth":
            LOGGER.debug("{}:shortPoll: Waiting for user to authorize...".format(self.address))
        else:
            # Must be waiting on our PIN Authorization
            LOGGER.debug("{}:shortPoll: Try to get tokens...".format(self.address))
            if self._getTokens(self.waiting_on_tokens):
                self.removeNoticesAll()
                LOGGER.info("shortPoll: Calling discover now that we have authorization...")
                self.discover()

    def longPoll(self):
        # Call discovery if it failed on startup
        LOGGER.debug("{}:longPoll".format(self.address))
        self.check_api() # In case the message went away...
        self.heartbeat()
        if not self.ready:
            LOGGER.debug("{}:longPoll: not run, not ready...".format(self.address))
            return False
        if self.waiting_on_tokens is not False:
            LOGGER.debug("{}:longPoll: not run, waiting for user to authorize...".format(self.address))
            return False
        if self.in_discover:
            LOGGER.debug("{}:longPoll: Skipping since discover is still running".format(self.address))
            return
        if self.discover_st is False:
            LOGGER.info("longPoll: Calling discover...")
            self.discover()
        self.updateThermostats()

    def heartbeat(self):
        LOGGER.debug('heartbeat hb={}'.format(self.hb))
        if self.hb == 0:
            self.reportCmd("DON",2)
            self.hb = 1
        else:
            self.reportCmd("DOF",2)
            self.hb = 0

    # sends a stop command for the nodeserver to Polyglot
    def exit(self):
        LOGGER.info('Asking Polyglot to stop me.')
        self.poly.send({"stop": {}})    # sends a stop command for the nodeserver to Polyglot

    def delete(self):
        LOGGER.warning("Nodeserver is being deleted...")
        # Ecobee delete tokens not working, need info from Ecobee
        #if self.ecobeeDelete():
        #    self.tokenData = {}

    def get_session(self):
        self.session = pgSession(self,self.name,LOGGER,ECOBEE_API_URL,debug_level=self.debug_level)

    def check_api(self):
        """
        Check if api being used is old and user needs to re-auth
        """
        msg = None
        if 'api_key' in self.polyConfig['customData']:
            if self.polyConfig['customData']['api_key'] != self.api_key:
                msg = 'You are using old api key "{}".'.format(self.polyConfig['customData']['api_key'])
                return False
        elif 'tokenData' in self.polyConfig['customData']:
            msg = 'No api key found, if you have previously authorized, '
        if msg is not None:
            msg += ' Please Signin to your Ecobee account. Click on Profile > My Apps > Remove App of the old Nodeserver, and resart this NodeServer'
            LOGGER.warning(msg)
            self.addNotice({'check_api': msg})
            return False
        return True

    def authorize(self):
        if self._cloud is True:
            self._getOAuth()
        else:
            self._getPin()

    def _reAuth(self, reason):
        # Need to re-auth!
        LOGGER.error('_reAuth because: {}'.format(reason))
        self.addNotice({'reauth': "Must Re-Authorize because {}".format(reason)})
        cdata = deepcopy(self.polyConfig['customData'])
        if not 'tokenData' in cdata:
            LOGGER.error('No tokenData in customData: {}'.format(cdata))
        self.saveCustomDataWait(cdata)
        self.authorize()

    def _getPin(self):
        # Ask Ecobee for our Pin and present it to the user in a notice
        res = self.session_get('authorize',
                              {
                                  'response_type':  'ecobeePin',
                                  'client_id':      self.api_key,
                                  'scope':          'smartWrite'
                              })
        if res is False:
            self.refreshingTokens = False
            return False
        res_data = res['data']
        res_code = res['code']
        if 'ecobeePin' in res_data:
            msg = 'Please Signin to your Ecobee account. Click on Profile > My Apps > Add Application and enter PIN: {} You have 10 minutes to complete this. The NodeServer will check every 60 seconds.'.format(res_data['ecobeePin'])
            LOGGER.info('_getPin: {}'.format(msg))
            self.addNotice({'getPin': msg})
            # cust_data = deepcopy(self.polyConfig['customData'])
            # cust_data['pinData'] = data
            # self.saveCustomDataWait(cust_data)
            # This will tell shortPoll to check for PIN
            self.waiting_on_tokens = res_data
        else:
            msg = 'ecobeePin Failed code={}: {}'.format(res_code,res_data)
            self.addNotice({'getPin': msg})

    def _getOAuthInit(self):
        """
        See if we have the oauth data stored already
        """
        sdata = {}
        if self._cloud:
            error = False
            if 'clientId' in self.poly.init['oauth']:
                sdata['api_client'] =  self.poly.init['oauth']['clientId']
            else:
                LOGGER.warning('Unable to find Client ID in the init oauth data: {}'.format(self.poly.init['oauth']))
                error = True
            if 'clientSecret' in self.poly.init['oauth']:
                sdata['api_key'] =  self.poly.init['oauth']['clientSecret']
            else:
                LOGGER.warning('Unable to find Client Secret in the init oauth data: {}'.format(self.poly.init['oauth']))
                error = True
            if error:
                return False
        return sdata

    def _getOAuth(self):
        # Do we have it?
        sdata = self._getOAuthInit()
        LOGGER.debug("_getOAuth: sdata={}".format(sdata))
        if sdata is not False:
            LOGGER.debug('Init={}'.format(sdata))
            self.serverdata['api_key'] = sdata['api_key']
            self.serverdata['api_client'] = sdata['api_client']
        else:
            #LOGGER.error(self.poly.init)
            url = 'https://{}/authorize?response_type=code&client_id={}&redirect_uri={}&state={}'.format(ECOBEE_API_URL,self.api_key,self.redirect_url,self.poly.init['worker'])
            msg = 'No existing Authorization found, Please Authorize access to your Ecobee Account'.format(url)
            self.addNotice({'oauth': msg})
            LOGGER.warning(msg)
            self.waiting_on_tokens = "OAuth"

    def oauth(self, oauth):
        LOGGER.info('OAUTH Received: {}'.format(oauth))
        if 'code' in oauth:
            if self._getTokens(oauth):
                self.removeNoticesAll()
                self.polyConfig['customData']['api_key'] = self.api_key
                self.discover()

    def _expire_delta(self):
        if not 'expires' in self.tokenData:
            return False
        ts_exp = datetime.strptime(self.tokenData['expires'], '%Y-%m-%dT%H:%M:%S')
        return ts_exp - datetime.now()

    def _checkTokens(self):
        if self.refreshingTokens:
            LOGGER.error('Waiting for token refresh to complete...')
            while self.refreshingTokens:
                time.sleep(.1)
        if 'access_token' in self.tokenData:
            exp_d = self._expire_delta()
            if exp_d is not False:
                # We allow for 10 long polls to refresh the token...
                if exp_d.total_seconds() < int(self.polyConfig['longPoll']) * 10:
                    self.l_info('_checkTokens','Tokens {} expires {} will expire in {} seconds, so refreshing now...'.format(self.tokenData['refresh_token'],self.tokenData['expires'],exp_d.total_seconds()))
                    return self._getRefresh()
                else:
                    # Only print this ones, then once a minute at most...
                    sd = True
                    if 'ctdt' in self.msgi:
                        md = datetime.now() - self.msgi['ctdt']
                        if md.total_seconds() < 60:
                            sd = False
                    if sd:
                        self.l_debug('_checkTokens',0,'Tokens valid until: {} ({} seconds, longPoll={})'.format(self.tokenData['expires'],exp_d.seconds,int(self.polyConfig['longPoll'])))
                    self.msgi['ctdt'] = datetime.now()
                    self.set_auth_st(True)
                    return True
            else:
                self.l_error('_checkTokens', 'No expires in tokenData:{}'.format(self.tokenData))
        else:
            self.set_auth_st(False)
            self.l_error('_checkTokens','tokenData or access_token not available')
            # self.saveCustomDataWait({})
            # this._getPin()
            return False

    def _startRefresh(self,test=False):
        # See if someone else already refreshed it?  Very small chance of this happening on PGC, but it could.
        if 'tokenData' in self.polyConfig['customData']:
            if 'refresh_token' in self.polyConfig['customData']['tokenData']:
                LOGGER.debug("{}= {}".format(self._data_tag,self.polyConfig['customData'][self._data_tag]))
                LOGGER.debug("_last_dtns={}".format(self._last_dtns))
                if self._last_dtns is not False:
                    # Check that it was ours
                    if not self._last_dtns == self.polyConfig['customData'][self._data_tag]:
                        LOGGER.error("Someone changed the db?")
                        LOGGER.error("{}= {}".format(self._data_tag,self.polyConfig['customData'][self._data_tag]))
                        LOGGER.error("_last_dtns={}".format(self._last_dtns))
                        try:
                            l_dt = datetime.fromtimestamp(self._last_dtns).strftime(self._lock_fmt)
                            c_dt = datetime.fromtimestamp(self.polyConfig['customData'][self._data_tag]).strftime(self._lock_fmt)
                            if c_dt < l_dt:
                                LOGGER.error("But it is older than what we wrote, so will ignore it...")
                            else:
                                LOGGER.error("And it's newer than what we wrote so will use it?")
                                LOGGER.error(" Mine:    {}".format(self.tokenData))
                                LOGGER.error(" Current: {}".format(self.polyConfig['customData']['tokenData']))
                                LOGGER.error("We will use the new tokens...")
                                self.tokenData = deepcopy(self.polyConfig['customData']['tokenData'])
                                return False
                        except:
                            LOGGER.error("Failed determing age of last db write by someone else, will ignore it.",exc_info=True)
        rval = self.lockCustomData()
        if (rval):
            self.refreshingTokens = True
        return rval

    # This is only called when refresh fails, when it works saveTokens clears
    # it, otherwise we get_ a race on who's customData is saved...
    def _endRefresh(self,refresh_data=False,test=False):
        cdata = deepcopy(self.polyConfig['customData'])
        if refresh_data is not False:
            if 'expires_in' in refresh_data:
                ts = time.time() + refresh_data['expires_in']
                refresh_data['expires'] = datetime.fromtimestamp(ts).strftime("%Y-%m-%dT%H:%M:%S")
            # Save new token data in customData
            cdata['tokenData'] = refresh_data
            # And save in our variable for checking, unless in test mode...
            if not test:
                self.tokenData = deepcopy(refresh_data)
            self.set_auth_st(True)
            self.removeNoticesAll()
        self.saveCustomDataWait(cdata)
        LOGGER.info('cleared lock')
        self.refreshingTokens = False

    # test option is passed in to force a refresh and save to db, but not our
    # locally saved self.tokenData.  This makes it look like someone else
    # refreshed the token, so we just grab from the db.
    def _getRefresh(self,test=False):
        if 'refresh_token' in self.tokenData:
            if not self._startRefresh(test=test):
                return False
            LOGGER.info('Attempting to refresh tokens...')
            res = self.session.post('token',
                params = {
                    'grant_type':    'refresh_token',
                    'client_id':     self.api_key,
                    'refresh_token': self.tokenData['refresh_token']
                })
            if res is False:
                self.set_ecobee_st(False)
                self._endRefresh(test=test)
                return False
            self.set_ecobee_st(True)
            res_data = res['data']
            res_code = res['code']
            if res_data is False:
                LOGGER.error('No data returned.')
            else:
                # https://www.ecobee.com/home/developer/api/documentation/v1/auth/auth-req-resp.shtml
                if 'error' in res_data:
                    self.set_ecobee_st(False)
                    self.addNotice({'grant_error': "{}: {} ".format(res_data['error'], res_data['error_description'])})
                    #self.addNotice({'grant_info': "For access_token={} refresh_token={} expires={}".format(self.tokenData['access_token'],self.tokenData['refresh_token'],self.tokenData['expires'])})
                    LOGGER.error('Requesting Auth: {} :: {}'.format(res_data['error'], res_data['error_description']))
                    LOGGER.error('For access_token={} refresh_token={} expires={}'.format(self.tokenData['access_token'],self.tokenData['refresh_token'],self.tokenData['expires']))
                    # Set auth to false for now, so user sees the error, even if we correct it later...
                    # JimBo: This can only happen if our refresh_token is bad, so we need to force a re-auth
                    if res_data['error'] == 'invalid_grant':
                        exp_d = self._expire_delta()
                        if exp_d is False:
                            self.addNotice({'grant_info': "No token expire data available, so will have to re-auth...".format(exp_d.total_seconds())})
                            LOGGER.error("No token expire data available, so will have to re-auth...".format(exp_d.total_seconds()))
                            self._reAuth('{} No token expire data available'.format(res_data['error']))
                        else:
                            # Only do this check for new authorization which tracks api_key in customData
                            if 'api_key' in self.polyConfig['customData']:
                                if exp_d.total_seconds() > 0:
                                    self.addNotice({'grant_info_2': "But token still has {} seconds to expire, so assuming this is an Ecobee server issue and will try to refresh on next poll...".format(exp_d.total_seconds())})
                                    LOGGER.error("But token still has {} seconds to expire, so assuming this is an Ecobee server issue and will try to refresh on next poll...".format(exp_d.total_seconds()))
                                else:
                                    self.addNotice({'grant_info_2': "Token expired {} seconds ago, so will have to re-auth...".format(exp_d.total_seconds())})
                                    LOGGER.error("Token expired {} seconds ago, so will have to re-auth...".format(exp_d.total_seconds()))
                                    # May need to remove the re-auth requirement because we get these and they don't seem to be real?
                                    self._reAuth('{} and Token expired'.format(res_data['error']))
                    elif res_data['error'] == 'invalid_client':
                        # We will Ignore it because it may correct itself on the next poll?
                        LOGGER.error('Ignoring invalid_client error, will try again later for now, but may need to mark it invalid if we see more than once?  See: https://github.com/Einstein42/udi-ecobee-poly/issues/60')
                    #elif res_data['error'] == 'authorization_expired':
                    #    self._reAuth('{}'.format(res_data['error']))
                    else:
                        # Should all other errors require re-auth?
                        #self._reAuth('{}'.format(res_data['error']))
                        LOGGER.error('Unknown error, not sure what to do here.  Please Generate Log Package and Notify Author with a github issue: https://github.com/Einstein42/udi-ecobee-poly/issues')
                    self._endRefresh(test=test)
                    return False
                elif 'access_token' in res_data:
                    self._endRefresh(res_data,test=test)
                    return True
        else:
            self._reAuth(' refresh_token not Found in tokenData={}'.format(self.tokenData))
        self._endRefresh(test=test)
        return False

    def _getTokens(self, pinData):
        LOGGER.debug('Attempting to get tokens for {}'.format(pinData))
        res = self.session.post('token',
            params = {
                        'grant_type':  self.grant_type,
                        'client_id':   self.api_key,
                        'code':        pinData['code'],
                        'redirect_uri': self.redirect_url
                    })
        if res is False:
            self.set_ecobee_st(False)
            self.set_auth_st(False)
            return False
        res_data = res['data']
        res_code = res['code']
        if res_data is False:
            LOGGER.error('_getTokens: No data returned.')
            self.set_auth_st(False)
            return False
        if 'error' in res_data:
            LOGGER.error('_getTokens: {} :: {}'.format(res_data['error'], res_data['error_description']))
            self.set_auth_st(False)
            if res_data['error'] == 'authorization_expired' or res_data['error'] == 'invalid_grant':
                msg = 'Nodeserver exiting because {}, please restart when you are ready to authorize.'.format(res_data['error'])
                LOGGER.error('_getTokens: {}'.format(msg))
                self.waiting_on_tokens = False
                self.removeNoticesAll()
                self.addNotice({'getTokens': msg})
                self.exit()
            return False
        if 'access_token' in res_data:
            self.waiting_on_tokens = False
            LOGGER.debug('Got tokens sucessfully.')
            self.removeNoticesAll()
            self.addNotice({'getTokens': 'Tokens obtained!'})
            self.polyConfig['customData']['api_key'] = self.api_key
            self.polyConfig['customData']['api_code'] = pinData['code']
            self._endRefresh(res_data)
            return True
        self.set_auth_st(False)

    def updateThermostats(self,force=False):
        LOGGER.debug("{}:updateThermostats: start".format(self.address))
        thermostats = self.getThermostats()
        if not isinstance(thermostats, dict):
            LOGGER.error('Thermostats instance wasn\'t dictionary. Skipping...')
            return
        for thermostatId, thermostat in thermostats.items():
            LOGGER.debug("{}:updateThermostats: {}".format(self.address,thermostatId))
            if self.checkRev(thermostat):
                address = self.thermostatIdToAddress(thermostatId)
                if address in self.nodes:
                    LOGGER.debug('Update detected in thermostat {}({}) doing full update.'.format(thermostat['name'], address))
                    fullData = self.getThermostatFull(thermostatId)
                    if fullData is not False:
                        self.nodes[address].update(thermostat, fullData)
                    else:
                        LOGGER.error('Failed to get updated data for thermostat: {}({})'.format(thermostat['name'], thermostatId))
                else:
                    LOGGER.error("Thermostat id '{}' address '{}' is not in our node list. thermostat: {}".format(thermostatId,address,thermostat))
            else:
                LOGGER.info("No {} '{}' update detected".format(thermostatId,thermostat['name']))
        LOGGER.debug("{}:updateThermostats: done".format(self.address))

    def checkRev(self, tstat):
        if tstat['thermostatId'] in self.revData:
            curData = self.revData[tstat['thermostatId']]
            if (tstat['thermostatRev'] != curData['thermostatRev']
                    or tstat['alertsRev'] != curData['alertsRev']
                    or tstat['runtimeRev'] != curData['runtimeRev']
                    or tstat['intervalRev'] != curData['intervalRev']):
                return True
        return False

    def query(self):
        self.reportDrivers()
        for node in self.nodes:
            self.nodes[node].reportDrivers()

    def stop(self):
        LOGGER.debug('NodeServer stoping...')
        self.set_ecobee_st(False)

    def thermostatIdToAddress(self,tid):
        return 't{}'.format(tid)

    def discover(self, *args, **kwargs):
        # True means we are in dsocvery
        if self.in_discover:
            LOGGER.info('Discovering Ecobee Thermostats already running?')
            return True
        self.in_discover = True
        self.discover_st = False
        try:
            self.discover_st = self._discover()
        except Exception as e:
            self.l_error('discover','failed: {}'.format(e),True)
            self.discover_st = False
        self.in_discover = False
        return self.discover_st

    def _discover(self, *args, **kwargs):
        LOGGER.info('Discovering Ecobee Thermostats')
        if not 'access_token' in self.tokenData:
            return False
        self.revData = {} # Intialize in case we fail
        thermostats = self.getThermostats()
        if thermostats is False:
            LOGGER.error("Discover Failed, No thermostats returned!  Will try again on next long poll")
            return False
        self.revData = deepcopy(thermostats)
        #
        # Build or update the profile first.
        #
        self.check_profile(thermostats)
        #
        # Now add our thermostats
        #
        for thermostatId, thermostat in thermostats.items():
            address = self.thermostatIdToAddress(thermostatId)
            if not address in self.nodes:
                fullData = self.getThermostatFull(thermostatId)
                if fullData is not False:
                    tstat = fullData['thermostatList'][0]
                    useCelsius = True if tstat['settings']['useCelsius'] else False
                    self.addNode(Thermostat(self, address, address, thermostatId,
                                            'Ecobee - {}'.format(get_valid_node_name(thermostat['name'])),
                                            thermostat, fullData, useCelsius))
        return True

    def check_profile(self,thermostats):
        self.profile_info = get_profile_info(LOGGER)
        #
        # First get all the climate programs so we can build the profile if necessary
        #
        climates = dict()
        for thermostatId, thermostat in thermostats.items():
            # Only get program data if we have the node.
            fullData = self.getThermostatSelection(thermostatId,includeProgram=True)
            if fullData is not False:
                programs = fullData['thermostatList'][0]['program']
                climates[thermostatId] = list()
                for climate in programs['climates']:
                    climates[thermostatId].append({'name': climate['name'], 'ref':climate['climateRef']})
        LOGGER.debug("check_profile: climates={}".format(climates))
        #
        # Set Default profile version if not Found
        #
        cdata = deepcopy(self.polyConfig['customData'])
        LOGGER.info('check_profile: profile_info={}'.format(self.profile_info))
        LOGGER.info('check_profile:   customData={}'.format(cdata))
        if not 'profile_info' in cdata:
            update_profile = True
        elif self.profile_info['version'] == cdata['profile_info']['version']:
            # Check if the climates are different
            update_profile = False
            LOGGER.info('check_profile: update_profile={} checking climates.'.format(update_profile))
            if 'climates' in cdata:
                current = cdata['climates']
                if not update_profile:
                    # Check if the climates have changed.
                    for id in climates:
                        if id in current:
                            if len(climates[id]) == len(current[id]):
                                for i in range(len(climates[id])):
                                    if climates[id][i] != current[id][i]:
                                        update_profile = True
                            else:
                                update_profile = True
                        else:
                            update_profile = True
            else:
                update_profile = True
        else:
            update_profile = True
        LOGGER.warning('check_profile: update_profile={}'.format(update_profile))
        if update_profile:
            self.write_profile(climates)
            self.poly.installprofile()
            cdata['profile_info'] = self.profile_info
            cdata['climates'] = climates
            self.saveCustomDataWait(cdata)

    def write_profile(self,climates):
      pfx = '{}:write_profile:'.format(self.address)
      #
      # Start the nls with the template data.
      #
      en_us_txt = "profile/nls/en_us.txt"
      make_file_dir(en_us_txt)
      LOGGER.info("{0} Writing {1}".format(pfx,en_us_txt))
      nls_tmpl = open("template/en_us.txt", "r")
      nls      = open(en_us_txt,  "w")
      for line in nls_tmpl:
        nls.write(line)
      nls_tmpl.close()
      # Open the nodedef custom for writing
      nodedef_f = 'profile/nodedef/custom.xml'
      LOGGER.info("{0} Writing {1}".format(pfx,nodedef_f))
      nodedef_h = open(nodedef_f, "w")
      nodedef_h.write('<nodedefs>\n')
      # Open the editor custom for writing
      editor_f = 'profile/editor/custom.xml'
      LOGGER.info("{0} Writing {1}".format(pfx,editor_f))
      editor_h = open(editor_f, "w")
      editor_h.write('<editors>\n')
      for id in climates:
        # Read thermostat template to write the custom version.
        in_h  = open('template/thermostat.xml','r')
        for line in in_h:
            nodedef_h.write(re.sub(r'tstatid',r'{0}'.format(id),line))
        in_h.close()
        # Read the editor template to write the custom version
        in_h  = open('template/editors.xml','r')
        for line in in_h:
            line = re.sub(r'tstatid',r'{0}'.format(id),line)
            line = re.sub(r'tstatcnta',r'{0}'.format(len(climateList)-1),line)
            # This is minus 3 because we don't allow selecting vacation or smartAway, ...
            # But not currently using this because we don't have different list for
            # status and programs?
            line = re.sub(r'tstatcnt',r'{0}'.format(len(climateList)-5),line)
            editor_h.write(line)
        in_h.close()
        # Then the NLS lines.
        nls.write("\n")
        nls.write('ND-EcobeeC_{0}-NAME = Ecobee Thermostat {0} (C)\n'.format(id))
        nls.write('ND-EcobeeC_{0}-ICON = Thermostat\n'.format(id))
        nls.write('ND-EcobeeF_{0}-NAME = Ecobee Thermostat {0} (F)\n'.format(id))
        nls.write('ND-EcobeeF_{0}-ICON = Thermostat\n'.format(id))
        # ucfirst them all
        customList = list()
        for i in range(len(climateList)):
            customList.append(climateList[i][0].upper() + climateList[i][1:])
        # Now see if there are custom names
        for i in range(len(climateList)):
            name = climateList[i]
            # Find this name in the map and replace with our name.
            for cli in climates[id]:
                if cli['ref'] == name:
                    customList[i] = cli['name']
        LOGGER.debug("{} customList={}".format(pfx,customList))
        for i in range(len(customList)):
            nls.write("CT_{}-{} = {}\n".format(id,i,customList[i]))
      nodedef_h.write('</nodedefs>\n')
      nodedef_h.close()
      editor_h.write('</editors>\n')
      editor_h.close()
      nls.close()
      LOGGER.info("{} done".format(pfx))

    # Calls session.get and converts params to weird ecobee formatting.
    def session_get (self,path,data):
        if path == 'authorize':
            # All calls before with have auth token, don't reformat with json
            return self.session.get(path,data)
        else:
            res = self.session.get(path,{ 'json': json.dumps(data) },
                                    auth='{} {}'.format(self.tokenData['token_type'], self.tokenData['access_token'])
                                    )
            if res is False:
                return res
            if res['data'] is False:
                return False
            self.l_debug('session_get', 0, 'res={}'.format(res))
            if not 'status' in res['data']:
                return res
            res_st_code = int(res['data']['status']['code'])
            if res_st_code == 0:
                return res
            LOGGER.error('Checking Bad Status Code {} for {}'.format(res_st_code,res))
            if res_st_code == 14:
                self.l_error('session_get', 'Token has expired, will refresh')
                # TODO: Should this be a loop instead ?
                if self._getRefresh() is True:
                    return self.session.get(path,{ 'json': json.dumps(data) },
                                     auth='{} {}'.format(self.tokenData['token_type'], self.tokenData['access_token']))
            elif res_st_code == 16:
                self._reAuth("session_get: Token deauthorized by user: {}".format(res))
            return False

    def getThermostats(self):
        if not self._checkTokens():
            LOGGER.debug('getThermostat failed. Couldn\'t get tokens.')
            return False
        LOGGER.debug('getThermostats: Getting Summary...')
        res = self.session_get('1/thermostatSummary',
                               {
                                    'selection': {
                                        'selectionType': 'registered',
                                        'selectionMatch': '',
                                        'includesEquipmentStatus': True
                                    },
                                })
        if res is False:
            self.set_ecobee_st(False)
            return False
        self.set_ecobee_st(True)
        thermostats = {}
        res_data = res['data']
        res_code = res['code']
        if res_data is False:
            self.l_error('getThermostats','Ecobee returned code {} but no data? ({})'.format(res_code,res_data))
            return thermostats
        if 'revisionList' in res_data:
            if res_data['revisionList'] is False:
                self.l_error('getThermostats','Ecobee returned code {} but no revisionList? ({})'.format(res_code,res_data['revisionList']))
            for thermostat in res_data['revisionList']:
                revisionArray = thermostat.split(':')
                thermostats['{}'.format(revisionArray[0])] = {
                    'name': revisionArray[1],
                    'thermostatId': revisionArray[0],
                    'connected': revisionArray[2],
                    'thermostatRev': revisionArray[3],
                    'alertsRev': revisionArray[4],
                    'runtimeRev': revisionArray[5],
                    'intervalRev': revisionArray[6]
                }
        return thermostats

    def getThermostatFull(self, id):
        return self.getThermostatSelection(id,True,True,True,True,True,True,True,True,True,True,True,True)

    def getThermostatSelection(self,id,
                               includeEvents=False,
                               includeProgram=False,
                               includeSettings=False,
                               includeRuntime=False,
                               includeExtendedRuntime=False,
                               includeLocation=False,
                               includeEquipmentStatus=False,
                               includeVersion=False,
                               includeUtility=False,
                               includeAlerts=False,
                               includeWeather=False,
                               includeSensors=False
                               ):
        if not self._checkTokens():
            LOGGER.error('getThermostat failed. Couldn\'t get tokens.')
            return False
        LOGGER.info('Getting Thermostat Data for {}'.format(id))
        res = self.session_get('1/thermostat',
                               {
                                   'selection': {
                                       'selectionType': 'thermostats',
                                       'selectionMatch': id,
                                       'includeEvents': includeEvents,
                                       'includeProgram': includeProgram,
                                       'includeSettings': includeSettings,
                                       'includeRuntime': includeRuntime,
                                       'includeExtendedRuntime': includeExtendedRuntime,
                                       'includeLocation': includeLocation,
                                       'includeEquipmentStatus': includeEquipmentStatus,
                                       'includeVersion': includeVersion,
                                       'includeUtility': includeUtility,
                                       'includeAlerts': includeAlerts,
                                       'includeWeather': includeWeather,
                                       'includeSensors': includeSensors
                                       }
                               }
                           )
        self.l_debug('getThermostatSelection',0,'done'.format(id))
        self.l_debug('getThermostatSelection',1,'data={}'.format(res))
        if res is False or res is None:
            return False
        return res['data']

    def ecobeePost(self, thermostatId, postData = {}):
        if not self._checkTokens():
            LOGGER.error('ecobeePost failed. Tokens not available.')
            return False
        LOGGER.info('Posting Update Data for Thermostat {}'.format(thermostatId))
        postData['selection'] = {
            'selectionType': 'thermostats',
            'selectionMatch': thermostatId
        }
        res = self.session.post('1/thermostat',params={'json': 'true'},payload=postData,
            auth='{} {}'.format(self.tokenData['token_type'], self.tokenData['access_token']),dump=True)
        if res is False:
            self.refreshingTokens = False
            self.set_ecobee_st(False)
            return False
        self.set_ecobee_st(True)
        if 'error' in res:
            LOGGER.error('ecobeePost: error="{}" {}'.format(res['error'], res['error_description']))
            return False
        res_data = res['data']
        res_code = res['code']
        if 'status' in res_data:
            if 'code' in res_data['status']:
                if res_data['status']['code'] == 0:
                    return True
                else:
                    LOGGER.error('Bad return code {}:{}'.format(res_data['status']['code'],res_data['status']['message']))
        return False

    def ecobeeDelete(self):
        if 'access_token' in self.tokenData:
            res = self.session.delete("/oauth2/acess_tokens/"+self.tokenData['access_token'])
            if res is False:
                return False
            if 'error' in res:
                LOGGER.error('ecobeePost: error="{}" {}'.format(res['error'], res['error_description']))
                return False
            res_data = res['data']
            res_code = res['code']
            if 'status' in res_data:
                if 'code' in res_data['status']:
                    if res_data['status']['code'] == 204:
                        LOGGER.info("Revoke successful")
                        return True
                    else:
                        LOGGER.error('Bad return code {}:{}'.format(res_data['status']['code'],res_data['status']['message']))
        else:
            LOGGER.warning("No access_token to revoke...")
        return False

    #
    # This method "locks" the DB by adding setting _data_lock to true.
    # There is no unlock method, currently it's done in saveCustomDataWait
    # because don't want to do multipe writes to save the data, then unlock it.
    #
    # TODO: Add a "wait" function with "timeout"?
    #
    # This holds the lock.
    _data_lock = '_data_lock'
    _lock_fmt  = '%Y-%m-%dT%H:%M:%S.%f'
    def lockCustomData(self):
        cdata = deepcopy(self.polyConfig['customData'])
        # Now see if someone is trying to refresh it.
        lock = cdata.get(self._data_lock)
        rval = False
        if lock is None or lock is False:
            rval = True
        else:
            LOGGER.error("Someone is already refreshing at {} will wait a while to see if it clears or take it...".format(lock))
            # See if it has expired
            ts_now = datetime.now()
            try:
                ts_start = datetime.strptime(lock,self._lock_fmt)
            except Exception as e:
                LOGGER.error('convert time {} failed {}, will grab the lock'.format(lock,e))
                rval = True
            else:
                ts_diff  = ts_now - ts_start
                if ts_diff.total_seconds() > 120:
                    LOGGER.error("But their attempt was {} seconds ago, so we will grab the lock...".format(ts_diff.total_seconds()))
                    rval = True
        if (rval):
            rval = self.saveCustomDataWait(cdata,lock=True)
        return rval

    # This holds the last date time we did saveCustomData
    _data_tag  = '_data_dtm'
    def saveCustomDataWait(self,ndata,lock=False,timeout=10):
        dtns = datetime.now().strftime(self._lock_fmt)
        ndata[self._data_tag] = dtns
        LOGGER.info("saveCustomData: {}".format(dtns))
        # Old stuff
        if 'pinData' in ndata:
            del ndata['pinData']
        if 'refresh_status' in ndata:
            del ndata['refresh_status']
        # lock=None means do nothing to the lock.
        if lock is not None:
            # If True then set to curent date/time
            # Otherwise set to what they desired, typically False
            ndata[self._data_lock] = dtns if lock else lock
        # Save it...
        done = False
        tcnt = 0
        while (not done):
            tcnt += 1
            LOGGER.info("Sending try={}\n{}={}\n{}={}\ntokenData=".format(tcnt,self._data_tag,ndata[self._data_tag],self._data_lock,ndata[self._data_lock]))
            if 'tokenData' in ndata:
                LOGGER.info("tokenData="+json.dumps(ndata['tokenData'],sort_keys=True,indent=2))
            self.saveCustomData(ndata)
            cnt = 0
            while (not done and cnt < 900):
                cd = deepcopy(self.polyConfig['customData'])
                if cd.get(self._data_tag) == dtns:
                    done = True
                else:
                    LOGGER.info("Waiting for custom data save to happen {}={} expecting {}".format(self._data_tag,cd.get(self._data_tag),dtns))
                    time.sleep(1)
                    cnt += 1
            if (done):
                self._last_dtns = dtns
                # IF we set auth false then set it back to true
                if (tcnt > 1):
                    self.set_auth_st(True)
            else:
                LOGGER.error("This may cause problems... timeout waiting for custom data save to happen {}={} expecting {}".format(self._data_tag,cd.get(self._data_tag),dtns))
                # No idea what to do in this case?  For now set auto false so can trigger a failure...
                self.set_auth_st(False)
        LOGGER.info("Done\n{}={}\n{}={}".format(self._data_tag,cd[self._data_tag],self._data_lock,cd[self._data_lock]))
        if 'tokenData' in cd:
            LOGGER.info("tokenData="+json.dumps(cd['tokenData'],sort_keys=True,indent=2))
        return True

    def cmd_poll(self,  *args, **kwargs):
        LOGGER.debug("{}:cmd_poll".format(self.address))
        self.updateThermostats(force=True)
        self.query()

    def cmd_query(self, *args, **kwargs):
        LOGGER.debug("{}:cmd_query".format(self.address))
        self.query()

    def cmd_upload_profile(self, *args, **kwargs):
        LOGGER.debug("{}:cmd_upload_profile".format(self.address))
        self.poly.installprofile()

    def cmd_debug_mode(self,command):
        val = int(command.get('value'))
        self.l_info("cmd_debug_mode",val)
        self.set_debug_mode(val)

    # This does a refresh, but doesn't record it to the local data structure so
    # the next time the NS tries to update it's runtime token will be invalid so
    # it will try to refresh, then see the DB has a new one and users it.
    # This is to manually test Issue #57
    def cmd_test_refresh(self,  *args, **kwargs):
        LOGGER.debug("{}".format(self.address))
        self._getRefresh(test=True)

    # This locks the DB, to test the auto-unlock timout feature for an old lock
    def cmd_test_lock(self,  *args, **kwargs):
        LOGGER.debug("{}".format(self.address))
        self.lockCustomData()

    def set_all_logs(self,level):
        self.l_info("set_all_logs",level)
        LOGGER.setLevel(level)
        #logging.getLogger('requests').setLevel(level)
        #logging.getLogger('urllib3').setLevel(level)

    def set_debug_mode(self,level=None):
        self.l_info("set_debug_mode",level)
        if level is None:
            try:
                level = self.getDriver('GV2')
            except:
                self.l_error('set_debug_mode','getDriver(GV2) failed',False)
            if level is None:
                level = 20
        level = int(level)
        self.debug_mode = level
        try:
            self.setDriver('GV2', level)
        except:
            self.l_error('set_debug_mode','setDriver(GV2) failed',True)
        self.debug_level = 0
        if level < 20:
            self.set_all_logs(logging.DEBUG)
            # 9 & 8 incrase pgsession debug level
            if level == 9:
                self.debug_level = 1
            elif level == 8:
                self.debug_level = 2
        elif level <= 20:
            self.set_all_logs(logging.INFO)
        elif level <= 30:
            self.set_all_logs(logging.WARNING)
        elif level <= 40:
            self.set_all_logs(logging.ERROR)
        elif level <= 50:
            self.set_all_logs(logging.CRITICAL)
        else:
            self.l_error("set_debug_mode","Unknown level {0}".format(level))
        self.l_info("set_debug_mode"," session debug_level={}".format(self.debug_level))

    def set_ecobee_st(self,val):
      ival = 1 if val else 0
      LOGGER.debug("{}:set_ecobee_st: {}={}".format(self.address,val,ival))
      self.setDriver('GV1',ival)

    def set_auth_st(self,val):
      ival = 1 if val else 0
      LOGGER.debug("{}:set_auth_st: {}={}".format(self.address,val,ival))
      self.setDriver('GV3',ival)

    def l_info(self, name, string):
        LOGGER.info("%s:%s:%s: %s" %  (self.id,self.name,name,string))

    def l_error(self, name, string, exc_info=False):
        LOGGER.error("%s:%s:%s: %s" % (self.id,self.name,name,string), exc_info=exc_info)

    def l_warning(self, name, string):
        LOGGER.warning("%s:%s:%s: %s" % (self.id,self.name,name,string))

    def l_debug(self, name, level, string, exc_info=False):
        if level <= self.debug_level:
            LOGGER.debug("%s:%s:%s: %s" % (self.id,self.name,name,string), exc_info=exc_info)

    id = 'ECO_CTR'
    commands = {
        'DISCOVER': discover,
        'QUERY': cmd_query,
        'POLL': cmd_poll,
        'DEBUG': cmd_debug_mode,
        'UPLOAD_PROFILE': cmd_upload_profile,
        'TESTREFRESH': cmd_test_refresh,
        'TESTLOCK': cmd_test_lock
    }
    drivers = [
        {'driver': 'ST', 'value': 1, 'uom': 2},
        {'driver': 'GV1', 'value': 0, 'uom': 2},
        {'driver': 'GV2', 'value': 30, 'uom': 25},
        {'driver': 'GV3', 'value': 0, 'uom': 2}
    ]


from pathlib import Path

HOME_DIR = Path.home()

BASE_DIR = HOME_DIR / 'covid_phylo_data'
BASE_DIR.mkdir(exist_ok=True)

CACHE_DIR = BASE_DIR / 'cache'

RAW_SEQUENCE_SHELVE_FNAME = 'raw_seqs.shelve'


""" Simple youtube downloader best resolution video by URL """
from pytube import YouTube

def print_progress_bar(file_size: int, bytes_remaining: int) -> None:
    """ Visualizes the download progress """
    # rows, columns = os.popen('stty size', 'r').read().split()
    fill_empty = ' '
    fill_full = '#'
    fill_width = 48

    current_percent = (file_size - bytes_remaining) / file_size

    filled_length = int(fill_width * current_percent)
    empty_length = fill_width - filled_length
    # bar = f'| {fill_full*filled_length + fill_empty*empty_length} | {bytes_remaining} bytes of {file_size} bytes.'
    bar = f'| {fill_full * filled_length + fill_empty * empty_length} | '

    print(f'Downloaded:{bar} {current_percent:.0%}', end='\n' if current_percent >= 1.0 else '\r')

def get_video(url: str, path: str = 'video/') -> None:
    """
    Download the best resolution video by URL

    :param url: video url address
    :param path: video save folder (default = 'video/')
    """

    def on_progress(stream, chunk, bytes_remaining: int):
        print_progress_bar(file_size, bytes_remaining)

    def on_complete(stream, path: str):
        print("File saved as:\n" + path)

    # print('\n============== Video info ==============')
    print('\nVideo and audio info:')
    yt = YouTube(url, on_progress_callback=on_progress, on_complete_callback=on_complete)
    print(f'Title: {yt.title}')
    print(f'Author: {yt.author}')

    # print('\n============ Start download ============')
    file_size = int(yt.streams.filter(progressive=True, file_extension='mp4')
                    .order_by('resolution').desc().first().filesize)
    print(f"File size {file_size: _} bytes")
    # yt.streams.get_by_itag(i_tag[video_number]).download('video/')
    yt.streams.filter(progressive=True, file_extension='mp4').order_by('resolution').desc().first().download(path)
    # print('================ Done! =================')

def main():
    url = input('Enter the YouTube URL: ')
    get_video(url)

if __name__ == '__main__':
    main()


import math
import numpy as np
import pandas as pd
from tqdm import tqdm
from scipy import sparse
from sklearn.decomposition import TruncatedSVD
from tools.utils import normalize_adjacency
from tools.Config import Config
class GraRep(object):
    """
    GraRep Model Object.
    A sparsity aware implementation of GraRep.
    For details see the paper: https://dl.acm.org/citation.cfm?id=2806512
    """
    def __init__(self, graph, dimension, configs = None):
        """
        :param A: Adjacency matrix.
        :param configs: Arguments object.
        """
        self.graph = graph
        self.node2idx = {}  # 节点对应的编号
        self.idx2node = {}
        idx=0
        for node in graph.nodes():
            self.node2idx[node] = idx
            self.idx2node[idx] = node
            idx += 1
        self.A = normalize_adjacency(graph.edges(), self.node2idx)

        self._setup_base_target_matrix()

        self.configs = Config({
            'output_path' : './output/GraRep_embedding.csv',
            'dimensions' : dimension,
            'order' : 1,
            'seed' : 42,
            'epochs' : 20
        })
        if configs:
            self.configs = self.configs.copy(vars(configs))

    def _setup_base_target_matrix(self):
        """
        Creating a base matrix to multiply.
        创建的是单位矩阵
        """
        values = [1.0 for i in range(self.graph.number_of_nodes())]
        indices = [i for i in range(self.graph.number_of_nodes())]
        self.A_hat = sparse.coo_matrix((values, (indices,indices)),shape=self.A.shape,dtype=np.float32)

    def _create_target_matrix(self):
        """
        Creating a log transformed target matrix.
        :return target_matrix: Matrix to decompose with SVD.
        """
        self.A_hat = sparse.coo_matrix(self.A_hat.dot(self.A))
        scores = np.log(self.A_hat.data)-math.log(self.A.shape[0])
        rows = self.A_hat.row[scores<0]
        cols = self.A_hat.col[scores<0]
        scores = scores[scores<0]
        target_matrix = sparse.coo_matrix((scores, (rows,cols)),shape=self.A.shape,dtype=np.float32)
        return target_matrix

    def train(self):
        """
        Learning an embedding.
        """
        print("\nOptimization started.\n")
        self.embeddings = []
        for step in tqdm(range(self.configs.order)):
            target_matrix = self._create_target_matrix()
            svd = TruncatedSVD(n_components=self.configs.dimensions, n_iter=self.configs.epochs, random_state=self.configs.seed)
            svd.fit(target_matrix)
            embedding = svd.transform(target_matrix)
            self.embeddings.append(embedding)

    def save_embeddings(self, file):
        """
        Saving the embedding.
        """
        print("\nSave embedding.\n")
        embeddings = self.get_embeddings()
        embeddings.to_pickle(file)

    def get_embeddings(self,):
        self.embeddings = np.concatenate(self.embeddings,axis=1)
        # column_count = self.configs.order*self.configs.dimensions
        # columns = ["ID"] + ["x_" + str(col) for col in range(column_count)]
        # ids = np.array([i for i in range(self.A.shape[0])]).reshape(-1,1)
        embedding_dict = {}
        for idx in range(self.graph.number_of_nodes()):
            embedding_dict[self.idx2node[idx]] = self.embeddings[idx]
        data = pd.DataFrame.from_dict(embedding_dict, orient ='index')
        #print(data.shape[0])
        return data


import pandas as pd
import numpy as np
import os
import random
import pywt
import pickle
from tqdm import tqdm
import torch
from random import sample
from dataloaders.utils import Normalizer,components_selection_one_signal,mag_3_signals,PrepareWavelets,FiltersExtention
from sklearn.utils import class_weight
from skimage.transform import resize
freq1 = 0.3
freq2 = 20

# ========================================       Data loader Base class               =============================
class BASE_DATA():

    def __init__(self, args):
        """
        root_path                      : Root directory of the data set
        freq_save_path                 : The path to save genarated Spectrogram. If the file has already been generated, Load it directly
        window_save_path               : The path to save genarated Window index. If the file has already been generated, Load it directly
                                         This could save time by avoiding generate them again.

        data_name (Str)                : the name of data set
                                       --->[TODO]

        freq (int)                     :Sampling Frequency of the correponding dataset
        representation_type (Str)      :  What kind of representation should be load
                                       --->[time, freq, time_freq]

        difference  (Bool)             : Whether to calculate the first order derivative of the original data
        datanorm_type (Str)            : How to normalize the data
                                       --->[standardization, minmax, per_sample_std, per_sample_minmax]

        load_all (Bool)                : This is for Freq representation data. Whether load all files in one time. this could save time by training, but it needs a lot RAM
        train_vali_quote (float)       : train vali split quote , default as 0.8

        windowsize                     :  the size of Sliding Window
        -------------------------------------------------------
		if training mode, The sliding step is 50% of the windowsize
        if test mode, The step is 10% of the windowsize. (It should be as one, But it results in to many window samples, it is difficult to generate the spectrogram)
        -------------------------------------------------------
        drop_transition  (Bool)        : Whether to drop the transition parts between different activities
        wavelet_function (Str)         : Method to generate Spectrogram
                                       ---> []

        """
        self.root_path              = args.root_path
        self.freq_save_path         = args.freq_save_path
        self.window_save_path       = args.window_save_path
        self.data_name              = args.data_name

        window_save_path = os.path.join(self.window_save_path,self.data_name)
        if not os.path.exists(window_save_path):
            os.mkdir(window_save_path)
        self.window_save_path       = window_save_path
        self.representation_type    = args.representation_type
        #assert self.data_name in []
        self.freq                   = args.sampling_freq

        self.difference             = args.difference
        self.filtering              = args.filtering
        self.magnitude              = args.magnitude
        self.datanorm_type          = args.datanorm_type
        self.load_all               = args.load_all
        self.train_vali_quote       = args.train_vali_quote
        self.windowsize             = args.windowsize
        self.drop_transition        = args.drop_transition
        self.wavelet_function       = args.wavelet_function
        #self.wavelet_filtering      = args.wavelet_filtering
        #self.number_wavelet_filtering = args.number_wavelet_filtering

        # ======================= Load the Data =================================
        self.data_x, self.data_y = self.load_all_the_data(self.root_path)
        # data_x : sub_id, sensor_1, sensor_2,..., sensor_n , sub
        # data_y : activity_id   index:sub_id
        # update col_names
        self.col_names = list(self.data_x.columns)[1:-1]

        # ======================= Differencing the Data =================================
        if self.difference:
            print("Channel Augmentation : Differencing")
            self.data_x = self.differencing(self.data_x.set_index('sub_id').copy())

        # data_x : sub_id, sensor_1, sensor_2,..., sensor_n , sub

        # ======================== Filtering the Data =========================================
        if self.filtering:
            print("Channel Augmentation : Acc Gyro Filtering")
            self.data_x = self.Sensor_data_noise_grav_filtering(self.data_x.set_index('sub_id').copy())

        # ======================== Mag the Data =====================================

        if self.magnitude:
            print("Channel Augmentation : Magnitute Calculating for acc and Gyro")
            self.data_x.columns , columns_groups = self.regroup_and_reindex_all_cols(self.data_x.set_index('sub_id').copy())

            temp_columns = list(self.data_x.columns[1:-1])
            for cols in columns_groups:
                if len(cols)== 3:
                    col1, col2, col3 = cols
                    col = "_".join(col1.split("_")[:-1])+"_mag"
                    temp_columns.append(col)
                    self.data_x[col] = mag_3_signals(np.array(self.data_x[col1]),
                                                     np.array(self.data_x[col2]),
                                                     np.array(self.data_x[col3]))
            self.data_x = self.data_x[["sub_id"]+temp_columns+["sub"] ]

        # ======================= Generate the Sliding window for Train/Test =================================
        # train/test is different by the sliding step.
        self.train_slidingwindows = self.get_the_sliding_index(self.data_x.copy(), self.data_y.copy(), "train")
        if self.exp_mode not in ["SOCV","FOCV"]:
            self.test_slidingwindows  = self.get_the_sliding_index(self.data_x.copy(), self.data_y.copy(), "test")

        # ----------------------- TODO ----------------------------------------
        if self.representation_type in ["freq", "time_freq"]:
            print("------------freq representation is needed -----------------")
            assert self.freq_save_path is not None
            # train/test is different by the sliding step.
            self.genarate_spectrogram(flag = "train")
            self.genarate_spectrogram(flag = "test")

            if self.load_all:
                print("-------load all freq DATA --------")
                self.data_freq = {}
                for file in tqdm(self.train_freq_file_name):
                    with open(os.path.join(self.train_freq_path,"{}.pickle".format(file)), 'rb') as handle:
                        sample_x = pickle.load(handle)
                    self.data_freq[file] = sample_x

        # ----------------------- TODO ----------------------------------------
        if self.exp_mode in ["SOCV","FOCV"]:
            self.num_of_cv = 5
            self.index_of_cv = 0
            self.step = int(len(self.train_slidingwindows)/self.num_of_cv)
            self.window_index_list = list(np.arange(len(self.train_slidingwindows)))
            random.shuffle(self.window_index_list)
            if self.datanorm_type is not None:
                self.normalized_data_x = self.normalization(self.data_x.copy())
            else:
                self.normalized_data_x = self.data_x.copy()

        elif self.exp_mode == "LOCV":
            self.num_of_cv = len(self.LOCV_keys)
            self.index_of_cv = 0

        else:
            self.num_of_cv = 1

    def update_train_val_test_keys(self):
        """
        It should be called at the begin of each iteration
        it will update:
        1. train_window_index
        2. vali_window_index
        3. test_window_index
        it will also:
        normalize the data , because each iteration uses different training data
        calculate the weights of each class
        """
        if self.exp_mode in ["Given", "LOCV"]:
            if self.exp_mode == "LOCV":
                print("Leave one Out Experiment : The {} Part as the test".format(self.index_of_cv+1))

                self.test_keys =  self.LOCV_keys[self.index_of_cv]
                self.train_keys = [key for key in self.all_keys if key not in self.test_keys]
                # update the index_of_cv for the next iteration
                self.index_of_cv = self.index_of_cv + 1

            # Normalization the data
            if self.datanorm_type is not None:
                train_vali_x = pd.DataFrame()
                for sub in self.train_keys:
                    temp = self.data_x[self.data_x[self.split_tag]==sub]
                    train_vali_x = pd.concat([train_vali_x,temp])

                test_x = pd.DataFrame()
                for sub in self.test_keys:
                    temp = self.data_x[self.data_x[self.split_tag]==sub]
                    test_x = pd.concat([test_x,temp])

                train_vali_x, test_x = self.normalization(train_vali_x, test_x)

                self.normalized_data_x = pd.concat([train_vali_x,test_x])
                self.normalized_data_x.sort_index(inplace=True)
            else:
                self.normalized_data_x = self.data_x.copy()

            # 根据test的keys  筛选出 window的第一个element有哪些
            all_test_keys = []
            if self.split_tag == "sub":
                for sub in self.test_keys:
                    all_test_keys.extend(self.sub_ids_of_each_sub[sub])
            else:
                all_test_keys = self.test_keys.copy()

            # -----------------test_window_index---------------------
            test_file_name = os.path.join(self.window_save_path,
                                          "{}_droptrans_{}_windowsize_{}_{}_test_ID_{}.pickle".format(self.data_name,
                                                                                self.drop_transition,
                                                                                self.exp_mode,
                                                                                self.windowsize,
                                                                                self.index_of_cv-1))
            if os.path.exists(test_file_name):
                with open(test_file_name, 'rb') as handle:
                    self.test_window_index = pickle.load(handle)
            else:
                self.test_window_index = []
                for index, window in enumerate(self.test_slidingwindows):
                    sub_id = window[0]
                    if sub_id in all_test_keys:
                        self.test_window_index.append(index)
                with open(test_file_name, 'wb') as handle:
                    pickle.dump(self.test_window_index, handle, protocol=pickle.HIGHEST_PROTOCOL)

            # -----------------train_vali_window_index---------------------

            train_file_name = os.path.join(self.window_save_path,
                                           "{}_droptrans_{}_windowsize_{}_{}_train_ID_{}.pickle".format(self.data_name,
                                                                                self.drop_transition,
                                                                                self.exp_mode,
                                                                                self.windowsize,
                                                                                self.index_of_cv-1))
            if os.path.exists(train_file_name):
                with open(train_file_name, 'rb') as handle:
                    train_vali_window_index = pickle.load(handle)
            else:
                train_vali_window_index = []
                for index, window in enumerate(self.train_slidingwindows):
                    sub_id = window[0]
                    if sub_id not in all_test_keys:
                        train_vali_window_index.append(index)
                with open(train_file_name, 'wb') as handle:
                    pickle.dump(train_vali_window_index, handle, protocol=pickle.HIGHEST_PROTOCOL)

            random.shuffle(train_vali_window_index)
            self.train_window_index = train_vali_window_index[:int(self.train_vali_quote*len(train_vali_window_index))]
            self.vali_window_index = train_vali_window_index[int(self.train_vali_quote*len(train_vali_window_index)):]

        elif self.exp_mode in ["SOCV","FOCV"]:
            print("Overlapping random Experiment : The {} Part as the test".format(self.index_of_cv+1))
            start = self.index_of_cv * self.step
            if self.index_of_cv < self.num_of_cv-1:
                end = (self.index_of_cv+1) * self.step
            else:
                end = len(self.train_slidingwindows)

            train_vali_index = self.window_index_list[0:start] + self.window_index_list[end:len(self.window_index_list)]
            self.test_window_index = self.window_index_list[start:end]
            # copy shuffle
            self.train_window_index = train_vali_index[:int(self.train_vali_quote*len(train_vali_index))]
            self.vali_window_index = train_vali_index[int(self.train_vali_quote*len(train_vali_index)):]

            self.index_of_cv = self.index_of_cv + 1

        else:
            raise NotImplementedError

        self.act_weights = self.update_classes_weight()

    def update_classes_weight(self):
        class_transform = {x: i for i, x in enumerate(self.no_drop_activites)}

        y_of_all_windows  = []
        # get all labels of all windows
        for index in self.train_window_index:

            start_index = self.train_slidingwindows[index][1]
            end_index = self.train_slidingwindows[index][2]

            y_of_all_windows.append(class_transform[self.data_y.iloc[start_index:end_index].mode().loc[0]])
        y_of_all_windows = np.array(y_of_all_windows)

        target_count = np.array([np.sum(y_of_all_windows == label) for label in set(y_of_all_windows)])
        weight_target = 1.0 / target_count
        weight_samples = np.array([weight_target[t] for t in y_of_all_windows])
        weight_samples = torch.from_numpy(weight_samples)
        weight_samples = weight_samples.double()

        print("[-] Target sampling weights: ", weight_target)

        return weight_samples

    def load_all_the_data(self, root_path):
        raise NotImplementedError

    def Sensor_data_noise_grav_filtering(self, df):
        """
        df : sensor_1, sensor_2, sub
           index=sub_id
        """
        all_columns = list(df.columns)[:-1]
        #rest_columns = list(set(all_columns) - set(self.col_names))

        filtered_data = []
        for sub_id in df.index.unique():
            temp = df.loc[sub_id,all_columns]
            filtered_temp = pd.DataFrame()

            for col in temp.columns:
                t_signal=np.array(temp[col]) # copie the signal values in 1D numpy array

                if 'acc' in col and "diff" not in col:
                    # the 2nd output DC_component is the gravity_acc
                    # The 3rd one is the body_component which in this case the body_acc
                    grav_acc,body_acc=components_selection_one_signal(t_signal,freq1,freq2,self.freq) # apply components selection

                    filtered_temp[col]=body_acc
                    filtered_temp['grav'+col]= grav_acc

                elif 'gyro' in col and "diff" not in col:

                    # The 3rd output of components_selection is the body_component which in this case the body_gyro component
                    _,        body_gyro=components_selection_one_signal(t_signal,freq1,freq2,self.freq)  # apply components selection
                    filtered_temp[col]=body_gyro # t_body_acc storing with the appropriate axis selected
                else:

                    filtered_temp[col] = t_signal

            #filtered_temp = filtered_temp[sorted(list(filtered_temp.columns))]
            filtered_temp.index = temp.index
            filtered_data.append(filtered_temp)

        filtered_data = pd.concat(filtered_data)
        #filtered_data = pd.concat([df.loc[:,rest_columns], filtered_data], axis=1)
        #filtered_data = filtered_data[sorted(list(filtered_data.columns))]
        filtered_data = pd.concat([filtered_data, df.iloc[:,-1]], axis=1)

        return filtered_data.reset_index()

    def differencing(self, df):
        # columns = [, "acc_x"..."acc_y", "sub"], index is sub_id
        # define the name for differenced columns
        all_columns = list(df.columns)[:-1]
        rest_columns = list(set(all_columns) - set(self.col_names))

        columns = ["diff_"+i for i in self.col_names]

        # The original data has been divided into segments by sub_id: a segment belongs to a same user
        # There is no continuity between different segments, so diffrecne is only done within each segment

        # ALL data Diff
        diff_data = []
        for id in df.index.unique():
            diff_data.append(df.loc[id,self.col_names].diff())
        diff_data = pd.concat(diff_data)
        diff_data.columns = columns
        diff_data.fillna(method ="backfill",inplace=True)

        data = pd.concat([df.iloc[:,:-1],  diff_data], axis=1)
        #data = data[sorted(list(data.columns))]
        data = pd.concat([data, df.iloc[:,-1]], axis=1)

        return data.reset_index()

    def normalization(self, train_vali, test=None):
        train_vali_sensors = train_vali.iloc[:,1:-1]
        self.normalizer = Normalizer(self.datanorm_type)
        self.normalizer.fit(train_vali_sensors)
        train_vali_sensors = self.normalizer.normalize(train_vali_sensors)
        train_vali_sensors = pd.concat([train_vali.iloc[:,0],train_vali_sensors,train_vali.iloc[:,-1]], axis=1)
        if test is None:
            return train_vali_sensors
        else:
            test_sensors  = test.iloc[:,1:-1]
            test_sensors  = self.normalizer.normalize(test_sensors)
            test_sensors  =  pd.concat([test.iloc[:,0],test_sensors,test.iloc[:,-1]], axis=1)
            return train_vali_sensors, test_sensors

    def get_the_sliding_index(self, data_x, data_y , flag = "train"):
        """
        Because of the large amount of data, it is not necessary to store all the contents of the slidingwindow,
        but only to access the index of the slidingwindow
        Each window consists of three parts: sub_ID , start_index , end_index
        The sub_ID ist used for train test split, if the subject train test split is applied
        """
        if os.path.exists(os.path.join(self.window_save_path,
                                       "{}_{}_drop_trans_{}_windowsize{}.pickle".format(self.data_name,
                                                                                flag,
                                                                                self.drop_transition,
                                                                                self.windowsize))):
            print("-----------------------Sliding file are generated -----------------------")
            with open(os.path.join(self.window_save_path,
                                   "{}_{}_drop_trans_{}_windowsize{}.pickle".format(self.data_name,
                                                                                flag,
                                                                                self.drop_transition,
                                                                                self.windowsize)), 'rb') as handle:
                window_index = pickle.load(handle)
        else:
            print("----------------------- Get the Sliding Window -------------------")

            data_y = data_y.reset_index()
            data_x["activity_id"] = data_y["activity_id"]

            if self.drop_transition:
                data_x['act_block'] = ((data_x['activity_id'].shift(1) != data_x['activity_id']) | (data_x['sub_id'].shift(1) != data_x['sub_id'])).astype(int).cumsum()
            else:
                data_x['act_block'] = (data_x['sub_id'].shift(1) != data_x['sub_id']).astype(int).cumsum()

            freq         = self.freq
            windowsize   = self.windowsize

            if flag == "train":
                displacement = int(0.5 * self.windowsize)
                #drop_for_augmentation = int(0.2 * self.windowsize)
            elif flag == "test":
                displacement = int(0.1 * self.windowsize)
                #drop_for_augmentation = 1

            window_index = []
            for index in data_x.act_block.unique():

                temp_df = data_x[data_x["act_block"]==index]
                assert len(temp_df["sub_id"].unique()) == 1
                sub_id = temp_df["sub_id"].unique()[0]
                start = temp_df.index[0]# + drop_for_augmentation
                end   = start+windowsize

                while end <= temp_df.index[-1]+1:# + drop_for_augmentation :

                    if temp_df.loc[start:end-1,"activity_id"].mode().loc[0] not in self.drop_activities:
                        window_index.append([sub_id, start, end])

                    start = start + displacement
                    end   = start + windowsize

            with open(os.path.join(self.window_save_path,"{}_{}_drop_trans_{}_windowsize{}.pickle".format(self.data_name, flag, self.drop_transition,windowsize)), 'wb') as handle:
                pickle.dump(window_index, handle, protocol=pickle.HIGHEST_PROTOCOL)

        return window_index

    def genarate_spectrogram(self, flag="train"):
        save_path = os.path.join(self.freq_save_path,self.data_name)
        if not os.path.exists(save_path):
            os.mkdir(save_path)

        if flag == "train":
            displacement = int(0.5 * self.windowsize)
            slidingwindows = self.train_slidingwindows
            self.train_freq_path = os.path.join(save_path,"diff_{}_window_{}_step_{}_drop_trans_{}".format(self.difference, self.windowsize,displacement, self.drop_transition))
            freq_path = self.train_freq_path
        elif flag == "test":
            displacement = int(0.1 * self.windowsize)
            slidingwindows = self.test_slidingwindows
            self.test_freq_path = os.path.join(save_path,"diff_{}_window_{}_step_{}_drop_trans_{}".format(self.difference, self.windowsize, displacement, self.drop_transition))
            freq_path = self.test_freq_path

        if os.path.exists(freq_path):
            print("----------------------- file are generated -----------------------")
            if flag == "train":
                with open(os.path.join(freq_path,"freq_file_name.pickle"), 'rb') as handle:
                    self.train_freq_file_name = pickle.load(handle)
            else:
                with open(os.path.join(freq_path,"freq_file_name.pickle"), 'rb') as handle:
                    self.test_freq_file_name = pickle.load(handle)

        else:
            print("----------------------- spetrogram generating for {} -----------------------".format(flag))
            os.mkdir(freq_path)

            scales1 = np.arange(1, self.freq + 1)

            totalscal = self.freq
            fc = pywt.central_frequency(self.wavename)#计算小波函数的中心频率
            cparam = 2 * fc * totalscal  #常数c
            scales2 = cparam/np.arange(totalscal,0,-1) #为使转换后的频率序列是一等差序列，尺度序列必须取为这一形式（也即小波尺度）

            if self.windowsize >= 60 and "synthetic" not in self.data_name:
                l_scale = 2
            else:
                l_scale = 1

            if self.freq >=40 and "synthetic" not in self.data_name:
                f_scale = 2
            else:
                f_scale = 1

            if f_scale>1 or l_scale>1:
                resize_flag = True
            else:
                resize_flag = False

            freq_file_name = []

            temp_data = self.normalization(self.data_x.copy())
            for window in slidingwindows:
                sub_id = window[0]
                start_index = window[1]
                end_index = window[2]

                name = "{}_{}_{}".format(sub_id,start_index,end_index)
                freq_file_name.append(name)

                sample_x = temp_data.iloc[start_index:end_index,1:-1].values
                scalogram = []

                for j in range(sample_x.shape[1]):
                    if self.difference and j>= int(sample_x.shape[1]/2):
                        [cwtmatr, frequencies] = pywt.cwt(sample_x[:,j],   scales2,  self.wavename, sampling_period = 1.0/self.freq)#连续小波变换模块
                    else:
                        [cwtmatr, frequencies] = pywt.cwt(sample_x[:,j],   scales1,  self.wavename, sampling_period = 1.0/self.freq)#连续小波变换模块
                    if resize_flag:
                        cwtmatr = resize(cwtmatr, (int(self.freq/f_scale), int(self.windowsize/l_scale)), mode = 'constant')
                    scalogram.append(cwtmatr)

                scalogram = np.stack(scalogram)

                with open(os.path.join(freq_path,"{}.pickle".format(name)), 'wb') as handle:
                    pickle.dump(scalogram, handle, protocol=pickle.HIGHEST_PROTOCOL)

            with open(os.path.join(freq_path,"freq_file_name.pickle"), 'wb') as handle:
                pickle.dump(freq_file_name, handle, protocol=pickle.HIGHEST_PROTOCOL)
            if flag == "train":
                self.train_freq_file_name = freq_file_name
            else:
                self.test_freq_file_name = freq_file_name

    def Sensor_filter_acoording_to_pos_and_type(self, select, filter, all_col_names, filtertype):
        """
        select  (list or None): What location should be chosen
        filter  (list or None): whether all sensors can be filtered
        all_col_names     (list)  : Columns currently available for Filtering
        """
        if select is not None:
            if filter is None:
                raise Exception('This dataset cannot be selected by sensor {}!'.format(filtertype))
            else:
                col_names = []
                for col in all_col_names:
                    selected = False
                    for one_select in select:
                        assert one_select in filter
                        if one_select in col:
                            selected = True
                    if selected:
                        col_names.append(col)
                return col_names
        else:
            return None

    def regroup_and_reindex_all_cols(self, df):
        columns = df.columns[:-1]
        # big gourps
        groups = {}
        for col in columns:
            index = col.split("_")[-1]
            if index in groups.keys():
                groups[index].append(col)
            else:
                groups[index] = [col]
        # sub_groups
        index = 1
        columns_mapping = {}
        columns_groups = []
        for key in groups.keys():
            cols = groups[key]
            cols_set = []
            for col in cols:
                cols_set.append(col.split("_")[0])
            cols_set = set(cols_set)

            for col_begin in cols_set:
                sub_groups= []
                for col in cols:
                    if col.split("_")[0]==col_begin:
                        columns_mapping[col] = "_".join(col.split("_")[:-1])+"_"+str(index)
                        sub_groups.append("_".join(col.split("_")[:-1])+"_"+str(index))
                index= index+1
                if col_begin in ["acc","gyro","gravacc"]:
                    columns_groups.append(sub_groups)
        columns = ["sub_id"]+[columns_mapping[col] for col in df.columns[:-1]] + ["sub"]
        return columns,columns_groups


from pymongo import MongoClient
from celery import Celery
import redis
from task_config import CACHE_REDIS_HOST
from task_config import CELERY_BROKER_URL
from task_config import CELERY_RESULT_BACKEND
from task_config import MONGODB_URL

#------- redis---------#
class RedisWrapper:
    def __init__(self):
        self.redis_instance = redis.Redis(host=CACHE_REDIS_HOST,port=6379,decode_responses=True)

redis_db = RedisWrapper()

#------- mongodb -----#
class MongoWrapper:
    def __init__(self):
        self.client = MongoClient(
            f"mongodb+srv://{MONGODB_URL}/?retryWrites=true&w=majority")
        self.db = self.client.macroseat

mongo_db = MongoWrapper()

app = Celery("task",broker=CELERY_BROKER_URL, backend=CELERY_RESULT_BACKEND)

#删除我的食物快取
@app.task()
def delmyfoodCache(key,page):
    redis_db.redis_instance.hdel(key,str(page))

#删除我的饮食计划快取
@app.task()
def delmyplanCache(key,page):
    redis_db.redis_instance.hdel(key,str(page))

#删除我的日纪录快取
@app.task()
def delmyrecordCache(key,timestamp):
    redis_db.redis_instance.hdel(key,str(timestamp))

#-------- 聊天task ---------#
@app.task()
def push_message(key,message_obj):
    collection =mongo_db.db.message_history
    result = collection.update_one({"history_id":key},
                            {"$setOnInsert":
                                {"message":
                                [message_obj]}
                            },upsert=True)
    if result.matched_count == 1: #如果是1,代表已经有这个document
        collection.update_one({"history_id" : key}, {"$push":{"message":message_obj}})

#----- 使用者传讯息给营养师 ----#

#更新使用者的"已读时间"
@app.task()
def update_user_read(key,message_time):
    collection = mongo_db.db.message_history
    #一定已经有这个document因为前面就已经先存讯息了
    collection.update_one({"history_id" : key}, {"$max":{"user_read":message_time}})

#更新使用者和营养师的"对话者列表"
@app.task()
def update_history_list(user_key,nutri_key):
    collection = mongo_db.db.user_history
    result1 = collection.update_one({"user_id" : user_key},
                            {"$setOnInsert":
                                {"chat_list":
                                [nutri_key]}
                            },upsert=True)
    if result1.matched_count == 1: #如果是1,代表已经有这个document
        collection.update_one({"user_id" : user_key}, {"$addToSet":{"chat_list":nutri_key}})
    collection = mongo_db.db.nutri_history
    result2 = collection.update_one({"nutri_id" : nutri_key},
                            {"$setOnInsert":
                                {"chat_list":
                                [user_key]}
                            },upsert=True)
    if result2.matched_count == 1: #如果是1,代表已经有这个document
        collection.update_one({"nutri_id" : nutri_key}, {"$addToSet":{"chat_list":user_key}})

#更新营养师的"已读时间"
@app.task()
def update_nutri_read(key,message_time):
    collection = mongo_db.db.message_history
    #一定已经有这个document因为是使用者先传讯息给营养师
    collection.update_one({"history_id" : key}, {"$max":{"nutri_read":message_time}})

#更新营养师的"未读时间"
@app.task()
def update_nutri_unread(key,message_time):
    collection = mongo_db.db.message_history
    #一定已经有这个document因为是使用者先传讯息给营养师
    collection.update_one({"history_id" : key}, {"$max":{"nutri_unread":message_time}})

#----- 营养师传讯息给使用者 ----#

#更新使用者的"未读时间"
@app.task()
def update_user_unread(key,message_time):
    collection = mongo_db.db.message_history
    #一定已经有这个document因为前面就已经先存讯息了
    collection.update_one({"history_id" : key}, {"$max":{"user_unread":message_time}})

#------- 使用者取完已读对话纪录 ------#
@app.task()
def update_user_read_unread(key,user_read,user_unread):
    collection = mongo_db.db.message_history
    #一定已经有这个document因为前面就已经先存讯息了
    collection.update_one({"history_id" : key}, {"$set":{"user_read":user_read,"user_unread":user_unread}})

#------- 营养师取完未读对话纪录 ------#
@app.task()
def update_nutri_read_unread(key,nutri_read,nutri_unread):
    collection = mongo_db.db.message_history
    #一定已经有这个document因为前面就已经先存讯息了
    collection.update_one({"history_id" : key}, {"$set":{"nutri_read":nutri_read,"nutri_unread":nutri_unread}})

#------- 更新营养师的未读数量 ------#
@app.task()
def update_nutri_unread_count(key,count):
    collection = mongo_db.db.message_history
    collection.update_one({"history_id" : key}, {"$set":{"nutri_unread_count":count}})

@app.task()
def incr_nutri_unread_count(key):
    collection = mongo_db.db.message_history
    collection.update_one({"history_id" : key}, {"$inc":{"nutri_unread_count":1}})

#------- 更新使用者的未读数量 ------#
@app.task()
def update_user_unread_count(key,count):
    collection = mongo_db.db.message_history
    collection.update_one({"history_id" : key}, {"$set":{"user_unread_count":count}})

@app.task()
def incr_user_unread_count(key):
    collection = mongo_db.db.message_history
    collection.update_one({"history_id" : key}, {"$inc":{"user_unread_count":1}})


#
# @lc app=leetcode id=23 lang=python3
#
# [23] Merge k Sorted Lists
#
from typing import List, Optional
# @lc code=start
# Definition for singly-linked list.

class ListNode:
    def __init__(self, val=0, next=None):
        self.val = val
        self.next = next
class Solution:
    def mergeKLists(self, lists: List[Optional[ListNode]]) -> Optional[ListNode]:
        return self.mergeKLists_smarter(lists)
    def mergeKLists_brutal(self, lists: List[Optional[ListNode]]) -> Optional[ListNode]:
        if self.edge_cases(lists) != False:
            return self.edge_cases(lists)

        head=ListNode()
        pointer=head
        while True:

            min=(10001,0)

            for i in range(len(lists)):
                if lists[i] is None:
                    continue
                if lists[i].val<min[0]:
                    min=(lists[i].val,i)

            if min[0]==10001:
                break
            pointer.next=ListNode(lists[min[1]].val)
            lists[min[1]]=lists[min[1]].next
            pointer=pointer.next

        return head.next

        pass
    def edge_cases(self, lists: List[Optional[ListNode]]) -> Optional[ListNode]:
        if len(lists)==0:
            return None
        if len(lists)==1:
            if lists[0]==None:
                return None
            else:
                return lists[0]

        else:
            return False

    def connect(self, lists: List[Optional[ListNode]],index1,pointer):

        """
        pointer
        |
        1   4
        | / |
        |/  |
        3    5
        |
        |
        index1

        """
        pointer.next=lists[index1]
        lists[index1]=lists[index1].next
        pointer=pointer.next

        pass

    def merge_sort(self,list_to_sort):

        if len(list_to_sort)<=6:
            return sorted(list_to_sort)

        part1=self.merge_sort(list_to_sort[0:int(len(list_to_sort)/2)])
        part2=self.merge_sort(list_to_sort[int(len(list_to_sort)/2):])

        resultlist=[]

        while len(part1)>0 or len(part2)>0:
            if len(part1)==0:
                resultlist.extend(part2)
            if len(part2)==0:
                resultlist.extend(part1)

            if part1[0]<part2[0]:
                resultlist.append(part1.pop(0))
            else:
                resultlist.append(part2.pop(0))

    def initial_compare(self,lists):
        initial_compare_dict={}
        for i in range(0,len(lists)):
            #if lists[i]==None:
                #initial_compare_dict[None].append(i)
            #else:

            if lists[i] !=None:
                if lists[i].val in initial_compare_dict:
                    initial_compare_dict[lists[i].val].append(i)
                else:
                    initial_compare_dict[lists[i].val]=[i]

        return initial_compare_dict

    def mergeKLists_smarter(self, lists: List[Optional[ListNode]]) -> Optional[ListNode]:
        if self.edge_cases(lists) != False:
            return self.edge_cases(lists)

        root=ListNode()
        pointer=root

        value_dict=self.initial_compare(lists)

        while len(value_dict) > 0:

            minval=min(value_dict.keys())

            index=value_dict[minval].pop()

            # connect the pointer to the listnode with minval, with index=index
            pointer.next=lists[index]
            lists[index]=lists[index].next
            pointer=pointer.next

            # add the next value to the value_dict
            if lists[index]!= None:
                if lists[index].val in value_dict:
                    value_dict[lists[index].val].append(index)
                else:
                    value_dict[lists[index].val]=[index]

            # detect after pop the minval, the dictionary is empty

            if len(value_dict[minval])==0:
                value_dict.pop(minval)

        return root.next

# @lc code=end


from django.contrib import admin
from .models import NewsItem, Comment

class CommentAdmin(admin.ModelAdmin):
    list_display = ['name_author', 'news', 'text', 'check_admin', 'show_piece_text']
    list_filter = ['name_author']
    search_fields = ['name_author']

    actions = ['switch_to_status_deleted']

    def show_piece_text(self, queryset):
        text = queryset.text

        if len(text) >= 15:
            queryset.text = f'{text[:15]}...'

        return queryset

    def switch_to_status_deleted(self, request, queryset):
        queryset.update(check_admin=False, text='Удалено администратором')

    switch_to_status_deleted.short_description = 'Перевести в статус - Удалено администратором'
    show_piece_text.short_description = 'Текст комментария'

class CommentInLine(admin.TabularInline):
    model = Comment

class NewsItemAdmin(admin.ModelAdmin):
    model = NewsItem

    list_display = ['title', 'create_date', 'edit_date', 'flag_active']
    list_filter = ['flag_active']
    search_fields = ['title']
    inlines = [CommentInLine]

    actions = ['mark_as_active', 'mark_as_inactive']

    def mark_as_active(self, request, queryset):
        queryset.update(flag_active=True)

    def mark_as_inactive(self, request, queryset):
        queryset.update(flag_active=False)

    mark_as_active.short_description = 'Перевести в статус - Активно'
    mark_as_inactive.short_description = 'Перевести в статус - Неактивно'

admin.site.register(NewsItem, NewsItemAdmin)
admin.site.register(Comment, CommentAdmin)


# -*- coding: utf-8 -*-
"""
Created on Fri Mar 29 17:52:45 2019

@author: qinzhen
"""

import numpy as np
import matplotlib.pyplot as plt

def readData(images_file, labels_file):
    x = np.loadtxt(images_file, delimiter=',')
    y = np.loadtxt(labels_file, delimiter=',')
    return x, y

def softmax(x):
    """
    Compute softmax function for input.
    Use tricks from previous assignment to avoid overflow
    """
	### YOUR CODE HERE
    s1 = np.exp(x)
    s = s1 / np.sum(s1, axis=1).reshape(-1, 1)
	### END YOUR CODE
    return s

def sigmoid(x):
    """
    Compute the sigmoid function for the input here.
    """
    ### YOUR CODE HERE
    s = 1 / (1 + np.exp(-x))
    ### END YOUR CODE
    return s

def forward_prop(data, labels, params):
    """
    return hidder layer, output(softmax) layer and loss
    """
    W1 = params['W1']
    b1 = params['b1']
    W2 = params['W2']
    b2 = params['b2']

    ### YOUR CODE HERE
    #隐藏层
    s1 = data.dot(W1) + b1
    h = sigmoid(s1)
    #输出层
    s2 = h.dot(W2) + b2
    y = softmax(s2)
    cost = - np.mean(labels * np.log(y))
    ### END YOUR CODE
    return h, y, cost

def loss(data, labels, params, Lambda=0):
    """
    计算梯度和损失
    """
    #### Compute the forward pass
    W1 = params['W1']
    b1 = params['b1']
    W2 = params['W2']
    b2 = params['b2']

    #隐藏层
    s1 = data.dot(W1) + b1
    h = sigmoid(s1)
    #输出层
    s2 = h.dot(W2) + b2
    y = softmax(s2)
    cost = - np.mean(labels * np.log(y))

    #判断有无正则项
    if Lambda != 0:
        cost += Lambda * (np.linalg.norm(W1) ** 2 + np.linalg.norm(W2) ** 2)

    #### Backward pass: compute gradients
    #第二层梯度
    N2, D2 = y.shape
    t2 = y - labels
    db2 = np.sum(t2, axis=0) / N2
    dW2 = h.T.dot(t2) / N2
    if Lambda != 0:
        dW2 += 2 * Lambda * W2
    dX2 = t2.dot(W2.T)

    #第一层梯度
    N2, D2 = data.shape
    t1 = h * (1 - h)
    dW1 = data.T.dot(t1 * dX2) / N2
    if Lambda != 0:
        dW1 += 2 * Lambda * W1
    db1 = np.sum(t1, axis=0) / N2

    grad = {}
    grad['W1'] = dW1
    grad['W2'] = dW2
    grad['b1'] = db1
    grad['b2'] = db2

    return cost, grad

def nn_train(trainData, trainLabels, devData, devLabels,
             num_hidden = 300, learning_rate = 5, epoch=30, Lambda=0):
    (m, n) = trainData.shape
    d = trainLabels.shape[1]
    params = {}

    ### YOUR CODE HERE
    batch = 1000
    num = m // batch
    #epoch = 50
    #记录损失和正确率
    Cost = []
    Train_accuracy = []
    Dev_accuracy = []
    #初始化
    params['W1'] = np.random.randn(n, num_hidden)
    params['b1'] = np.zeros(num_hidden)
    params['W2'] = np.random.randn(num_hidden, d)
    params['b2'] = np.zeros(d)
    for _ in range(epoch):
        print(_)
        for i in range(num):
            #计算损失和梯度
            cost, grad = loss(trainData[i*batch: (i+1)*batch, :], trainLabels[i*batch: (i+1)*batch, :], params, Lambda=Lambda)
            #更新
            params['W1'] -= learning_rate * grad['W1']
            params['b1'] -= learning_rate * grad['b1']
            params['W2'] -= learning_rate * grad['W2']
            params['b2'] -= learning_rate * grad['b2']
            if i == num - 1:
                Cost.append(cost)
                Train_accuracy.append(nn_test(trainData, trainLabels, params))
                Dev_accuracy.append(nn_test(devData, devLabels, params))

    plt.plot(range(epoch), Cost)
    plt.title("loss VS epoch")
    plt.show()

    plt.plot(range(epoch), Train_accuracy, label="Train_accuracy")
    plt.plot(range(epoch), Dev_accuracy, label="Dev_accuracy")
    plt.show()

    ### END YOUR CODE

    return params

def nn_test(data, labels, params):
    h, output, cost = forward_prop(data, labels, params)
    accuracy = compute_accuracy(output, labels)
    return accuracy

def compute_accuracy(output, labels):
    accuracy = (np.argmax(output,axis=1) == np.argmax(labels,axis=1)).sum() * 1. / labels.shape[0]
    return accuracy

def one_hot_labels(labels):
    one_hot_labels = np.zeros((labels.size, 10))
    one_hot_labels[np.arange(labels.size),labels.astype(int)] = 1
    return one_hot_labels

np.random.seed(100)
trainData, trainLabels = readData('images_train.csv', 'labels_train.csv')
trainLabels = one_hot_labels(trainLabels)
p = np.random.permutation(60000)
trainData = trainData[p,:]
trainLabels = trainLabels[p,:]

devData = trainData[0:10000,:]
devLabels = trainLabels[0:10000,:]
trainData = trainData[10000:,:]
trainLabels = trainLabels[10000:,:]

mean = np.mean(trainData)
std = np.std(trainData)
trainData = (trainData - mean) / std
devData = (devData - mean) / std

testData, testLabels = readData('images_test.csv', 'labels_test.csv')
testLabels = one_hot_labels(testLabels)
testData = (testData - mean) / std

####(a)
params1 = nn_train(trainData, trainLabels, devData, devLabels)

####(b)
Lambda = 0.0001
params2 = nn_train(trainData, trainLabels, devData, devLabels, Lambda=Lambda)

####(c)
readyForTesting = True
if readyForTesting:
    accuracy1 = nn_test(testData, testLabels, params1)
    print('Test accuracy: %f' % accuracy1)

    accuracy2 = nn_test(testData, testLabels, params2)
    print('Test accuracy: %f' % accuracy2)