# system imports
import logging
import sys
import os
import io
import concurrent.futures
import shutil
from time import sleep
#import keyring

# google and http imports
from google.auth.transport.requests import Request
from google.oauth2.credentials import Credentials
from google_auth_oauthlib.flow import InstalledAppFlow
from googleapiclient.discovery import build
from googleapiclient.errors import HttpError
from googleapiclient.http import MediaIoBaseDownload
from googleapiclient.http import MediaFileUpload
import googleapiclient
import google_auth_httplib2
import httplib2
from googleapiclient import discovery

# application imports
current = os.path.dirname(os.path.realpath(__file__))
parent = os.path.dirname(current)
sys.path.append(parent)
from libdata.data_types import *
from libdata.sqlite_store import *
from lib import mods
from lib import keyring
from lib import filewatcher
from config import config as cfg
#from lib.mods import *

# data structure for queueing changes
"""
class Change:
    def __init__(self, change: str = "", src_object=None, dst_object=None, type="", retry=0):
        if change not in ['modified', 'created', 'deleted', 'moved', 'closed']:
            raise "Invalid change type '%s'" % change
        if type not in ['file', 'directory']:
            raise "Invalid change type '%s'" % type
        self.change_type = change
        self.object_type=type
        self.change_object = src_object
        self.dst_object = dst_object
        self.retry = retry
"""

def test_func():
    print("test function called")

def login_to_drive():
    logging.info("initializing application credentials")
    creds = None
    # The file token.json stores the user's access and refresh tokens, and is
    # created automatically when the authorization flow completes for the first
    # time.
    if cfg.USE_KEYRING == True:
        kr = keyring.Keyring()
    else:
        kr = None

    if cfg.USE_KEYRING == True:
        logging.debug("looking for and existing token in the OS keyring")
        try:
            tokenStr = kr.get_data("gdrive", "token")
            if tokenStr is not None and tokenStr != "":
                tokenStr = json.loads(tokenStr)
                if tokenStr['scopes'] != cfg.TARGET_SCOPES:
                    creds = None
                    kr.delete_data("gdrive", "token")
                else:
                    creds = Credentials.from_authorized_user_info(tokenStr, cfg.TARGET_SCOPES)
        except Exception as err:
            logging.error("Unable to fetch the oauth token from the OS keyring. %s" % str(err))
    else:
        logging.debug("looking for an existing token in" + cfg.TOKEN_CACHE)
        if os.path.exists(cfg.TOKEN_CACHE):
            creds = Credentials.from_authorized_user_file(cfg.TOKEN_CACHE, cfg.TARGET_SCOPES)
            with open(cfg.TOKEN_CACHE, 'r') as tokenFile:
                token = json.loads(tokenFile.read())
                if token['scopes'] != cfg.TARGET_SCOPES:
                    logging.warning("token cache scopes are not valid, removing token")
                    creds = None
                    os.remove(cfg.TOKEN_CACHE)

    # If there are no (valid) credentials available, let the user log in.
    if not creds or not creds.valid:
        logging.warning("valid credentials weren't found, initializing oauth consent from in default browser.")
        try:
            if creds and creds.expired and creds.refresh_token:
                try:
                    creds.refresh(Request())
                except HttpError as err:
                    logging.error("error logging in to google drive. %s" % str(err))
                except Exception as err:
                    # if refersh token expired, remove the token cache and rerun self
                    if 'invalid_grant: Token has been expired or revoked.' in err.args[0]:
                        logging.warning("oauth refresh token expired, clearing token cache.")
                        if cfg.USE_KEYRING == True:
                            kr.delete_data("gdrive", "token")
                        else:
                            os.remove(cfg.TOKEN_CACHE)
                        login_to_drive()
                        return
                    logging.error("error logging in to Google Drive. %s" % str(err))
            else:
                flow = InstalledAppFlow.from_client_secrets_file(cfg.APP_CREDS, cfg.TARGET_SCOPES)
                creds = flow.run_local_server(port=0)
            # Save the credentials for the next run
            if cfg.USE_KEYRING == True:
                kr.store_data("gdrive", "token", creds.to_json())
            else:
                with open(cfg.TOKEN_CACHE, 'w+') as token:
                    logging.debug("saving credentials to " + cfg.TOKEN_CACHE)
                    token.write(creds.to_json())
        except HttpError as err:
            print(err)

    return creds

# Create a new Http() object for every request
# https://googleapis.github.io/google-api-python-client/docs/thread_safety.html
# overrides the constructor of the http2 object
def build_request(http, *args, **kwargs):
    new_http = google_auth_httplib2.AuthorizedHttp(cfg.CREDENTIALS, http=httplib2.Http())
    return googleapiclient.http.HttpRequest(new_http, *args, **kwargs)

# get the root folder
def get_root_folder(service) -> gFolder:
    logging.debug("fetching the root folder")
    rootFolder = None
    try:
        gServiceFiles = service.files()
        params = { "fileId": 'root'
        }
        request = gServiceFiles.get(**params)
        rootFolderResult =request.execute()
        rootFolder = gFolder(rootFolderResult)
    except HttpError as err:
        logging.error("error fetching the root folder." + str(err))
        print(err)
    return rootFolder

# print out the google drive folder tree (won't be used in production)
def print_folder_tree(folders = None):
    # grab the root folder
    rootFolder = list(filter(lambda rf: rf.id == cfg.ROOT_FOLDER_ID, folders))
    #print(rootFolder[0]['name'])

    def printTree(parent, level=0):
        print("-" * level + parent.name)
        for child in parent.children:
            printTree(child, level+1)

    #printTree(folders, rootFolder[0], 0)
    printTree(rootFolder[0], 0)
    return

def get_full_folder_path(service, folder: gFolder)-> str:
    full_path = str(folder.name)
    try:
        if 'parents' in folder.properties.keys():
            gServiceFiles = service.files()
            params = { "fileId": folder.properties['parents'][0], "fields": "parents, mimeType, id, name, ownedByMe"}
            request = gServiceFiles.get(**params)
            parent = request.execute()
            full_path = parent['name'] + "/" + full_path
            while 'parents' in parent.keys():
                params = { "fileId": parent['parents'][0], "fields": "parents, mimeType, id, name, ownedByMe"}
                request = gServiceFiles.get(**params)
                parent = request.execute()
                full_path = parent['name'] + "/" + full_path
            if parent['ownedByMe'] == False:
                # a folder shared outside of the current owner for the drive object.
                # stick in the root folder
                full_path = "_shared_withme/" + full_path

        else:
            if folder.properties['ownedByMe'] == False:
                full_path = "_shared_withme/" + full_path

    except Exception as err:
        logging.error("Error getting full local path for folder id %s. %s" % (folder.id, str(err)))
        print(str(err))

    return full_path

# download a single file (will be called multi-threaded)
def download_file(service, file: gFile, targetPath:str, threadSafeDB:sqlite_store = None):
    logging.debug("beginning to download file %s", file.name)
    sReturn = ""
    try:
        gServiceFiles = service.files()
        params = { "fileId": file.id,
                    "acknowledgeAbuse": True
        }
        request = gServiceFiles.get_media(**params)
        fileData = io.BytesIO()
        downloader = MediaIoBaseDownload(fileData, request)
        done = False
        logging.info("downloading file %s." % targetPath)
        #print("downloading file %s." % targetPath)

        fileDir = os.path.dirname(targetPath)
        if not os.path.exists(fileDir):
            logging.debug("file's parent directory '%s' doesn't exist, creating." % fileDir)
            os.makedirs(os.path.expanduser(fileDir))

        while done is False:
            status, done = downloader.next_chunk()
            #print(F'Download {int(status.progress() * 100)}.')

        with open(targetPath, "wb+") as f:
            f.write(fileData.getbuffer())

        file.localPath = targetPath
        file.md5 = mods.hash_file(targetPath)

        # update the file timestamp to match what's in Drive
        mod_time = int(datetime.datetime.strptime(file.properties['modifiedTime'][:-5], '%Y-%m-%dT%H:%M:%S').strftime("%s"))
        os.utime(targetPath, (mod_time, mod_time))

        if threadSafeDB is not None:
            threadSafeDB.insert_gObject(file=file)
        else:
            cfg.DATABASE.insert_gObject(file=file)

        fileSize = os.path.getsize(targetPath)
        sReturn = "file %s written %d byes." % (targetPath, fileSize)

    except HttpError as err:
        logging.error("error downloading file. %s" % str(err))
        print(err)
        sReturn = "file %s download failed with %s" % (targetPath, str(err))
    except Exception as err:
        logging.error("error downloading file. %s" % str(err))
        print(err)
        sReturn = "file %s download failed with %s" % (targetPath, str(err))
    return sReturn

# export a native google document format (can't be downloaded)
def export_native_file(service, file: gFile, targetPath: str)-> bool:
    logging.debug("exporting the native google application file %s.", file.name)
    bSuccess = False
    try:
        gServiceFiles = service.files()
        # get type of application
        targetMimeType = None
        if file.properties['mimeType'] in cfg.MEDIA_EXPORT_MATRIX.keys():
            targetMimeType = cfg.MEDIA_EXPORT_MATRIX[file.properties['mimeType']]["targetMimeType"]
            targetExtension = cfg.MEDIA_EXPORT_MATRIX[file.properties['mimeType']]["extension"]
            targetPath = targetPath + targetExtension
        if targetMimeType is None:
            return False
        params = { "fileId": file.id,
                    "mimeType": targetMimeType
        }
        request = gServiceFiles.export_media(**params)
        fileData = io.BytesIO()
        downloader = MediaIoBaseDownload(fileData, request)
        done = False
        while done is False:
            status, done = downloader.next_chunk()
            print(F'Download {int(status.progress() * 100)}.')

        with open(targetPath, "wb+") as f:
            f.write(fileData.getbuffer())

    except HttpError as err:
        logging.error("error exporting google application file. %s", str(err))
        print(err)
        bSuccess = False
    return bSuccess

# return a listing of files in a directory (non-recursive)
def list_files_in_dir(service, folder:gFolder, maxFiles = 1000) -> List[gFile]:
    logging.debug("listing files in %s directory", folder.name)
    files = []
    try:
        gServiceFiles = service.files()
        params = { "q": "mimeType!='application/vnd.google-apps.folder' and '" +
                    folder.id + "' in parents",
                    "pageSize": cfg.PAGE_SIZE,
                    "fields": "nextPageToken," + cfg.FILE_FIELDS
        }
        request = gServiceFiles.list(**params)

        while (request is not None) and len(files) <= maxFiles:
            files_page = request.execute()
            fs = files_page.get('files', [])
            for f in fs:
                objFile = gFile(f)
                objFile.md5 = None
                files.append(objFile)

            request = gServiceFiles.list_next(request, files_page)
    except HttpError as err:
        logging.error("error listing files." + str(err))
        print(err)
    return files

# download all files in a folder (non-recursive)
def download_files_from_folder(service, folder: gFolder, targetDir: str) -> bool:
    logging.debug("starting to download files from %s to %s" % (folder.name, targetDir))
    bResult = False
    try:
        files = list_files_in_dir(service, folder)

        # the google api module isn't thread safe, since it's based on http2 which also isn't thread safe
        # https://googleapis.github.io/google-api-python-client/docs/thread_safety.html
        with concurrent.futures.ThreadPoolExecutor(max_workers=cfg.MAX_THREADS) as executor:
            futures = []
            for f in files:
                if not "application/vnd.google-apps" in f.properties['mimeType']:
                    filePath = os.path.join(targetDir, folder.name, f.name)

                    # build a new http2 object to enable thread safety.  gets passed to each thread
                    credentials = Credentials.from_authorized_user_file(cfg.TOKEN_CACHE, cfg.TARGET_SCOPES)
                    authorized_http = google_auth_httplib2.AuthorizedHttp(credentials, http=httplib2.Http())
                    service = discovery.build('drive', 'v3', requestBuilder=build_request, http=authorized_http)

                    # build new database object for multi-threading too
                    threadSafeDB = sqlite_store()
                    threadSafeDB.open(cfg.DATABASE_PATH)

                    futures.append(executor.submit(
                            download_file, service, f, filePath, threadSafeDB
                        ))
            for future in concurrent.futures.as_completed(futures):
                result = future.result()
                logging.debug("Download result: %s" % str(result))
            #wait(futures) # want to make sure we don't start too many threads
    except Exception as err:
        logging.error("error downloading directory %s. %s." % (folder.name, str(err)))
        bResult = False
    return bResult

def write_folder_cache(service, localCachePath:str = cfg.FOLDERS_CACHE_PATH):
    logging.debug("writing local folder cache to %s." % str(localCachePath))
    try:
        # get the root folder
        gServiceFiles = service.files()
        if not cfg.ROOT_FOLDER_OBJECT:
            request = gServiceFiles.get(fileId = 'root')
            rootFolder = request.execute()

        else:
            rootFolder = cfg.ROOT_FOLDER_OBJECT.properties

        fRootFolder = open(cfg.FOLDERS_CACHE_PATH + "_root", "w+")
        fRootFolder.write(json.dumps(rootFolder, indent = 4))
        fRootFolder.close()

        #global ROOT_FOLDER_ID
        if cfg.ROOT_FOLDER_ID == '':
            cfg.ROOT_FOLDER_ID = rootFolder['id']

        #print('List files')

        pageToken = None
        params = { "q": "mimeType='application/vnd.google-apps.folder'",
                    "pageSize": cfg.PAGE_SIZE,
                    "fields": "nextPageToken," + cfg.FOLDER_FIELDS
        }
        request = gServiceFiles.list(**params)

        while request is not None:
            files_page = request.execute()
            fs = files_page.get('files', [])
            for f in fs:
                #print(f)
                with open(cfg.FOLDERS_CACHE_PATH + f['id'], 'w+') as folder_data:
                    folderObj = gFolder(f)
                    folderObj.localPath = os.path.join(cfg.DRIVE_CACHE_PATH, get_full_folder_path(service, folderObj))
                    cfg.DATABASE.insert_gObject(folder=folderObj)
                    if 'parents' in folderObj.properties.keys():
                        cfg.DATABASE.insert_parents(folderObj.id, folderObj.properties['parents'])
                    folder_data.write(json.dumps(f, indent=5))
                    folder_data.close()

            request = gServiceFiles.list_next(request, files_page)

    except HttpError as err:
        logging.error("error writing local folder cache. %s", str(err))
        print(err)

# full sync down
def do_full_download(service, folder: gFolder, targetPath:str):
    logging.debug("starting full download from google drive to %s" % targetPath)
    try:
        download_files_from_folder(service, folder, os.path.join(targetPath))
        if folder.children is not None:
            for child in folder.children:
                do_full_download(service, child, os.path.join(targetPath, folder.name))

    except Exception as err:
        logging.error("error writing local folder cache. %s" % str(err))
        print(str(err))

# retrieve the metadata for Google object (file or folder)
def get_drive_object(service, id:str):
    return_object = None
    try:
        gServiceFiles = service.files()
        params = { "fileId": id,
                    "fields": "*"
        }
        request = gServiceFiles.get(**params)
        object = request.execute()
        if object is not None:
            if object['mimeType'] == 'application/vnd.google-apps.folder':
                return_object = gFolder(object)
            else:
                return_object = gFile(object)

    except HttpError as err:
        logging.error("Unable to fetch metadata from google drive for object id %s. %s" % (id, str(err)))
    except Exception as err:
        logging.error("Unable to fetch metadata from google drive for object id %s. %s" % (id, str(err)))

    return return_object

# create folder in Google Drive
def create_drive_folder(service, folderName:str, localPath:str, parentId:str=None) -> gFolder:
    folder = None
    try:

        if parentId is None or parentId == "":
            parentId = cfg.ROOT_FOLDER_ID
        file_metadata = {
            'name': folderName,
            'mimeType': 'application/vnd.google-apps.folder',
            'parents': parentId
        }

        logging.info("creating folder %s in Google Drive" % folderName)
        f = service.files().create(body=file_metadata, fields='*').execute()
        folder = gFolder(f)
        folder.localPath = localPath
        cfg.DATABASE.insert_gObject(folder=folder)

    except HttpError as err:
        logging.error("error creating Google Drive folder. %s" % str(err))
    except Exception as err:
        logging.error("error creating Google Drive folder. %s" % str(err))

    return folder

# create the entire folder tree, if any part doesn't exist
def create_drive_folder_tree(service, folderPath:str) -> gFolder:
    parentFolder = None
    try:
        if cfg.DRIVE_CACHE_PATH not in cfg.ROOT_FOLDER_OBJECT.localPath:
            raise "folder path isn't the defined drive cache path."
            return
        folderPath = folderPath.replace(cfg.ROOT_FOLDER_OBJECT.localPath + "/", "")
        folders = folderPath.split(os.sep)
        parent = cfg.ROOT_FOLDER_OBJECT
        currentFolder = cfg.ROOT_FOLDER_OBJECT.localPath
        for folder in folders:
            currentFolder = os.path.join(currentFolder, folder)
            c, dbFolders = cfg.DATABASE.fetch_gObjectSet(searchField="local_path", \
                            searchCriteria=currentFolder)
            if len(dbFolders) == 0:
                parent = create_drive_folder(service, folder, currentFolder, parent.id)
            else:
                parent = dbFolders[0]

        parentFolder = parent

    except HttpError as err:
        logging.error("error creating Google Drive folder tree. %s" % str(err))
    except Exception as err:
        logging.error("error creating Google Drive folder tree. %s" % str(err))
    return parentFolder

def upload_drive_file(service, filePath:str, parentId:str = None)-> gFile:
    file = None
    attempt = 1
    try:
        while attempt <= cfg.UPLOAD_RETRIES_MAX:
            # if file is under 5 mb perform a simple upload
            fileSize = os.path.getsize(filePath)
            fileHash = mods.hash_file(filePath)
            if fileSize <= (5 * 1024 * 1024):
                f = upload_drive_file_simple(service, filePath, parentId)
            else:
                # we need to figure out how to resumable uploads at some point later
                f = upload_drive_file_simple(service, filePath, parentId)
            file = gFile(f)
            if fileHash != file.properties['md5Checksum']:
                logging.warning("File upload resulted in a hash mismatch.")
                # remove the file
                file = delete_drive_file(service, file)
                attempt += 1
            else:
                file.localPath = filePath
                file.md5 = fileHash
                cfg.DATABASE.insert_gObject(file=file)
                break
        if attempt == cfg.UPLOAD_RETRIES_MAX:
            logging.error("Exceeded max retries to upload file '%s'" % filePath)
    except HttpError as err:
        logging.error("error uploading file to Google Drive. %s" % str(err))
    except Exception as err:
        logging.error("error uploading file to Google Drive. %s" % str(err))
    return file

def upload_drive_file_simple(service, filePath:str, parentId:str=None)->gFile:
    file = None
    try:
        if parentId is None or parentId == "":
            parentId = cfg.ROOT_FOLDER_ID
        fileDir = os.path.dirname(filePath)
        fileName = os.path.basename(filePath)
        logging.info("performing simple upload of file '%s'" % filePath)
        file_metadata = {'name': fileName, 'parents': parentId}
        media = MediaFileUpload(filePath, resumable=True)
        file = service.files().create(body=file_metadata, media_body=media,
                                      fields='*').execute()
    except HttpError as err:
        logging.error("error downing a simple file upload to Google Drive. %s" % str(err))
    except Exception as err:
        logging.error("error downing a simple file upload to Google Drive. %s" % str(err))
    return file

# uploads new files that have been identified as missing from the cloud post reconciliation
def upload_new_local_files(service):
    logging.debug("starting to upload new local files to the cloud.")
    try:
        records, new_local_files = cfg.DATABASE.fetch_newLocalFiles()
        recordsParsed = 0
        while records > 0:
            for f in new_local_files:
                if cfg.ROOT_FOLDER_OBJECT.localPath in f.localPath:
                    if f.mimeType == cfg.TYPE_GOOGLE_FOLDER:
                        return
                    else:
                        parentFolder = os.path.dirname(f.localPath)
                        c, db_parentFolders = cfg.DATABASE.fetch_gObjectSet(searchField = "local_path", \
                                                    searchCriteria=parentFolder)
                        db_parentFolder = db_parentFolders[0]
                        if db_parentFolder is not None:
                            f.properties['parents'] = [db_parentFolder.id]
                        else:
                            parent = create_drive_folder_tree(service, parentFolder)
                            f.properties['parents'] = parent.id
                        #file = upload_drive_file(service, f.localPath, f.properties['parents'][0])
                        change = filewatcher.Change('created', f.localPath, None, 'file')
                        cfg.LOCAL_QUEUE.put(change)
                else:
                    logging.warning("skipping file '%s'. path not in local cache directory." % f.localPath)
                recordsParsed += 1
            records, new_local_files = cfg.DATABASE.fetch_newLocalFiles(offset=recordsParsed)
    except Exception as err:
        logging.error("error uploading new files to the cloud. %s" % str(err))

def update_drive_file(service, file:gFile, localPath:str):
    logging.info("updating Google drive file %s." % file.name)
    updated_file = None
    try:
        media_body = MediaFileUpload(localPath, resumable=True)

        # Send the request to the API.
        updated_file = service.files().update(
            fileId=file.id,
            #body=file.properties,
            media_body=media_body).execute()
        # get the encriched full metadata
        updated_file = get_drive_object(service, updated_file['id'])
        updated_file.localPath = file.localPath
        updated_file.md5 = file.md5

        cfg.DATABASE.update_gObject(file=updated_file)

    except HttpError as err:
        logging.error("error updating Google drive file '%s'. %s" % (file.name, str(err)))
    except Exception as err:
        logging.error("error updating Google drive file '%s'. %s" % (file.name, str(err)))
    return updated_file

def update_drive_files(service):
    logging.debug("starting to update changed local files to the cloud.")
    try:
        c, changed_local_files = cfg.DATABASE.fetch_changedLocalFiles()
        recordsParsed = 0
        while c > 0:
            for f in changed_local_files:
                if cfg.ROOT_FOLDER_OBJECT.localPath in f.localPath:
                    if f.mimeType == cfg.TYPE_GOOGLE_FOLDER:
                        return
                    else:
                        #parentFolder = os.path.dirname(f.localPath)
                        #db_parentFolders, c = cfg.DATABASE.fetch_gObjectSet(searchField = "local_path", \
                        #                            searchCriteria=parentFolder)
                        #db_parentFolder = db_parentFolders[0]
                        #if db_parentFolder is not None:
                        #    f.properties['parents'] = [db_parentFolder.id]
                        #else:
                        #    parent = create_drive_folder_tree(service, parentFolder)
                        #    f.properties['parents'] = parent.id
                        #file = update_drive_file(service, f, f.localPath)
                        change = filewatcher.Change('modified', f.localPath, None, 'file')
                        cfg.LOCAL_QUEUE.put(change)
                else:
                    logging.warning("skipping file '%s'. path not in local cache directory." % f.localPath)
                recordsParsed += 1
            c, changed_local_files = cfg.DATABASE.fetch_newLocalFiles(offset=recordsParsed)
    except Exception as err:
        logging.error("error updating cloudfile '%s'. %s" % (f.name, str(err)))

def move_drive_file(service, file:gFile, newParent_id: str=None, newName:str = None) -> gFile:
    try:
        if file.properties['parents'][0] != newParent_id and newParent_id is not None:
            prev_parents = ','.join(file.properties['parents'])
            file = service.files().update(fileId=file.id, addParents=newParent_id,
                                        removeParents=prev_parents,
                                        fields='id, parents').execute()
            file = get_drive_object(service, file['id'])
            logging.info("moved file ID '%s' to new parent ID '%s'" % (file.id, newParent_id))
        else:
            if file.name != newName:
                file = service.files().update(fileId=file.id, body={'name': newName}).execute()
                file = get_drive_object(service, file['id'])
            else:
                logging.warning("Unable to process file '%s' move.  Can't parse the change." % file.id)
    except HttpError as err:
        logging.error("error moving file '%s' in Google Drive. %s" % (file.name, str(err)))
    except Exception as err:
        logging.error("error moving file '%s' in Google Drive. %s" % (file.name, str(err)))
    return file

def delete_drive_file(service, file:gFile):
    #file = None
    try:
        file.properties['trashed'] = True
        gSerivceFiles = service.files()
        gSerivceFiles.delete(fileId = file.id).execute()
        logging.info("deleted Google Drive file with id %s" % file.id)
        #cfg.DATABASE.update_gObject(file=file)
        cfg.DATABASE.delete_gObject(id=file.id) # do we want to delete the file?  or just mark it as trashed?

    except HttpError as err:
        # 404, delete from db since it's gone from Drive
        if err.resp.status == 404:
            logging.info("File not found in Drive, removing from db.")
            cfg.DATABASE.delete_gObject(id=file.id) # do we want to delete the file?  or just mark it as trashed?
        else:
            logging.error("error deleteing file '%s' from Google Drive. %s" % (file.name, str(err)))
    except Exception as err:
        logging.error("error deleteing file '%s' from Google Drive. %s" % (file.name, str(err)))
    return file

# region : Change tracking in Google drive

# gets the change token for changes in the drive since last sync
# https://developers.google.com/drive/api/guides/manage-changes
def get_drive_changes_token(service):
    logging.info("fetching the start changes token from Google Drive.")
    startToken = None
    try:
        response = service.changes().getStartPageToken().execute()
        startToken = response.get("startPageToken")
    except HttpError as err:
        logging.error("error getting changes start token. %s", str(err))
        print(err)
    except Exception as err:
        logging.error("error getting changes start token. %s", str(err))
        print(str(err))

    return startToken

# get changes since the last change token fetch
# https://developers.google.com/drive/api/guides/manage-changes
def get_drive_changes(service, changeToken):
    changes = []
    try:
        while changeToken is not None:
            response = service.changes().list(pageToken=changeToken,
                                              spaces='drive').execute()
            for change in response.get('changes'):
                # Process change
                changes.append(change)
            if 'newStartPageToken' in response:
                # Last page, save this token for the next polling interval
                #global CHANGES_TOKEN
                cfg.CHANGES_TOKEN = response.get('newStartPageToken')
            changeToken = response.get('nextPageToken')
    except HttpError as err:
        logging.error("error getting changes from Drive. %s", str(err))
        print(err)
    except Exception as err:
        logging.error("error getting changes from Drive. %s", str(err))
        print(str(err))

    return changes

# handles any sort of change in a file in google drive (create, update, delete)
def handle_changed_file(service, file:gFile = None):
    try:
        parents = []
        if file is not None:
            # ******************************************
            #      create or update an existing file
            # ******************************************
            dbFiles = cfg.DATABASE.fetch_gObject(file.id)
            # **** handle file creation ****
            if len(dbFiles) > 1:
                logging.warn("file id %s has multiple entries in the database. skipping." % file.id)
            elif len(dbFiles) == 0:
                # **** handle new files from Google Drive ****
                logging.debug("file id %s isn't in the database, assuming a new object." % file.id)
                if 'parents' in file.properties.keys():
                    for parent_id in file.properties['parents']:
                        parent_folder = get_drive_object(service, parent_id)
                        full_path = os.path.join(cfg.DRIVE_CACHE_PATH, \
                            get_full_folder_path(service, parent_folder), \
                            file.name)
                        full_path = os.path.expanduser(full_path)
                        file.localPath = full_path
                        cfg.DATABASE.insert_gObject(file=file)

                        if file.properties['trashed'] == False:
                            cfg.LQUEUE_IGNORE.append(full_path)
                            download_file(service, file, full_path)

            else:
                # **** handle file updates ****
                dbFile  = dbFiles[0]
                if file.properties != dbFile.properties and int(file.properties['version']) > int(dbFile.properties['version']):
                    # if the md5 is different for the file, then we are going to remove the local version and re-download
                    logging.info("file id %s is newer in the cloud and has changes, processing." % file.id)
                    if (file.properties['trashed'] == False):
                        file.localPath = dbFile.localPath
                        if (file.properties['md5Checksum'] != dbFile.md5 or file.name != dbFile.name):
                            file.md5 = dbFile.md5 # we'll download it later if we need to
                            try:
                                # delete the existing files and redownload for each instance of the file
                                if 'parents' in file.properties.keys():
                                    for parent_id in file.properties['parents']:
                                        for db_parent_id in dbFile.properties['parents']:
                                            parent_folder = get_drive_object(service, parent_id)
                                            root_path = os.path.join(cfg.DRIVE_CACHE_PATH, \
                                                get_full_folder_path(service, parent_folder))
                                            full_path = os.path.join(root_path, file.name)
                                            full_path = os.path.expanduser(full_path)

                                            parent_folder = get_drive_object(service, db_parent_id)
                                            root_path_old = os.path.join(cfg.DRIVE_CACHE_PATH, \
                                                get_full_folder_path(service, parent_folder))
                                            full_path_old = os.path.join(root_path_old, dbFile.name)
                                            full_path_old = os.path.expanduser(full_path_old)

                                            # do the the redownload if the md5 doesn't match
                                            if file.properties['md5Checksum'] != dbFile.md5:
                                                logging.info("file id %s checksum is different and cloud version is newer, redownloading." % file.id)

                                                if file.properties['trashed'] == False:
                                                    cfg.LQUEUE_IGNORE.append(full_path)
                                                    if os.path.exists(full_path):
                                                        logging.info("removing outdated file '%s'." % full_path)
                                                        os.remove(full_path)
                                                    download_file(service, file, full_path)

                                            # do the rename
                                            if file.name != dbFile.name:
                                                if root_path_old == root_path:
                                                    if file.properties['trashed'] == False:
                                                        cfg.LQUEUE_IGNORE.append(full_path_old)
                                                        cfg.LQUEUE_IGNORE.append(full_path)
                                                        os.rename(full_path_old, full_path)
                                                        cfg.DATABASE.update_gObject(file=file)
                                                        #sleep(0.2) # give the Watchdog service time to catch up
                                                        #cfg.LQUEUE_IGNORE.remove(full_path_old)
                                                        #cfg.LQUEUE_IGNORE.remove(full_path)

                            except Exception as err:
                                logging.error("unable to update file id %s. %s" % (file.id, str(err)))
                        else:
                            file.md5 = dbFile.md5
                            cfg.DATABASE.update_gObject(file=file)

                    # ***** delete a local file ******
                    elif file.properties['trashed'] == True:
                        file.md5 = dbFile.md5
                        file.localPath = dbFile.localPath
                        cfg.DATABASE.update_gObject(file=file)
                        if 'parents' in file.properties.keys():
                            for parent_id in file.properties['parents']:
                                try:
                                    parent_folder = get_drive_object(service, parent_id)
                                    full_path = os.path.join(cfg.DRIVE_CACHE_PATH, \
                                        get_full_folder_path(service, parent_folder), \
                                        file.name)
                                    full_path = os.path.expanduser(full_path)
                                    if os.path.exists(full_path):
                                        logging.info("removing trashed file '%s'" % full_path)
                                        cfg.LQUEUE_IGNORE.append(full_path)
                                        os.remove(full_path)
                                        #sleep(0.2) # give the Watchdog service time to catch up
                                        #cfg.LQUEUE_IGNORE.remove(full_path)
                                except Exception as err:
                                    logging.error("unable to remove local file %s. %s" % (full_path, str(err)))

    except Exception as err:
        logging.error("error processing Google object change. %s" % str(err))
    except HttpError as err:
        logging.error("error processing Google object change. %s" % str(err))
    return

# handles any sort of folder change in google drive (create, update, delete)
def handle_changed_folder(service, folder: gFolder = None):
    try:

        parents = []

        if folder is not None:
            # *************************************************************************
            #    create or update an existing folder
            # *************************************************************************
            dbFolders = cfg.DATABASE.fetch_gObject(folder.id)
            if len(dbFolders) > 1:
                logging.warn("folder id %s has multiple entries in the database. skipping." % folder.id)
            elif len(dbFolders) == 0:
                logging.debug("folder %s isn't in the database, assuming a new object." % folder.id)
                folder.localPath = os.path.join(cfg.DRIVE_CACHE_PATH, get_full_folder_path(service, folder))
                cfg.DATABASE.insert_gObject(folder=folder)
                if 'parents' in folder.properties.keys():
                    for parent_id in folder.properties['parents']:
                        parent_folder = get_drive_object(service, parent_id)
                        full_path = os.path.join(cfg.DRIVE_CACHE_PATH, \
                            get_full_folder_path(service, parent_folder), \
                            folder.name)
                        full_path = os.path.expanduser(full_path)
                        if not os.path.exists(full_path):
                            cfg.LQUEUE_IGNORE.append(full_path)
                            logging.info("creating new local folder '%s'" % full_path)
                            os.mkdir(os.path.expanduser(full_path))

            else:
                # if folder name is different, rename it.  if it's trashed, remove it.  only changes possible for folders
                dbFolder = dbFolders[0]
                if folder.properties != dbFolder.properties and int(folder.properties['version']) > int(dbFolder.properties['version']):
                    logging.info("folder id %s has a later version and different properties in Google Drive, applying changes" % folder.id)
                    # update the folder properties in the db
                    folder.localPath = os.path.join(cfg.DRIVE_CACHE_PATH, get_full_folder_path(service, folder))
                    cfg.DATABASE.update_gObject(folder=folder)
                    # **** rename the local folder(s) ****
                    if folder.name != dbFolder.name and folder.properties['trashed'] == False:
                        for parent_id in folder.properties['parents']:
                            for db_parent_id in dbFolder.properties['parents']:
                                parent_folder = get_drive_object(service, parent_id)
                                root_path_new = os.path.join(cfg.DRIVE_CACHE_PATH, \
                                    get_full_folder_path(service, parent_folder))
                                full_path_new = os.path.join(root_path_new, folder.name)
                                full_path_new = os.path.expanduser(full_path_new)

                                parent_folder = get_drive_object(service, db_parent_id)
                                root_path_old = os.path.join(cfg.DRIVE_CACHE_PATH, \
                                    get_full_folder_path(service, parent_folder))
                                full_path_old = os.path.join(root_path_old, dbFolder.name)
                                full_path_old = os.path.expanduser(full_path_old)

                                if root_path_old == root_path_new:
                                    cfg.LQUEUE_IGNORE.append(full_path_new)
                                    cfg.LQUEUE_IGNORE.append(full_path_old)
                                    os.rename(full_path_old, full_path_new)

                    # ***** delete a local folder ******
                    if folder.properties['trashed'] == True:
                        if 'parents' in folder.properties.keys():
                            for parent_id in folder.properties['parents']:
                                parent_folder = get_drive_object(service, parent_id)
                                full_path = os.path.join(cfg.DRIVE_CACHE_PATH, \
                                    get_full_folder_path(service, parent_folder), \
                                    folder.name)
                                full_path = os.path.expanduser(full_path)
                                if os.path.exists(full_path):
                                    cfg.LQUEUE_IGNORE.append(full_path)
                                    logging.info("removing trashed directory '%s'" % full_path)
                                    shutil.rmtree(full_path)

    except Exception as err:
        logging.error("error processing Google object change. %s" % str(err))
    except HttpError as err:
        logging.error("error processing Google object change. %s" % str(err))
    return

# scans all files in Google drive that aren't in the db.  that's our change set.
# this only needs to happen during startup.  otherwise, change notifications will do the job
def get_gdrive_changes(service) -> List:
    # loop through the pages of files from google drive
    # return the md5Checksum property, along with name, id, mimeType, version, parents
    # compare files by id with the db.  look where the md5Checksum != md5 stored in the db
    # also look for files not in the db
    # important: if the file in google drive is a later version, it's authoritative
    # this will be the changes from the side of google drive

    logging.info("scanning google drive files, looking for files and folders that have changed.")
    differences = []
    try:
        gServiceFiles = service.files()
        params = { "q": "'me' in owners",
                    "pageSize": cfg.PAGE_SIZE,
                    "fields": "nextPageToken," + "files(id, name, mimeType, version, md5Checksum, parents, ownedByMe)"
        }
        request = gServiceFiles.list(**params)

        while (request is not None):
            files_page = request.execute()
            fs = files_page.get('files', [])
            for f in fs:
                dbFile = None
                rows = cfg.DATABASE.fetch_gObject(f['id'])
                if len(rows) > 0:
                    dbFile = rows[0]
                if f['mimeType'] == cfg.TYPE_GOOGLE_FOLDER:
                    googleFolder = gFolder(f)
                    if dbFile is not None and 'version' in dbFile.properties.keys():
                        # if (dbFile.id != googleFolder.id or \
                        #            dbFile.name != googleFolder.name) and \
                        #            dbFile.properties['version'] < googleFolder.properties['version']:
                        if (int(dbFile.properties['version']) < int(googleFolder.properties['version'])):
                            # fetch full metadata of the file
                            get_params = {"fileId": googleFolder.id, "fields": "*"}
                            get_req = gServiceFiles.get(**get_params)
                            full_folder = gFolder(get_req.execute())
                            full_folder.localPath = get_full_folder_path(service, full_folder)
                            #differences.append(full_folder)
                            cfg.REMOTE_QUEUE.put(full_folder)
                    else:
                        get_params = {"fileId": googleFolder.id, "fields": "*"}
                        get_req = gServiceFiles.get(**get_params)
                        full_folder = gFolder(get_req.execute())
                        #differences.append(full_folder)
                        cfg.REMOTE_QUEUE.put(full_folder)

                else:
                    googleFile = gFile(f)
                    if dbFile is not None and 'version' in dbFile.properties.keys():
                        #if (dbFile.md5 != googleFile.properties['md5Checksum'] or \
                        #    dbFile.mimeType != googleFile.mimeType) and \
                        #    dbFile.properties['version'] < googleFile.properties['version']:
                        if (int(dbFile.properties['version']) < int(googleFile.properties['version'])):
                                # fetch full metadata of the file
                                get_params = {"fileId": googleFile.id, "fields": "*"}
                                get_req = gServiceFiles.get(**get_params)
                                full_file = gFile(get_req.execute())
                                #differences.append(full_file)
                                cfg.REMOTE_QUEUE.put(full_file)
                    else:
                        if cfg.TYPE_GOOGLE_APPS not in googleFile.mimeType:
                            get_params = {"fileId": googleFile.id, "fields": "*"}
                            get_req = gServiceFiles.get(**get_params)
                            full_file = gFile(get_req.execute())
                            #differences.append(full_file)
                            cfg.REMOTE_QUEUE.put(full_file)
            request = gServiceFiles.list_next(request, files_page)
    except HttpError as err:
        #exc_type, exc_obj, exc_tb = sys.exc_info()
        #fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
        #logging.error(exc_type, fname, exc_tb.tb_lineno)
        logging.error("error scanning google drive files. %s" % str(err))
        print(err)
    except Exception as err:
        #exc_type, exc_obj, exc_tb = sys.exc_info()
        #fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
        #logging.error(exc_type, fname, exc_tb.tb_lineno)
        logging.error("error scanning google drive files. %s" % str(err))
        print(err)
    return differences

# endregion


"""Adapted from:
https://github.com/hugovk/random-street-view/blob/main/random_street_view.py
"""

import argparse
import json
import os
import random
import sys
import requests
from urllib.request import urlopen, urlretrieve
from datetime import datetime
import contextlib
import yaml
import traceback

import shapefile  # pip install pyshp
import mapillary.interface as mly  # pip install mapillary

# Optional, http://stackoverflow.com/a/1557906/724176
try:
    import timing
    assert timing  # avoid flake8 warning
except ImportError:
    pass

IMG_SUFFIX = "jpg"
MAX_TRIES = 10  # Used to set number of maximum attempts at finding a non-filtered image

with open("api_key.yaml", "r") as ymlfile:
    key = yaml.load(ymlfile, Loader=yaml.FullLoader)
token = key['mly_token']
mly.set_access_token(token)

parser = argparse.ArgumentParser(
    description="Get random Street View images from within the borders of a given country. http://bboxfinder.com may "
    "be helpful for creating box coordinates and https://www.mapillary.com/app may be helpful for checking those boxes "
    "contain any images. By default, images are filtered out if they have any traffic signs as detected by Mapillary's "
    "systems; this should not be trusted absolutely, images should be manually checked for signs.",
    formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser.add_argument("country", help="ISO 3166-1 Alpha-3 Country Code, 'none', or 'near_global'")
help_str = "For default from country borders, enter 0. Ignored if 'near_global'."
parser.add_argument("min_lon", type=float, help=help_str)
parser.add_argument("min_lat", type=float, help=help_str)
parser.add_argument("max_lon", type=float, help=help_str)
parser.add_argument("max_lat", type=float, help=help_str)
parser.add_argument("-n", "--images-wanted", type=int, default=100, help="Total number of images wanted.")
parser.add_argument("-b", "--burst", type=int, default=10, help="The maximum number of nearby images downloaded from "
                    "any random geographical point that hits. >1 (e.g. 10) is recommended if using 'near_global'. Note "
                    "that nearby images may be captured by the same camera on the same day, so there is a trade-off "
                    "between speed of image retrieval and diversity of images to chosen here. I have so far been "
                    "unable to determine the definition of 'nearby' from the Mapillary SDK documentation, but each "
                    "point hit will often return 100s-1000s of nearby images as shown by '--save-to-json'.")
parser.add_argument("-j", "--save-to-json", action="store_true", help="Save to a JSON file metadata of images found "
                    "near a point.")
parser.add_argument("-N", "--no-filter", action="store_true", help="Turn off filtering of images with traffic signs.")
args = parser.parse_args()

# TODO: "--all-in-box", '-A' mode where every single image in the box is downloaded, not just random ones; ADD A WARNING

if args.images_wanted < 1:
    raise ValueError("Number of images wanted must be at least 1.")
# TODO: Allow for -n 0 for only downloading metadata for images near point (require that -j must also be specified)

# Determine if a point is inside a given polygon or not
# Polygon is a list of (x,y) pairs.
# http://www.ariel.com.au/a/python-point-int-poly.html
def point_inside_polygon(x, y, poly):
    n = len(poly)
    inside = False
    p1x, p1y = poly[0]
    for i in range(n + 1):
        p2x, p2y = poly[i % n]
        if y > min(p1y, p2y):
            if y <= max(p1y, p2y):
                if x <= max(p1x, p2x):
                    if p1y != p2y:
                        xinters = (y - p1y) * (p2x - p1x) / (p2y - p1y) + p1x
                    if p1x == p2x or x <= xinters:
                        inside = not inside
        p1x, p1y = p2x, p2y
    return inside

print("Loading borders...")
shape_file = "TM_WORLD_BORDERS-0.3.shp"
if not os.path.exists(shape_file):
    sys.exit(
        f"Cannot find {shape_file}. Please download it from "
        "http://thematicmapping.org/downloads/world_borders.php and try again."
    )

sf = shapefile.Reader(shape_file, encoding="latin1")
shapes = sf.shapes()

if args.country.lower() == "none":
    if args.min_lon == 0 or args.min_lat == 0 or args.max_lon == 0 or args.max_lat == 0:
        sys.exit("A valid bounding box must be entered if no country is specified.")
    min_lon = args.min_lon
    min_lat = args.min_lat
    max_lon = args.max_lon
    max_lat = args.max_lat
    borders = []
elif args.country.lower() == "near_global":
    min_lon = -160
    min_lat = -56
    max_lon = 180
    max_lat = 71
    borders = []
else:
    print("Finding country...")
    for i, record in enumerate(sf.records()):
        if record[2] == args.country.upper():
            print(record[2], record[4])
            print(shapes[i].bbox)
            min_lon = shapes[i].bbox[0] if args.min_lon == 0 else args.min_lon
            min_lat = shapes[i].bbox[1] if args.min_lat == 0 else args.min_lat
            max_lon = shapes[i].bbox[2] if args.max_lon == 0 else args.max_lon
            max_lat = shapes[i].bbox[3] if args.max_lat == 0 else args.max_lat
            borders = shapes[i].points
            break

print("Getting images...")
attempts, country_hits, point_hits, point_misses, imagery_hits, imagery_misses, imagery_filtered = 0, 0, 0, 0, 0, 0, 0

dtime = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
outdir = f"mly_random_{args.country.upper()}_{dtime}"

img_ids = []
try:
    while True:
        attempts += 1
        rand_lat = random.uniform(min_lat, max_lat)
        rand_lon = random.uniform(min_lon, max_lon)

        # Check if (lon, lat) is inside country borders
        point_inside = True
        if borders != []:
            point_inside = point_inside_polygon(rand_lon, rand_lat, borders)

        if point_inside:
            # print("In country")
            country_hits += 1

            try:
                # We will only retrieve flat images, although the MTSD contains 1138 flattened 360° panorama images
                # NOTE: I couldn't figure out how to suppress the GET prints from the Mapillary SDK
                print()
                images = mly.get_image_close_to(longitude=rand_lon, latitude=rand_lat, image_type='flat',
                                                fields=['thumb_original_url']).to_dict()
                point_hits += 1
            except IndexError:
                point_misses += 1
                print(f"  No images found close to point")
                continue

            hit_dir = f"hits_{args.country.upper()}_{dtime}"
            if args.save_to_json:
                os.makedirs(hit_dir, exist_ok=True)
                with open(os.path.join(hit_dir, f"hit_#{point_hits}_with_{len(images['features'])}_imgs_"
                                       f"{rand_lon}_{rand_lat}.json"), mode="w") as f:
                    json.dump(images, f, indent=4)

            print('\n\n  lat,lon: ' + str(rand_lat) + ',' + str(rand_lon))
            ii = 0
            found = 0
            while found < args.burst and ii < (found + 1) * MAX_TRIES:
                if ii >= len(images['features']):
                    break
                img_id = images['features'][ii]['properties']['id']
                if img_id in img_ids:
                    imagery_misses += 1
                    ii += 1
                    continue
                url_request = f"https://graph.mapillary.com/{img_id}?access_token={token}&fields=thumb_original_url"
                response = requests.get(url_request).json()  # Query the API for the original image URL
                try:
                    url = response['thumb_original_url']
                except KeyError:
                    print(f"  Error retrieving image URL for {img_id}")
                    imagery_misses += 1
                    ii += 1
                    continue

                # Filter out images with traffic signs detected by Mapillary
                passed_filter = True
                if not args.no_filter:
                    # NOTE: I couldn't figure out how to suppress the GET prints from the Mapillary SDK
                    detections = mly.get_detections_with_image_id(img_id).to_dict()
                    def is_sign(detection):
                        v = detection['properties']['value']
                        return (
                            "complementary" in v or
                            "information" in v or
                            "regulatory" in v or
                            "warning" in v
                        )
                    signs = [d['properties']['value'] for d in detections['features'] if not is_sign(d)]
                    if signs != []:
                        imagery_filtered += 1
                        passed_filter = False
                        print("  ----- Skipped image with traffic sign detections -----")

                if passed_filter:
                    os.makedirs(outdir, exist_ok=True)
                    outfile = os.path.join(outdir, f"{img_id}.{IMG_SUFFIX}")
                    try:
                        # Download the image
                        data = requests.get(url)
                        with open(outfile, "wb") as f:
                            f.write(data.content)
                    except KeyboardInterrupt:
                        sys.exit("exit")

                    if os.path.isfile(outfile):
                        print(f"  ========== Got one! Taken from this point: {found + 1} "
                              f"(from {ii + 1} attempts) ==========")
                        img_ids.append(img_id)
                        imagery_hits += 1
                        found += 1
                        if imagery_hits >= args.images_wanted:
                            break
                    else:
                        imagery_misses += 1
                ii += 1
            if imagery_hits >= args.images_wanted:
                break
        else:
            # print("  Point outside country")
            pass
except KeyboardInterrupt:
    print("Keyboard interrupt")
except Exception:
    # Make sure that stats are still printed and saved
    traceback.print_exc()

stats_str = f"Attempts:\t{attempts}\n"
if borders != []:
    stats_str += f"Country hits:\t{country_hits}\n"
stats_str += f"Point misses:\t{point_misses}\n"
stats_str += f"Point hits:\t{point_hits}\n"
stats_str += f"Imagery misses:\t{imagery_misses}\n"
if not args.no_filter:
    stats_str += f"Filtered out:\t{imagery_filtered}\n"
stats_str += f"Imagery hits:\t{imagery_hits}"
print(f"\n{stats_str}")
with open(os.path.join(outdir, "stats.txt"), mode="w") as f:
    f.write(stats_str)


import numpy as np
from scipy.stats import ttest_ind

np.random.seed(0)

# set up simulation parameters
num_simulations = 15000  # number of simulations to run
sample_size = 20  # sample size for each group
mean_1 = 0  # true mean for group 1
std_dev = 1  # standard deviation for both groups
corr = 0.5  # correlation between dependent variables

# generate dependent variables with specified correlation
cov_matrix = np.array([[1, corr], [corr, 1]])
# dependent_vars = np.random.multivariate_normal([0, 0], cov_matrix, size=sample_size)

# initialize lists to store results
p_values = []

# run simulations
for i in range(num_simulations):
    # generate random data for both groups
    group_1 = np.random.multivariate_normal([mean_1, mean_1], cov_matrix, size=sample_size)
    null_hypothesis = np.random.normal(loc=mean_1, scale=std_dev, size=sample_size)
    # this null hypothesis has the same mean and standard deviation as the dependent variables we
    # obtained from multivariate normal distribution
    dep_var1 = group_1[:, 0]
    dep_var2 = group_1[:, 1]
    dep_mean = (dep_var1 + dep_var2) / 2

    # calculate t-test p-value for each situation
    p_value_1 = ttest_ind(dep_var1, null_hypothesis)[1]
    if p_value_1 < 0.05:
        p_values.append(p_value_1)
    else:
        p_value_2 = ttest_ind(dep_var2, null_hypothesis)[1]
        if p_value_2 < 0.05:
            p_values.append(p_value_2)
        else:
            p_value_3 = ttest_ind(dep_mean, null_hypothesis)[1]
            if p_value_3 < 0.05:
                p_values.append(p_value_3)

# print the results
print(len(p_values))
print("Percentage of significant results", round(len(p_values) / num_simulations, 3))


from zeep.client import Client
import zeep

settings = zeep.Settings(strict=False, xml_huge_tree=True)
wsdl = 'http://localhost:7000/ws/EstudianteWebServices?wsdl'
cliente = Client(wsdl)

ListEstudiantes = cliente.service.getListaEstudiante()

def consultar(matricula):
    Estudiante = cliente.service.getEstudiante(matricula)
    return Estudiante

def CrearEstudiante(matricula, nombre, carrera):
    factory = cliente.type_factory('http://soap.eict.pucmm.edu/')
    nuevoEstudiante = factory.estudiante(carrera=carrera, matricula=carrera, nombre=nombre)
    creado = cliente.service.crearEstudiante(nuevoEstudiante)
    return creado

def EliminarEstudiante(matricula):

    return cliente.service.eliminarEstudiante(matricula)

print("[INFO] LISTA DE ESTUDIANTES: ")
for est in ListEstudiantes:
    print('[Nombre]: ' + est.nombre + ' [Matricula]: ' + str(est.matricula))

print('[INFO] Ingrese matricula del Estudiante que desea consultar:  {DEFAULT= 20011136} \n')
id = input()
print('[INFO] ESTUDIANTE CONSULTADO: \n' + str(consultar(matricula=id)))

print('[INFO] Ingrese Información del Nuevo Estudiante que desea CREAR: ')
print('[CARRERA]: ')
carrera = input()
print('[matricula]: ')
matricula = input()
print('[nombre]: ')
nombre = input()

print('[INFO] ESTUDIANTE CREADO: ' + str(CrearEstudiante(matricula=matricula,nombre=nombre,carrera=carrera)))

print('[INFO] Matricular del Estudiante que desea ELIMINAR: ')
id = input()
if(EliminarEstudiante(matricula=id)):
    print('[INFO] ESTUDIANTE Eliminado!')

print('~(^_^)~ --[HAPPY CODING!!]-- ~(^_^)~ ')


import torch.nn.functional as F
import torch
import numpy as np

def mae(input, target):
    return torch.mean(torch.abs(input - target))

def logmae_wav(model, output_dict, target):
    loss = torch.log10(torch.clamp(mae(output_dict['wav'], target), 1e-8, np.inf))
    return loss

def max_si_snr(input, target, eps = 1e-8):
    assert input.size() == target.size()
    B, C, T = target.size()
    # print(B, C, T)

    # zero-mean norm
    mean_target = torch.sum(target, dim = 2, keepdim=True) / T
    # print(mean_target.size())
    mean_input = torch.sum(input, dim = 2, keepdim=True) / T
    # print(mean_input.size())
    zero_mean_target = target - mean_target
    zero_mean_input = input - mean_input
    # print(zero_mean_target.size())
    # print(zero_mean_input.size())

    # Si-SNR
    s_target = torch.unsqueeze(zero_mean_target, dim = 1)
    # print(s_target.size())
    s_input = torch.unsqueeze(zero_mean_input, dim = 2)
    # print(s_input.size())
    pair_wise_dot = torch.sum(s_input * s_target, dim = 3, keepdim=True)
    # print(pair_wise_dot.size())
    s_target_energy = torch.sum(s_target ** 2, dim = 3, keepdim=True)
    # print(s_target_energy.size())
    pair_wise_proj = pair_wise_dot * s_target / s_target_energy
    # print(pair_wise_proj.size())
    e_noise = s_input - pair_wise_proj
    # print(e_noise.size())
    pair_wise_si_snr = torch.sum(pair_wise_proj ** 2, dim = 3) / (torch.sum(e_noise ** 2, dim = 3) + eps)

    pair_wise_si_snr = 10 * torch.log10(pair_wise_si_snr + eps)
    # print(pair_wise_si_snr.size())

    k = pair_wise_si_snr.squeeze(1).squeeze(1)
    # print(k.size())
    loss = 0 - torch.mean(k)

    return loss

def get_loss_func(loss_type):
    if loss_type == 'logmae_wav':
        return logmae_wav
    elif loss_type == 'mae':
        return mae
    elif loss_type == 'si_snr':
        return max_si_snr
    elif loss_type == 'mse':
        return torch.nn.MSELoss()
    else:
        raise Exception('Incorrect loss_type!')


# -*- coding: utf-8 -*-
"""
A new file.
"""
import numpy as np
from numba import jit, vectorize

from utils import timeit
from loops import loop1

@timeit
def loop(m, n):
    s = 0
    for i in range(1, m + 1):
        for j in range(1, n + 1):
            s += 1.0 / i + 1.0 / j
    return s

@timeit
def loopcxx(m, n):
    return loop1(m, n)

@timeit
@jit
def loop_jit(m, n):
    s = 0
    for i in range(1, m + 1):
        for j in range(1, n + 1):
            s += 1.0 / i + 1.0 / j
    return s

@timeit
@vectorize(['float64(int64, int64)'])
def loop_vec(m, n):
    s = 0
    for i in range(1, m + 1):
        for j in range(1, n + 1):
            s += 1.0 / i + 1.0 / j
    return s

def main():
    m = 10000
    n = 1000
    r = loop(m, n)
    r1 = loopcxx(m, n)
    r_jit = loop_jit(m, n)
    r_vec = loop_vec(m, n)
    print(r, r1, r_jit, r_vec)

if __name__ == '__main__':
    print('running...')
    main()


from kivymd.uix.button import MDIconButton
from kivy.app import App
from kivy.properties import (
    AliasProperty,
    BooleanProperty,
    ListProperty,
    NumericProperty,
    ObjectProperty,
    StringProperty,
)

class MDIconButtonTwoPosition(MDIconButton):
    enabled = True
    sourceEnabled = StringProperty()
    sourceDisabled = StringProperty()
    triggerFunction = ObjectProperty()
    arguments = ObjectProperty()

    def on_release(self):
        if self.enabled == True:
            self.enabled = False
        else:
            self.enabled = True

        self.change_source()

        if self.triggerFunction != None:
            self.triggerFunction(self.arguments,self.enabled)

    def change_source(self):
        if self.enabled == True:
            self.icon = self.sourceEnabled
        else:
            self.icon = self.sourceDisabled


from django.shortcuts import render, redirect
from django.http import JsonResponse
from django.views.decorators.csrf import csrf_exempt

from .models import *
from realtime.models import InAndOut
from .decorators import user_required, get_park_time

'''用户端 - 我的'''

@user_required
def personal(request, user):
	ctx = { 'menu': 'personal'}
	print(user.wechat_user().head_img)
	ctx['user'] = user

	return render(request, 'public_count/personal/my.html', ctx)

@user_required
def mybill(request, user, id=None):
	ctx = {}

	if id:
		ctx['r'] = r = InAndOut.objects.filter(id=id).first()
		if r:
			ctx['hours1'] = get_park_time(r.in_time, r.out_time)
			ctx['hours2'] = get_park_time(r.out_time, r.final_out_time)

		return render(request, 'public_count/personal/bill-detail.html', ctx)

	return render(request, 'public_count/personal/mybill.html', ctx)

@user_required
def mycard(request, user):
	ctx = {}

	from chargerule.models import Card

	plates = MyPlate.objects.filter(user=user)
	plates = [i.plate for i in plates]
	if plates:
		ctx['cards'] = Card.objects.filter(car_number__in=plates)

	return render(request, 'public_count/personal/mycard.html', ctx)

@user_required
def mycoupon(request, user):
	ctx = {}

	return render(request, 'public_count/personal/mycoupon.html', ctx)

@user_required
def myplate(request, user):
	ctx = {}

	plates = MyPlate.objects.filter(user=user)

	if request.method == 'POST':
		action = request.POST.get('action')
		if action == 'bound':
			plate = request.POST.get('plate', '')
			MyPlate.objects.get_or_create(user=user, plate=plate)
		elif action == 'unbound':
			id = request.POST.get('id', '')
			MyPlate.objects.filter(id=id).delete()

		return redirect('/wechat/personal/myplate/')

	ctx['plates'] = plates
	return render(request, 'public_count/personal/myplate.html', ctx)

@user_required
def bound(request, user):
	return render(request, 'public_count/personal/bound.html')

@user_required
def problem(request, user):
	if request.method == 'POST':
		car_number = request.POST.get('car_number', '')
		gate_id = request.POST.get('gate', '')
		parkinglot_id = request.POST.get('parkinglot', '')

		p = Problem(user=user, plate=car_number, parkinglot_id=parkinglot_id)
		if gate_id:
			p.gate_id = gate_id
		p.save()

		return render(request, 'public_count/problem.html')

@user_required
def scan_coupon(request, user, id, code):
	from chargerule.models import TicketRecord, UserCoupon

	ctx = {}

	r = TicketRecord.objects.filter(id=id)
	if r.exists():
		r = r.first()
		if r.qrrandom == code:
			from business.views import ran
			r.qrrandom = ran()
			r.save()

			UserCoupon.objects.create(user=user, ticket_record=r.first())
		else:
			ctx['error'] = '二维码已过期'

	coupons = UserCoupon.objects.select_related('ticket_record', 'ticket_record__coupons').filter(status=0).order_by('-create_time')

	return render(request, 'public_count/personal/mycoupon.html', ctx)


class Nellix2ssdf:

    def __init__(self,dicom_basedir,ptcode,ctcode,basedir):

        import imageio
        #import easygui

        from stentseg.utils.datahandling import loadvol
        from stentseg.utils.datahandling import savecropvols, saveaveraged

        ## Select base directory for LOADING DICOM data

        #dicom_basedir = easygui.diropenbox()
        print('DICOM Path = ', dicom_basedir)

        #ctcode = '12months'  # 'pre', 'post_x', '12months'
        stenttype = 'nellix'

        ## Select base directory to SAVE SSDF
        #basedir = easygui.diropenbox()
        print('Base Path = ', basedir)

        # Set which crops to save
        cropnames = ['prox'] #,'stent'] # ['ring'] or ['ring','stent'] or ..

        #===============================================================================
        ## Step A: read single volumes to get vols:
        #  folder1 = '10%'
        #  folder2 = '60%'
        #  vol1 = imageio.volread(os.path.join(dicom_basedir, folder1), 'dicom')
        #  vol2 = imageio.volread(os.path.join(dicom_basedir, folder2), 'dicom')
        #  print(  )
        #
        #  if vol1.meta.SeriesDescription[:2] < vol2.meta.SeriesDescription[:2]:
        #      vols4078 = [vol1,vol2]
        #  else:
        #      vols4078 = [vol2,vol1]
        #
        #  vols = vols4078.copy()
        #
        #  for vol in vols:
        #      vol.meta.PatientName = ptcode # anonimyze
        #      vol.meta.PatientID = 'anonymous'
        #     print(vol.meta.SeriesDescription,'-', vol.meta.sampling)
        #===============================================================================

        ##Orginele code
        #===============================================================================
        #
        # folder1 = '40% iDose'
        # folder2 = '78 iDose'
        # vol1 = imageio.volread(os.path.join(dicom_basedir, folder1), 'dicom')
        # vol2 = imageio.volread(os.path.join(dicom_basedir, folder2), 'dicom')
        # print(  )
        #
        # if vol1.meta.SeriesDescription[:2] < vol2.meta.SeriesDescription[:2]:
        #     vols4078 = [vol1,vol2]
        # else:
        #     vols4078 = [vol2,vol1]
        #
        # vols = vols4078.copy()
        #
        # for vol in vols:
        #     vol.meta.PatientName = ptcode # anonimyze
        #     vol.meta.PatientID = 'anonymous'
        #     print(vol.meta.SeriesDescription,'-', vol.meta.sampling)
        #===============================================================================

        ## Step A: read 10 volumes to get vols
        # Deze zoekt alle mappen en dat zijn er dus 10 maar niet in de goede volgorde
        vols2 = [vol2 for vol2 in imageio.get_reader(dicom_basedir, 'DICOM', 'V')]

        vols = [None] * len(vols2)
        for i, vol in enumerate(vols2):
        #    print(vol.meta.sampling)
            print(vol.meta.SeriesDescription)
            phase = int(vol.meta.SeriesDescription[:1])
            # use phase to fix order of phases
            vols[phase] = vol
            #vols[phase].meta.ImagePositionPatient = (0.0,0.0,0.0)

        for i,vol in enumerate(vols): #wat ik heb veranderd is i, en enumerate()
            print(vol.meta.SeriesDescription)
            assert vol.shape == vols[0].shape
            assert str(i*10) in vol.meta.SeriesDescription # 0% , 10% etc.

        ## Step B: Crop and Save SSDF
        # 1 of 2 cropnames opgeven voor opslaan 1 of 2 crpos.
        # Het eerste volume wordt geladen in MIP, crop met marges van minimaal 30 mm
        for cropname in cropnames:
            savecropvols(vols, basedir, ptcode, ctcode, cropname, stenttype)
        #    saveaveraged(basedir, ptcode, ctcode, cropname, range(0,100,10))

        ## Visualize result

        #s1 = loadvol(basedir, ptcode, ctcode, cropnames[0], what ='10avgreg')
        #s2 = loadvol(basedir, ptcode, ctcode, cropnames[0], what ='10phases')
        s1 = loadvol(basedir, ptcode, ctcode, cropnames[0], what ='phases')
        #s2 = loadvol(basedir, ptcode, ctcode, cropnames[0], what = 'avg010')
        #vol1 = s1.vol
        vol1 = s1.vol40

        # Visualize and compare
        colormap = {'r': [(0.0, 0.0), (0.17727272, 1.0)],
         'g': [(0.0, 0.0), (0.27272728, 1.0)],
         'b': [(0.0, 0.0), (0.34545454, 1.0)],
         'a': [(0.0, 1.0), (1.0, 1.0)]}

        import visvis as vv

        fig = vv.figure(1); vv.clf()
        fig.position = 0, 22, 1366, 706
        a1 = vv.subplot(111)
        a1.daspect = 1, 1, -1
        # t1 = vv.volshow(vol1, clim=(0, 3000), renderStyle='iso') # iso or mip
        # t1.isoThreshold = 600 # stond op 400 maar je moet hoger zetten als je alleen stent wil
        # t1.colormap = colormap
        a1 = vv.volshow2(vol1, clim=(-500, 1500),renderStyle='mip')
        vv.xlabel('x'), vv.ylabel('y'), vv.zlabel('z')
        # vv.title('One volume at %i procent of cardiac cycle' % phase )
        vv.title('Vol40' )


import datetime
import sys
import time
import traceback

# logger
from pandacommon.pandalogger.PandaLogger import PandaLogger
from pandajedi.jediconfig import jedi_config
from pandajedi.jedicore import Interaction, JediException
from pandajedi.jedicore.FactoryBase import FactoryBase
from pandajedi.jedicore.JediDatasetSpec import JediDatasetSpec
from pandajedi.jedicore.JediTaskSpec import JediTaskSpec
from pandajedi.jedicore.MsgWrapper import MsgWrapper
from pandajedi.jedicore.ThreadUtils import ListWithLock, ThreadPool, WorkerThread
from pandajedi.jedirefine import RefinerUtils

from .JediKnight import JediKnight

logger = PandaLogger().getLogger(__name__.split(".")[-1])

# worker class to refine TASK_PARAM to fill JEDI tables
class TaskRefiner(JediKnight, FactoryBase):
    # constructor
    def __init__(self, commuChannel, taskBufferIF, ddmIF, vos, prodSourceLabels):
        self.vos = self.parseInit(vos)
        self.prodSourceLabels = self.parseInit(prodSourceLabels)
        JediKnight.__init__(self, commuChannel, taskBufferIF, ddmIF, logger)
        FactoryBase.__init__(self, self.vos, self.prodSourceLabels, logger, jedi_config.taskrefine.modConfig)

    # main
    def start(self):
        # start base classes
        JediKnight.start(self)
        FactoryBase.initializeMods(self, self.taskBufferIF, self.ddmIF)
        # go into main loop
        while True:
            startTime = datetime.datetime.utcnow()
            try:
                # get logger
                tmpLog = MsgWrapper(logger)
                tmpLog.debug("start")
                # loop over all vos
                for vo in self.vos:
                    # loop over all sourceLabels
                    for prodSourceLabel in self.prodSourceLabels:
                        # get the list of tasks to refine
                        tmpList = self.taskBufferIF.getTasksToRefine_JEDI(vo, prodSourceLabel)
                        if tmpList is None:
                            # failed
                            tmpLog.error("failed to get the list of tasks to refine")
                        else:
                            tmpLog.debug(f"got {len(tmpList)} tasks")
                            # put to a locked list
                            taskList = ListWithLock(tmpList)
                            # make thread pool
                            threadPool = ThreadPool()
                            # get work queue mapper
                            workQueueMapper = self.taskBufferIF.getWorkQueueMap()
                            # make workers
                            nWorker = jedi_config.taskrefine.nWorkers
                            for iWorker in range(nWorker):
                                thr = TaskRefinerThread(taskList, threadPool, self.taskBufferIF, self.ddmIF, self, workQueueMapper)
                                thr.start()
                            # join
                            threadPool.join()
            except Exception:
                errtype, errvalue = sys.exc_info()[:2]
                tmpLog.error(f"failed in {self.__class__.__name__}.start() with {errtype.__name__} {errvalue}")
                tmpLog.error(f"Traceback: {traceback.format_exc()}")
            # sleep if needed
            loopCycle = jedi_config.taskrefine.loopCycle
            timeDelta = datetime.datetime.utcnow() - startTime
            sleepPeriod = loopCycle - timeDelta.seconds
            if sleepPeriod > 0:
                time.sleep(sleepPeriod)
            # randomize cycle
            self.randomSleep(max_val=loopCycle)

# thread for real worker
class TaskRefinerThread(WorkerThread):
    # constructor
    def __init__(self, taskList, threadPool, taskbufferIF, ddmIF, implFactory, workQueueMapper):
        # initialize woker with no semaphore
        WorkerThread.__init__(self, None, threadPool, logger)
        # attributres
        self.taskList = taskList
        self.taskBufferIF = taskbufferIF
        self.ddmIF = ddmIF
        self.implFactory = implFactory
        self.workQueueMapper = workQueueMapper
        self.msgType = "taskrefiner"

    # main
    def runImpl(self):
        while True:
            try:
                # get a part of list
                nTasks = 10
                taskList = self.taskList.get(nTasks)
                # no more datasets
                if len(taskList) == 0:
                    self.logger.info(f"{self.__class__.__name__} terminating since no more items")
                    return
                # loop over all tasks
                for jediTaskID, splitRule, taskStatus, parent_tid in taskList:
                    # make logger
                    tmpLog = MsgWrapper(self.logger, f"< jediTaskID={jediTaskID} >")
                    tmpLog.debug("start")
                    tmpStat = Interaction.SC_SUCCEEDED
                    errStr = ""
                    # read task parameters
                    try:
                        taskParam = None
                        taskParam = self.taskBufferIF.getTaskParamsWithID_JEDI(jediTaskID)
                        taskParamMap = RefinerUtils.decodeJSON(taskParam)
                    except Exception:
                        errtype, errvalue = sys.exc_info()[:2]
                        errStr = f"conversion to map from json failed with {errtype.__name__}:{errvalue}"
                        tmpLog.debug(taskParam)
                        tmpLog.error(errStr)
                        continue
                        tmpStat = Interaction.SC_FAILED
                    # get impl
                    if tmpStat == Interaction.SC_SUCCEEDED:
                        tmpLog.info("getting Impl")
                        try:
                            # get VO and sourceLabel
                            vo = taskParamMap["vo"]
                            prodSourceLabel = taskParamMap["prodSourceLabel"]
                            taskType = taskParamMap["taskType"]
                            tmpLog.info(f"vo={vo} sourceLabel={prodSourceLabel} taskType={taskType}")
                            # get impl
                            impl = self.implFactory.instantiateImpl(vo, prodSourceLabel, taskType, self.taskBufferIF, self.ddmIF)
                            if impl is None:
                                # task refiner is undefined
                                errStr = f"task refiner is undefined for vo={vo} sourceLabel={prodSourceLabel}"
                                tmpLog.error(errStr)
                                tmpStat = Interaction.SC_FAILED
                        except Exception:
                            errtype, errvalue = sys.exc_info()[:2]
                            errStr = f"failed to get task refiner with {errtype.__name__}:{errvalue}"
                            tmpLog.error(errStr)
                            tmpStat = Interaction.SC_FAILED
                    # extract common parameters
                    if tmpStat == Interaction.SC_SUCCEEDED:
                        tmpLog.info("extracting common")
                        try:
                            # initalize impl
                            impl.initializeRefiner(tmpLog)
                            impl.oldTaskStatus = taskStatus
                            # extract common parameters
                            impl.extractCommon(jediTaskID, taskParamMap, self.workQueueMapper, splitRule)
                            # set parent tid
                            if parent_tid not in [None, jediTaskID]:
                                impl.taskSpec.parent_tid = parent_tid
                        except Exception:
                            errtype, errvalue = sys.exc_info()[:2]
                            # on hold in case of external error
                            if errtype == JediException.ExternalTempError:
                                tmpErrStr = f"pending due to external problem. {errvalue}"
                                setFrozenTime = True
                                impl.taskSpec.status = taskStatus
                                impl.taskSpec.setOnHold()
                                impl.taskSpec.setErrDiag(tmpErrStr)
                                # not to update some task attributes
                                impl.taskSpec.resetRefinedAttrs()
                                tmpLog.info(tmpErrStr)
                                self.taskBufferIF.updateTask_JEDI(
                                    impl.taskSpec,
                                    {"jediTaskID": impl.taskSpec.jediTaskID},
                                    oldStatus=[taskStatus],
                                    insertUnknown=impl.unknownDatasetList,
                                    setFrozenTime=setFrozenTime,
                                )
                                continue
                            errStr = f"failed to extract common parameters with {errtype.__name__}:{errvalue} {traceback.format_exc()}"
                            tmpLog.error(errStr)
                            tmpStat = Interaction.SC_FAILED
                    # check attribute length
                    if tmpStat == Interaction.SC_SUCCEEDED:
                        tmpLog.info("checking attribute length")
                        if not impl.taskSpec.checkAttrLength():
                            tmpLog.error(impl.taskSpec.errorDialog)
                            tmpStat = Interaction.SC_FAILED
                    # staging
                    if tmpStat == Interaction.SC_SUCCEEDED:
                        if "toStaging" in taskParamMap and taskStatus not in ["staged", "rerefine"]:
                            errStr = "wait until staging is done"
                            impl.taskSpec.status = "staging"
                            impl.taskSpec.oldStatus = taskStatus
                            impl.taskSpec.setErrDiag(errStr)
                            # not to update some task attributes
                            impl.taskSpec.resetRefinedAttrs()
                            tmpLog.info(errStr)
                            self.taskBufferIF.updateTask_JEDI(
                                impl.taskSpec, {"jediTaskID": impl.taskSpec.jediTaskID}, oldStatus=[taskStatus], updateDEFT=False, setFrozenTime=False
                            )
                            continue
                    # check parent
                    noWaitParent = False
                    parentState = None
                    if tmpStat == Interaction.SC_SUCCEEDED:
                        if parent_tid not in [None, jediTaskID]:
                            tmpLog.info("check parent task")
                            try:
                                tmpStat = self.taskBufferIF.checkParentTask_JEDI(parent_tid)
                                parentState = tmpStat
                                if tmpStat == "completed":
                                    # parent is done
                                    tmpStat = Interaction.SC_SUCCEEDED
                                elif tmpStat is None or tmpStat == "running":
                                    if not impl.taskSpec.noWaitParent():
                                        # parent is running
                                        errStr = f"pending until parent task {parent_tid} is done"
                                        impl.taskSpec.status = taskStatus
                                        impl.taskSpec.setOnHold()
                                        impl.taskSpec.setErrDiag(errStr)
                                        # not to update some task attributes
                                        impl.taskSpec.resetRefinedAttrs()
                                        tmpLog.info(errStr)
                                        self.taskBufferIF.updateTask_JEDI(
                                            impl.taskSpec, {"jediTaskID": impl.taskSpec.jediTaskID}, oldStatus=[taskStatus], setFrozenTime=False
                                        )
                                        continue
                                    else:
                                        # not wait for parent
                                        tmpStat = Interaction.SC_SUCCEEDED
                                        noWaitParent = True
                                else:
                                    # parent is corrupted
                                    tmpStat = Interaction.SC_FAILED
                                    tmpErrStr = f"parent task {parent_tid} failed to complete"
                                    impl.taskSpec.setErrDiag(tmpErrStr)
                            except Exception:
                                errtype, errvalue = sys.exc_info()[:2]
                                errStr = f"failed to check parent task with {errtype.__name__}:{errvalue}"
                                tmpLog.error(errStr)
                                tmpStat = Interaction.SC_FAILED
                    # refine
                    if tmpStat == Interaction.SC_SUCCEEDED:
                        tmpLog.info(f"refining with {impl.__class__.__name__}")
                        try:
                            tmpStat = impl.doRefine(jediTaskID, taskParamMap)
                        except Exception:
                            errtype, errvalue = sys.exc_info()[:2]
                            # wait unknown input if noWaitParent or waitInput
                            toFinish = False
                            if (
                                ((impl.taskSpec.noWaitParent() or impl.taskSpec.waitInput()) and errtype == JediException.UnknownDatasetError)
                                or parentState == "running"
                                or errtype == Interaction.JEDITemporaryError
                            ):
                                if impl.taskSpec.noWaitParent() and errtype == JediException.UnknownDatasetError and parentState != "running":
                                    if impl.taskSpec.allowEmptyInput():
                                        tmpErrStr = f"finishing due to missing input while parent is {parentState}"
                                        toFinish = True
                                        setFrozenTime = False
                                    else:
                                        tmpErrStr = f"pending due to missing input while parent is {parentState}"
                                        setFrozenTime = True
                                elif impl.taskSpec.noWaitParent() or parentState == "running":
                                    tmpErrStr = f"pending until parent produces input. parent is {parentState}"
                                    setFrozenTime = False
                                elif errtype == Interaction.JEDITemporaryError:
                                    tmpErrStr = f"pending due to DDM problem. {errvalue}"
                                    setFrozenTime = True
                                else:
                                    tmpErrStr = "pending until input is staged"
                                    setFrozenTime = True
                                if toFinish:
                                    impl.taskSpec.status = "finishing"
                                else:
                                    impl.taskSpec.status = taskStatus
                                    impl.taskSpec.setOnHold()
                                impl.taskSpec.setErrDiag(tmpErrStr)
                                # not to update some task attributes
                                impl.taskSpec.resetRefinedAttrs()
                                tmpLog.info(tmpErrStr)
                                self.taskBufferIF.updateTask_JEDI(
                                    impl.taskSpec,
                                    {"jediTaskID": impl.taskSpec.jediTaskID},
                                    oldStatus=[taskStatus],
                                    insertUnknown=impl.unknownDatasetList,
                                    setFrozenTime=setFrozenTime,
                                )
                                continue
                            elif (
                                not (impl.taskSpec.noWaitParent() or impl.taskSpec.waitInput())
                                and errtype == JediException.UnknownDatasetError
                                and impl.taskSpec.allowEmptyInput()
                            ):
                                impl.taskSpec.status = "finishing"
                                tmpErrStr = f"finishing due to missing input after parent is {parentState}"
                                impl.taskSpec.setErrDiag(tmpErrStr)
                                # not to update some task attributes
                                impl.taskSpec.resetRefinedAttrs()
                                tmpLog.info(tmpErrStr)
                                self.taskBufferIF.updateTask_JEDI(
                                    impl.taskSpec, {"jediTaskID": impl.taskSpec.jediTaskID}, oldStatus=[taskStatus], insertUnknown=impl.unknownDatasetList
                                )
                                continue
                            else:
                                errStr = f"failed to refine task with {errtype.__name__}:{errvalue}"
                                tmpLog.error(errStr)
                                tmpStat = Interaction.SC_FAILED
                    # register
                    if tmpStat != Interaction.SC_SUCCEEDED:
                        tmpLog.error("failed to refine the task")
                        if impl is None or impl.taskSpec is None:
                            tmpTaskSpec = JediTaskSpec()
                            tmpTaskSpec.jediTaskID = jediTaskID
                        else:
                            tmpTaskSpec = impl.taskSpec
                        tmpTaskSpec.status = "tobroken"
                        if errStr != "":
                            tmpTaskSpec.setErrDiag(errStr, True)
                        self.taskBufferIF.updateTask_JEDI(tmpTaskSpec, {"jediTaskID": tmpTaskSpec.jediTaskID}, oldStatus=[taskStatus])
                    else:
                        tmpLog.info("registering")
                        # fill JEDI tables
                        try:
                            # enable protection against task duplication
                            if "uniqueTaskName" in taskParamMap and taskParamMap["uniqueTaskName"] and not impl.taskSpec.checkPreProcessed():
                                uniqueTaskName = True
                            else:
                                uniqueTaskName = False
                            strTaskParams = None
                            if impl.updatedTaskParams is not None:
                                strTaskParams = RefinerUtils.encodeJSON(impl.updatedTaskParams)
                            if taskStatus in ["registered", "staged"]:
                                # unset pre-process flag
                                if impl.taskSpec.checkPreProcessed():
                                    impl.taskSpec.setPostPreProcess()
                                # full registration
                                tmpStat, newTaskStatus = self.taskBufferIF.registerTaskInOneShot_JEDI(
                                    jediTaskID,
                                    impl.taskSpec,
                                    impl.inMasterDatasetSpec,
                                    impl.inSecDatasetSpecList,
                                    impl.outDatasetSpecList,
                                    impl.outputTemplateMap,
                                    impl.jobParamsTemplate,
                                    strTaskParams,
                                    impl.unmergeMasterDatasetSpec,
                                    impl.unmergeDatasetSpecMap,
                                    uniqueTaskName,
                                    taskStatus,
                                )
                                if not tmpStat:
                                    tmpErrStr = "failed to register the task to JEDI in a single shot"
                                    tmpLog.error(tmpErrStr)
                                    tmpTaskSpec = JediTaskSpec()
                                    tmpTaskSpec.status = newTaskStatus
                                    tmpTaskSpec.errorDialog = impl.taskSpec.errorDialog
                                    tmpTaskSpec.setErrDiag(tmpErrStr, True)
                                    self.taskBufferIF.updateTask_JEDI(tmpTaskSpec, {"jediTaskID": impl.taskSpec.jediTaskID}, oldStatus=[taskStatus])
                                tmpMsg = f"set task_status={newTaskStatus}"
                                tmpLog.info(tmpMsg)
                                tmpLog.sendMsg(tmpMsg, self.msgType)
                                # send message to contents feeder if the task is registered
                                if tmpStat and impl.taskSpec.is_msg_driven():
                                    push_ret = self.taskBufferIF.push_task_trigger_message("jedi_contents_feeder", jediTaskID)
                                    if push_ret:
                                        tmpLog.debug("pushed trigger message to jedi_contents_feeder")
                                    else:
                                        tmpLog.warning("failed to push trigger message to jedi_contents_feeder")
                            else:
                                # disable scouts if previous attempt didn't use it
                                if not impl.taskSpec.useScout(splitRule):
                                    impl.taskSpec.setUseScout(False)
                                # disallow to reset some attributes
                                impl.taskSpec.reserve_old_attributes()
                                # update task with new params
                                self.taskBufferIF.updateTask_JEDI(impl.taskSpec, {"jediTaskID": impl.taskSpec.jediTaskID}, oldStatus=[taskStatus])
                                # appending for incremental execution
                                tmpStat = self.taskBufferIF.appendDatasets_JEDI(jediTaskID, impl.inMasterDatasetSpec, impl.inSecDatasetSpecList)
                                if not tmpStat:
                                    tmpLog.error("failed to append datasets for incexec")
                        except Exception:
                            errtype, errvalue = sys.exc_info()[:2]
                            tmpErrStr = f"failed to register the task to JEDI with {errtype.__name__}:{errvalue}"
                            tmpLog.error(tmpErrStr)
                        else:
                            tmpLog.info("done")
            except Exception:
                errtype, errvalue = sys.exc_info()[:2]
                logger.error(f"{self.__class__.__name__} failed in runImpl() with {errtype.__name__}:{errvalue}")

# lauch

def launcher(commuChannel, taskBufferIF, ddmIF, vos=None, prodSourceLabels=None):
    p = TaskRefiner(commuChannel, taskBufferIF, ddmIF, vos, prodSourceLabels)
    p.start()


from typing import Tuple

import numpy as np
from env import EnvWithModel
from policy import Policy

def value_prediction(env: EnvWithModel, pi: Policy, initV: np.array, theta: float) -> Tuple[np.array, np.array]:
    """
    inp:
        env: environment with model information, i.e. you know transition dynamics and reward function
        pi: policy
        initV: initial V(s); numpy array shape of [nS,]
        theta: exit criteria
    return:
        V: $v_\pi$ function; numpy array shape of [nS]
        Q: $q_\pi$ function; numpy array shape of [nS,nA]
    """

    #####################
    # Implement Value Prediction Algorithm (Hint: Sutton Book p.75)
    #####################

    # Cache the action prob matrix
    action_prob_mat = np.array([[pi.action_prob(s, a) for s in range(env.spec.nS)] for a in range(env.spec.nA)])

    # Init optimization variables
    V = initV.copy()

    # Loop to update the value prediction
    while True:
        Delta = 0  # maximal update error
        for s in range(env.spec.nS):
            v = V[s]
            pi_vec = action_prob_mat[:, s]
            p_mat = env.TD[s, :, :]
            r_mat = env.R[s, :, :]
            return_mat = pi_vec[:, np.newaxis] * p_mat * (r_mat + env.spec.gamma * V[np.newaxis, :])
            V[s] = return_mat.sum()
            Delta = max(Delta, np.abs(V[s] - v))

        if Delta < theta:
            break

    # Now use the value prediction to estimate action-value
    Q = (env.TD * (env.R + env.spec.gamma * V[np.newaxis, np.newaxis, :])).sum(axis=-1)

    return V, Q

def value_iteration(env: EnvWithModel, initV: np.array, theta: float) -> Tuple[np.array, Policy]:
    """
    inp:
        env: environment with model information, i.e. you know transition dynamics and reward function
        initV: initial V(s); numpy array shape of [nS,]
        theta: exit criteria
    return:
        value: optimal value function; numpy array shape of [nS]
        policy: optimal deterministic policy; instance of Policy class
    """

    #####################
    # Implement Value Iteration Algorithm (Hint: Sutton Book p.83)
    #####################

    # Init optimization variables
    V = initV.copy()

    # Loop to update the value prediction of optimal policy
    while True:
        Delta = 0  # maximal update error
        for s in range(env.spec.nS):
            v = V[s]
            p_mat = env.TD[s, :, :]
            r_mat = env.R[s, :, :]
            return_mat = (p_mat * (r_mat + env.spec.gamma * V[np.newaxis, :])).sum(axis=1).max()
            V[s] = return_mat.sum()
            Delta = max(Delta, np.abs(V[s] - v))

        if Delta < theta:
            break

    # Compute the optimal policy that yields the value estimate
    class MyPolicy(Policy):
        def __init__(self, pi_mat):
            self.pi_mat = pi_mat

        def action_prob(self, state: int, action: int) -> float:
            return self.pi_mat[state, action]

        def action(self, state: int) -> int:
            return self.pi_mat[state, :].argmax()

    pi_mat = (env.TD * (env.R + env.spec.gamma * V[np.newaxis, np.newaxis, :])).sum(axis=2)
    pi = MyPolicy(pi_mat)

    return V, pi