import copy
import itertools
import logging
import os
from pathlib import Path
import html
import boto3
import time

import json

import gradio
import requests
import base64
from urllib.parse import urljoin

import gradio as gr

import utils
from aws_extension.auth_service.simple_cloud_auth import cloud_auth_manager
from aws_extension.cloud_api_manager.api_manager import api_manager
from aws_extension.sagemaker_ui_utils import create_refresh_button_by_user

from modules import shared, scripts
from modules.ui import create_refresh_button
from modules.shared import opts
from modules.ui_components import FormRow, FormColumn, FormGroup, ToolButton, FormHTML
from utils import get_variable_from_json
from utils import upload_file_to_s3_by_presign_url, upload_multipart_files_to_s3_by_signed_url
from requests.exceptions import JSONDecodeError
from datetime import datetime
import math
import re

import asyncio
import nest_asyncio

from utils import cp, tar, rm

logger = logging.getLogger(__name__)
logger.setLevel(utils.LOGGING_LEVEL)

None_Option_For_On_Cloud_Model = "don't use on cloud inference"

inference_job_dropdown = None
textual_inversion_dropdown = None
hyperNetwork_dropdown = None
lora_dropdown = None
# sagemaker_endpoint = None
modelmerger_merge_on_cloud = None
interrogate_clip_on_cloud_button = None
interrogate_deep_booru_on_cloud_button = None

primary_model_name = None
secondary_model_name = None
tertiary_model_name = None

# TODO: convert to dynamically init the following variables
txt2img_inference_job_ids = []

sd_checkpoints = []
textual_inversion_list = []
lora_list = []
hyperNetwork_list = []
ControlNet_model_list = []

# Initial checkpoints information
checkpoint_info = {}
checkpoint_type = ["Stable-diffusion", "embeddings", "Lora", "hypernetworks", "ControlNet", "VAE"]
checkpoint_name = ["stable_diffusion", "embeddings", "lora", "hypernetworks", "controlnet", "VAE"]
stable_diffusion_list = []
embeddings_list = []

hypernetworks_list = []
controlnet_list = []
for ckpt_type, ckpt_name in zip(checkpoint_type, checkpoint_name):
    checkpoint_info[ckpt_type] = {}
# get api_gateway_url
api_gateway_url = get_variable_from_json('api_gateway_url')
api_key = get_variable_from_json('api_token')

start_time_picker_img_value = None
end_time_picker_img_value = None
start_time_picker_txt_value = None
end_time_picker_txt_value = None

txt_task_type = None
txt_status = None
txt_endpoint = None
txt_checkpoint = None

img_task_type = None
img_status = None
img_endpoint = None
img_checkpoint = None

show_all_inference_job = False

modelTypeMap = {
    'SD Checkpoints': 'Stable-diffusion',
    'Textual Inversion': 'embeddings',
    'LoRA model': 'Lora',
    'ControlNet model': 'ControlNet',
    'Hypernetwork': 'hypernetworks',
    'VAE': 'VAE'
}

def plaintext_to_html(text):
    text = "
" + "
\n".join([f"{html.escape(x)}" for x in text.split('\n')]) + "
"
    return text

def server_request_post(path, params):
    api_gateway_url = get_variable_from_json('api_gateway_url')
    # Check if api_url ends with '/', if not append it
    if not api_gateway_url.endswith('/'):
        api_gateway_url += '/'
    api_key = get_variable_from_json('api_token')
    headers = {
        "x-api-key": api_key,
        "Content-Type": "application/json"
    }
    list_endpoint_url = f'{api_gateway_url}{path}'
    response = requests.post(list_endpoint_url, json=params, headers=headers)
    return response

def get_s3_file_names(bucket, folder):
    """Get a list of file names from an S3 bucket and folder."""
    s3 = boto3.resource('s3')
    bucket = s3.Bucket(bucket)
    objects = bucket.objects.filter(Prefix=folder)
    names = [obj.key for obj in objects]
    return names

def get_current_date():
    today = datetime.today()
    formatted_date = today.strftime('%Y-%m-%d')
    return formatted_date

def server_request(path):
    api_gateway_url = get_variable_from_json('api_gateway_url')
    # Check if api_url ends with '/', if not append it
    if not api_gateway_url.endswith('/'):
        api_gateway_url += '/'
    api_key = get_variable_from_json('api_token')
    headers = {
        "x-api-key": api_key,
        "Content-Type": "application/json"
    }
    list_endpoint_url = f'{api_gateway_url}{path}'
    response = requests.get(list_endpoint_url, headers=headers)
    return response

def datetime_to_short_form(datetime_str):
    dt = datetime.strptime(datetime_str, "%Y-%m-%d %H:%M:%S.%f")
    short_form = dt.strftime("%Y-%m-%d-%H-%M-%S")
    return short_form

def query_page_inference_job_list(task_type: str, status: str, endpoint: str, checkpoint: str, is_img2img: bool,
                                  show_all: bool):
    global show_all_inference_job
    show_all_inference_job = show_all
    if is_img2img:
        global img_task_type
        img_task_type = task_type
        global img_status
        img_status = status
        global img_endpoint
        img_endpoint = endpoint
        global img_checkpoint
        img_checkpoint = checkpoint
        opt_type = 'img2img'
        logger.debug(f"{opt_type} {img_task_type} {img_status} {img_endpoint} {img_checkpoint} {show_all}")
        return query_inference_job_list(task_type, status, endpoint, checkpoint, opt_type)
    else:
        global txt_task_type
        txt_task_type = task_type
        global txt_status
        txt_status = txt_status
        global txt_endpoint
        txt_endpoint = endpoint
        global txt_checkpoint
        txt_checkpoint = checkpoint
        opt_type = 'txt2img'
        logger.debug(f"{opt_type} {txt_task_type} {txt_status} {txt_endpoint} {txt_checkpoint} {show_all}")
        return query_inference_job_list(task_type, status, endpoint, checkpoint, opt_type)

def query_img_inference_job_list(task_type: str, status: str, endpoint: str, checkpoint: str):
    opt_type = 'img2img'
    global img_task_type
    img_task_type = task_type
    global img_status
    img_status = status
    global img_endpoint
    img_endpoint = endpoint
    global img_checkpoint
    img_checkpoint = checkpoint
    return query_inference_job_list(task_type, status, endpoint, checkpoint, opt_type)

def query_txt_inference_job_list(task_type: str, status: str, endpoint: str, checkpoint: str):
    opt_type = 'txt2img'
    global txt_task_type
    txt_task_type = task_type
    global txt_status
    txt_status = status
    global txt_endpoint
    txt_endpoint = endpoint
    global txt_checkpoint
    txt_checkpoint = checkpoint

    return query_inference_job_list(task_type, status, endpoint, checkpoint, opt_type)

def query_inference_job_list(task_type: str = '', status: str = '',
                             endpoint: str = '', checkpoint: str = '', opt_type: str = ''):
    logger.debug(
        f"query_inference_job_list start！！{status},{task_type},{endpoint},{checkpoint},{start_time_picker_txt_value},{end_time_picker_txt_value} {show_all_inference_job}")
    try:
        body_params = {}
        if status:
            body_params['status'] = status
        if task_type:
            body_params['task_type'] = task_type
        if opt_type == 'txt2img':
            if start_time_picker_txt_value:
                body_params['start_time'] = start_time_picker_txt_value
            if end_time_picker_txt_value:
                body_params['end_time'] = end_time_picker_txt_value
        elif opt_type == 'img2img':
            if start_time_picker_img_value:
                body_params['start_time'] = start_time_picker_img_value
            if end_time_picker_img_value:
                body_params['end_time'] = end_time_picker_img_value
        if endpoint:
            endpoint_name_array = endpoint.split("+")
            if len(endpoint_name_array) > 0:
                body_params['endpoint'] = endpoint_name_array[0]
        if checkpoint:
            body_params['checkpoint'] = checkpoint
        body_params['limit'] = -1 if show_all_inference_job else 10
        response = server_request_post(f'inference/query-inference-jobs', body_params)
        r = response.json()
        logger.debug(r)
        if r:
            txt2img_inference_job_ids.clear()  # Clear the existing list before appending new values
            temp_list = []
            for obj in r:
                if obj.get('completeTime') is None:
                    complete_time = obj.get('startTime')
                else:
                    complete_time = obj.get('completeTime')
                status = obj.get('status')
                task_type = obj.get('taskType', 'txt2img')
                inference_job_id = obj.get('InferenceJobId')
                combined_string = f"{complete_time}-->{task_type}-->{status}-->{inference_job_id}"
                temp_list.append((complete_time, combined_string))
            # Sort the list based on completeTime in ascending order
            sorted_list = sorted(temp_list, key=lambda x: x[0], reverse=False)
            # Append the sorted combined strings to the txt2img_inference_job_ids list
            for item in sorted_list:
                txt2img_inference_job_ids.append(item[1])
            # inference_job_dropdown.update(choices=txt2img_inference_job_ids)
            return gr.Dropdown.update(choices=txt2img_inference_job_ids)
        else:
            logger.info("The API response is empty.")
            return gr.Dropdown.update(choices=[])
    except Exception as e:
        logger.error("Exception occurred when fetching inference_job_ids")
        logger.error(e)
        return gr.Dropdown.update(choices=[])

# def get_inference_job_list(txt2img_type_checkbox=True, img2img_type_checkbox=True, interrogate_type_checkbox=True):
#     global txt2img_inference_job_ids
#     try:
#         txt2img_inference_job_ids.clear()  # Clear the existing list before appending new values
#         response = server_request('inference/list-inference-jobs')
#         r = response.json()
#         logger.debug(f"response: {response.json()}")
#         filter_checkbox = False
#         selected_types = []
#         if txt2img_type_checkbox:
#             selected_types.append('txt2img')
#             filter_checkbox = True
#         if img2img_type_checkbox:
#             selected_types.append('img2img')
#             filter_checkbox = True
#         if interrogate_type_checkbox:
#             selected_types.append('interrogate_deepbooru')
#             filter_checkbox = True
#         logging.debug(f"selected_types: {selected_types}")
#         if r:
#             txt2img_inference_job_ids.clear()  # Clear the existing list before appending new values
#             temp_list = []
#             for obj in r:
#                 if obj.get('completeTime') is None:
#                     complete_time = obj.get('startTime')
#                 else:
#                     complete_time = obj.get('completeTime')
#                 status = obj.get('status')
#                 task_type = obj.get('taskType', 'txt2img')
#                 inference_job_id = obj.get('InferenceJobId')
#                 if filter_checkbox and task_type not in selected_types:
#                     continue
#                 combined_string = f"{complete_time}-->{task_type}-->{status}-->{inference_job_id}"
#                 temp_list.append((complete_time, combined_string))
#
#             # Sort the list based on completeTime in ascending order
#             sorted_list = sorted(temp_list, key=lambda x: x[0], reverse=False)
#
#             # Append the sorted combined strings to the txt2img_inference_job_ids list
#             for item in sorted_list:
#                 txt2img_inference_job_ids.append(item[1])
#             # inference_job_dropdown.update(choices=txt2img_inference_job_ids)
#             return gr.Dropdown.update(choices=txt2img_inference_job_ids)
#         else:
#             logger.debug("The API response is empty.")
#             return gr.Dropdown.update(choices=[])
#
#     except Exception as e:
#         logger.error("Exception occurred when fetching inference_job_ids")
#         return gr.Dropdown.update(choices=[])

def get_inference_job(inference_job_id):
    response = server_request(f'inference/get-inference-job?jobID={inference_job_id}')
    return response.json()

def get_inference_job_image_output(inference_job_id):
    try:
        response = server_request(f'inference/get-inference-job-image-output?jobID={inference_job_id}')
        r = response.json()
        txt2img_inference_job_image_list = []
        for obj in r:
            obj_value = str(obj)
            txt2img_inference_job_image_list.append(obj_value)
        return txt2img_inference_job_image_list
    except Exception as e:
        logger.error(f"An error occurred while getting inference job image output: {e}")
        return []

def get_inference_job_param_output(inference_job_id):
    try:
        response = server_request(f'inference/get-inference-job-param-output?jobID={inference_job_id}')
        r = response.json()
        txt2img_inference_job_param_list = []
        for obj in r:
            obj_value = str(obj)
            txt2img_inference_job_param_list.append(obj_value)
        return txt2img_inference_job_param_list
    except Exception as e:
        logger.error(f"An error occurred while getting inference job param output: {e}")
        return []

def download_images(image_urls: list, local_directory: str):
    if not os.path.exists(local_directory):
        os.makedirs(local_directory)

    image_list = []
    for url in image_urls:
        try:
            response = requests.get(url)
            response.raise_for_status()
            image_name = os.path.basename(url).split('?')[0]
            local_path = os.path.join(local_directory, image_name)

            with open(local_path, 'wb') as f:
                f.write(response.content)
            image_list.append(local_path)
        except requests.exceptions.RequestException as e:
            logger.error(f"Error downloading image {url}: {e}")
    return image_list

def get_model_list_by_type(model_type):
    api_gateway_url = get_variable_from_json('api_gateway_url')
    api_key = get_variable_from_json('api_token')

    # check if api_gateway_url and api_key are set
    if api_gateway_url is None or api_key is None:
        logger.info("api_gateway_url or api_key is not set")
        return []

    # Check if api_url ends with '/', if not append it
    if not api_gateway_url.endswith('/'):
        api_gateway_url += '/'

    url = api_gateway_url + f"checkpoints?status=Active"
    if isinstance(model_type, list):
        url += "&types=" + "&types=".join(model_type)
    else:
        url += f"&types={model_type}"

    try:
        response = requests.get(url=url, headers={'x-api-key': api_key})
        response.raise_for_status()
        json_response = response.json()

        if "checkpoints" not in json_response.keys():
            return []

        checkpoint_list = []
        for ckpt in json_response["checkpoints"]:
            if "name" not in ckpt:
                continue
            if ckpt["name"] is None:
                continue
            if ckpt["type"] is None:
                continue
            ckpt_type = ckpt["type"]
            for ckpt_name in ckpt["name"]:
                ckpt_s3_pos = f"{ckpt['s3Location']}/{ckpt_name}"
                checkpoint_info[ckpt_type][ckpt_name] = ckpt_s3_pos
                checkpoint_list.append(ckpt_name)

        unique_list = list(set(checkpoint_list))
        return unique_list
    except Exception as e:
        logger.error(f"Error fetching model list: {e}")
        return []

def get_checkpoints_by_type(model_type):
    url = "checkpoints?status=Active"
    if isinstance(model_type, list):
        url += "&types=" + "&types=".join(model_type)
    else:
        url += f"&types={model_type}"
    try:
        response = server_request(url)
        response.raise_for_status()
        json_response = response.json()
        if "checkpoints" not in json_response.keys():
            return []
        checkpoint_dict = {}
        for ckpt in json_response["checkpoints"]:
            if "name" not in ckpt:
                continue
            if ckpt["name"] is None:
                continue
            create_time = ckpt['created']
            created = datetime.fromtimestamp(create_time)
            for ckpt_name in ckpt["name"]:
                checkpoint = [ckpt_name, created]
                if ckpt_name not in checkpoint_dict:
                    checkpoint_dict[ckpt_name] = checkpoint
        checkpoint_list = list(checkpoint_dict.values())
        return checkpoint_list
    except Exception as e:
        logging.error(f"Error fetching model list: {e}")
        return []

def update_sd_checkpoints():
    model_type = ["Stable-diffusion"]
    return get_model_list_by_type(model_type)

def get_texual_inversion_list():
    model_type = "embeddings"
    return get_model_list_by_type(model_type)

def get_lora_list():
    model_type = "Lora"
    return get_model_list_by_type(model_type)

def get_hypernetwork_list():
    model_type = "hypernetworks"
    return get_model_list_by_type(model_type)

def get_controlnet_model_list():
    model_type = "ControlNet"
    return get_model_list_by_type(model_type)

def refresh_all_models():
    api_gateway_url = get_variable_from_json('api_gateway_url')
    api_key = get_variable_from_json('api_token')
    try:
        for rp, name in zip(checkpoint_type, checkpoint_name):
            url = api_gateway_url + f"checkpoints?status=Active&types={rp}"
            response = requests.get(url=url, headers={'x-api-key': api_key})
            json_response = response.json()
            logger.debug(f"response url json for model {rp} is {json_response}")
            checkpoint_info[rp] = {}
            if "checkpoints" not in json_response.keys():
                continue
            for ckpt in json_response["checkpoints"]:
                if "name" not in ckpt:
                    continue
                if ckpt["name"] is None:
                    continue
                ckpt_type = ckpt["type"]
                checkpoint_info[ckpt_type] = {}
                for ckpt_name in ckpt["name"]:
                    ckpt_s3_pos = f"{ckpt['s3Location']}/{ckpt_name.split(os.sep)[-1]}"
                    checkpoint_info[ckpt_type][ckpt_name] = ckpt_s3_pos
    except Exception as e:
        logger.error(f"Error refresh all models: {e}")

def sagemaker_upload_model_s3(sd_checkpoints_path, textual_inversion_path, lora_path, hypernetwork_path,
                              controlnet_model_path, vae_path, pr: gradio.Request):
    log = "start upload model to s3:"

    local_paths = [sd_checkpoints_path, textual_inversion_path, lora_path, hypernetwork_path, controlnet_model_path,
                   vae_path]

    logger.info(f"Refresh checkpoints before upload to get rid of duplicate uploads...")
    refresh_all_models()

    for lp, rp in zip(local_paths, checkpoint_type):
        if lp == "" or not lp:
            continue
        logger.debug(f"lp is {lp}")
        model_name = lp.split(os.sep)[-1]

        exist_model_list = list(checkpoint_info[rp].keys())

        if model_name in exist_model_list:
            logger.info(f"!!!skip to upload duplicate model {model_name}")
            continue

        part_size = 1000 * 1024 * 1024
        file_size = os.stat(lp)
        parts_number = math.ceil(file_size.st_size / part_size)
        logger.info(f'!!!!!!!!!!{file_size} {parts_number}')

        # local_tar_path = f'{model_name}.tar'
        local_tar_path = model_name
        payload = {
            "checkpoint_type": rp,
            "filenames": [{
                "filename": local_tar_path,
                "parts_number": parts_number
            }],
            "params": {
                "message": "placeholder for chkpts upload test",
                "creator": pr.username
            }
        }
        api_gateway_url = get_variable_from_json('api_gateway_url')
        # Check if api_url ends with '/', if not append it
        if not api_gateway_url.endswith('/'):
            api_gateway_url += '/'
        api_key = get_variable_from_json('api_token')
        logger.info(f'!!!!!!api_gateway_url {api_gateway_url}')

        url = str(api_gateway_url) + "checkpoint"

        logger.debug(f"Post request for upload s3 presign url: {url}")

        response = requests.post(url=url, json=payload, headers={'x-api-key': api_key})

        try:
            response.raise_for_status()
            json_response = response.json()
            logger.debug(f"Response json {json_response}")
            s3_base = json_response["checkpoint"]["s3_location"]
            checkpoint_id = json_response["checkpoint"]["id"]
            logger.debug(f"Upload to S3 {s3_base}")
            logger.debug(f"Checkpoint ID: {checkpoint_id}")

            s3_signed_urls_resp = json_response["s3PresignUrl"][local_tar_path]
            # Upload src model to S3.
            if rp != "embeddings":
                local_model_path_in_repo = os.sep.join(['models', rp, model_name])
            else:
                local_model_path_in_repo = os.sep.join([rp, model_name])
            logger.debug("Pack the model file.")
            cp(lp, local_model_path_in_repo, recursive=True)
            if rp == "Stable-diffusion":
                model_yaml_name = model_name.split('.')[0] + ".yaml"
                local_model_yaml_path = os.sep.join([*lp.split(os.sep)[:-1], model_yaml_name])
                local_model_yaml_path_in_repo = os.sep.join(["models", rp, model_yaml_name])
                if os.path.isfile(local_model_yaml_path):
                    cp(local_model_yaml_path, local_model_yaml_path_in_repo, recursive=True)
                    tar(mode='c', archive=local_tar_path,
                        sfiles=[local_model_path_in_repo, local_model_yaml_path_in_repo], verbose=True)
                else:
                    tar(mode='c', archive=local_tar_path, sfiles=[local_model_path_in_repo], verbose=True)
            else:
                tar(mode='c', archive=local_tar_path, sfiles=[local_model_path_in_repo], verbose=True)

            multiparts_tags = upload_multipart_files_to_s3_by_signed_url(
                local_tar_path,
                s3_signed_urls_resp,
                part_size
            )

            payload = {
                "checkpoint_id": checkpoint_id,
                "status": "Active",
                "multi_parts_tags": {local_tar_path: multiparts_tags}
            }
            # Start creating model on cloud.
            response = requests.put(url=url, json=payload, headers={'x-api-key': api_key})
            s3_input_path = s3_base
            logger.debug(response)

            log = f"\n finish upload {local_tar_path} to {s3_base}"

            # os.system(f"rm {local_tar_path}")
            rm(local_tar_path, recursive=True)
        except Exception as e:
            logger.error(f"fail to upload model {lp}, error: {e}")

    logger.debug(f"Refresh checkpoints after upload...")
    refresh_all_models()
    return log, None, None, None, None, None, None

def sagemaker_upload_model_s3_local():
    log = "Start upload:"
    return log

def sagemaker_upload_model_s3_url(model_type: str, url_list: str, params: str, pr: gradio.Request):
    model_type = modelTypeMap.get(model_type)
    if not model_type:
        return "Please choose the model type."
    url_pattern = r'(https?|ftp)://[^\s/$.?#].[^\s]*'
    if re.match(f'^{url_pattern}$', url_list):
        url_list = url_list.split(',')
    else:
        return "Please fill in right url list."
    if params:
        params_dict = json.loads(params)
    else:
        params_dict = {}

    params_dict['creator'] = pr.username
    body_params = {'checkpointType': model_type, 'modelUrl': url_list, 'params': params_dict}
    response = server_request_post('upload_checkpoint', body_params)
    response_data = response.json()
    logging.info(f"sagemaker_upload_model_s3_url response:{response_data}")
    log = "uploading……"
    if 'checkpoint' in response_data:
        if response_data['checkpoint'].get('status') == 'Active':
            log = "upload success!"
    return log

def generate_on_cloud(sagemaker_endpoint):
    logger.info(f"checkpiont_info {checkpoint_info}")
    logger.info(f"sagemaker endpoint {sagemaker_endpoint}")
    text = "failed to check endpoint"
    return plaintext_to_html(text)

# create a global event loop and apply the patch to allow nested event loops in single thread
loop = asyncio.get_event_loop()
nest_asyncio.apply()

MAX_RUNNING_LIMIT = 10

def async_loop_wrapper(f):
    global loop
    # check if there are any running or queued tasks inside the event loop
    if loop.is_running():
        # Calculate the number of running tasks
        while len([task for task in asyncio.all_tasks(loop) if not task.done()]) > MAX_RUNNING_LIMIT:
            logger.debug(f'Waiting for {MAX_RUNNING_LIMIT} running tasks to complete')
            time.sleep(1)
    else:
        # check if loop is closed and create a new one
        if loop.is_closed():
            loop = asyncio.new_event_loop()
            asyncio.set_event_loop(loop)
            # log this event since it should never happen
            logger.debug('Event loop was closed, created a new one')

    # Add new task to the event loop
    result = loop.run_until_complete(f())
    return result

def async_loop_wrapper_with_input(sagemaker_endpoint, type):
    global loop
    # check if there are any running or queued tasks inside the event loop
    if loop.is_running():
        # Calculate the number of running tasks
        while len([task for task in asyncio.all_tasks(loop) if not task.done()]) > MAX_RUNNING_LIMIT:
            logger.debug(f'Waiting for {MAX_RUNNING_LIMIT} running tasks to complete')
            time.sleep(1)
    else:
        # check if loop is closed and create a new one
        if loop.is_closed():
            loop = asyncio.new_event_loop()
            asyncio.set_event_loop(loop)
            # log this event since it should never happen
            logger.debug('Event loop was closed, created a new one')

    # Add new task to the event loop
    result = loop.run_until_complete(call_remote_inference(sagemaker_endpoint, type))
    return result

def call_interrogate_clip(sagemaker_endpoint, init_img, sketch, init_img_with_mask, inpaint_color_sketch,
                          init_img_inpaint, init_mask_inpaint):
    return async_loop_wrapper_with_input(sagemaker_endpoint, 'interrogate_clip')

def call_interrogate_deepbooru(sagemaker_endpoint, init_img, sketch, init_img_with_mask, inpaint_color_sketch,
                               init_img_inpaint, init_mask_inpaint):
    return async_loop_wrapper_with_input(sagemaker_endpoint, 'interrogate_deepbooru')

async def call_remote_inference(sagemaker_endpoint, type):
    logger.debug(f"chosen ep {sagemaker_endpoint}")
    logger.debug(f"inference type is {type}")

    if sagemaker_endpoint == '':
        image_list = []  # Return an empty list if selected_value is None
        info_text = ''
        infotexts = "Failed! Please choose the endpoint in 'InService' states "
        return image_list, info_text, plaintext_to_html(infotexts)
    elif sagemaker_endpoint == 'FAILURE':
        image_list = []  # Return an empty list if selected_value is None
        info_text = ''
        infotexts = "Failed upload the config to cloud  "
        return image_list, info_text, plaintext_to_html(infotexts)

    sagemaker_endpoint_status = sagemaker_endpoint.split("+")[1]

    if sagemaker_endpoint_status != "InService":
        image_list = []  # Return an empty list if selected_value is None
        info_text = ''
        infotexts = "Failed! Please choose the endpoint in 'InService' states "
        return image_list, info_text, plaintext_to_html(infotexts)

    api_gateway_url = get_variable_from_json('api_gateway_url')
    # Check if api_url ends with '/', if not append it
    if not api_gateway_url.endswith('/'):
        api_gateway_url += '/'
    api_key = get_variable_from_json('api_token')

    # stage 2: inference using endpoint_name
    headers = {
        "x-api-key": api_key,
        "Content-Type": "application/json"
    }
    checkpoint_info['sagemaker_endpoint'] = sagemaker_endpoint.split("+")[0]
    payload = checkpoint_info
    payload['task_type'] = type
    logger.info(f"checkpointinfo is {payload}")

    inference_url = f"{api_gateway_url}inference/run-sagemaker-inference"
    response = requests.post(inference_url, json=payload, headers=headers)
    logger.info(f"Raw server response: {response.text}")
    try:
        r = response.json()
    except JSONDecodeError as e:
        logger.error(f"Failed to decode JSON response: {e}")
        logger.error(f"Raw server response: {response.text}")
    else:
        inference_id = r.get('inference_id')  # Assuming the response contains 'inference_id' field
        try:
            return process_result_by_inference_id(inference_id)
        except Exception as e:
            logger.error(f"Failed to get inference job {inference_id}, error: {e}")

def process_result_by_inference_id(inference_id):
    image_list = []  # Return an empty list if selected_value is None
    info_text = ''
    infotexts = f"Inference id is {inference_id}, please check all historical inference result in 'Inference Job' dropdown list"
    json_list = []
    prompt_txt = ''

    resp = get_inference_job(inference_id)
    if resp is None:
        return image_list, info_text, plaintext_to_html(infotexts)
    else:
        if resp['taskType'] in ['txt2img', 'img2img', 'interrogate_clip', 'interrogate_deepbooru']:
            while resp['status'] == "inprogress":
                time.sleep(3)
                resp = get_inference_job(inference_id)
            if resp['status'] == "failed":
                infotexts = f"Inference job {inference_id} is failed, error message: {resp['sagemakerRaw']}"
                return image_list, info_text, plaintext_to_html(infotexts), infotexts
            elif resp['status'] == "succeed":
                if resp['taskType'] in ['interrogate_clip', 'interrogate_deepbooru']:
                    prompt_txt = resp['caption']
                    # return with default value, including image_list, info_text, infotexts
                    return image_list, info_text, plaintext_to_html(infotexts), prompt_txt
                images = get_inference_job_image_output(inference_id.strip())
                inference_param_json_list = get_inference_job_param_output(inference_id)
                # todo: these not need anymore
                if resp['taskType'] == "txt2img":
                    image_list = download_images(images, f"outputs/txt2img-images/{get_current_date()}/{inference_id}/")
                    json_list = download_images(inference_param_json_list,
                                                f"outputs/txt2img-images/{get_current_date()}/{inference_id}/")
                    json_file = f"outputs/txt2img-images/{get_current_date()}/{inference_id}/{inference_id}_param.json"
                elif resp['taskType'] == "img2img":
                    image_list = download_images(images, f"outputs/img2img-images/{get_current_date()}/{inference_id}/")
                    json_list = download_images(inference_param_json_list,
                                                f"outputs/img2img-images/{get_current_date()}/{inference_id}/")
                    json_file = f"outputs/img2img-images/{get_current_date()}/{inference_id}/{inference_id}_param.json"
                if os.path.isfile(json_file):
                    with open(json_file) as f:
                        log_file = json.load(f)
                        info_text = log_file["info"]
                        infotexts = f"Inference id is {inference_id}\n" + json.loads(info_text)["infotexts"][0]
                else:
                    logger.debug(f"File {json_file} does not exist.")
                    info_text = 'something wrong when trying to download the inference parameters'
                    infotexts = info_text
                return image_list, info_text, plaintext_to_html(infotexts), infotexts
        else:
            return image_list, info_text, plaintext_to_html(infotexts), infotexts

def modelmerger_on_cloud_func(primary_model_name, secondary_model_name, teritary_model_name):
    logger.debug(f"function under development, current checkpoint_info is {checkpoint_info}")
    api_gateway_url = get_variable_from_json('api_gateway_url')
    # Check if api_url ends with '/', if not append it
    if not api_gateway_url.endswith('/'):
        api_gateway_url += '/'
    api_key = get_variable_from_json('api_token')

    if api_gateway_url is None:
        logger.debug(f"modelmerger: failed to get the api-gateway url, can not fetch remote data")
        return []
    modelmerge_url = f"{api_gateway_url}inference/run-model-merge"

    payload = {
        "primary_model_name": primary_model_name,
        "secondary_model_name": secondary_model_name,
        "tertiary_model_name": teritary_model_name
    }

    headers = {
        "x-api-key": api_key,
        "Content-Type": "application/json"
    }

    response = requests.post(modelmerge_url, json=payload, headers=headers)

    try:
        r = response.json()
    except JSONDecodeError as e:
        logger.error(f"Failed to decode JSON response: {e}")
        logger.error(f"Raw server response: {response.text}")
    else:
        logger.debug(f"response for rest api {r}")

# def txt2img_config_save():
#     # placeholder for saving txt2img config
#     pass

def displayEndpointInfo(input_string: str):
    logger.debug(f"selected value is {input_string}")
    if not input_string:
        return
    parts = input_string.split('+')

    if len(parts) < 2:
        return plaintext_to_html("")

    endpoint_job_id, status = parts[0], parts[1]

    if status == 'failed':
        response = server_request(f'inference/get-endpoint-deployment-job?jobID={endpoint_job_id}')
        # Do something with the response
        r = response.json()
        if "error" in r:
            return plaintext_to_html(r["error"])
        else:
            return plaintext_to_html(r["EndpointDeploymentJobId"])
    else:
        return plaintext_to_html("")

def update_txt2imgPrompt_from_TextualInversion(selected_items, txt2img_prompt):
    return update_txt2imgPrompt_from_model_select(selected_items, txt2img_prompt, 'embeddings', False)

def update_txt2imgPrompt_from_Hypernetworks(selected_items, txt2img_prompt):
    return update_txt2imgPrompt_from_model_select(selected_items, txt2img_prompt, 'hypernetworks', True)

def update_txt2imgPrompt_from_Lora(selected_items, txt2img_prompt):
    return update_txt2imgPrompt_from_model_select(selected_items, txt2img_prompt, 'Lora', True)

def update_txt2imgPrompt_from_model_select(selected_items, txt2img_prompt, model_name='embeddings',
                                           with_angle_brackets=False):
    logger.debug(selected_items)  # example ['FastNegativeV2.pt']
    logger.debug(txt2img_prompt)
    logger.debug(get_model_list_by_type('embeddings'))
    full_dropdown_items = get_model_list_by_type(model_name)  # example ['FastNegativeV2.pt', 'okuryl3nko.pt']

    # Remove extensions from selected_items and full_dropdown_items
    selected_items = [item.split('.')[0] for item in selected_items]
    full_dropdown_items = [item.split('.')[0] for item in full_dropdown_items]

    # Loop over each item in full_dropdown_items and remove it from txt2img_prompt
    type_str = ''
    if model_name == 'Lora':
        type_str = 'lora:'
    elif model_name == 'hypernetworks':
        type_str = 'hypernet:'
    for item in full_dropdown_items:
        if with_angle_brackets:
            txt2img_prompt = re.sub(f'<{type_str}{item}:\d+>', "", txt2img_prompt).strip()
        else:
            txt2img_prompt = txt2img_prompt.replace(item, "").strip()

    # Loop over each item in selected_items and append it to txt2img_prompt
    for item in selected_items:
        if with_angle_brackets:
            txt2img_prompt += ' ' + '<' + type_str + item + ':1>'
        else:
            txt2img_prompt += ' ' + item

    # Remove any leading or trailing whitespace
    txt2img_prompt = txt2img_prompt.strip()

    return txt2img_prompt

def fake_gan(selected_value, original_prompt):
    logger.debug(f"selected value is {selected_value}")
    logger.debug(f"original prompt is {original_prompt}")
    if selected_value is not None:
        delimiter = "-->"
        parts = selected_value.split(delimiter)
        # Extract the InferenceJobId value
        inference_job_id = parts[3].strip()
        inference_job_status = parts[2].strip()
        inference_job_taskType = parts[1].strip()
        if inference_job_status == 'inprogress':
            return [], [], plaintext_to_html('inference still in progress')

        if inference_job_taskType in ["txt2img", "img2img"]:
            prompt_txt = original_prompt
            images = get_inference_job_image_output(inference_job_id)
            image_list = []
            json_list = []
            inference_pram_json_list = get_inference_job_param_output(inference_job_id)
            # output directory mapping to task type
            if inference_job_taskType == "txt2img":
                image_list = download_images(images, f"outputs/txt2img-images/{get_current_date()}/{inference_job_id}/")
                json_list = download_images(inference_pram_json_list,
                                            f"outputs/txt2img-images/{get_current_date()}/{inference_job_id}/")
                json_file = f"outputs/txt2img-images/{get_current_date()}/{inference_job_id}/{inference_job_id}_param.json"
            elif inference_job_taskType == "img2img":
                image_list = download_images(images, f"outputs/img2img-images/{get_current_date()}/{inference_job_id}/")
                json_list = download_images(inference_pram_json_list,
                                            f"outputs/img2img-images/{get_current_date()}/{inference_job_id}/")
                json_file = f"outputs/img2img-images/{get_current_date()}/{inference_job_id}/{inference_job_id}_param.json"
            logger.debug(f"{str(images)}")
            logger.debug(f"{str(inference_pram_json_list)}")
            if os.path.isfile(json_file):
                with open(json_file) as f:
                    log_file = json.load(f)
                    info_text = log_file["info"]
                    infotexts = json.loads(info_text)["infotexts"][0]
            else:
                logger.debug(f"File {json_file} does not exist.")
                info_text = 'something wrong when trying to download the inference parameters'
                infotexts = 'something wrong when trying to download the inference parameters'
        elif inference_job_taskType in ["interrogate_clip", "interrogate_deepbooru"]:
            job_status = get_inference_job(inference_job_id)
            logger.debug(job_status)
            caption = job_status['caption']
            prompt_txt = caption
            image_list = []  # Return an empty list if selected_value is None
            json_list = []
            info_text = ''
            infotexts = ''
    else:
        prompt_txt = original_prompt
        image_list = []  # Return an empty list if selected_value is None
        json_list = []
        info_text = ''
        infotexts = ''
    return image_list, info_text, plaintext_to_html(infotexts), prompt_txt

def display_inference_result(inference_id: str):
    logger.debug(f"selected value is {inference_id}")
    if inference_id is not None:
        # Extract the InferenceJobId value
        inference_job_id = inference_id
        images = get_inference_job_image_output(inference_job_id)
        image_list = []
        image_list = download_images(images, f"outputs/txt2img-images/{get_current_date()}/{inference_job_id}/")

        inference_pram_json_list = get_inference_job_param_output(inference_job_id)
        json_list = []
        json_list = download_images(inference_pram_json_list,
                                    f"outputs/txt2img-images/{get_current_date()}/{inference_job_id}/")

        logger.debug(f"{str(images)}")
        logger.debug(f"{str(inference_pram_json_list)}")

        json_file = f"outputs/txt2img-images/{get_current_date()}/{inference_job_id}/{inference_job_id}_param.json"

        f = open(json_file)

        log_file = json.load(f)

        info_text = log_file["info"]

        infotexts = json.loads(info_text)["infotexts"][0]
    else:
        image_list = []  # Return an empty list if selected_value is None
        json_list = []
        info_text = ''

    return image_list, info_text, plaintext_to_html(infotexts)

def init_refresh_resource_list_from_cloud():
    logger.debug(f"start refreshing resource list from cloud")
    if get_variable_from_json('api_gateway_url') is not None:
        if not cloud_auth_manager.enableAuth:
            # api_manager.list_all_sagemaker_endpoints()
            refresh_all_models()
            get_texual_inversion_list()
            get_lora_list()
            get_hypernetwork_list()
            get_controlnet_model_list()
            # get_inference_job_list()
        else:
            logger.debug('auth enabled, not preload load any model')
    else:
        logger.debug(f"there is no api-gateway url and token in local file,")

def on_txt_time_change(start_time, end_time):
    logger.debug(f"!!!!!!!!!on_txt_time_change!!!!!!{start_time},{end_time}")
    global start_time_picker_txt_value
    global end_time_picker_txt_value
    start_time_picker_txt_value = start_time
    end_time_picker_txt_value = end_time
    return query_inference_job_list(txt_task_type, txt_status, txt_endpoint, txt_checkpoint, "txt2img")

def on_img_time_change(start_time, end_time):
    logger.debug(f"!!!!!!!!!on_img_time_change!!!!!!{start_time},{end_time}")
    global start_time_picker_img_value
    global end_time_picker_img_value
    start_time_picker_img_value = start_time
    end_time_picker_img_value = end_time
    return query_inference_job_list(img_task_type, img_status, img_endpoint, img_checkpoint, "img2img")

def load_inference_job_list(username, usertoken):
    inference_jobs = [None_Option_For_On_Cloud_Model]
    inferences_jobs_list = api_manager.list_all_inference_jobs_on_cloud(username, usertoken)

    temp_list = []
    for obj in inferences_jobs_list:
        if obj.get('completeTime') is None:
            complete_time = obj.get('startTime')
        else:
            complete_time = obj.get('completeTime')
        status = obj.get('status')
        task_type = obj.get('taskType', 'txt2img')
        inference_job_id = obj.get('InferenceJobId')
        # if filter_checkbox and task_type not in selected_types:
        #     continue
        temp_list.append((complete_time, f"{complete_time}-->{task_type}-->{status}-->{inference_job_id}"))
    # Sort the list based on completeTime in ascending order
    sorted_list = sorted(temp_list, key=lambda x: x[0], reverse=False)
    # Append the sorted combined strings to the txt2img_inference_job_ids list
    for item in sorted_list:
        inference_jobs.append(item[1])

    return inference_jobs

def load_model_list(username, user_token):
    models_on_cloud = [None_Option_For_On_Cloud_Model]
    models_on_cloud += list(set([model['name'] for model in api_manager.list_models_on_cloud(username, user_token)]))
    return models_on_cloud

def load_lora_models(username, user_token):
    return list(set([model['name'] for model in api_manager.list_models_on_cloud(username, user_token, types='Lora')]))

def load_hypernetworks_models(username, user_token):
    return list(set([model['name'] for model in api_manager.list_models_on_cloud(username, user_token, types='hypernetworks')]))

def load_vae_list(username, user_token):
    vae_model_on_cloud = ['Automatic', 'None']
    vae_model_on_cloud += list(set([model['name'] for model in api_manager.list_models_on_cloud(username, user_token, types='VAE')]))
    return vae_model_on_cloud

def load_controlnet_list(username, user_token):
    vae_model_on_cloud = ['None']
    vae_model_on_cloud += list(set([model['name'] for model in api_manager.list_models_on_cloud(username, user_token, types='ControlNet')]))
    return vae_model_on_cloud

def load_embeddings_list(username, user_token):
    # vae_model_on_cloud = ['None']
    vae_model_on_cloud = list(set([model['name'] for model in api_manager.list_models_on_cloud(username, user_token, types='embeddings')]))
    return vae_model_on_cloud

def create_ui(is_img2img):
    global txt2img_gallery, txt2img_generation_info
    import modules.ui

    init_refresh_resource_list_from_cloud()

    with gr.Blocks() as sagemaker_inference_tab:
        gr.HTML('
Amazon SageMaker Inference
')
        sagemaker_html_log = gr.HTML(elem_id=f'html_log_sagemaker')
        with gr.Column():
            with gr.Row():
                lora_and_hypernet_models_state = gr.State({})
                sd_model_on_cloud_dropdown = gr.Dropdown(choices=[], value=None_Option_For_On_Cloud_Model,
                                                         label='Stable Diffusion Checkpoint Used on Cloud')

                create_refresh_button_by_user(sd_model_on_cloud_dropdown,
                                              lambda *args: None,
                                              lambda username: {
                                                  'choices': load_model_list(username, username)
                                              }, 'refresh_cloud_model_down')

            with gr.Row():
                sd_vae_on_cloud_dropdown = gr.Dropdown(choices=[], value='Automatic',
                                                       label='SD Vae on Cloud')

                create_refresh_button_by_user(sd_vae_on_cloud_dropdown,
                                              lambda *args: None,
                                              lambda username: {
                                                  'choices': load_vae_list(username, username)
                                              }, 'refresh_cloud_vae_down')
            with gr.Row(visible=is_img2img):
                gr.HTML('
')

            with gr.Row(visible=is_img2img):
                global generate_on_cloud_button_with_js
                # if not is_img2img:
                #     generate_on_cloud_button_with_js = gr.Button(value="Generate on Cloud", variant='primary', elem_id="generate_on_cloud_with_cloud_config_button",queue=True, show_progress=True)
                global generate_on_cloud_button_with_js_img2img
                global interrogate_clip_on_cloud_button
                global interrogate_deep_booru_on_cloud_button

                interrogate_clip_on_cloud_button = gr.Button(value="Interrogate CLIP",  variant='primary',
                                                             elem_id="interrogate_clip_on_cloud_button")
                interrogate_deep_booru_on_cloud_button = gr.Button(value="Interrogte DeepBooru",  variant='primary',
                                                                   elem_id="interrogate_deep_booru_on_cloud_button")
            with gr.Row():
                gr.HTML('
')

            with gr.Row():
                global inference_job_dropdown
                # global txt2img_inference_job_ids

                inference_job_dropdown = gr.Dropdown(choices=[], value=None_Option_For_On_Cloud_Model,
                                                     label="Inference Job: Time-Type-Status-Uuid")
                create_refresh_button_by_user(inference_job_dropdown,
                                              lambda *args: None,
                                              lambda username: {
                                                  'choices': load_inference_job_list(username, username)
                                              }, 'refresh_inference_job_down')
                # inference_job_dropdown = gr.Dropdown(choices=txt2img_inference_job_ids,
                #                                      label="Inference Job: Time-Type-Status-Uuid",
                #                                      elem_id="txt2img_inference_job_ids_dropdown"
                #                                      )
                # txt2img_inference_job_ids_refresh_button = create_refresh_button(inference_job_dropdown,
                #                                                                  query_inference_job_list,
                #                                                                  lambda: {
                #                                                                     "choices": txt2img_inference_job_ids,
                #                                                                     "value": None},
                #                                                                  "refresh_txt2img_inference_job_ids")
            # fixme: inference filters need to be fixed
            # with gr.Row():
            #     inference_job_filter = gr.Checkbox(
            #         label="Advanced Inference Job filter", value=False, visible=True
            #     )
            #     inference_job_page = gr.Checkbox(label="Show All(unchecked: max 10 items)",
            #                                      elem_id="inference_job_page_checkbox", value=False)
            # with gr.Row(variant='panel', visible=False) as filter_row:
            #     with gr.Column(scale=1):
            #         gr.HTML(value="Inference Job type filters")
            #     with gr.Column(scale=2):
            #         with gr.Row():
            #             task_type_choices = ["txt2img", "img2img", "interrogate_clip", "interrogate_deepbooru"]
            #             task_type_dropdown = gr.Dropdown(label="Task Type", choices=task_type_choices,
            #                                              elem_id="task_type_ids_dropdown")
            #             status_choices = ["succeed", "inprogress", "failed"]
            #             status_dropdown = gr.Dropdown(label="Status", choices=status_choices,
            #                                           elem_id="task_status_dropdown")
            #         # with gr.Row():
            #         #     sagemaker_endpoint_filter = gr.Dropdown(api_manager.list_all_sagemaker_endpoints(),
            #         #                                             label="SageMaker Endpoint",
            #         #                                             elem_id="sagemaker_endpoint_dropdown")
            #         #     modules.ui.create_refresh_button(sagemaker_endpoint_filter, lambda: None,
            #         #                                      lambda: {"choices": api_manager.list_all_sagemaker_endpoints()},
            #         #                                      "refresh_sagemaker_endpoints")
            #
            #         with gr.Row():
            #             sd_checkpoint_filter = gr.Dropdown(label="Checkpoint", choices=sorted(update_sd_checkpoints()),
            #                                                elem_id="stable_diffusion_checkpoint_dropdown")
            #             modules.ui.create_refresh_button(sd_checkpoint_filter, update_sd_checkpoints,
            #                                              lambda: {"choices": sorted(update_sd_checkpoints())},
            #                                              "refresh_sd_checkpoints")
            #         if is_img2img:
            #             with gr.Row():
            #                 start_time_picker_img = gr.HTML(elem_id="start_timepicker_img_e",
            #                                                 value="""
            #
            #                                                     Start Time
            #                                                     <input type="date"
            #                                                            lang="en"
            #                                                            id="start_timepicker_img"
            #                                                            min="2023-01-01"
            #                                                            max="2033-12-31"
            #                                                            class="wrap svelte-aqlk7e"
            #                                                            style="color: #6B7280"
            #                                                            onchange="inference_job_timepicker_img_change()" />
            #
            #                                                 """)
            #                 end_time_picker_img = gr.HTML(elem_id="end_timepicker_img_e",
            #                                               value="""
            #
            #                                                 End Time
            #                                                 <input type="date"
            #                                                        lang="en"
            #                                                        id="end_timepicker_img"
            #                                                        min="2023-01-01" max="2033-12-31"
            #                                                        class="wrap svelte-aqlk7e"
            #                                                        style="color: #6B7280"
            #                                                        onchange="inference_job_timepicker_img_change()">
            #
            #                                               """)
            #                 start_time_picker_img_hidden = gr.Button(elem_id="start_time_picker_img_hidden",
            #                                                          visible=True)
            #                 end_time_picker_img_hidden = gr.Button(elem_id="end_time_picker_img_hidden",
            #                                                        visible=True)
            #             start_time_picker_img_hidden.click(fn=on_img_time_change,
            #                                                _js='get_time_img_value',
            #                                                inputs=[start_time_picker_img, end_time_picker_img],
            #                                                outputs=inference_job_dropdown)
            #             end_time_picker_img_hidden.click(fn=on_img_time_change,
            #                                              _js='get_time_img_value',
            #                                              inputs=[start_time_picker_img, end_time_picker_img],
            #                                              outputs=inference_job_dropdown
            #                                              )
            #             task_type_dropdown.change(fn=query_img_inference_job_list,
            #                                       inputs=[task_type_dropdown, status_dropdown,
            #                                               sagemaker_endpoint_filter,
            #                                               sd_checkpoint_filter], outputs=inference_job_dropdown)
            #             status_dropdown.change(fn=query_img_inference_job_list,
            #                                    inputs=[task_type_dropdown, status_dropdown, sagemaker_endpoint_filter,
            #                                            sd_checkpoint_filter], outputs=inference_job_dropdown)
            #
            #             sagemaker_endpoint_filter.change(fn=query_img_inference_job_list,
            #                                              inputs=[task_type_dropdown, status_dropdown,
            #                                                      sagemaker_endpoint_filter,
            #                                                      sd_checkpoint_filter], outputs=inference_job_dropdown)
            #
            #             sd_checkpoint_filter.change(fn=query_img_inference_job_list,
            #                                         inputs=[task_type_dropdown, status_dropdown,
            #                                                 sagemaker_endpoint_filter,
            #                                                 sd_checkpoint_filter], outputs=inference_job_dropdown)
            #         else:
            #             with gr.Row():
            #                 start_time_picker_text = gr.HTML(elem_id="start_timepicker_text_e",
            #                                                  value="""
            #
            #                                                     Start Time
            #                                                     <input type="date" lang="en" id="start_timepicker_text"
            #                                                            min="2023-01-01" max="2033-12-31"
            #                                                            class="wrap svelte-aqlk7e" style="color: #6B7280"
            #                                                            onchange="inference_job_timepicker_text_change()">
            #
            #                                                  """)
            #                 end_time_picker_text = gr.HTML(elem_id="end_timepicker_text_e",
            #                                                value="""
            #
            #                                                     End Time
            #                                                     <input type="date" lang="en" id="end_timepicker_text"
            #                                                             min="2023-01-01" max="2033-12-31"
            #                                                             class="wrap svelte-aqlk7e"
            #                                                             style="color: #6B7280"
            #                                                             onchange="inference_job_timepicker_text_change()">
            #
            #                                                """)
            #                 start_time_picker_button_hidden = gr.Button(elem_id="start_time_picker_button_hidden",
            #                                                             visible=False)
            #                 end_time_picker_button_hidden = gr.Button(elem_id="end_time_picker_button_hidden",
            #                                                           visible=False)
            #             start_time_picker_button_hidden.click(fn=on_txt_time_change,
            #                                                   _js='get_time_button_value',
            #                                                   inputs=[start_time_picker_text, end_time_picker_text],
            #                                                   outputs=inference_job_dropdown)
            #             end_time_picker_button_hidden.click(fn=on_txt_time_change,
            #                                                 _js='get_time_button_value',
            #                                                 inputs=[start_time_picker_text, end_time_picker_text],
            #                                                 outputs=inference_job_dropdown)
            #             task_type_dropdown.change(fn=query_txt_inference_job_list,
            #                                       inputs=[task_type_dropdown, status_dropdown,
            #                                               sagemaker_endpoint_filter, sd_checkpoint_filter],
            #                                       outputs=inference_job_dropdown)
            #             status_dropdown.change(fn=query_txt_inference_job_list,
            #                                    inputs=[task_type_dropdown, status_dropdown,
            #                                            sagemaker_endpoint_filter,
            #                                            sd_checkpoint_filter], outputs=inference_job_dropdown)
            #             sagemaker_endpoint_filter.change(fn=query_txt_inference_job_list,
            #                                              inputs=[task_type_dropdown, status_dropdown,
            #                                                      sagemaker_endpoint_filter,
            #                                                      sd_checkpoint_filter], outputs=inference_job_dropdown)
            #             sd_checkpoint_filter.change(fn=query_txt_inference_job_list,
            #                                         inputs=[task_type_dropdown, status_dropdown,
            #                                                 sagemaker_endpoint_filter, sd_checkpoint_filter],
            #                                         outputs=inference_job_dropdown)
            #
            #         def toggle_new_rows(create_from):
            #             global start_time_picker_txt_value
            #             start_time_picker_txt_value = None
            #             global end_time_picker_txt_value
            #             end_time_picker_txt_value = None
            #             return [gr.update(visible=create_from), None, None, None, None]
            #
            #         inference_job_filter.change(
            #             fn=toggle_new_rows,
            #             inputs=[inference_job_filter],
            #             outputs=[filter_row, task_type_dropdown, status_dropdown, sagemaker_endpoint_filter,
            #                      sd_checkpoint_filter],
            #         )
            #         hidden_check_type = gr.Textbox(elem_id="hidden_check_type", value=is_img2img, visible=False)
            #         inference_job_page.change(fn=query_page_inference_job_list,
            #                                   inputs=[task_type_dropdown, status_dropdown,
            #                                           sagemaker_endpoint_filter,
            #                                           sd_checkpoint_filter, hidden_check_type,
            #                                           inference_job_page],
            #
            #                                   outputs=inference_job_dropdown)

                def setup_inference_for_plugin(pr: gr.Request):
                    models_on_cloud = load_model_list(pr.username, pr.username)
                    vae_model_on_cloud = load_vae_list(pr.username, pr.username)
                    inference_jobs = load_inference_job_list(pr.username, pr.username)
                    lora_models_on_cloud = load_lora_models(username=pr.username, user_token=pr.username)
                    hypernetworks_models_on_cloud = load_hypernetworks_models(pr.username, pr.username)
                    controlnet_list = load_controlnet_list(pr.username, pr.username)
                    lora_hypernets = {
                        'lora': lora_models_on_cloud,
                        'hypernet': hypernetworks_models_on_cloud,
                        'controlnet': controlnet_list,
                    }

                    return lora_hypernets, \
                        gr.update(choices=models_on_cloud), \
                        gr.update(choices=inference_jobs), \
                        gr.update(choices=vae_model_on_cloud)

                sagemaker_inference_tab.load(fn=setup_inference_for_plugin, inputs=[],
                                             outputs=[
                                                 lora_and_hypernet_models_state,
                                                 sd_model_on_cloud_dropdown,
                                                 inference_job_dropdown,
                                                 sd_vae_on_cloud_dropdown]
                                             )
    with gr.Group():
        with gr.Accordion("Open for Checkpoint Merge in the Cloud!", visible=False, open=False):
            sagemaker_html_log = gr.HTML(elem_id=f'html_log_sagemaker')
            with FormRow(elem_id="modelmerger_models_in_the_cloud"):
                global primary_model_name
                primary_model_name = gr.Dropdown(choices=sorted(update_sd_checkpoints()),
                                                 elem_id="modelmerger_primary_model_name_in_the_cloud",
                                                 label="Primary model (A) in the cloud")
                create_refresh_button(primary_model_name, update_sd_checkpoints,
                                      lambda: {"choices": sorted(update_sd_checkpoints())},
                                      "refresh_checkpoint_A_in_the_cloud")

                global secondary_model_name
                secondary_model_name = gr.Dropdown(choices=sorted(update_sd_checkpoints()),
                                                   elem_id="modelmerger_secondary_model_name_in_the_cloud",
                                                   label="Secondary model (B) in the cloud")
                create_refresh_button(secondary_model_name, update_sd_checkpoints,
                                      lambda: {"choices": sorted(update_sd_checkpoints())},
                                      "refresh_checkpoint_B_in_the_cloud")

                global tertiary_model_name
                tertiary_model_name = gr.Dropdown(choices=sorted(update_sd_checkpoints()),
                                                  elem_id="modelmerger_tertiary_model_name_in_the_cloud",
                                                  label="Tertiary model (C) in the cloud")
                create_refresh_button(tertiary_model_name, update_sd_checkpoints,
                                      lambda: {"choices": sorted(update_sd_checkpoints())},
                                      "refresh_checkpoint_C_in_the_cloud")
            with gr.Row():
                global modelmerger_merge_on_cloud
                modelmerger_merge_on_cloud = gr.Button(elem_id="modelmerger_merge_in_the_cloud", value="Merge on Cloud",
                                                       variant='primary')

    return sd_model_on_cloud_dropdown, sd_vae_on_cloud_dropdown, inference_job_dropdown, primary_model_name, secondary_model_name, tertiary_model_name, modelmerger_merge_on_cloud, lora_and_hypernet_models_state


import base64
import locale
import sys

"""
def encoder(content):
    print('編碼類型：{:10}|{}'.format('default',content.encode()))
    print('編碼類型：{:10}|{}'.format('UTF-8',content.encode(encoding='utf8')))
    print('編碼類型：{:10}|{}'.format('UTF-16',content.encode(encoding='utf16')))
    print('編碼類型：{:10}|{}'.format('UTF-32',content.encode(encoding='utf32')))
    print('編碼類型：{:10}|{}'.format('cp950',content.encode(encoding='cp950')))
"""
"""
def encoder(content):

    enc = ['utf8','utf32','cp950']
    for _ in enc:
        print('編碼類型：{:10}|{}'.format(_,content.encode(encoding = _)))
"""

"""
def decoder(content):

    enc = ['base64','utf8','utf32','cp950']
    cypher = []
    for _ in enc:
        if(_ == 'base64'):
            cypher.append(base64.b64encode(content.encode()))
            print('編碼類型：{:10}|{}'.format(_,base64.b64encode(content.encode())))

        else:
            cypher.append(content.encode(encoding = _))
            print('編碼類型：{:10}|{}'.format(_,content.encode(encoding = _)))

    print("-"*60)
    index = 0
    for _ in enc:
        if(_ == 'base64'):
            cypher.append(base64.b64encode(content.encode()))
            print('編碼類型：{:10}|{}'.format(_,base64.b64decode(cypher[index]).decode()))

        else:
            print('解碼類型：{:10}|{}'.format(_,cypher[index].decode(encoding = _)))

        index +=1
"""
def bash64Decoder(url):
    content = base64.b64decode(url)
    print(content)
    with open('a.png','wb') as f:
        f.write(content)

    #content.decode('utf-8')
    #print(content)

#print(sys.getfilesystemencoding())
#print(locale.getpreferredencoding())
url = "Oz4rPj0+LDovPiwsKDAtOw=="
bash64Decoder(url)


from PIL import Image
import os, sys

def topng(path):
    for dirpath, dirs, files in os.walk(path):
        #ignore finished files
        if "finished" in dirs:
            dirs.remove("finished")
        for file in files:
            try:
                imgpath = os.path.join(dirpath, file)
                im = Image.open(imgpath)
                alpha = im.split()[-1]
                if not im.mode == 'RGB':
                    print("convert to png…" + imgpath)
                    rgb_im = im.convert('RGB')
                    rgb_im.putalpha(alpha)
                    rgb_im.save(imgpath, "png")
                    # Rename the file from PNG to JPG
                    thisFile = imgpath
                    base = os.path.splitext(thisFile)[0]
                    os.rename(thisFile, base + ".png")
            except:
                pass
    print("\n所有图片修改为PNG\n")


import codecs
import os

import setuptools

with open("README.md", "r") as fh:
    long_description = fh.read()

def read(rel_path):
    here = os.path.abspath(os.path.dirname(__file__))
    with codecs.open(os.path.join(here, rel_path), "r", encoding="utf-8") as fp:
        return fp.read()

def get_version(rel_path):
    for line in read(rel_path).splitlines():
        if line.startswith("__version__"):
            delim = '"' if '"' in line else "'"
            return line.split(delim)[1]
    raise RuntimeError("Unable to find version string.")

install_requires = [
    "numpy>=1.18.0",
    "tqdm",
    "six",
    "setuptools",
    "torch",
    "torchvision",
    "opencv-python",
    "scikit-image",
    "tensorflow",
    "tensorrt",
    "matplotlib",
    "pandas",
    "click",
    "progressbar",
]

setuptools.setup(
    name='whitebox-adversarial-toolbox',
    version=get_version(os.path.join("what", "__init__.py")),
    author="wuhanstudio",
    author_email="wuhanstudios@gmail.com",
    maintainer="wuhanstudio",
    maintainer_email="wuhanstudios@gmail.com",
    description='White-box Adversarial Toolbox (WHAT) - Python Library for Deep Learning Security',
    url="https://github.com/wuhanstudio/whitebox-adversarial-toolbox",
    license="MIT",
    install_requires=install_requires,
    extra_require = {
        "dev": [    "progressbar",
        ]
    },
    entry_points={
        'console_scripts': [
            'what=what._main:main',
        ],
    },
    packages=setuptools.find_packages(),
    long_description=long_description,
    long_description_content_type="text/markdown",
    classifiers=[
        "Intended Audience :: Science/Research",
        "License :: OSI Approved :: MIT License",
        "Programming Language :: Python :: 3.6",
        "Programming Language :: Python :: 3.7",
        "Programming Language :: Python :: 3.8",
        "Programming Language :: Python :: 3.9",
        "Topic :: Scientific/Engineering :: Artificial Intelligence",
    ],
)


import math
import glob
import re
import os
import logging

import os
pat = os.getcwd()
pat = pat + '\word.txt'

kgram_len_lst = []
dct_lst = {}
filename_lst = []
path = os.getcwd()

'''
List of stop words inserted in a list.
'''

f = open(pat,'r')
stop_word = set(f.read().split())
f.close()

'''
To read files to create lists. Copy them in the current directory\
and remove any other .txt files from current directory.
'''

for filename in glob.glob(os.path.join(path, '*.txt')):
	if filename != pat:
		f = open(filename,'r')
		fw = f.read()
		f.close()

		'''
		Lower casing and removing special characters. Assuming words to be\
		with alphabets, under_score and numeric characters only.
		'''

		fw = fw.lower()
		fw = re.sub(r'[^a-z0-9_ \n]', '', fw).split()

		'''
		remove stop words and make a string of all rest of the words.
		'''
		lw = []
		for i in fw:
			if i not in stop_word:
				lw.append(i)

		l = "".join(lw)

		'''
		Storing in dct_lst with file name as key an value a string.\
		Also storing filenames in a list.
		'''

		file_name = filename.split('\\')
		path_ = path.split('\\')
		file = ''
		for i in file_name:
			if i not in path_:
				file = file + i

		filename_lst.append(file)
		dct_lst[file] = l

'''
Taking k as 5 to calculate k-grams and p as 99991 to get 0-mod p.
'''
k = 5
p = 10

class finger_printing:

	def hash(x):
		'''
		Hash function to calculate hash values of k-grams.
		'''
		h = 0
		for i in range(0,5):
			h = h + (ord(x[i]))*(k**(5-i))
		return h

	def k_Gram(l):
		'''
		k gram function takes a string and returns a list of k grams \
		with length 5.
		'''
		ls = []
		for i in range(0,len(l)-4):
			ls.append(l[i:i+5])
		return ls

	def p_mod(ls):
		'''
		p mod function takes a list of hash values and gives us p mod values\
		(p = 991) list.
		'''
		pm = []
		for i in ls:
			if i%p==0:
				pm.append(i)
		return pm

	def Fngprnt(dct1, dct2, den):
		'''
		This finger print function takes two lists of p-mods and \
		and compare those to give the match percentage.
		'''
		z = 0
		i = 0
		c = 0
		for i in range(0,len(dct1)):
			k = i
			j = 0
			c = 0
			while(k<len(dct1) and j<len(dct2)):
				while ((k<len(dct1) and j<len(dct2)) and  dct1[k]==dct2[j]):
					c += 1
					k += 1
					j += 1
				if(k<len(dct1) and j<len(dct2)):
					if dct1[k] != dct2[j]:
						k = i
						if c>z:
							z = c
						c = 0
						if dct1[k]!=dct2[j]:
							j += 1
			if c>z:
				z = c
		if c>z:
			z = c
		return ((2*z)/(den))

'''
Converting strings to k-grams.
'''
for i in dct_lst:
	dct_lst[i] = finger_printing.k_Gram(dct_lst[i])
	kgram_len_lst.append(len(dct_lst[i]))

'''
Converting k-grams to hashes.
'''
for i in dct_lst:
	for j in range(0,len(dct_lst[i])):
		dct_lst[i][j] = finger_printing.hash(dct_lst[i][j])

'''
Stores signatures in lists for each file.
'''
for i in dct_lst:
	dct_lst[i] = finger_printing.p_mod(dct_lst[i])

'''
Compare file signatures as Longest common sequence.
'''
p = ''
for i in range(0,(len(filename_lst)-1)):
	for j in range((i+1),len(filename_lst)):
		try:
			d = kgram_len_lst[i] + kgram_len_lst[j]
			s = ('Fingerprint match between '+str(filename_lst[i])+' and '+str(filename_lst[j])+\
				' is: '+str(finger_printing.Fngprnt(dct_lst[filename_lst[i]],\
				 dct_lst[filename_lst[j]],d)*100))
		except ZeroDivisionError:
			s = (str(filename_lst[i])+' and '+str(filename_lst[j])+' are either empty or are too small with little length.')
		p = p + '\n' + s + '\n'

'''
Loging into a log file.txt
'''

log = "log file.log"
logging.basicConfig(filename=log,level=logging.DEBUG,\
	format='%(asctime)s %(message)s', datefmt='%d/%m/%Y %H:%M:%S')
logging.info('FINGER PRINT SIMILARITIES \n' + p)

print(p)


import json
import sys
import uuid
from dataclasses import dataclass
from datetime import date, datetime
from decimal import Decimal
from pathlib import Path
from typing import (
    Any,
    Dict,
    FrozenSet,
    Iterable,
    Iterator,
    List,
    Mapping,
    Optional,
    Tuple,
    Union,
)
from urllib.parse import unquote

from deltalake.fs import DeltaStorageHandler

from ._util import encode_partition_value

if sys.version_info >= (3, 8):
    from typing import Literal
else:
    from typing_extensions import Literal

import pyarrow as pa
import pyarrow.dataset as ds
import pyarrow.fs as pa_fs
from pyarrow.lib import RecordBatchReader

from deltalake.schema import delta_arrow_schema_from_pandas

from ._internal import DeltaDataChecker as _DeltaDataChecker
from ._internal import batch_distinct
from ._internal import write_new_deltalake as _write_new_deltalake
from .exceptions import DeltaProtocolError, TableNotFoundError
from .table import MAX_SUPPORTED_WRITER_VERSION, DeltaTable

try:
    import pandas as pd  # noqa: F811
except ModuleNotFoundError:
    _has_pandas = False
else:
    _has_pandas = True

PYARROW_MAJOR_VERSION = int(pa.__version__.split(".", maxsplit=1)[0])

@dataclass
class AddAction:
    path: str
    size: int
    partition_values: Mapping[str, Optional[str]]
    modification_time: int
    data_change: bool
    stats: str

def write_deltalake(
    table_or_uri: Union[str, Path, DeltaTable],
    data: Union[
        "pd.DataFrame",
        pa.Table,
        pa.RecordBatch,
        Iterable[pa.RecordBatch],
        RecordBatchReader,
    ],
    *,
    schema: Optional[pa.Schema] = None,
    partition_by: Optional[Union[List[str], str]] = None,
    filesystem: Optional[pa_fs.FileSystem] = None,
    mode: Literal["error", "append", "overwrite", "ignore"] = "error",
    file_options: Optional[ds.ParquetFileWriteOptions] = None,
    max_partitions: Optional[int] = None,
    max_open_files: int = 1024,
    max_rows_per_file: int = 10 * 1024 * 1024,
    min_rows_per_group: int = 64 * 1024,
    max_rows_per_group: int = 128 * 1024,
    name: Optional[str] = None,
    description: Optional[str] = None,
    configuration: Optional[Mapping[str, Optional[str]]] = None,
    overwrite_schema: bool = False,
    storage_options: Optional[Dict[str, str]] = None,
    partition_filters: Optional[List[Tuple[str, str, Any]]] = None,
    large_dtypes: bool = False,
) -> None:
    """Write to a Delta Lake table

    If the table does not already exist, it will be created.

    This function only supports writer protocol version 2 currently. When
    attempting to write to an existing table with a higher min_writer_version,
    this function will throw DeltaProtocolError.

    Note that this function does NOT register this table in a data catalog.

    A locking mechanism is needed to prevent unsafe concurrent writes to a
    delta lake directory when writing to S3. DynamoDB is the only available
    locking provider at the moment in delta-rs. To enable DynamoDB as the
    locking provider, you need to set the `AWS_S3_LOCKING_PROVIDER` to 'dynamodb'
    as a storage_option or as an environment variable.

    Additionally, you must create a DynamoDB table with the name 'delta_rs_lock_table'
    so that it can be automatically discovered by delta-rs. Alternatively, you can
    use a table name of your choice, but you must set the `DYNAMO_LOCK_TABLE_NAME`
    variable to match your chosen table name. The required schema for the DynamoDB
    table is as follows:

    - Key Schema: AttributeName=key, KeyType=HASH
    - Attribute Definitions: AttributeName=key, AttributeType=S

    Please note that this locking mechanism is not compatible with any other
    locking mechanisms, including the one used by Spark.

    Args:
        table_or_uri: URI of a table or a DeltaTable object.
        data: Data to write. If passing iterable, the schema must also be given.
        schema: Optional schema to write.
        partition_by: List of columns to partition the table by. Only required
            when creating a new table.
        filesystem: Optional filesystem to pass to PyArrow. If not provided will
            be inferred from uri. The file system has to be rooted in the table root.
            Use the pyarrow.fs.SubTreeFileSystem, to adopt the root of pyarrow file systems.
        mode: How to handle existing data. Default is to error if table already exists.
            If 'append', will add new data.
            If 'overwrite', will replace table with new data.
            If 'ignore', will not write anything if table already exists.
        file_options: Optional write options for Parquet (ParquetFileWriteOptions).
            Can be provided with defaults using ParquetFileWriteOptions().make_write_options().
            Please refer to https://github.com/apache/arrow/blob/master/python/pyarrow/_dataset_parquet.pyx#L492-L533
            for the list of available options
        max_partitions: the maximum number of partitions that will be used.
        max_open_files: Limits the maximum number of
            files that can be left open while writing. If an attempt is made to open
            too many files then the least recently used file will be closed.
            If this setting is set too low you may end up fragmenting your
            data into many small files.
        max_rows_per_file: Maximum number of rows per file.
            If greater than 0 then this will limit how many rows are placed in any single file.
            Otherwise there will be no limit and one file will be created in each output directory
            unless files need to be closed to respect max_open_files
            min_rows_per_group: Minimum number of rows per group. When the value is set,
            the dataset writer will batch incoming data and only write the row groups to the disk
            when sufficient rows have accumulated.
        max_rows_per_group: Maximum number of rows per group.
            If the value is set, then the dataset writer may split up large incoming batches into multiple row groups.
            If this value is set, then min_rows_per_group should also be set.
        name: User-provided identifier for this table.
        description: User-provided description for this table.
        configuration: A map containing configuration options for the metadata action.
        overwrite_schema: If True, allows updating the schema of the table.
        storage_options: options passed to the native delta filesystem. Unused if 'filesystem' is defined.
        partition_filters: the partition filters that will be used for partition overwrite.
        large_dtypes: If True, the table schema is checked against large_dtypes
    """
    if _has_pandas and isinstance(data, pd.DataFrame):
        if schema is not None:
            data = pa.Table.from_pandas(data, schema=schema)
        else:
            data, schema = delta_arrow_schema_from_pandas(data)

    table, table_uri = try_get_table_and_table_uri(table_or_uri, storage_options)

    # We need to write against the latest table version
    if table:
        table.update_incremental()

    if schema is None:
        if isinstance(data, RecordBatchReader):
            schema = data.schema
        elif isinstance(data, Iterable):
            raise ValueError("You must provide schema if data is Iterable")
        else:
            schema = data.schema

    if filesystem is not None:
        raise NotImplementedError("Filesystem support is not yet implemented.  #570")

    if table is not None:
        storage_options = table._storage_options or {}
        storage_options.update(storage_options or {})

    filesystem = pa_fs.PyFileSystem(DeltaStorageHandler(table_uri, storage_options))

    __enforce_append_only(table=table, configuration=configuration, mode=mode)

    if isinstance(partition_by, str):
        partition_by = [partition_by]

    if table:  # already exists
        if schema != table.schema().to_pyarrow(as_large_types=large_dtypes) and not (
            mode == "overwrite" and overwrite_schema
        ):
            raise ValueError(
                "Schema of data does not match table schema\n"
                f"Data schema:\n{schema}\nTable Schema:\n{table.schema().to_pyarrow(as_large_types=large_dtypes)}"
            )

        if mode == "error":
            raise AssertionError("DeltaTable already exists.")
        elif mode == "ignore":
            return

        current_version = table.version()

        if partition_by:
            assert partition_by == table.metadata().partition_columns
        else:
            partition_by = table.metadata().partition_columns

        if table.protocol().min_writer_version > MAX_SUPPORTED_WRITER_VERSION:
            raise DeltaProtocolError(
                "This table's min_writer_version is "
                f"{table.protocol().min_writer_version}, "
                "but this method only supports version 2."
            )
    else:  # creating a new table
        current_version = -1

    dtype_map = {
        pa.large_string(): pa.string(),  # type: ignore
    }

    def _large_to_normal_dtype(dtype: pa.DataType) -> pa.DataType:
        try:
            return dtype_map[dtype]
        except KeyError:
            return dtype

    if partition_by:
        if PYARROW_MAJOR_VERSION < 12:
            partition_schema = pa.schema(
                [
                    pa.field(name, _large_to_normal_dtype(schema.field(name).type))
                    for name in partition_by
                ]
            )
        else:
            partition_schema = pa.schema([schema.field(name) for name in partition_by])
        partitioning = ds.partitioning(partition_schema, flavor="hive")
    else:
        partitioning = None

    add_actions: List[AddAction] = []

    def visitor(written_file: Any) -> None:
        path, partition_values = get_partitions_from_path(written_file.path)
        stats = get_file_stats_from_metadata(written_file.metadata)

        # PyArrow added support for written_file.size in 9.0.0
        if PYARROW_MAJOR_VERSION >= 9:
            size = written_file.size
        elif filesystem is not None:
            size = filesystem.get_file_info([path])[0].size
        else:
            size = 0

        add_actions.append(
            AddAction(
                path,
                size,
                partition_values,
                int(datetime.now().timestamp() * 1000),
                True,
                json.dumps(stats, cls=DeltaJSONEncoder),
            )
        )

    if table is not None:
        # We don't currently provide a way to set invariants
        # (and maybe never will), so only enforce if already exist.
        invariants = table.schema().invariants
        checker = _DeltaDataChecker(invariants)

        def check_data_is_aligned_with_partition_filtering(
            batch: pa.RecordBatch,
        ) -> None:
            if table is None:
                return
            existed_partitions: FrozenSet[
                FrozenSet[Tuple[str, Optional[str]]]
            ] = table._table.get_active_partitions()
            allowed_partitions: FrozenSet[
                FrozenSet[Tuple[str, Optional[str]]]
            ] = table._table.get_active_partitions(partition_filters)
            partition_values = pa.RecordBatch.from_arrays(
                [
                    batch.column(column_name)
                    for column_name in table.metadata().partition_columns
                ],
                table.metadata().partition_columns,
            )
            partition_values = batch_distinct(partition_values)
            for i in range(partition_values.num_rows):
                # Map will maintain order of partition_columns
                partition_map = {
                    column_name: encode_partition_value(
                        batch.column(column_name)[i].as_py()
                    )
                    for column_name in table.metadata().partition_columns
                }
                partition = frozenset(partition_map.items())
                if (
                    partition not in allowed_partitions
                    and partition in existed_partitions
                ):
                    partition_repr = " ".join(
                        f"{key}={value}" for key, value in partition_map.items()
                    )
                    raise ValueError(
                        f"Data should be aligned with partitioning. "
                        f"Data contained values for partition {partition_repr}"
                    )

        def validate_batch(batch: pa.RecordBatch) -> pa.RecordBatch:
            checker.check_batch(batch)

            if mode == "overwrite" and partition_filters:
                check_data_is_aligned_with_partition_filtering(batch)

            return batch

        if isinstance(data, RecordBatchReader):
            batch_iter = data
        elif isinstance(data, pa.RecordBatch):
            batch_iter = [data]
        elif isinstance(data, pa.Table):
            batch_iter = data.to_batches()
        else:
            batch_iter = data

        data = RecordBatchReader.from_batches(
            schema, (validate_batch(batch) for batch in batch_iter)
        )

    ds.write_dataset(
        data,
        base_dir="/",
        basename_template=f"{current_version + 1}-{uuid.uuid4()}-{{i}}.parquet",
        format="parquet",
        partitioning=partitioning,
        # It will not accept a schema if using a RBR
        schema=schema if not isinstance(data, RecordBatchReader) else None,
        file_visitor=visitor,
        existing_data_behavior="overwrite_or_ignore",
        file_options=file_options,
        max_open_files=max_open_files,
        max_rows_per_file=max_rows_per_file,
        min_rows_per_group=min_rows_per_group,
        max_rows_per_group=max_rows_per_group,
        filesystem=filesystem,
        max_partitions=max_partitions,
    )

    if table is None:
        _write_new_deltalake(
            table_uri,
            schema,
            add_actions,
            mode,
            partition_by or [],
            name,
            description,
            configuration,
            storage_options,
        )
    else:
        table._table.create_write_transaction(
            add_actions,
            mode,
            partition_by or [],
            schema,
            partition_filters,
        )
        table.update_incremental()

def __enforce_append_only(
    table: Optional[DeltaTable],
    configuration: Optional[Mapping[str, Optional[str]]],
    mode: str,
) -> None:
    """Throw ValueError if table configuration contains delta.appendOnly and mode is not append"""
    if table:
        configuration = table.metadata().configuration
    config_delta_append_only = (
        configuration and configuration.get("delta.appendOnly", "false") == "true"
    )
    if config_delta_append_only and mode != "append":
        raise ValueError(
            "If configuration has delta.appendOnly = 'true', mode must be 'append'."
            f" Mode is currently {mode}"
        )

class DeltaJSONEncoder(json.JSONEncoder):
    def default(self, obj: Any) -> Any:
        if isinstance(obj, bytes):
            return obj.decode("unicode_escape", "backslashreplace")
        elif isinstance(obj, date):
            return obj.isoformat()
        elif isinstance(obj, datetime):
            return obj.isoformat()
        elif isinstance(obj, Decimal):
            return str(obj)
        # Let the base class default method raise the TypeError
        return json.JSONEncoder.default(self, obj)

def try_get_table_and_table_uri(
    table_or_uri: Union[str, Path, DeltaTable],
    storage_options: Optional[Dict[str, str]] = None,
) -> Tuple[Optional[DeltaTable], str]:
    """Parses the `table_or_uri`.
    Raises [ValueError] If `table_or_uri` is not of type str, Path or DeltaTable.

    Args:
        table_or_uri: URI of a table or a DeltaTable object.
        storage_options: Options passed to the native delta filesystem.

    Returns:
        (DeltaTable object, URI of the table)
    """
    if not isinstance(table_or_uri, (str, Path, DeltaTable)):
        raise ValueError("table_or_uri must be a str, Path or DeltaTable")

    if isinstance(table_or_uri, (str, Path)):
        table = try_get_deltatable(table_or_uri, storage_options)
        table_uri = str(table_or_uri)
    else:
        table = table_or_uri
        table_uri = table._table.table_uri()

    return (table, table_uri)

def try_get_deltatable(
    table_uri: Union[str, Path], storage_options: Optional[Dict[str, str]]
) -> Optional[DeltaTable]:
    try:
        return DeltaTable(table_uri, storage_options=storage_options)
    except TableNotFoundError:
        return None

def get_partitions_from_path(path: str) -> Tuple[str, Dict[str, Optional[str]]]:
    if path[0] == "/":
        path = path[1:]
    parts = path.split("/")
    parts.pop()  # remove filename
    out: Dict[str, Optional[str]] = {}
    for part in parts:
        if part == "":
            continue
        key, value = part.split("=", maxsplit=1)
        if value == "__HIVE_DEFAULT_PARTITION__":
            out[key] = None
        else:
            out[key] = unquote(value)
    return path, out

def get_file_stats_from_metadata(
    metadata: Any,
) -> Dict[str, Union[int, Dict[str, Any]]]:
    stats = {
        "numRecords": metadata.num_rows,
        "minValues": {},
        "maxValues": {},
        "nullCount": {},
    }

    def iter_groups(metadata: Any) -> Iterator[Any]:
        for i in range(metadata.num_row_groups):
            yield metadata.row_group(i)

    for column_idx in range(metadata.num_columns):
        name = metadata.row_group(0).column(column_idx).path_in_schema
        # If stats missing, then we can't know aggregate stats
        if all(
            group.column(column_idx).is_stats_set for group in iter_groups(metadata)
        ):
            stats["nullCount"][name] = sum(
                group.column(column_idx).statistics.null_count
                for group in iter_groups(metadata)
            )

            # Min / max may not exist for some column types, or if all values are null
            if any(
                group.column(column_idx).statistics.has_min_max
                for group in iter_groups(metadata)
            ):
                # Min and Max are recorded in physical type, not logical type
                # https://stackoverflow.com/questions/66753485/decoding-parquet-min-max-statistics-for-decimal-type
                # TODO: Add logic to decode physical type for DATE, DECIMAL
                logical_type = (
                    metadata.row_group(0)
                    .column(column_idx)
                    .statistics.logical_type.type
                )

                if PYARROW_MAJOR_VERSION < 8 and logical_type not in [
                    "STRING",
                    "INT",
                    "TIMESTAMP",
                    "NONE",
                ]:
                    continue

                minimums = (
                    group.column(column_idx).statistics.min
                    for group in iter_groups(metadata)
                )
                # If some row groups have all null values, their min and max will be null too.
                stats["minValues"][name] = min(
                    minimum for minimum in minimums if minimum is not None
                )
                maximums = (
                    group.column(column_idx).statistics.max
                    for group in iter_groups(metadata)
                )
                stats["maxValues"][name] = max(
                    maximum for maximum in maximums if maximum is not None
                )
    return stats


from django.urls import path
from .views import (ShopIndexView, GroupsListView,
                    ProductDetailsView, ProductsListView,
                    OrdersListView, OrdersDetailsView,
                    CreateProductView, UpdateProductView,
                    DeleteProductView, CreateOrderView,
                    UpdateOrderView, DeleteOrderView)

app_name = "shopapp"

urlpatterns = [
    path("", ShopIndexView.as_view(), name="index"),
    path("groups/", GroupsListView.as_view(), name="groups_list"),

    path("products/", ProductsListView.as_view(), name="products_list"),
    path("products/create/", CreateProductView.as_view(), name="create_product"),
    path("products/<int:pk>/", ProductDetailsView.as_view(), name="products_details"),
    path("products/<int:pk>/update", UpdateProductView.as_view(), name="product_update"),
    path("products/<int:pk>/delete", DeleteProductView.as_view(), name="product_delete"),

    path("orders/", OrdersListView.as_view(), name="orders_list"),
    path("orders/create/", CreateOrderView.as_view(), name="create_order"),
    path("orders/<int:pk>/", OrdersDetailsView.as_view(), name="order_details"),
    path("orders/<int:pk>/update", UpdateOrderView.as_view(), name="order_update"),
    path("orders/<int:pk>/delete", DeleteOrderView.as_view(), name="order_delete"),
]