Spaces:
Sleeping
Sleeping
Upload dollar.py
Browse files
dollar.py
ADDED
|
@@ -0,0 +1,771 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# dollar_agent_system_updated.py
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
import asyncio
|
| 5 |
+
import time
|
| 6 |
+
import json
|
| 7 |
+
from collections import deque
|
| 8 |
+
from typing import Any, List, Dict, Optional, Callable
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
# --- Dummy Litellm for Local Execution Simulation ---
|
| 12 |
+
# (Remains the same as in the previous complete code block)
|
| 13 |
+
class DummyLitellm:
|
| 14 |
+
async def acompletion(self, model: str, messages: List[Dict[str, Any]], max_tokens: Optional[int] = None,
|
| 15 |
+
temperature: Optional[float] = None, api_base: Optional[str] = None, **kwargs) -> Dict[str, Any]:
|
| 16 |
+
|
| 17 |
+
content_val = messages[0]['content']
|
| 18 |
+
prompt_preview = ""
|
| 19 |
+
if isinstance(content_val, str):
|
| 20 |
+
prompt_preview = content_val[:50]
|
| 21 |
+
elif isinstance(content_val, list):
|
| 22 |
+
for item in content_val:
|
| 23 |
+
if item.get("type") == "text":
|
| 24 |
+
prompt_preview = item["text"][:50]
|
| 25 |
+
break
|
| 26 |
+
elif item.get("image_url"):
|
| 27 |
+
prompt_preview = f"Multimodal (images): {item['image_url']['url'][:20]}..."
|
| 28 |
+
break
|
| 29 |
+
if not prompt_preview:
|
| 30 |
+
prompt_preview = str(content_val)[:50]
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
print(f" (Dummy litellm acompletion: model='{model}', prompt='{prompt_preview}'...)")
|
| 34 |
+
await asyncio.sleep(0.1) # Simulate network latency
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
# Simulate success responses based on model/tool type
|
| 38 |
+
if "flux" in model or "dalle" in model or "ideogram" in model or "stable-diffusion" in model:
|
| 39 |
+
return {'choices': [{'message': {'content': f"https://image.url/{model.replace('/', '_')}/{abs(hash(str(messages)))}.png"}}], 'model': model}
|
| 40 |
+
elif "sora" in model or "runway" in model or "kling" in model or "luma" in model or "pika" in model:
|
| 41 |
+
# Simulate slight variation in video quality for critique loop
|
| 42 |
+
video_hash = abs(hash(str(messages)))
|
| 43 |
+
quality_suffix = ""
|
| 44 |
+
if kwargs.get("attempt", 1) < 3: # Lower quality for first few attempts
|
| 45 |
+
quality_suffix = "_low_res_noisy"
|
| 46 |
+
return {'choices': [{'message': {'content': f"https://video.url/{model.replace('/', '_')}/{video_hash}{quality_suffix}.mp4"}}], 'model': model}
|
| 47 |
+
elif "perplexity" in model or "tavily" in model:
|
| 48 |
+
return {'choices': [{'message': {'content': f"Search results from {model} for '{prompt_preview}...'"}}], 'model': model}
|
| 49 |
+
elif "claude" in model or "gpt-4o" in model:
|
| 50 |
+
if "rewrite this prompt" in prompt_preview.lower():
|
| 51 |
+
original_p = kwargs.get('original_prompt', prompt_preview)
|
| 52 |
+
attempt_num = kwargs.get('attempt_number', 1)
|
| 53 |
+
# Simulate an LLM attempting to improve the prompt based on feedback
|
| 54 |
+
if attempt_num == 1:
|
| 55 |
+
rewritten_prompt = f"Optimized by {model.split('/')[-1]} (Attempt 1): {original_p} - Refined for vivid details and smooth camera movement."
|
| 56 |
+
elif attempt_num == 2:
|
| 57 |
+
rewritten_prompt = f"Optimized by {model.split('/')[-1]} (Attempt 2): {original_p} - Focusing on object consistency and cinematic color grading, as per critique."
|
| 58 |
+
else:
|
| 59 |
+
rewritten_prompt = f"Optimized by {model.split('/')[-1]} (Attempt {attempt_num}): {original_p} - Further adjustments based on refined critique."
|
| 60 |
+
return {'choices': [{'message': {'content': rewritten_prompt}}], 'model': model}
|
| 61 |
+
elif "analyze feedback" in prompt_preview.lower():
|
| 62 |
+
feedback = kwargs.get('feedback', '')
|
| 63 |
+
if "low consistency" in feedback.lower():
|
| 64 |
+
return {'choices': [{'message': {'content': f"LLM Planner: Feedback analyzed. Suggest modifying prompt to explicitly request 'stable camera' and 'consistent character appearance'. Consider trying 'runwayml/gen-2' for next attempt due to its reputation for consistency."}}], 'model': model}
|
| 65 |
+
elif "script misalignment" in feedback.lower():
|
| 66 |
+
return {'choices': [{'message': {'content': f"LLM Planner: Feedback analyzed. Suggest refining prompt to emphasize key elements from the script: '{feedback}'. Recommend reviewing individual scene descriptions."}}], 'model': model}
|
| 67 |
+
return {'choices': [{'message': {'content': f"LLM Planner: Feedback analyzed. No clear re-planning needed from: {feedback}"}}], 'model': model}
|
| 68 |
+
elif "does the video's content" in prompt_preview.lower():
|
| 69 |
+
video_url = kwargs.get('video_url', '')
|
| 70 |
+
# Simulate critique getting better or worse based on simulated video quality
|
| 71 |
+
if "_low_res_noisy.mp4" in video_url:
|
| 72 |
+
return {'choices': [{'message': {'content': f"Critique from {model}: Score 5/10. Low consistency, grainy visuals. Key script elements are present but overall quality is poor."}}], 'model': model}
|
| 73 |
+
else:
|
| 74 |
+
return {'choices': [{'message': {'content': f"Critique from {model}: Score 8/10. Good visual consistency. Final scene could better capture the 'untouched by time' serenity from script."}}], 'model': model}
|
| 75 |
+
else:
|
| 76 |
+
return {'choices': [{'message': {'content': f"Hello from {model}! You asked about: {prompt_preview}"}}], 'model': model}
|
| 77 |
+
elif "elevenlabs" in model or "audiocraft" in model or "suno" in model or "bark" in model:
|
| 78 |
+
return {'choices': [{'message': {'content': f"https://audio.url/{model.replace('/', '_')}/{abs(hash(str(messages)))}.mp3"}}], 'model': model}
|
| 79 |
+
elif "topaz-labs" in model or "ultimate-upscaler-api" in model:
|
| 80 |
+
original_url = kwargs.get('video_url', 'unknown_url')
|
| 81 |
+
return {'choices': [{'message': {'content': f"{original_url.replace('_low_res_noisy.mp4', '_upscaled.mp4').replace('.mp4', '_upscaled.mp4')}"}}], 'model': model}
|
| 82 |
+
elif "internal-cv-model" in model:
|
| 83 |
+
video_url = kwargs.get('video_url', '')
|
| 84 |
+
if "_low_res_noisy.mp4" in video_url:
|
| 85 |
+
return {'choices': [{'message': {'content': {"score": 0.55, "feedback": "Low frame-to-frame consistency, noisy."}}}]}
|
| 86 |
+
else:
|
| 87 |
+
return {'choices': [{'message': {'content': {"score": 0.85, "feedback": "Good consistency."}}}]}
|
| 88 |
+
else:
|
| 89 |
+
return {'choices': [{'message': {'content': f"Hello from {model}! You asked about: {prompt_preview}"}}], 'model': model}
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
# --- Tool Definitions (Remains the same as in the previous complete code block) ---
|
| 95 |
+
TOOL_CONFIG = {
|
| 96 |
+
"web_search": {
|
| 97 |
+
"name": "web_search",
|
| 98 |
+
"description": "Searches the web for information.",
|
| 99 |
+
"input_schema": {"query": "str"},
|
| 100 |
+
"models": ["perplexity/online"],
|
| 101 |
+
"cost_per_use": 0.001,
|
| 102 |
+
"latency_ms": 200,
|
| 103 |
+
"is_autonomous": True
|
| 104 |
+
},
|
| 105 |
+
"generate_image": {
|
| 106 |
+
"name": "generate_image",
|
| 107 |
+
"description": "Generates an image from a text prompt.",
|
| 108 |
+
"input_schema": {"prompt": "str", "size": "str", "quality": "str", "style": "str"},
|
| 109 |
+
"models": ["dalle-3", "stability-ai/stable-diffusion-xl-turbo"],
|
| 110 |
+
"cost_per_use": {"dalle-3": 0.04, "stability-ai/stable-diffusion-xl-turbo": 0.005},
|
| 111 |
+
"latency_ms": {"dalle-3": 2000, "stability-ai/stable-diffusion-xl-turbo": 500},
|
| 112 |
+
"is_autonomous": False,
|
| 113 |
+
},
|
| 114 |
+
"generate_video": {
|
| 115 |
+
"name": "generate_video",
|
| 116 |
+
"description": "Generates a video from a text prompt.",
|
| 117 |
+
"input_schema": {"prompt": "str", "resolution": "str", "duration": "int", "custom_params": "dict"},
|
| 118 |
+
"models": ["kling", "sora", "runwayml/gen-2", "pika-labs"],
|
| 119 |
+
"cost_per_use": {"kling": 5.00, "sora": 10.00, "runwayml/gen-2": 0.50, "pika-labs": 0.30},
|
| 120 |
+
"latency_ms": {"kling": 30000, "sora": 60000, "runwayml/gen-2": 10000, "pika-labs": 8000},
|
| 121 |
+
"is_autonomous": False
|
| 122 |
+
},
|
| 123 |
+
"optimize_prompt": {
|
| 124 |
+
"name": "optimize_prompt",
|
| 125 |
+
"description": "Rewrites and enhances a prompt for optimal generation.",
|
| 126 |
+
"input_schema": {"original_prompt": "str", "target_model_type": "str"},
|
| 127 |
+
"models": ["claude-3-5-sonnet", "gpt-4o"],
|
| 128 |
+
"cost_per_use": 0.01,
|
| 129 |
+
"latency_ms": 1500,
|
| 130 |
+
"is_autonomous": True
|
| 131 |
+
},
|
| 132 |
+
"upscale_video": {
|
| 133 |
+
"name": "upscale_video",
|
| 134 |
+
"description": "Upscales a given video URL to higher resolution/quality.",
|
| 135 |
+
"input_schema": {"video_url": "str", "target_resolution": "str"},
|
| 136 |
+
"models": ["topaz-labs/video-ai", "ultimate-upscaler-api"],
|
| 137 |
+
"cost_per_use": 0.20,
|
| 138 |
+
"latency_ms": 15000,
|
| 139 |
+
"is_autonomous": True
|
| 140 |
+
},
|
| 141 |
+
"generate_audio": {
|
| 142 |
+
"name": "generate_audio",
|
| 143 |
+
"description": "Generates audio (speech, music, sound effects) from a text prompt.",
|
| 144 |
+
"input_schema": {"prompt": "str", "audio_type": "str"},
|
| 145 |
+
"models": ["elevenlabs/speech", "audiocraft/musicgen", "suno/bark"],
|
| 146 |
+
"cost_per_use": 0.05,
|
| 147 |
+
"latency_ms": 5000,
|
| 148 |
+
"is_autonomous": True
|
| 149 |
+
},
|
| 150 |
+
"compare_frames": {
|
| 151 |
+
"name": "compare_frames",
|
| 152 |
+
"description": "Compares first and last frames of a video for consistency.",
|
| 153 |
+
"input_schema": {"video_url": "str"},
|
| 154 |
+
"models": ["internal-cv-model"],
|
| 155 |
+
"cost_per_use": 0.005,
|
| 156 |
+
"latency_ms": 500,
|
| 157 |
+
"is_autonomous": True
|
| 158 |
+
},
|
| 159 |
+
"local_animatediff": {
|
| 160 |
+
"name": "local_animatediff",
|
| 161 |
+
"description": "Generates video locally using ComfyUI + AnimateDiff.",
|
| 162 |
+
"input_schema": {"prompt": "str", "resolution": "str", "duration": "int"},
|
| 163 |
+
"models": ["local"],
|
| 164 |
+
"cost_per_use": 0.0,
|
| 165 |
+
"latency_ms": 60000,
|
| 166 |
+
"is_autonomous": True
|
| 167 |
+
},
|
| 168 |
+
"llm_replan": { # New internal tool for re-planning based on critique
|
| 169 |
+
"name": "llm_replan",
|
| 170 |
+
"description": "Analyzes critique feedback and suggests modifications to prompt or strategy for next generation attempt.",
|
| 171 |
+
"input_schema": {"current_prompt": "str", "feedback": "str", "previous_model": "str", "attempt_number": "int"},
|
| 172 |
+
"models": ["claude-3-5-sonnet", "gpt-4o"],
|
| 173 |
+
"cost_per_use": 0.015,
|
| 174 |
+
"latency_ms": 2000,
|
| 175 |
+
"is_autonomous": True
|
| 176 |
+
}
|
| 177 |
+
}
|
| 178 |
+
|
| 179 |
+
|
| 180 |
+
# --- Default Configuration from dollar.yaml (Conceptual) ---
|
| 181 |
+
DEFAULT_CONFIG = {
|
| 182 |
+
"budget": {
|
| 183 |
+
"hard_cap": 100.0,
|
| 184 |
+
"warn_at": 75.0
|
| 185 |
+
},
|
| 186 |
+
"preferences": {
|
| 187 |
+
"default_video_model": "kling",
|
| 188 |
+
"fallback_chain": ["kling", "sora", "runwayml/gen-2", "pika-labs", "local_animatediff"],
|
| 189 |
+
"upscaling": "always",
|
| 190 |
+
"audio": True,
|
| 191 |
+
"self_critique": "strict", # "strict", "moderate", "none"
|
| 192 |
+
"style_consistency_enforcement": "high",
|
| 193 |
+
"default_style": "cinematic",
|
| 194 |
+
"max_generation_attempts": 3 # New config item
|
| 195 |
+
},
|
| 196 |
+
"agent_settings": {
|
| 197 |
+
"memory_size": 20,
|
| 198 |
+
"cost_weight": 1000,
|
| 199 |
+
"latency_weight": 1
|
| 200 |
+
}
|
| 201 |
+
}
|
| 202 |
+
|
| 203 |
+
|
| 204 |
+
|
| 205 |
+
|
| 206 |
+
# --- AgenticDollar Class ---
|
| 207 |
+
class AgenticDollar:
|
| 208 |
+
def __init__(self, config: Dict[str, Any] = None, litellm_client: Any = None):
|
| 209 |
+
self.config = config if config is not None else DEFAULT_CONFIG
|
| 210 |
+
self.litellm = litellm_client if litellm_client else DummyLitellm()
|
| 211 |
+
self.tools = TOOL_CONFIG
|
| 212 |
+
self.working_memory = deque(maxlen=self.config["agent_settings"].get("memory_size", 10))
|
| 213 |
+
self.budget_hard_cap = self.config["budget"]["hard_cap"]
|
| 214 |
+
self.budget_warn_at = self.config["budget"]["warn_at"]
|
| 215 |
+
self.current_spend = 0.0
|
| 216 |
+
self.user_confirmation_callback: Optional[Callable[[str], bool]] = None
|
| 217 |
+
|
| 218 |
+
|
| 219 |
+
print(f"AgenticDollar initialized with hard budget cap: ${self.budget_hard_cap:.2f}")
|
| 220 |
+
print(f"Preferences: {json.dumps(self.config['preferences'], indent=2)}")
|
| 221 |
+
|
| 222 |
+
|
| 223 |
+
async def _log_action(self, action_type: str, details: Dict[str, Any]):
|
| 224 |
+
"""Logs actions for transparency and future self-improvement."""
|
| 225 |
+
log_entry = {"timestamp": time.time(), "action_type": action_type, "details": details}
|
| 226 |
+
self.working_memory.append(log_entry)
|
| 227 |
+
print(f" [LOG] {action_type}: {details.get('tool', '')} - {details.get('message', '')[:50]}...")
|
| 228 |
+
|
| 229 |
+
|
| 230 |
+
async def _ask_for_user_confirmation(self, message: str) -> bool:
|
| 231 |
+
"""Asks the user for confirmation for high-cost or critical actions."""
|
| 232 |
+
if self.user_confirmation_callback:
|
| 233 |
+
return await asyncio.to_thread(self.user_confirmation_callback, message)
|
| 234 |
+
print(f" [USER CONFIRMATION REQUIRED]: {message} (Simulating 'yes' for now)")
|
| 235 |
+
return True
|
| 236 |
+
|
| 237 |
+
|
| 238 |
+
async def _select_best_tool_and_model(self, task_description: str, tool_type: str,
|
| 239 |
+
max_cost_for_step: float = float('inf')) -> Optional[Dict]:
|
| 240 |
+
"""
|
| 241 |
+
Dynamically selects the most appropriate tool and model based on task,
|
| 242 |
+
cost, and available options, respecting preference for `fallback_chain`.
|
| 243 |
+
Latency is now handled more implicitly by the fallback order.
|
| 244 |
+
"""
|
| 245 |
+
|
| 246 |
+
eligible_tools_defs = {t_name: t_def for t_name, t_def in self.tools.items()
|
| 247 |
+
if t_def.get("name") == tool_type}
|
| 248 |
+
|
| 249 |
+
|
| 250 |
+
if not eligible_tools_defs:
|
| 251 |
+
return None
|
| 252 |
+
|
| 253 |
+
|
| 254 |
+
best_option = None
|
| 255 |
+
|
| 256 |
+
# Determine the order of models to try based on preferences or default
|
| 257 |
+
# For video generation, use the explicit fallback chain
|
| 258 |
+
if tool_type == "generate_video":
|
| 259 |
+
model_selection_order = self.config["preferences"]["fallback_chain"]
|
| 260 |
+
else:
|
| 261 |
+
# For other tools, find all models and sort by a simple heuristic (e.g., lowest cost first)
|
| 262 |
+
all_possible_models = []
|
| 263 |
+
for t_def in eligible_tools_defs.values():
|
| 264 |
+
for m in t_def.get("models", []):
|
| 265 |
+
cost_val = t_def["cost_per_use"].get(m, t_def["cost_per_use"]) if isinstance(t_def["cost_per_use"], dict) else t_def["cost_per_use"]
|
| 266 |
+
all_possible_models.append((m, cost_val))
|
| 267 |
+
model_selection_order = [m for m, _ in sorted(all_possible_models, key=lambda x: x[1])]
|
| 268 |
+
|
| 269 |
+
for model_name in model_selection_order:
|
| 270 |
+
tool_found_name = None
|
| 271 |
+
tool_def = None
|
| 272 |
+
for t_name, t_def_candidate in eligible_tools_defs.items():
|
| 273 |
+
if model_name in t_def_candidate.get("models", []):
|
| 274 |
+
tool_found_name = t_name
|
| 275 |
+
tool_def = t_def_candidate
|
| 276 |
+
break
|
| 277 |
+
|
| 278 |
+
if not tool_def:
|
| 279 |
+
continue
|
| 280 |
+
|
| 281 |
+
|
| 282 |
+
cost = tool_def["cost_per_use"].get(model_name, tool_def["cost_per_use"]) if isinstance(tool_def["cost_per_use"], dict) else tool_def["cost_per_use"]
|
| 283 |
+
latency = tool_def["latency_ms"].get(model_name, tool_def["latency_ms"]) if isinstance(tool_def["latency_ms"], dict) else tool_def["latency_ms"]
|
| 284 |
+
|
| 285 |
+
|
| 286 |
+
# Check if this model exceeds budget for this step or overall hard cap
|
| 287 |
+
if cost <= max_cost_for_step and (self.current_spend + cost <= self.budget_hard_cap):
|
| 288 |
+
best_option = {"tool_name": tool_found_name, "model_name": model_name, "cost": cost, "latency": latency}
|
| 289 |
+
# Since the fallback chain implies preference/order, take the first one that fits
|
| 290 |
+
break # Found the best available model in the preferred order within budget
|
| 291 |
+
|
| 292 |
+
# We only update current_spend upon successful _execution_ of the tool
|
| 293 |
+
return best_option
|
| 294 |
+
|
| 295 |
+
|
| 296 |
+
async def _execute_tool(self, tool_option: Dict, **kwargs) -> Any:
|
| 297 |
+
"""Executes the chosen tool and handles potential retries/fallbacks."""
|
| 298 |
+
tool_name = tool_option["tool_name"]
|
| 299 |
+
model_name = tool_option["model_name"]
|
| 300 |
+
cost = tool_option["cost"]
|
| 301 |
+
|
| 302 |
+
|
| 303 |
+
# Special handling for local fallback tool
|
| 304 |
+
if tool_name == "local_animatediff":
|
| 305 |
+
print(f" [LOCAL EXECUTION] Executing {tool_name} locally with params: {kwargs}")
|
| 306 |
+
await asyncio.sleep(tool_option["latency"])
|
| 307 |
+
return {"result_url": f"https://local.server/{tool_name}_{abs(hash(str(kwargs)))}.mp4", "status": "completed"}
|
| 308 |
+
|
| 309 |
+
|
| 310 |
+
messages = [{"role": "user", "content": json.dumps(kwargs)}]
|
| 311 |
+
|
| 312 |
+
|
| 313 |
+
try:
|
| 314 |
+
response = await self.litellm.acompletion(model=model_name, messages=messages, **kwargs)
|
| 315 |
+
content = response['choices'][0]['message']['content']
|
| 316 |
+
self._log_action("TOOL_EXECUTION", {"tool": tool_name, "model": model_name, "cost": cost, "result_preview": content[:100]})
|
| 317 |
+
self.current_spend += cost
|
| 318 |
+
return {"result": content, "status": "completed"}
|
| 319 |
+
except Exception as e:
|
| 320 |
+
self._log_action("TOOL_EXECUTION_FAILURE", {"tool": tool_name, "model": model_name, "error": str(e)})
|
| 321 |
+
print(f" [ERROR] Tool execution failed for {tool_name}/{model_name}: {e}")
|
| 322 |
+
return {"result": None, "status": "failed", "error": str(e)}
|
| 323 |
+
|
| 324 |
+
|
| 325 |
+
async def _optimize_prompt(self, original_prompt: str, target_model_type: str, current_spend_context: float, attempt_number: int) -> str:
|
| 326 |
+
"""Uses an LLM to rewrite and enhance a prompt for optimal generation."""
|
| 327 |
+
if self.config["preferences"]["self_critique"] == "none":
|
| 328 |
+
print(" [PROMPT OPTIMIZATION] Skipping prompt optimization based on preferences.")
|
| 329 |
+
return original_prompt
|
| 330 |
+
|
| 331 |
+
|
| 332 |
+
print(f" [PROMPT OPTIMIZATION] Optimizing prompt for {target_model_type} (Attempt {attempt_number})...")
|
| 333 |
+
remaining_budget = self.budget_hard_cap - current_spend_context
|
| 334 |
+
# Cap optimization cost to a small fraction of remaining budget or a fixed small amount
|
| 335 |
+
max_opt_cost = min(0.05, remaining_budget * 0.1) # Max 5 cents or 10% of remaining budget
|
| 336 |
+
|
| 337 |
+
opt_tool = await self._select_best_tool_and_model(
|
| 338 |
+
"Optimize prompt", "optimize_prompt", max_cost_for_step=max_opt_cost
|
| 339 |
+
)
|
| 340 |
+
if not opt_tool:
|
| 341 |
+
print(" [WARNING] No prompt optimization tool available or within budget. Using original prompt.")
|
| 342 |
+
return original_prompt
|
| 343 |
+
|
| 344 |
+
|
| 345 |
+
optimized_response = await self._execute_tool(
|
| 346 |
+
opt_tool,
|
| 347 |
+
original_prompt=original_prompt,
|
| 348 |
+
target_model_type=target_model_type,
|
| 349 |
+
attempt_number=attempt_number # Pass attempt number for dummy to vary output
|
| 350 |
+
)
|
| 351 |
+
return optimized_response["result"] if optimized_response["status"] == "completed" else original_prompt
|
| 352 |
+
|
| 353 |
+
|
| 354 |
+
async def _analyze_critique_and_replan(self, current_prompt: str, feedback: Dict[str, Any], previous_model: str, attempt_number: int) -> Dict[str, Any]:
|
| 355 |
+
"""
|
| 356 |
+
Uses an LLM to analyze critique feedback and suggest modifications
|
| 357 |
+
to the prompt or strategy for the next generation attempt.
|
| 358 |
+
Returns a dict with 'new_prompt' and optionally 'suggested_model'.
|
| 359 |
+
"""
|
| 360 |
+
print(f" [REPLANNING] Analyzing critique for attempt {attempt_number}...")
|
| 361 |
+
replan_tool = await self._select_best_tool_and_model("Analyze critique and replan", "llm_replan")
|
| 362 |
+
if not replan_tool:
|
| 363 |
+
print(" [WARNING] No replanning tool available. Cannot intelligently re-plan.")
|
| 364 |
+
return {"new_prompt": current_prompt, "suggested_model": None}
|
| 365 |
+
|
| 366 |
+
|
| 367 |
+
# Structure the feedback for the LLM
|
| 368 |
+
feedback_str = json.dumps(feedback)
|
| 369 |
+
|
| 370 |
+
llm_response = await self._execute_tool(
|
| 371 |
+
replan_tool,
|
| 372 |
+
current_prompt=current_prompt,
|
| 373 |
+
feedback=feedback_str,
|
| 374 |
+
previous_model=previous_model,
|
| 375 |
+
attempt_number=attempt_number
|
| 376 |
+
)
|
| 377 |
+
|
| 378 |
+
|
| 379 |
+
if llm_response["status"] == "completed":
|
| 380 |
+
# Simulate parsing LLM's suggested new prompt and model
|
| 381 |
+
response_content = llm_response["result"]
|
| 382 |
+
new_prompt = current_prompt # Default to no change
|
| 383 |
+
suggested_model = None
|
| 384 |
+
|
| 385 |
+
|
| 386 |
+
if "Suggest modifying prompt to explicitly request" in response_content:
|
| 387 |
+
# Simple parsing of dummy LLM response
|
| 388 |
+
parts = response_content.split("'")
|
| 389 |
+
if len(parts) > 1:
|
| 390 |
+
new_prompt = current_prompt + " (Refined based on critique: " + parts[1] + ")"
|
| 391 |
+
if "Consider trying 'runwayml/gen-2'" in response_content:
|
| 392 |
+
suggested_model = "runwayml/gen-2"
|
| 393 |
+
elif "refining prompt to emphasize key elements" in response_content:
|
| 394 |
+
new_prompt = current_prompt + " (Emphasizing script elements based on critique)"
|
| 395 |
+
|
| 396 |
+
|
| 397 |
+
print(f" [REPLANNING] LLM suggested new prompt: '{new_prompt}'")
|
| 398 |
+
if suggested_model:
|
| 399 |
+
print(f" [REPLANNING] LLM suggested new model: '{suggested_model}'")
|
| 400 |
+
|
| 401 |
+
|
| 402 |
+
return {"new_prompt": new_prompt, "suggested_model": suggested_model}
|
| 403 |
+
|
| 404 |
+
print(" [WARNING] Replanning LLM failed. No changes suggested.")
|
| 405 |
+
return {"new_prompt": current_prompt, "suggested_model": None}
|
| 406 |
+
|
| 407 |
+
|
| 408 |
+
|
| 409 |
+
|
| 410 |
+
async def _assess_video_quality(self, video_url: str, script: str) -> Dict[str, Any]:
|
| 411 |
+
"""Performs self-critique: frame consistency and script adherence."""
|
| 412 |
+
self_critique_level = self.config["preferences"]["self_critique"]
|
| 413 |
+
if self_critique_level == "none":
|
| 414 |
+
print(" [QUALITY ASSESSMENT] Skipping self-critique based on preferences.")
|
| 415 |
+
return {"frame_consistency": {"score": 1.0, "feedback": "skipped"}, "script_adherence": "skipped", "overall_pass": True, "feedback_messages": []}
|
| 416 |
+
|
| 417 |
+
|
| 418 |
+
print(f" [QUALITY ASSESSMENT] Assessing video quality for {video_url} (Level: {self_critique_level})...")
|
| 419 |
+
overall_pass = True
|
| 420 |
+
feedback_messages = []
|
| 421 |
+
frame_consistency_result = {"score": 1.0, "feedback": "skipped"}
|
| 422 |
+
script_adherence_result = "skipped"
|
| 423 |
+
|
| 424 |
+
|
| 425 |
+
# 1. Frame Consistency Scoring
|
| 426 |
+
if self_critique_level in ["strict", "moderate"]:
|
| 427 |
+
consistency_tool = await self._select_best_tool_and_model("Check video frame consistency", "compare_frames",
|
| 428 |
+
max_cost_for_step=self.budget_hard_cap - self.current_spend)
|
| 429 |
+
if consistency_tool:
|
| 430 |
+
consistency_response = await self._execute_tool(consistency_tool, video_url=video_url)
|
| 431 |
+
if consistency_response["status"] == "completed":
|
| 432 |
+
content = consistency_response["result"] # This is the dict from DummyLitellm
|
| 433 |
+
if isinstance(content, dict) and "score" in content:
|
| 434 |
+
frame_consistency_result = content
|
| 435 |
+
if content["score"] < 0.7: # Example threshold for failure
|
| 436 |
+
overall_pass = False
|
| 437 |
+
feedback_messages.append(f"Low frame consistency ({content['score']}): {content['feedback']}")
|
| 438 |
+
else:
|
| 439 |
+
feedback_messages.append("Could not parse frame consistency score.")
|
| 440 |
+
else:
|
| 441 |
+
feedback_messages.append("Frame consistency tool execution failed.")
|
| 442 |
+
print(f" Frame consistency check: {frame_consistency_result}")
|
| 443 |
+
else:
|
| 444 |
+
feedback_messages.append("Frame consistency tool unavailable or out of budget.")
|
| 445 |
+
print(" Frame consistency tool unavailable.")
|
| 446 |
+
else:
|
| 447 |
+
print(" Frame consistency check skipped (self_critique level).")
|
| 448 |
+
|
| 449 |
+
|
| 450 |
+
# 2. Script Adherence (using an LLM to compare video description to script)
|
| 451 |
+
if self_critique_level == "strict" and script:
|
| 452 |
+
llm_critique_tool = await self._select_best_tool_and_model("Critique video against script", "optimize_prompt",
|
| 453 |
+
max_cost_for_step=self.budget_hard_cap - self.current_spend)
|
| 454 |
+
if llm_critique_tool:
|
| 455 |
+
critique_response = await self._execute_tool(
|
| 456 |
+
llm_critique_tool,
|
| 457 |
+
original_prompt=f"Critique video at {video_url} against script: '{script}'",
|
| 458 |
+
target_model_type="video_critique_llm",
|
| 459 |
+
video_url=video_url # Pass for DummyLitellm's quality simulation
|
| 460 |
+
)
|
| 461 |
+
if critique_response["status"] == "completed":
|
| 462 |
+
script_adherence_result = critique_response["result"]
|
| 463 |
+
if "score 8/10" not in script_adherence_result: # Example parsing of failure for strict mode
|
| 464 |
+
overall_pass = False
|
| 465 |
+
feedback_messages.append(f"LLM critique indicated script misalignment: {script_adherence_result}")
|
| 466 |
+
else:
|
| 467 |
+
feedback_messages.append("LLM critique tool execution failed.")
|
| 468 |
+
print(f" Script adherence critique: {script_adherence_result}")
|
| 469 |
+
else:
|
| 470 |
+
feedback_messages.append("LLM critique tool unavailable or out of budget.")
|
| 471 |
+
print(" Script adherence critique tool unavailable.")
|
| 472 |
+
else:
|
| 473 |
+
print(" Script adherence critique skipped (self_critique level or no script).")
|
| 474 |
+
|
| 475 |
+
|
| 476 |
+
return {
|
| 477 |
+
"frame_consistency": frame_consistency_result,
|
| 478 |
+
"script_adherence": script_adherence_result,
|
| 479 |
+
"overall_pass": overall_pass,
|
| 480 |
+
"feedback_messages": feedback_messages
|
| 481 |
+
}
|
| 482 |
+
|
| 483 |
+
|
| 484 |
+
async def generate_advanced_video(self, user_request_prompt: str, script: str = "",
|
| 485 |
+
target_resolution: Optional[str] = None, max_duration: Optional[int] = None,
|
| 486 |
+
max_cost_per_job: Optional[float] = None,
|
| 487 |
+
request_audio: Optional[bool] = None,
|
| 488 |
+
request_upscaling: Optional[str] = None
|
| 489 |
+
) -> Dict[str, Any]:
|
| 490 |
+
"""
|
| 491 |
+
Orchestrates the generation of a high-quality video with advanced features,
|
| 492 |
+
respecting config and command-line overrides, including an iterative self-critique loop.
|
| 493 |
+
"""
|
| 494 |
+
print(f"\n--- Initiating Advanced Video Generation ---")
|
| 495 |
+
print(f"User Request: '{user_request_prompt}'")
|
| 496 |
+
if script:
|
| 497 |
+
print(f"Script provided: '{script[:100]}...'")
|
| 498 |
+
|
| 499 |
+
|
| 500 |
+
# --- Apply Config Defaults and Command-Line Overrides ---
|
| 501 |
+
actual_max_cost_per_job = max_cost_per_job if max_cost_per_job is not None else self.budget_hard_cap
|
| 502 |
+
actual_resolution = target_resolution if target_resolution is not None else "1080p"
|
| 503 |
+
actual_audio_preference = request_audio if request_audio is not None else self.config["preferences"]["audio"]
|
| 504 |
+
actual_upscaling_preference = request_upscaling if request_upscaling is not None else self.config["preferences"]["upscaling"]
|
| 505 |
+
actual_self_critique_rigor = self.config["preferences"]["self_critique"]
|
| 506 |
+
max_attempts = self.config["preferences"]["max_generation_attempts"]
|
| 507 |
+
|
| 508 |
+
current_video_prompt = user_request_prompt # This prompt will be refined in the loop
|
| 509 |
+
generated_video_url = None
|
| 510 |
+
video_generator_used = "N/A"
|
| 511 |
+
final_quality_assessment = {}
|
| 512 |
+
all_attempt_details = []
|
| 513 |
+
|
| 514 |
+
|
| 515 |
+
# --- Budget Checks ---
|
| 516 |
+
if self.current_spend + actual_max_cost_per_job > self.budget_hard_cap:
|
| 517 |
+
print(f" [BUDGET ALERT] Job would exceed hard cap of ${self.budget_hard_cap:.2f}. Current spend: ${self.current_spend:.2f}. Requested job cost: ${actual_max_cost_per_job:.2f}.")
|
| 518 |
+
return {"status": "aborted", "message": "Job exceeds overall budget cap."}
|
| 519 |
+
elif self.current_spend + actual_max_cost_per_job > self.budget_warn_at:
|
| 520 |
+
print(f" [BUDGET WARNING] Job will cause total spend to exceed warning threshold of ${self.budget_warn_at:.2f}. Current spend: ${self.current_spend:.2f}.")
|
| 521 |
+
|
| 522 |
+
|
| 523 |
+
# --- User Confirmation (for high-cost/critical tasks) ---
|
| 524 |
+
if not await self._ask_for_user_confirmation(
|
| 525 |
+
f"This video generation could cost up to ${actual_max_cost_per_job:.2f} and take several minutes. Proceed?"
|
| 526 |
+
):
|
| 527 |
+
return {"status": "aborted", "message": "User cancelled operation."}
|
| 528 |
+
|
| 529 |
+
|
| 530 |
+
# --- Iterative Self-Critique & Refinement Loop ---
|
| 531 |
+
for attempt in range(1, max_attempts + 1):
|
| 532 |
+
print(f"\n--- GENERATION ATTEMPT {attempt}/{max_attempts} ---")
|
| 533 |
+
|
| 534 |
+
|
| 535 |
+
# 1. Auto-Prompt Optimization (or Re-optimization based on critique)
|
| 536 |
+
optimized_prompt_for_attempt = await self._optimize_prompt(current_video_prompt, "video_generator", self.current_spend, attempt)
|
| 537 |
+
print(f"Prompt for this attempt: '{optimized_prompt_for_attempt}'")
|
| 538 |
+
|
| 539 |
+
# 2. Dynamic Tool/Model Selection for Video Generation
|
| 540 |
+
# If the replanning LLM suggested a model, prioritize it for this attempt
|
| 541 |
+
suggested_model_for_attempt = None
|
| 542 |
+
if attempt > 1 and all_attempt_details[-1].get("replan_suggestions"):
|
| 543 |
+
suggested_model_for_attempt = all_attempt_details[-1]["replan_suggestions"].get("suggested_model")
|
| 544 |
+
if suggested_model_for_attempt:
|
| 545 |
+
print(f" [MODEL OVERRIDE] Prioritizing LLM-suggested model: {suggested_model_for_attempt}")
|
| 546 |
+
|
| 547 |
+
|
| 548 |
+
# Calculate remaining budget for this generation attempt
|
| 549 |
+
remaining_budget_for_generation = actual_max_cost_per_job - (self.current_spend - (self.current_spend if self.current_spend < actual_max_cost_per_job else 0)) # simplified
|
| 550 |
+
|
| 551 |
+
|
| 552 |
+
video_tool_option = await self._select_best_tool_and_model(
|
| 553 |
+
f"Generate video for: {optimized_prompt_for_attempt}",
|
| 554 |
+
"generate_video",
|
| 555 |
+
max_cost_for_step=remaining_budget_for_generation
|
| 556 |
+
)
|
| 557 |
+
# If a model was explicitly suggested by the replanning LLM, try to use it if available
|
| 558 |
+
if suggested_model_for_attempt and video_tool_option and video_tool_option["model_name"] != suggested_model_for_attempt:
|
| 559 |
+
# Re-evaluate, forcing the suggested model if possible
|
| 560 |
+
forced_tool_option = None
|
| 561 |
+
for t_name, t_def in self.tools.items():
|
| 562 |
+
if t_def.get("name") == "generate_video" and suggested_model_for_attempt in t_def.get("models", []):
|
| 563 |
+
cost = t_def["cost_per_use"].get(suggested_model_for_attempt, t_def["cost_per_use"]) if isinstance(t_def["cost_per_use"], dict) else t_def["cost_per_use"]
|
| 564 |
+
latency = t_def["latency_ms"].get(suggested_model_for_attempt, t_def["latency_ms"]) if isinstance(t_def["latency_ms"], dict) else t_def["latency_ms"]
|
| 565 |
+
if cost <= remaining_budget_for_generation and (self.current_spend + cost <= self.budget_hard_cap):
|
| 566 |
+
forced_tool_option = {"tool_name": t_name, "model_name": suggested_model_for_attempt, "cost": cost, "latency": latency}
|
| 567 |
+
break
|
| 568 |
+
if forced_tool_option:
|
| 569 |
+
video_tool_option = forced_tool_option
|
| 570 |
+
print(f" [MODEL OVERRIDE] Successfully switched to LLM-suggested model: {suggested_model_for_attempt}")
|
| 571 |
+
|
| 572 |
+
|
| 573 |
+
|
| 574 |
+
|
| 575 |
+
if not video_tool_option:
|
| 576 |
+
all_attempt_details.append({"attempt": attempt, "status": "failed_no_model", "message": "Could not find a suitable video generation model within budget/constraints."})
|
| 577 |
+
print(f" [FAILURE] Attempt {attempt} failed: No suitable video generation model.")
|
| 578 |
+
break # Cannot proceed without a model
|
| 579 |
+
|
| 580 |
+
|
| 581 |
+
video_generator_used = video_tool_option["model_name"]
|
| 582 |
+
print(f"Selected video generator for attempt {attempt}: {video_generator_used}")
|
| 583 |
+
|
| 584 |
+
|
| 585 |
+
# 3. Video Generation Attempt
|
| 586 |
+
video_result = await self._execute_tool(
|
| 587 |
+
video_tool_option,
|
| 588 |
+
prompt=optimized_prompt_for_attempt,
|
| 589 |
+
resolution=actual_resolution,
|
| 590 |
+
duration=max_duration,
|
| 591 |
+
attempt=attempt # Pass attempt number for dummy to vary output
|
| 592 |
+
)
|
| 593 |
+
|
| 594 |
+
|
| 595 |
+
if video_result["status"] != "completed" or not video_result["result"]:
|
| 596 |
+
all_attempt_details.append({"attempt": attempt, "status": "failed_generation", "message": video_result.get("error", "Generation failed.")})
|
| 597 |
+
print(f" [FAILURE] Attempt {attempt} failed: {video_result.get('error', 'Generation failed.')}")
|
| 598 |
+
# Decide if we can retry with a different model or if it's a hard failure
|
| 599 |
+
if attempt == max_attempts:
|
| 600 |
+
return {"status": "failed", "message": "Video generation failed after all attempts."}
|
| 601 |
+
else:
|
| 602 |
+
# For a real system, here we'd analyze the error and decide to replan or try next model in fallback
|
| 603 |
+
print(" [RETRY] Attempting next iteration without explicit replanning for now.")
|
| 604 |
+
continue # Try next iteration
|
| 605 |
+
|
| 606 |
+
|
| 607 |
+
generated_video_url = video_result["result"]
|
| 608 |
+
print(f"Generated Video URL (Attempt {attempt}): {generated_video_url}")
|
| 609 |
+
|
| 610 |
+
|
| 611 |
+
# 4. Self-Critique Loop
|
| 612 |
+
final_quality_assessment = await self._assess_video_quality(generated_video_url, script)
|
| 613 |
+
all_attempt_details.append({
|
| 614 |
+
"attempt": attempt,
|
| 615 |
+
"status": "completed",
|
| 616 |
+
"video_url": generated_video_url,
|
| 617 |
+
"assessment": final_quality_assessment,
|
| 618 |
+
"optimized_prompt": optimized_prompt_for_attempt,
|
| 619 |
+
"video_generator": video_generator_used
|
| 620 |
+
})
|
| 621 |
+
|
| 622 |
+
|
| 623 |
+
if final_quality_assessment["overall_pass"]:
|
| 624 |
+
print(f" [SUCCESS] Video passed quality assessment on attempt {attempt}!")
|
| 625 |
+
break # Exit loop, quality met
|
| 626 |
+
else:
|
| 627 |
+
print(f" [FAILURE] Video did NOT pass quality assessment on attempt {attempt}: {final_quality_assessment['feedback_messages']}")
|
| 628 |
+
if attempt < max_attempts:
|
| 629 |
+
# 5. Analyze Critique and Re-plan for next attempt
|
| 630 |
+
replan_result = await self._analyze_critique_and_replan(
|
| 631 |
+
current_prompt=current_video_prompt, # Use the user's initial prompt as base for replanning
|
| 632 |
+
feedback=final_quality_assessment,
|
| 633 |
+
previous_model=video_generator_used,
|
| 634 |
+
attempt_number=attempt
|
| 635 |
+
)
|
| 636 |
+
current_video_prompt = replan_result["new_prompt"] # Update prompt for next attempt
|
| 637 |
+
all_attempt_details[-1]["replan_suggestions"] = replan_result # Store replan suggestions
|
| 638 |
+
print(f" [RETRYING] Retrying with refined prompt: '{current_video_prompt}'")
|
| 639 |
+
# The suggested model from replan will be prioritized in the next loop iteration
|
| 640 |
+
else:
|
| 641 |
+
print(f" [FAILURE] Max attempts ({max_attempts}) reached. Video did not meet quality standards.")
|
| 642 |
+
return {"status": "failed", "message": "Video did not meet quality standards after max attempts."}
|
| 643 |
+
|
| 644 |
+
|
| 645 |
+
if not generated_video_url:
|
| 646 |
+
return {"status": "failed", "message": "No video could be successfully generated."}
|
| 647 |
+
|
| 648 |
+
|
| 649 |
+
|
| 650 |
+
|
| 651 |
+
# --- Post-Processing Steps (executed only after a quality-approved video is generated) ---
|
| 652 |
+
|
| 653 |
+
# 6. Video Upscaling
|
| 654 |
+
final_video_url = generated_video_url
|
| 655 |
+
if actual_upscaling_preference == "always" or \
|
| 656 |
+
(actual_upscaling_preference == "if_needed" and actual_resolution not in final_video_url): # Simplified heuristic
|
| 657 |
+
|
| 658 |
+
print(" [UPSCALING] Attempting to upscale video...")
|
| 659 |
+
upscale_tool_option = await self._select_best_tool_and_model("Upscale video", "upscale_video",
|
| 660 |
+
max_cost_for_step=self.budget_hard_cap - self.current_spend)
|
| 661 |
+
if upscale_tool_option:
|
| 662 |
+
upscaled_video_result = await self._execute_tool(
|
| 663 |
+
upscale_tool_option,
|
| 664 |
+
video_url=generated_video_url,
|
| 665 |
+
target_resolution=actual_resolution
|
| 666 |
+
)
|
| 667 |
+
if upscaled_video_result["status"] == "completed":
|
| 668 |
+
final_video_url = upscaled_video_result["result"]
|
| 669 |
+
print(f"Upscaled Video URL: {final_video_url}")
|
| 670 |
+
else:
|
| 671 |
+
print(" [WARNING] Video upscaling failed. Using raw generated video.")
|
| 672 |
+
else:
|
| 673 |
+
print(" [WARNING] Video upscaling tool not available or out of budget. Using raw generated video.")
|
| 674 |
+
else:
|
| 675 |
+
print(f" [UPSCALING] Upscaling preference set to '{actual_upscaling_preference}', skipping.")
|
| 676 |
+
|
| 677 |
+
|
| 678 |
+
# 7. Audio Generation
|
| 679 |
+
generated_audio_url = None
|
| 680 |
+
if script and actual_audio_preference:
|
| 681 |
+
print(" [AUDIO GENERATION] Generating audio for script...")
|
| 682 |
+
audio_tool_option = await self._select_best_tool_and_model("Generate audio from script", "generate_audio",
|
| 683 |
+
max_cost_for_step=self.budget_hard_cap - self.current_spend)
|
| 684 |
+
if audio_tool_option:
|
| 685 |
+
audio_result = await self._execute_tool(
|
| 686 |
+
audio_tool_option,
|
| 687 |
+
prompt=script,
|
| 688 |
+
audio_type="speech"
|
| 689 |
+
)
|
| 690 |
+
if audio_result["status"] == "completed":
|
| 691 |
+
generated_audio_url = audio_result["result"]
|
| 692 |
+
print(f"Generated Audio URL: {generated_audio_url}")
|
| 693 |
+
else:
|
| 694 |
+
print(" [WARNING] Audio generation failed.")
|
| 695 |
+
else:
|
| 696 |
+
print(" [WARNING] Audio generation tool not available or out of budget.")
|
| 697 |
+
else:
|
| 698 |
+
print(" [AUDIO GENERATION] Audio generation skipped based on preferences or no script.")
|
| 699 |
+
|
| 700 |
+
|
| 701 |
+
return {
|
| 702 |
+
"status": "completed",
|
| 703 |
+
"final_video_url": final_video_url,
|
| 704 |
+
"generated_audio_url": generated_audio_url,
|
| 705 |
+
"details": {
|
| 706 |
+
"initial_prompt": user_request_prompt,
|
| 707 |
+
"final_optimized_prompt": current_video_prompt,
|
| 708 |
+
"video_generator_used": video_generator_used,
|
| 709 |
+
"quality_assessment_final": final_quality_assessment,
|
| 710 |
+
"total_cost_incurred_for_job": self.current_spend,
|
| 711 |
+
"generation_attempts": all_attempt_details
|
| 712 |
+
}
|
| 713 |
+
}
|
| 714 |
+
|
| 715 |
+
|
| 716 |
+
# --- Example Usage (How you would interact with Dollar via CLI or API) ---
|
| 717 |
+
async def main():
|
| 718 |
+
dollar_instance = AgenticDollar()
|
| 719 |
+
|
| 720 |
+
|
| 721 |
+
async def my_user_confirm_func(message: str) -> bool:
|
| 722 |
+
response = await asyncio.to_thread(input, f"Confirm (y/n): {message} ")
|
| 723 |
+
return response.lower() == 'y'
|
| 724 |
+
dollar_instance.user_confirmation_callback = my_user_confirm_func
|
| 725 |
+
|
| 726 |
+
|
| 727 |
+
|
| 728 |
+
|
| 729 |
+
# --- Scenario 1: Simulate CLI command with a prompt that might need refinement
|
| 730 |
+
print("\n" + "="*80)
|
| 731 |
+
print("SCENARIO 1: Cyberpunk Samurai Video (requiring refinement)")
|
| 732 |
+
print("="*80)
|
| 733 |
+
|
| 734 |
+
|
| 735 |
+
cli_result = await dollar_instance.generate_advanced_video(
|
| 736 |
+
user_request_prompt="A cyberpunk samurai walking through neon rain, slow motion. Make it look cool.",
|
| 737 |
+
target_resolution="4K",
|
| 738 |
+
max_cost_per_job=20.0, # Increased budget to allow for retries
|
| 739 |
+
request_audio=True
|
| 740 |
+
)
|
| 741 |
+
print(f"\n--- SCENARIO 1 Result ---")
|
| 742 |
+
print(json.dumps(cli_result, indent=2))
|
| 743 |
+
print(f"Current Total Spend: ${dollar_instance.current_spend:.2f}")
|
| 744 |
+
|
| 745 |
+
|
| 746 |
+
# --- Scenario 2: Simple video with script that might pass on first try or one retry
|
| 747 |
+
print("\n" + "="*80)
|
| 748 |
+
print("SCENARIO 2: Serene Forest Scene with Script")
|
| 749 |
+
print("="*80)
|
| 750 |
+
|
| 751 |
+
|
| 752 |
+
dollar_instance_2 = AgenticDollar() # New instance for a clean budget
|
| 753 |
+
dollar_instance_2.user_confirmation_callback = my_user_confirm_func
|
| 754 |
+
|
| 755 |
+
|
| 756 |
+
api_result = await dollar_instance_2.generate_advanced_video(
|
| 757 |
+
user_request_prompt="A serene forest scene with a hidden waterfall, gentle sunlight filtering through dense canopy, ancient trees.",
|
| 758 |
+
script="The ancient trees whispered secrets as crystal waters tumbled into a hidden pool, untouched by time, a haven of tranquility.",
|
| 759 |
+
target_resolution="1080p",
|
| 760 |
+
max_cost_per_job=15.0,
|
| 761 |
+
request_audio=True
|
| 762 |
+
)
|
| 763 |
+
print(f"\n--- SCENARIO 2 Result ---")
|
| 764 |
+
print(json.dumps(api_result, indent=2))
|
| 765 |
+
print(f"Current Total Spend (for this second job): ${dollar_instance_2.current_spend:.2f}")
|
| 766 |
+
|
| 767 |
+
|
| 768 |
+
|
| 769 |
+
|
| 770 |
+
if __name__ == "__main__":
|
| 771 |
+
asyncio.run(main())
|