aaross1223 commited on
Commit
492a6f7
·
verified ·
1 Parent(s): a806c64

Upload dollar.py

Browse files
Files changed (1) hide show
  1. dollar.py +771 -0
dollar.py ADDED
@@ -0,0 +1,771 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # dollar_agent_system_updated.py
2
+
3
+
4
+ import asyncio
5
+ import time
6
+ import json
7
+ from collections import deque
8
+ from typing import Any, List, Dict, Optional, Callable
9
+
10
+
11
+ # --- Dummy Litellm for Local Execution Simulation ---
12
+ # (Remains the same as in the previous complete code block)
13
+ class DummyLitellm:
14
+ async def acompletion(self, model: str, messages: List[Dict[str, Any]], max_tokens: Optional[int] = None,
15
+ temperature: Optional[float] = None, api_base: Optional[str] = None, **kwargs) -> Dict[str, Any]:
16
+
17
+ content_val = messages[0]['content']
18
+ prompt_preview = ""
19
+ if isinstance(content_val, str):
20
+ prompt_preview = content_val[:50]
21
+ elif isinstance(content_val, list):
22
+ for item in content_val:
23
+ if item.get("type") == "text":
24
+ prompt_preview = item["text"][:50]
25
+ break
26
+ elif item.get("image_url"):
27
+ prompt_preview = f"Multimodal (images): {item['image_url']['url'][:20]}..."
28
+ break
29
+ if not prompt_preview:
30
+ prompt_preview = str(content_val)[:50]
31
+
32
+
33
+ print(f" (Dummy litellm acompletion: model='{model}', prompt='{prompt_preview}'...)")
34
+ await asyncio.sleep(0.1) # Simulate network latency
35
+
36
+
37
+ # Simulate success responses based on model/tool type
38
+ if "flux" in model or "dalle" in model or "ideogram" in model or "stable-diffusion" in model:
39
+ return {'choices': [{'message': {'content': f"https://image.url/{model.replace('/', '_')}/{abs(hash(str(messages)))}.png"}}], 'model': model}
40
+ elif "sora" in model or "runway" in model or "kling" in model or "luma" in model or "pika" in model:
41
+ # Simulate slight variation in video quality for critique loop
42
+ video_hash = abs(hash(str(messages)))
43
+ quality_suffix = ""
44
+ if kwargs.get("attempt", 1) < 3: # Lower quality for first few attempts
45
+ quality_suffix = "_low_res_noisy"
46
+ return {'choices': [{'message': {'content': f"https://video.url/{model.replace('/', '_')}/{video_hash}{quality_suffix}.mp4"}}], 'model': model}
47
+ elif "perplexity" in model or "tavily" in model:
48
+ return {'choices': [{'message': {'content': f"Search results from {model} for '{prompt_preview}...'"}}], 'model': model}
49
+ elif "claude" in model or "gpt-4o" in model:
50
+ if "rewrite this prompt" in prompt_preview.lower():
51
+ original_p = kwargs.get('original_prompt', prompt_preview)
52
+ attempt_num = kwargs.get('attempt_number', 1)
53
+ # Simulate an LLM attempting to improve the prompt based on feedback
54
+ if attempt_num == 1:
55
+ rewritten_prompt = f"Optimized by {model.split('/')[-1]} (Attempt 1): {original_p} - Refined for vivid details and smooth camera movement."
56
+ elif attempt_num == 2:
57
+ rewritten_prompt = f"Optimized by {model.split('/')[-1]} (Attempt 2): {original_p} - Focusing on object consistency and cinematic color grading, as per critique."
58
+ else:
59
+ rewritten_prompt = f"Optimized by {model.split('/')[-1]} (Attempt {attempt_num}): {original_p} - Further adjustments based on refined critique."
60
+ return {'choices': [{'message': {'content': rewritten_prompt}}], 'model': model}
61
+ elif "analyze feedback" in prompt_preview.lower():
62
+ feedback = kwargs.get('feedback', '')
63
+ if "low consistency" in feedback.lower():
64
+ return {'choices': [{'message': {'content': f"LLM Planner: Feedback analyzed. Suggest modifying prompt to explicitly request 'stable camera' and 'consistent character appearance'. Consider trying 'runwayml/gen-2' for next attempt due to its reputation for consistency."}}], 'model': model}
65
+ elif "script misalignment" in feedback.lower():
66
+ return {'choices': [{'message': {'content': f"LLM Planner: Feedback analyzed. Suggest refining prompt to emphasize key elements from the script: '{feedback}'. Recommend reviewing individual scene descriptions."}}], 'model': model}
67
+ return {'choices': [{'message': {'content': f"LLM Planner: Feedback analyzed. No clear re-planning needed from: {feedback}"}}], 'model': model}
68
+ elif "does the video's content" in prompt_preview.lower():
69
+ video_url = kwargs.get('video_url', '')
70
+ # Simulate critique getting better or worse based on simulated video quality
71
+ if "_low_res_noisy.mp4" in video_url:
72
+ return {'choices': [{'message': {'content': f"Critique from {model}: Score 5/10. Low consistency, grainy visuals. Key script elements are present but overall quality is poor."}}], 'model': model}
73
+ else:
74
+ return {'choices': [{'message': {'content': f"Critique from {model}: Score 8/10. Good visual consistency. Final scene could better capture the 'untouched by time' serenity from script."}}], 'model': model}
75
+ else:
76
+ return {'choices': [{'message': {'content': f"Hello from {model}! You asked about: {prompt_preview}"}}], 'model': model}
77
+ elif "elevenlabs" in model or "audiocraft" in model or "suno" in model or "bark" in model:
78
+ return {'choices': [{'message': {'content': f"https://audio.url/{model.replace('/', '_')}/{abs(hash(str(messages)))}.mp3"}}], 'model': model}
79
+ elif "topaz-labs" in model or "ultimate-upscaler-api" in model:
80
+ original_url = kwargs.get('video_url', 'unknown_url')
81
+ return {'choices': [{'message': {'content': f"{original_url.replace('_low_res_noisy.mp4', '_upscaled.mp4').replace('.mp4', '_upscaled.mp4')}"}}], 'model': model}
82
+ elif "internal-cv-model" in model:
83
+ video_url = kwargs.get('video_url', '')
84
+ if "_low_res_noisy.mp4" in video_url:
85
+ return {'choices': [{'message': {'content': {"score": 0.55, "feedback": "Low frame-to-frame consistency, noisy."}}}]}
86
+ else:
87
+ return {'choices': [{'message': {'content': {"score": 0.85, "feedback": "Good consistency."}}}]}
88
+ else:
89
+ return {'choices': [{'message': {'content': f"Hello from {model}! You asked about: {prompt_preview}"}}], 'model': model}
90
+
91
+
92
+
93
+
94
+ # --- Tool Definitions (Remains the same as in the previous complete code block) ---
95
+ TOOL_CONFIG = {
96
+ "web_search": {
97
+ "name": "web_search",
98
+ "description": "Searches the web for information.",
99
+ "input_schema": {"query": "str"},
100
+ "models": ["perplexity/online"],
101
+ "cost_per_use": 0.001,
102
+ "latency_ms": 200,
103
+ "is_autonomous": True
104
+ },
105
+ "generate_image": {
106
+ "name": "generate_image",
107
+ "description": "Generates an image from a text prompt.",
108
+ "input_schema": {"prompt": "str", "size": "str", "quality": "str", "style": "str"},
109
+ "models": ["dalle-3", "stability-ai/stable-diffusion-xl-turbo"],
110
+ "cost_per_use": {"dalle-3": 0.04, "stability-ai/stable-diffusion-xl-turbo": 0.005},
111
+ "latency_ms": {"dalle-3": 2000, "stability-ai/stable-diffusion-xl-turbo": 500},
112
+ "is_autonomous": False,
113
+ },
114
+ "generate_video": {
115
+ "name": "generate_video",
116
+ "description": "Generates a video from a text prompt.",
117
+ "input_schema": {"prompt": "str", "resolution": "str", "duration": "int", "custom_params": "dict"},
118
+ "models": ["kling", "sora", "runwayml/gen-2", "pika-labs"],
119
+ "cost_per_use": {"kling": 5.00, "sora": 10.00, "runwayml/gen-2": 0.50, "pika-labs": 0.30},
120
+ "latency_ms": {"kling": 30000, "sora": 60000, "runwayml/gen-2": 10000, "pika-labs": 8000},
121
+ "is_autonomous": False
122
+ },
123
+ "optimize_prompt": {
124
+ "name": "optimize_prompt",
125
+ "description": "Rewrites and enhances a prompt for optimal generation.",
126
+ "input_schema": {"original_prompt": "str", "target_model_type": "str"},
127
+ "models": ["claude-3-5-sonnet", "gpt-4o"],
128
+ "cost_per_use": 0.01,
129
+ "latency_ms": 1500,
130
+ "is_autonomous": True
131
+ },
132
+ "upscale_video": {
133
+ "name": "upscale_video",
134
+ "description": "Upscales a given video URL to higher resolution/quality.",
135
+ "input_schema": {"video_url": "str", "target_resolution": "str"},
136
+ "models": ["topaz-labs/video-ai", "ultimate-upscaler-api"],
137
+ "cost_per_use": 0.20,
138
+ "latency_ms": 15000,
139
+ "is_autonomous": True
140
+ },
141
+ "generate_audio": {
142
+ "name": "generate_audio",
143
+ "description": "Generates audio (speech, music, sound effects) from a text prompt.",
144
+ "input_schema": {"prompt": "str", "audio_type": "str"},
145
+ "models": ["elevenlabs/speech", "audiocraft/musicgen", "suno/bark"],
146
+ "cost_per_use": 0.05,
147
+ "latency_ms": 5000,
148
+ "is_autonomous": True
149
+ },
150
+ "compare_frames": {
151
+ "name": "compare_frames",
152
+ "description": "Compares first and last frames of a video for consistency.",
153
+ "input_schema": {"video_url": "str"},
154
+ "models": ["internal-cv-model"],
155
+ "cost_per_use": 0.005,
156
+ "latency_ms": 500,
157
+ "is_autonomous": True
158
+ },
159
+ "local_animatediff": {
160
+ "name": "local_animatediff",
161
+ "description": "Generates video locally using ComfyUI + AnimateDiff.",
162
+ "input_schema": {"prompt": "str", "resolution": "str", "duration": "int"},
163
+ "models": ["local"],
164
+ "cost_per_use": 0.0,
165
+ "latency_ms": 60000,
166
+ "is_autonomous": True
167
+ },
168
+ "llm_replan": { # New internal tool for re-planning based on critique
169
+ "name": "llm_replan",
170
+ "description": "Analyzes critique feedback and suggests modifications to prompt or strategy for next generation attempt.",
171
+ "input_schema": {"current_prompt": "str", "feedback": "str", "previous_model": "str", "attempt_number": "int"},
172
+ "models": ["claude-3-5-sonnet", "gpt-4o"],
173
+ "cost_per_use": 0.015,
174
+ "latency_ms": 2000,
175
+ "is_autonomous": True
176
+ }
177
+ }
178
+
179
+
180
+ # --- Default Configuration from dollar.yaml (Conceptual) ---
181
+ DEFAULT_CONFIG = {
182
+ "budget": {
183
+ "hard_cap": 100.0,
184
+ "warn_at": 75.0
185
+ },
186
+ "preferences": {
187
+ "default_video_model": "kling",
188
+ "fallback_chain": ["kling", "sora", "runwayml/gen-2", "pika-labs", "local_animatediff"],
189
+ "upscaling": "always",
190
+ "audio": True,
191
+ "self_critique": "strict", # "strict", "moderate", "none"
192
+ "style_consistency_enforcement": "high",
193
+ "default_style": "cinematic",
194
+ "max_generation_attempts": 3 # New config item
195
+ },
196
+ "agent_settings": {
197
+ "memory_size": 20,
198
+ "cost_weight": 1000,
199
+ "latency_weight": 1
200
+ }
201
+ }
202
+
203
+
204
+
205
+
206
+ # --- AgenticDollar Class ---
207
+ class AgenticDollar:
208
+ def __init__(self, config: Dict[str, Any] = None, litellm_client: Any = None):
209
+ self.config = config if config is not None else DEFAULT_CONFIG
210
+ self.litellm = litellm_client if litellm_client else DummyLitellm()
211
+ self.tools = TOOL_CONFIG
212
+ self.working_memory = deque(maxlen=self.config["agent_settings"].get("memory_size", 10))
213
+ self.budget_hard_cap = self.config["budget"]["hard_cap"]
214
+ self.budget_warn_at = self.config["budget"]["warn_at"]
215
+ self.current_spend = 0.0
216
+ self.user_confirmation_callback: Optional[Callable[[str], bool]] = None
217
+
218
+
219
+ print(f"AgenticDollar initialized with hard budget cap: ${self.budget_hard_cap:.2f}")
220
+ print(f"Preferences: {json.dumps(self.config['preferences'], indent=2)}")
221
+
222
+
223
+ async def _log_action(self, action_type: str, details: Dict[str, Any]):
224
+ """Logs actions for transparency and future self-improvement."""
225
+ log_entry = {"timestamp": time.time(), "action_type": action_type, "details": details}
226
+ self.working_memory.append(log_entry)
227
+ print(f" [LOG] {action_type}: {details.get('tool', '')} - {details.get('message', '')[:50]}...")
228
+
229
+
230
+ async def _ask_for_user_confirmation(self, message: str) -> bool:
231
+ """Asks the user for confirmation for high-cost or critical actions."""
232
+ if self.user_confirmation_callback:
233
+ return await asyncio.to_thread(self.user_confirmation_callback, message)
234
+ print(f" [USER CONFIRMATION REQUIRED]: {message} (Simulating 'yes' for now)")
235
+ return True
236
+
237
+
238
+ async def _select_best_tool_and_model(self, task_description: str, tool_type: str,
239
+ max_cost_for_step: float = float('inf')) -> Optional[Dict]:
240
+ """
241
+ Dynamically selects the most appropriate tool and model based on task,
242
+ cost, and available options, respecting preference for `fallback_chain`.
243
+ Latency is now handled more implicitly by the fallback order.
244
+ """
245
+
246
+ eligible_tools_defs = {t_name: t_def for t_name, t_def in self.tools.items()
247
+ if t_def.get("name") == tool_type}
248
+
249
+
250
+ if not eligible_tools_defs:
251
+ return None
252
+
253
+
254
+ best_option = None
255
+
256
+ # Determine the order of models to try based on preferences or default
257
+ # For video generation, use the explicit fallback chain
258
+ if tool_type == "generate_video":
259
+ model_selection_order = self.config["preferences"]["fallback_chain"]
260
+ else:
261
+ # For other tools, find all models and sort by a simple heuristic (e.g., lowest cost first)
262
+ all_possible_models = []
263
+ for t_def in eligible_tools_defs.values():
264
+ for m in t_def.get("models", []):
265
+ cost_val = t_def["cost_per_use"].get(m, t_def["cost_per_use"]) if isinstance(t_def["cost_per_use"], dict) else t_def["cost_per_use"]
266
+ all_possible_models.append((m, cost_val))
267
+ model_selection_order = [m for m, _ in sorted(all_possible_models, key=lambda x: x[1])]
268
+
269
+ for model_name in model_selection_order:
270
+ tool_found_name = None
271
+ tool_def = None
272
+ for t_name, t_def_candidate in eligible_tools_defs.items():
273
+ if model_name in t_def_candidate.get("models", []):
274
+ tool_found_name = t_name
275
+ tool_def = t_def_candidate
276
+ break
277
+
278
+ if not tool_def:
279
+ continue
280
+
281
+
282
+ cost = tool_def["cost_per_use"].get(model_name, tool_def["cost_per_use"]) if isinstance(tool_def["cost_per_use"], dict) else tool_def["cost_per_use"]
283
+ latency = tool_def["latency_ms"].get(model_name, tool_def["latency_ms"]) if isinstance(tool_def["latency_ms"], dict) else tool_def["latency_ms"]
284
+
285
+
286
+ # Check if this model exceeds budget for this step or overall hard cap
287
+ if cost <= max_cost_for_step and (self.current_spend + cost <= self.budget_hard_cap):
288
+ best_option = {"tool_name": tool_found_name, "model_name": model_name, "cost": cost, "latency": latency}
289
+ # Since the fallback chain implies preference/order, take the first one that fits
290
+ break # Found the best available model in the preferred order within budget
291
+
292
+ # We only update current_spend upon successful _execution_ of the tool
293
+ return best_option
294
+
295
+
296
+ async def _execute_tool(self, tool_option: Dict, **kwargs) -> Any:
297
+ """Executes the chosen tool and handles potential retries/fallbacks."""
298
+ tool_name = tool_option["tool_name"]
299
+ model_name = tool_option["model_name"]
300
+ cost = tool_option["cost"]
301
+
302
+
303
+ # Special handling for local fallback tool
304
+ if tool_name == "local_animatediff":
305
+ print(f" [LOCAL EXECUTION] Executing {tool_name} locally with params: {kwargs}")
306
+ await asyncio.sleep(tool_option["latency"])
307
+ return {"result_url": f"https://local.server/{tool_name}_{abs(hash(str(kwargs)))}.mp4", "status": "completed"}
308
+
309
+
310
+ messages = [{"role": "user", "content": json.dumps(kwargs)}]
311
+
312
+
313
+ try:
314
+ response = await self.litellm.acompletion(model=model_name, messages=messages, **kwargs)
315
+ content = response['choices'][0]['message']['content']
316
+ self._log_action("TOOL_EXECUTION", {"tool": tool_name, "model": model_name, "cost": cost, "result_preview": content[:100]})
317
+ self.current_spend += cost
318
+ return {"result": content, "status": "completed"}
319
+ except Exception as e:
320
+ self._log_action("TOOL_EXECUTION_FAILURE", {"tool": tool_name, "model": model_name, "error": str(e)})
321
+ print(f" [ERROR] Tool execution failed for {tool_name}/{model_name}: {e}")
322
+ return {"result": None, "status": "failed", "error": str(e)}
323
+
324
+
325
+ async def _optimize_prompt(self, original_prompt: str, target_model_type: str, current_spend_context: float, attempt_number: int) -> str:
326
+ """Uses an LLM to rewrite and enhance a prompt for optimal generation."""
327
+ if self.config["preferences"]["self_critique"] == "none":
328
+ print(" [PROMPT OPTIMIZATION] Skipping prompt optimization based on preferences.")
329
+ return original_prompt
330
+
331
+
332
+ print(f" [PROMPT OPTIMIZATION] Optimizing prompt for {target_model_type} (Attempt {attempt_number})...")
333
+ remaining_budget = self.budget_hard_cap - current_spend_context
334
+ # Cap optimization cost to a small fraction of remaining budget or a fixed small amount
335
+ max_opt_cost = min(0.05, remaining_budget * 0.1) # Max 5 cents or 10% of remaining budget
336
+
337
+ opt_tool = await self._select_best_tool_and_model(
338
+ "Optimize prompt", "optimize_prompt", max_cost_for_step=max_opt_cost
339
+ )
340
+ if not opt_tool:
341
+ print(" [WARNING] No prompt optimization tool available or within budget. Using original prompt.")
342
+ return original_prompt
343
+
344
+
345
+ optimized_response = await self._execute_tool(
346
+ opt_tool,
347
+ original_prompt=original_prompt,
348
+ target_model_type=target_model_type,
349
+ attempt_number=attempt_number # Pass attempt number for dummy to vary output
350
+ )
351
+ return optimized_response["result"] if optimized_response["status"] == "completed" else original_prompt
352
+
353
+
354
+ async def _analyze_critique_and_replan(self, current_prompt: str, feedback: Dict[str, Any], previous_model: str, attempt_number: int) -> Dict[str, Any]:
355
+ """
356
+ Uses an LLM to analyze critique feedback and suggest modifications
357
+ to the prompt or strategy for the next generation attempt.
358
+ Returns a dict with 'new_prompt' and optionally 'suggested_model'.
359
+ """
360
+ print(f" [REPLANNING] Analyzing critique for attempt {attempt_number}...")
361
+ replan_tool = await self._select_best_tool_and_model("Analyze critique and replan", "llm_replan")
362
+ if not replan_tool:
363
+ print(" [WARNING] No replanning tool available. Cannot intelligently re-plan.")
364
+ return {"new_prompt": current_prompt, "suggested_model": None}
365
+
366
+
367
+ # Structure the feedback for the LLM
368
+ feedback_str = json.dumps(feedback)
369
+
370
+ llm_response = await self._execute_tool(
371
+ replan_tool,
372
+ current_prompt=current_prompt,
373
+ feedback=feedback_str,
374
+ previous_model=previous_model,
375
+ attempt_number=attempt_number
376
+ )
377
+
378
+
379
+ if llm_response["status"] == "completed":
380
+ # Simulate parsing LLM's suggested new prompt and model
381
+ response_content = llm_response["result"]
382
+ new_prompt = current_prompt # Default to no change
383
+ suggested_model = None
384
+
385
+
386
+ if "Suggest modifying prompt to explicitly request" in response_content:
387
+ # Simple parsing of dummy LLM response
388
+ parts = response_content.split("'")
389
+ if len(parts) > 1:
390
+ new_prompt = current_prompt + " (Refined based on critique: " + parts[1] + ")"
391
+ if "Consider trying 'runwayml/gen-2'" in response_content:
392
+ suggested_model = "runwayml/gen-2"
393
+ elif "refining prompt to emphasize key elements" in response_content:
394
+ new_prompt = current_prompt + " (Emphasizing script elements based on critique)"
395
+
396
+
397
+ print(f" [REPLANNING] LLM suggested new prompt: '{new_prompt}'")
398
+ if suggested_model:
399
+ print(f" [REPLANNING] LLM suggested new model: '{suggested_model}'")
400
+
401
+
402
+ return {"new_prompt": new_prompt, "suggested_model": suggested_model}
403
+
404
+ print(" [WARNING] Replanning LLM failed. No changes suggested.")
405
+ return {"new_prompt": current_prompt, "suggested_model": None}
406
+
407
+
408
+
409
+
410
+ async def _assess_video_quality(self, video_url: str, script: str) -> Dict[str, Any]:
411
+ """Performs self-critique: frame consistency and script adherence."""
412
+ self_critique_level = self.config["preferences"]["self_critique"]
413
+ if self_critique_level == "none":
414
+ print(" [QUALITY ASSESSMENT] Skipping self-critique based on preferences.")
415
+ return {"frame_consistency": {"score": 1.0, "feedback": "skipped"}, "script_adherence": "skipped", "overall_pass": True, "feedback_messages": []}
416
+
417
+
418
+ print(f" [QUALITY ASSESSMENT] Assessing video quality for {video_url} (Level: {self_critique_level})...")
419
+ overall_pass = True
420
+ feedback_messages = []
421
+ frame_consistency_result = {"score": 1.0, "feedback": "skipped"}
422
+ script_adherence_result = "skipped"
423
+
424
+
425
+ # 1. Frame Consistency Scoring
426
+ if self_critique_level in ["strict", "moderate"]:
427
+ consistency_tool = await self._select_best_tool_and_model("Check video frame consistency", "compare_frames",
428
+ max_cost_for_step=self.budget_hard_cap - self.current_spend)
429
+ if consistency_tool:
430
+ consistency_response = await self._execute_tool(consistency_tool, video_url=video_url)
431
+ if consistency_response["status"] == "completed":
432
+ content = consistency_response["result"] # This is the dict from DummyLitellm
433
+ if isinstance(content, dict) and "score" in content:
434
+ frame_consistency_result = content
435
+ if content["score"] < 0.7: # Example threshold for failure
436
+ overall_pass = False
437
+ feedback_messages.append(f"Low frame consistency ({content['score']}): {content['feedback']}")
438
+ else:
439
+ feedback_messages.append("Could not parse frame consistency score.")
440
+ else:
441
+ feedback_messages.append("Frame consistency tool execution failed.")
442
+ print(f" Frame consistency check: {frame_consistency_result}")
443
+ else:
444
+ feedback_messages.append("Frame consistency tool unavailable or out of budget.")
445
+ print(" Frame consistency tool unavailable.")
446
+ else:
447
+ print(" Frame consistency check skipped (self_critique level).")
448
+
449
+
450
+ # 2. Script Adherence (using an LLM to compare video description to script)
451
+ if self_critique_level == "strict" and script:
452
+ llm_critique_tool = await self._select_best_tool_and_model("Critique video against script", "optimize_prompt",
453
+ max_cost_for_step=self.budget_hard_cap - self.current_spend)
454
+ if llm_critique_tool:
455
+ critique_response = await self._execute_tool(
456
+ llm_critique_tool,
457
+ original_prompt=f"Critique video at {video_url} against script: '{script}'",
458
+ target_model_type="video_critique_llm",
459
+ video_url=video_url # Pass for DummyLitellm's quality simulation
460
+ )
461
+ if critique_response["status"] == "completed":
462
+ script_adherence_result = critique_response["result"]
463
+ if "score 8/10" not in script_adherence_result: # Example parsing of failure for strict mode
464
+ overall_pass = False
465
+ feedback_messages.append(f"LLM critique indicated script misalignment: {script_adherence_result}")
466
+ else:
467
+ feedback_messages.append("LLM critique tool execution failed.")
468
+ print(f" Script adherence critique: {script_adherence_result}")
469
+ else:
470
+ feedback_messages.append("LLM critique tool unavailable or out of budget.")
471
+ print(" Script adherence critique tool unavailable.")
472
+ else:
473
+ print(" Script adherence critique skipped (self_critique level or no script).")
474
+
475
+
476
+ return {
477
+ "frame_consistency": frame_consistency_result,
478
+ "script_adherence": script_adherence_result,
479
+ "overall_pass": overall_pass,
480
+ "feedback_messages": feedback_messages
481
+ }
482
+
483
+
484
+ async def generate_advanced_video(self, user_request_prompt: str, script: str = "",
485
+ target_resolution: Optional[str] = None, max_duration: Optional[int] = None,
486
+ max_cost_per_job: Optional[float] = None,
487
+ request_audio: Optional[bool] = None,
488
+ request_upscaling: Optional[str] = None
489
+ ) -> Dict[str, Any]:
490
+ """
491
+ Orchestrates the generation of a high-quality video with advanced features,
492
+ respecting config and command-line overrides, including an iterative self-critique loop.
493
+ """
494
+ print(f"\n--- Initiating Advanced Video Generation ---")
495
+ print(f"User Request: '{user_request_prompt}'")
496
+ if script:
497
+ print(f"Script provided: '{script[:100]}...'")
498
+
499
+
500
+ # --- Apply Config Defaults and Command-Line Overrides ---
501
+ actual_max_cost_per_job = max_cost_per_job if max_cost_per_job is not None else self.budget_hard_cap
502
+ actual_resolution = target_resolution if target_resolution is not None else "1080p"
503
+ actual_audio_preference = request_audio if request_audio is not None else self.config["preferences"]["audio"]
504
+ actual_upscaling_preference = request_upscaling if request_upscaling is not None else self.config["preferences"]["upscaling"]
505
+ actual_self_critique_rigor = self.config["preferences"]["self_critique"]
506
+ max_attempts = self.config["preferences"]["max_generation_attempts"]
507
+
508
+ current_video_prompt = user_request_prompt # This prompt will be refined in the loop
509
+ generated_video_url = None
510
+ video_generator_used = "N/A"
511
+ final_quality_assessment = {}
512
+ all_attempt_details = []
513
+
514
+
515
+ # --- Budget Checks ---
516
+ if self.current_spend + actual_max_cost_per_job > self.budget_hard_cap:
517
+ print(f" [BUDGET ALERT] Job would exceed hard cap of ${self.budget_hard_cap:.2f}. Current spend: ${self.current_spend:.2f}. Requested job cost: ${actual_max_cost_per_job:.2f}.")
518
+ return {"status": "aborted", "message": "Job exceeds overall budget cap."}
519
+ elif self.current_spend + actual_max_cost_per_job > self.budget_warn_at:
520
+ print(f" [BUDGET WARNING] Job will cause total spend to exceed warning threshold of ${self.budget_warn_at:.2f}. Current spend: ${self.current_spend:.2f}.")
521
+
522
+
523
+ # --- User Confirmation (for high-cost/critical tasks) ---
524
+ if not await self._ask_for_user_confirmation(
525
+ f"This video generation could cost up to ${actual_max_cost_per_job:.2f} and take several minutes. Proceed?"
526
+ ):
527
+ return {"status": "aborted", "message": "User cancelled operation."}
528
+
529
+
530
+ # --- Iterative Self-Critique & Refinement Loop ---
531
+ for attempt in range(1, max_attempts + 1):
532
+ print(f"\n--- GENERATION ATTEMPT {attempt}/{max_attempts} ---")
533
+
534
+
535
+ # 1. Auto-Prompt Optimization (or Re-optimization based on critique)
536
+ optimized_prompt_for_attempt = await self._optimize_prompt(current_video_prompt, "video_generator", self.current_spend, attempt)
537
+ print(f"Prompt for this attempt: '{optimized_prompt_for_attempt}'")
538
+
539
+ # 2. Dynamic Tool/Model Selection for Video Generation
540
+ # If the replanning LLM suggested a model, prioritize it for this attempt
541
+ suggested_model_for_attempt = None
542
+ if attempt > 1 and all_attempt_details[-1].get("replan_suggestions"):
543
+ suggested_model_for_attempt = all_attempt_details[-1]["replan_suggestions"].get("suggested_model")
544
+ if suggested_model_for_attempt:
545
+ print(f" [MODEL OVERRIDE] Prioritizing LLM-suggested model: {suggested_model_for_attempt}")
546
+
547
+
548
+ # Calculate remaining budget for this generation attempt
549
+ remaining_budget_for_generation = actual_max_cost_per_job - (self.current_spend - (self.current_spend if self.current_spend < actual_max_cost_per_job else 0)) # simplified
550
+
551
+
552
+ video_tool_option = await self._select_best_tool_and_model(
553
+ f"Generate video for: {optimized_prompt_for_attempt}",
554
+ "generate_video",
555
+ max_cost_for_step=remaining_budget_for_generation
556
+ )
557
+ # If a model was explicitly suggested by the replanning LLM, try to use it if available
558
+ if suggested_model_for_attempt and video_tool_option and video_tool_option["model_name"] != suggested_model_for_attempt:
559
+ # Re-evaluate, forcing the suggested model if possible
560
+ forced_tool_option = None
561
+ for t_name, t_def in self.tools.items():
562
+ if t_def.get("name") == "generate_video" and suggested_model_for_attempt in t_def.get("models", []):
563
+ cost = t_def["cost_per_use"].get(suggested_model_for_attempt, t_def["cost_per_use"]) if isinstance(t_def["cost_per_use"], dict) else t_def["cost_per_use"]
564
+ latency = t_def["latency_ms"].get(suggested_model_for_attempt, t_def["latency_ms"]) if isinstance(t_def["latency_ms"], dict) else t_def["latency_ms"]
565
+ if cost <= remaining_budget_for_generation and (self.current_spend + cost <= self.budget_hard_cap):
566
+ forced_tool_option = {"tool_name": t_name, "model_name": suggested_model_for_attempt, "cost": cost, "latency": latency}
567
+ break
568
+ if forced_tool_option:
569
+ video_tool_option = forced_tool_option
570
+ print(f" [MODEL OVERRIDE] Successfully switched to LLM-suggested model: {suggested_model_for_attempt}")
571
+
572
+
573
+
574
+
575
+ if not video_tool_option:
576
+ all_attempt_details.append({"attempt": attempt, "status": "failed_no_model", "message": "Could not find a suitable video generation model within budget/constraints."})
577
+ print(f" [FAILURE] Attempt {attempt} failed: No suitable video generation model.")
578
+ break # Cannot proceed without a model
579
+
580
+
581
+ video_generator_used = video_tool_option["model_name"]
582
+ print(f"Selected video generator for attempt {attempt}: {video_generator_used}")
583
+
584
+
585
+ # 3. Video Generation Attempt
586
+ video_result = await self._execute_tool(
587
+ video_tool_option,
588
+ prompt=optimized_prompt_for_attempt,
589
+ resolution=actual_resolution,
590
+ duration=max_duration,
591
+ attempt=attempt # Pass attempt number for dummy to vary output
592
+ )
593
+
594
+
595
+ if video_result["status"] != "completed" or not video_result["result"]:
596
+ all_attempt_details.append({"attempt": attempt, "status": "failed_generation", "message": video_result.get("error", "Generation failed.")})
597
+ print(f" [FAILURE] Attempt {attempt} failed: {video_result.get('error', 'Generation failed.')}")
598
+ # Decide if we can retry with a different model or if it's a hard failure
599
+ if attempt == max_attempts:
600
+ return {"status": "failed", "message": "Video generation failed after all attempts."}
601
+ else:
602
+ # For a real system, here we'd analyze the error and decide to replan or try next model in fallback
603
+ print(" [RETRY] Attempting next iteration without explicit replanning for now.")
604
+ continue # Try next iteration
605
+
606
+
607
+ generated_video_url = video_result["result"]
608
+ print(f"Generated Video URL (Attempt {attempt}): {generated_video_url}")
609
+
610
+
611
+ # 4. Self-Critique Loop
612
+ final_quality_assessment = await self._assess_video_quality(generated_video_url, script)
613
+ all_attempt_details.append({
614
+ "attempt": attempt,
615
+ "status": "completed",
616
+ "video_url": generated_video_url,
617
+ "assessment": final_quality_assessment,
618
+ "optimized_prompt": optimized_prompt_for_attempt,
619
+ "video_generator": video_generator_used
620
+ })
621
+
622
+
623
+ if final_quality_assessment["overall_pass"]:
624
+ print(f" [SUCCESS] Video passed quality assessment on attempt {attempt}!")
625
+ break # Exit loop, quality met
626
+ else:
627
+ print(f" [FAILURE] Video did NOT pass quality assessment on attempt {attempt}: {final_quality_assessment['feedback_messages']}")
628
+ if attempt < max_attempts:
629
+ # 5. Analyze Critique and Re-plan for next attempt
630
+ replan_result = await self._analyze_critique_and_replan(
631
+ current_prompt=current_video_prompt, # Use the user's initial prompt as base for replanning
632
+ feedback=final_quality_assessment,
633
+ previous_model=video_generator_used,
634
+ attempt_number=attempt
635
+ )
636
+ current_video_prompt = replan_result["new_prompt"] # Update prompt for next attempt
637
+ all_attempt_details[-1]["replan_suggestions"] = replan_result # Store replan suggestions
638
+ print(f" [RETRYING] Retrying with refined prompt: '{current_video_prompt}'")
639
+ # The suggested model from replan will be prioritized in the next loop iteration
640
+ else:
641
+ print(f" [FAILURE] Max attempts ({max_attempts}) reached. Video did not meet quality standards.")
642
+ return {"status": "failed", "message": "Video did not meet quality standards after max attempts."}
643
+
644
+
645
+ if not generated_video_url:
646
+ return {"status": "failed", "message": "No video could be successfully generated."}
647
+
648
+
649
+
650
+
651
+ # --- Post-Processing Steps (executed only after a quality-approved video is generated) ---
652
+
653
+ # 6. Video Upscaling
654
+ final_video_url = generated_video_url
655
+ if actual_upscaling_preference == "always" or \
656
+ (actual_upscaling_preference == "if_needed" and actual_resolution not in final_video_url): # Simplified heuristic
657
+
658
+ print(" [UPSCALING] Attempting to upscale video...")
659
+ upscale_tool_option = await self._select_best_tool_and_model("Upscale video", "upscale_video",
660
+ max_cost_for_step=self.budget_hard_cap - self.current_spend)
661
+ if upscale_tool_option:
662
+ upscaled_video_result = await self._execute_tool(
663
+ upscale_tool_option,
664
+ video_url=generated_video_url,
665
+ target_resolution=actual_resolution
666
+ )
667
+ if upscaled_video_result["status"] == "completed":
668
+ final_video_url = upscaled_video_result["result"]
669
+ print(f"Upscaled Video URL: {final_video_url}")
670
+ else:
671
+ print(" [WARNING] Video upscaling failed. Using raw generated video.")
672
+ else:
673
+ print(" [WARNING] Video upscaling tool not available or out of budget. Using raw generated video.")
674
+ else:
675
+ print(f" [UPSCALING] Upscaling preference set to '{actual_upscaling_preference}', skipping.")
676
+
677
+
678
+ # 7. Audio Generation
679
+ generated_audio_url = None
680
+ if script and actual_audio_preference:
681
+ print(" [AUDIO GENERATION] Generating audio for script...")
682
+ audio_tool_option = await self._select_best_tool_and_model("Generate audio from script", "generate_audio",
683
+ max_cost_for_step=self.budget_hard_cap - self.current_spend)
684
+ if audio_tool_option:
685
+ audio_result = await self._execute_tool(
686
+ audio_tool_option,
687
+ prompt=script,
688
+ audio_type="speech"
689
+ )
690
+ if audio_result["status"] == "completed":
691
+ generated_audio_url = audio_result["result"]
692
+ print(f"Generated Audio URL: {generated_audio_url}")
693
+ else:
694
+ print(" [WARNING] Audio generation failed.")
695
+ else:
696
+ print(" [WARNING] Audio generation tool not available or out of budget.")
697
+ else:
698
+ print(" [AUDIO GENERATION] Audio generation skipped based on preferences or no script.")
699
+
700
+
701
+ return {
702
+ "status": "completed",
703
+ "final_video_url": final_video_url,
704
+ "generated_audio_url": generated_audio_url,
705
+ "details": {
706
+ "initial_prompt": user_request_prompt,
707
+ "final_optimized_prompt": current_video_prompt,
708
+ "video_generator_used": video_generator_used,
709
+ "quality_assessment_final": final_quality_assessment,
710
+ "total_cost_incurred_for_job": self.current_spend,
711
+ "generation_attempts": all_attempt_details
712
+ }
713
+ }
714
+
715
+
716
+ # --- Example Usage (How you would interact with Dollar via CLI or API) ---
717
+ async def main():
718
+ dollar_instance = AgenticDollar()
719
+
720
+
721
+ async def my_user_confirm_func(message: str) -> bool:
722
+ response = await asyncio.to_thread(input, f"Confirm (y/n): {message} ")
723
+ return response.lower() == 'y'
724
+ dollar_instance.user_confirmation_callback = my_user_confirm_func
725
+
726
+
727
+
728
+
729
+ # --- Scenario 1: Simulate CLI command with a prompt that might need refinement
730
+ print("\n" + "="*80)
731
+ print("SCENARIO 1: Cyberpunk Samurai Video (requiring refinement)")
732
+ print("="*80)
733
+
734
+
735
+ cli_result = await dollar_instance.generate_advanced_video(
736
+ user_request_prompt="A cyberpunk samurai walking through neon rain, slow motion. Make it look cool.",
737
+ target_resolution="4K",
738
+ max_cost_per_job=20.0, # Increased budget to allow for retries
739
+ request_audio=True
740
+ )
741
+ print(f"\n--- SCENARIO 1 Result ---")
742
+ print(json.dumps(cli_result, indent=2))
743
+ print(f"Current Total Spend: ${dollar_instance.current_spend:.2f}")
744
+
745
+
746
+ # --- Scenario 2: Simple video with script that might pass on first try or one retry
747
+ print("\n" + "="*80)
748
+ print("SCENARIO 2: Serene Forest Scene with Script")
749
+ print("="*80)
750
+
751
+
752
+ dollar_instance_2 = AgenticDollar() # New instance for a clean budget
753
+ dollar_instance_2.user_confirmation_callback = my_user_confirm_func
754
+
755
+
756
+ api_result = await dollar_instance_2.generate_advanced_video(
757
+ user_request_prompt="A serene forest scene with a hidden waterfall, gentle sunlight filtering through dense canopy, ancient trees.",
758
+ script="The ancient trees whispered secrets as crystal waters tumbled into a hidden pool, untouched by time, a haven of tranquility.",
759
+ target_resolution="1080p",
760
+ max_cost_per_job=15.0,
761
+ request_audio=True
762
+ )
763
+ print(f"\n--- SCENARIO 2 Result ---")
764
+ print(json.dumps(api_result, indent=2))
765
+ print(f"Current Total Spend (for this second job): ${dollar_instance_2.current_spend:.2f}")
766
+
767
+
768
+
769
+
770
+ if __name__ == "__main__":
771
+ asyncio.run(main())