#!/usr/bin/env python3 """ Context & Codebase Dumper for VHectorLab 3D. Generates vhectorlab-context.txt containing full context, architecture specifications, developer protocol lessons, configuration, and source code for AI assistant analysis. Optimized to produce a compact context payload (~100k-180k tokens) suitable for LLM context windows. """ import os import sys import argparse from datetime import datetime from pathlib import Path # Included file extensions for general source code scanning SOURCE_EXTENSIONS = { ".py", ".js", ".ts", ".jsx", ".tsx", ".css", ".html", ".glsl", ".json", ".toml", ".yml", ".yaml", ".md", ".sh", ".mjs" } # Binary file extensions to strictly exclude BINARY_EXTENSIONS = { ".png", ".jpg", ".jpeg", ".gif", ".ico", ".pdf", ".zip", ".tar", ".gz", ".npz", ".pt", ".mp4", ".webm", ".mov", ".ttf", ".woff", ".woff2", ".eot" } # Directories to strictly exclude from code scan EXCLUDE_DIRS = { ".git", "node_modules", ".venv", "venv", "dist", ".pnpm-store", ".pytest_cache", "__pycache__", ".vite", "htmlcov", ".DS_Store", "demo", "archivo", ".agents", "skills", ".claude", ".gemini" } # Files to explicitly exclude from code scan EXCLUDE_FILES = { "vhectorlab-context.txt", "vhectorlab_dev.log", "vectorlab_dev.log", "public/vocab.txt", "vocab.txt", "vocab_embeddings.npz", "pnpm-lock.yaml", "package-lock.json", "CONTEXT_HAB.MD", "scripts/capture-galaxy-demo.mjs", "capture-galaxy-demo.mjs", "diagnose_group_separation.py" } # Explicitly ordered root context & specification files (placed at top of dump) PRIORITY_DOCS = [ "manifest.json", "README.md", "CONTEXT.md", "architecture_spec.md", "CHANGELOG.md", "roadmap/README.md", "roadmap/gui-art.md", "roadmap/sae-denoise.md", ] def is_binary_file(filepath: Path) -> bool: """Check if file is binary by checking extensions or initial bytes.""" if filepath.suffix.lower() in BINARY_EXTENSIONS: return True try: with open(filepath, "rb") as f: chunk = f.read(1024) if b"\x00" in chunk: return True except Exception: return True return False def collect_project_files(root_dir: Path, include_tests: bool = False): """Collect relevant project files grouped by relative path.""" collected = [] for dirpath, dirnames, filenames in os.walk(root_dir): # Filter directories in-place dirnames[:] = [ d for d in dirnames if d not in EXCLUDE_DIRS and not d.startswith(".venv") and (include_tests or d != "tests") ] rel_dir = Path(dirpath).relative_to(root_dir) # Skip tests directory unless explicitly included if not include_tests: if "tests" in rel_dir.parts or "backend/tests" in str(rel_dir): continue for filename in sorted(filenames): rel_file = rel_dir / filename if str(rel_dir) != "." else Path(filename) rel_path_str = str(rel_file) # Check exclusions if filename in EXCLUDE_FILES or rel_path_str in EXCLUDE_FILES: continue if filename.startswith(".") and filename != ".gitignore" and filename != ".env.example": continue if filename.startswith("PROMPT-"): continue if filename.endswith(".log") or filename.endswith(".npz") or filename.endswith(".pt"): continue if not include_tests and (filename.endswith(".test.js") or filename.startswith("test_")): continue filepath = root_dir / rel_file if filepath.suffix.lower() not in SOURCE_EXTENSIONS and filename not in {"Dockerfile", "LICENSE", "NOTICE", "setup.sh"}: continue if is_binary_file(filepath): continue collected.append(rel_file) # Sort files logically: priority docs first, then alphabetical by relative path def file_sort_key(p: Path): p_str = str(p) if p_str in PRIORITY_DOCS: return (0, PRIORITY_DOCS.index(p_str)) return (1, p_str) collected.sort(key=file_sort_key) return collected def generate_context_dump(root_dir: Path, output_path: Path, include_tests: bool = False, max_changelog_lines: int = 150): """Generate the complete context and codebase dump txt file.""" if output_path.exists(): print(f"Removing existing dump file: {output_path}") output_path.unlink() project_files = collect_project_files(root_dir, include_tests=include_tests) timestamp = datetime.now().isoformat() print(f"Gathering {len(project_files)} project files into {output_path.name} (include_tests={include_tests})...") total_lines = 0 file_count = 0 with open(output_path, "w", encoding="utf-8") as out: # Header block out.write("================================================================================\n") out.write("VHECTORLAB 3D - COMPLETE CODEBASE & CONTEXT DUMP FOR AI AGENT ANALYSIS\n") out.write(f"Generated: {timestamp}\n") out.write(f"Repository Root: {root_dir.resolve()}\n") out.write(f"Total Source Files Included: {len(project_files)}\n") out.write(f"Include Tests: {include_tests}\n") out.write("================================================================================\n\n") # Project Summary / Description out.write("--- 1. PROJECT OVERVIEW & ARCHITECTURE SUMMARY ---\n") out.write("VHectorLab 3D is a WebGL vector space visualizer & LLM embedding analytics workspace.\n") out.write("- Backend: FastAPI (Python 3.11+, `uv`), SentenceTransformers, PyTorch, SAE (Sparse Autoencoders).\n") out.write("- Frontend: Vanilla JS + Three.js WebGL (Vite, npm).\n") out.write("- Modes: ARITHMETIC (vector algebra) & COMPARE (cosine similarity & multi-token comparison).\n") out.write("- Views: ANALYSIS (vertical stack), NAVIGATION (3D corridor), GALAXY (2D/3D UMAP manifold).\n") out.write("- Render Modes: POINTS & RIBBONS (wide quad strips).\n") out.write("- Quality & Protocol: Managed via `.agents/skills/dev-protocol/` (TDD, deep modules, lessons-learned).\n\n") # Directory Tree Section out.write("--- 2. REPOSITORY FILE TREE ---\n") for pf in project_files: out.write(f"- {pf}\n") out.write("\n") # Detailed Files Content Section out.write("--- 3. DETAILED SOURCE CODE & CONTEXT FILES ---\n\n") for pf in project_files: full_path = root_dir / pf try: content = full_path.read_text(encoding="utf-8", errors="replace") lines = content.splitlines() # Compact long CHANGELOG.md if needed if pf.name == "CHANGELOG.md" and len(lines) > max_changelog_lines: lines = lines[:max_changelog_lines] + [f"\n[... Truncated older changelog history ({len(lines) - max_changelog_lines} lines omitted) ...]"] content = "\n".join(lines) num_lines = len(lines) total_lines += num_lines file_count += 1 out.write("================================================================================\n") out.write(f"FILE: {pf} ({num_lines} lines)\n") out.write("================================================================================\n") out.write(content) if not content.endswith("\n"): out.write("\n") out.write("\n") except Exception as e: out.write(f"ERROR: Could not read file {pf}: {e}\n\n") file_size_mb = output_path.stat().st_size / (1024 * 1024) estimated_tokens = total_lines * 8 # approximate ~8-9 tokens per line of code/markdown print(f"✓ Context dump successfully created at {output_path}") print(f" - Files: {file_count}") print(f" - Total Lines: {total_lines}") print(f" - File Size: {file_size_mb:.2f} MB (~{estimated_tokens // 1000}k tokens)") return output_path if __name__ == "__main__": parser = argparse.ArgumentParser(description="Dump codebase context to txt for AI analysis") parser.add_argument("--output", type=str, default="vhectorlab-context.txt", help="Output file path") parser.add_argument("--include-tests", action="store_true", help="Include test files in the context dump") args = parser.parse_args() root_path = Path(__file__).resolve().parent.parent out_file = Path(args.output) if not out_file.is_absolute(): out_file = root_path / out_file generate_context_dump(root_path, out_file, include_tests=args.include_tests)