comfyui/meridian_geometry.py

# Copyright 2026 Viggle AI. Licensed under the Apache License, Version 2.0 (see LICENSE-CODE).
# SPDX-License-Identifier: Apache-2.0
"""A ComfyUI node that produces Meridian's two reference videos from a clip and a camera move.

Drop this file into `ComfyUI/custom_nodes/` and **Meridian Geometry** appears under `video`. It shells
out to this repo's `inference/sample.py --preview-only`, which runs VGGT-Omega, unprojects frame
`--start` into a point cloud, moves the camera and rasterises it -- then stops before the VAE. What
comes back is what the sampler wants: `<Video 1>` the source crop, `<Video 2>` the warp.

A subprocess, not an import, because the geometry side pulls in VGGT-Omega, which is gated and
FAIR-NC-licensed: you accept that licence and install it yourself, and nothing of it ends up inside
ComfyUI's process. `python` must therefore be an interpreter that can import it -- not necessarily the
one ComfyUI runs on.

`args` is passed to `sample.py` verbatim, so the camera vocabulary is the one the README documents
(`--yaw --truck --boom --dolly --zoom --pivot --pivot-lock --aim --sweep --ease --bounce --freeze
--follow --cull --start --frames ...`). Nothing is re-declared here, so nothing can drift out of sync --
which is also where `--vggt` and `--vggt-repo` go if you have not exported `$VGGT_OMEGA_CKPT`/`$VGGT_OMEGA_DIR`.

What is read back is the `cond_*.mp4` pair, which `--preview-only` writes at the condition canvas --
the model is conditioned at the 480 class, so the sampler is handed exactly what `inference/sample.py`
feeds its own, with no second resampling in between.
`width`/`height` are the target canvas and belong on `MiniMaxH3ReferenceToVideo`: do not let it size the
references itself, its `adapt_canvas` would land on the 768 class and condition the model off-distribution.
"""
import os
import re
import shlex
import subprocess
import tempfile

import av
import numpy as np
import torch


def _frames(path):
    with av.open(path) as c:
        a = np.stack([f.to_ndarray(format="rgb24") for f in c.decode(video=0)])
    return torch.from_numpy(a).float().div_(255.0)


class MeridianGeometry:
    @classmethod
    def INPUT_TYPES(cls):
        return {"required": {"video": ("STRING", {"default": "clip.mp4"}),
                             "args": ("STRING", {"default": "--boom 0.35 --pivot 0.5,0.55 --ease --sweep",
                                                 "multiline": True}),
                             "repo": ("STRING", {"default": "/path/to/release_recam"}),
                             "python": ("STRING", {"default": "python"})}}

    RETURN_TYPES = ("IMAGE", "IMAGE", "INT", "INT", "INT")
    RETURN_NAMES = ("source", "render", "width", "height", "length")
    FUNCTION = "build"
    CATEGORY = "video"

    def build(self, video, args, repo, python):
        out = tempfile.mkdtemp(prefix="meridian_")
        cmd = [python, f"{repo}/inference/sample.py", "--video", video, "--out", out, "--preview-only"]
        cmd += shlex.split(args)
        print("Meridian geometry:", " ".join(cmd), flush=True)
        r = subprocess.run(cmd, cwd=repo, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
        print(r.stdout, flush=True)
        r.check_returncode()

        # "1280x960 -> content ... canvas (1184, 864), cond (736, 544)"
        w, h = (int(g) for g in re.search(r"canvas \((\d+), (\d+)\)", r.stdout).groups())
        source, render = _frames(f"{out}/cond_source.mp4"), _frames(f"{out}/cond_render.mp4")
        for f in os.listdir(out):
            os.remove(f"{out}/{f}")
        os.rmdir(out)
        return (source, render, w, h, source.shape[0])


NODE_CLASS_MAPPINGS = {"MeridianGeometry": MeridianGeometry}
NODE_DISPLAY_NAME_MAPPINGS = {"MeridianGeometry": "Meridian Geometry"}