Stealth clone from baidu/NAVA

.gitattributes

@@ -1,35 +1,18 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.ckpt           filter=lfs diff=lfs merge=lfs -text
+*.safetensors    filter=lfs diff=lfs merge=lfs -text
+*.pth            filter=lfs diff=lfs merge=lfs -text
+*.pt             filter=lfs diff=lfs merge=lfs -text
+*.bin            filter=lfs diff=lfs merge=lfs -text
+*.onnx           filter=lfs diff=lfs merge=lfs -text
+*.tar            filter=lfs diff=lfs merge=lfs -text
+*.tar.gz         filter=lfs diff=lfs merge=lfs -text
+*.h5             filter=lfs diff=lfs merge=lfs -text
+*.msgpack        filter=lfs diff=lfs merge=lfs -text
+*.npz            filter=lfs diff=lfs merge=lfs -text
+*.npy            filter=lfs diff=lfs merge=lfs -text
+*.model          filter=lfs diff=lfs merge=lfs -text
+*.mp4            filter=lfs diff=lfs merge=lfs -text
+*.mov            filter=lfs diff=lfs merge=lfs -text
+*.wav            filter=lfs diff=lfs merge=lfs -text
+*.png            filter=lfs diff=lfs merge=lfs -text
+tokenizer.json   filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,199 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for describing the origin of the Work and
+      reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Support. While redistributing the Work or
+      Derivative Works thereof, You may choose to offer, and charge a
+      fee for, acceptance of support, warranty, indemnity, or other
+      liability obligations and/or rights consistent with this License.
+      However, in accepting such obligations, You may act only on Your
+      own behalf and on Your sole responsibility, not on behalf of any
+      other Contributor, and only if You agree to indemnify, defend, and
+      hold each Contributor harmless for any liability incurred by, or
+      claims asserted against, such Contributor by reason of your accepting
+      any such warranty or support.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recognize an "as is"
+      basis for an "as if" basis if applicable.
+
+   Copyright 2026 Baidu Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+   implied. See the License for the specific language governing permissions
+   and limitations under the License.

NAVA.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:35afc60aa9b26c239cfab53868d525a95dee685133c4e8f8d5cda9b068b359e5
+size 25188108008

NAVA_fp8.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c8841f4c16311969d90fe908b5f2a75236b1ef0b8c153e1b04d0cfe4a1a3231
+size 6934862256

README.md

@@ -0,0 +1,317 @@
+---
+license: apache-2.0
+language:
+  - en
+  - zh
+tags:
+  - text-to-video
+  - text-to-audio-video
+  - audio-video-generation
+  - mmdit
+  - flow-matching
+  - wan2.2
+pipeline_tag: text-to-video
+library_name: custom
+base_model: Wan-AI/Wan2.2-TI2V-5B
+---
+
+<p align="center">
+  <img src="assets/logo.png" alt="NAVA" width="160">
+</p>
+
+<h1 align="center">NAVA — Native Audio-Visual Alignment for Generation</h1>
+
+<p align="center">
+  <em>State-of-the-art audio-visual synchronization with only <b>6.3 B</b> parameters.</em>
+</p>
+
+<p align="center">
+  <a href="https://arxiv.org/abs/2605.30073"><img alt="arXiv" src="https://img.shields.io/badge/Paper-arXiv-b31b1b.svg"></a>
+  <a href="https://github.com/ernie-research/NAVA"><img alt="Code" src="https://img.shields.io/badge/Code-GitHub-181717.svg"></a>
+  <a href="https://ernie-research.github.io/NAVA/"><img alt="Project Page" src="https://img.shields.io/badge/Project_Page-online-2c8ebb.svg"></a>
+  <img alt="License" src="https://img.shields.io/badge/license-Apache--2.0-green.svg">
+  <img alt="Params" src="https://img.shields.io/badge/params-6.3B-orange.svg">
+  <img alt="Base model" src="https://img.shields.io/badge/base-Wan2.2--TI2V--5B-7c5cff.svg">
+</p>
+
+<p align="center">
+  <b>ERNIE Team</b> · Baidu Inc. · arXiv 2026
+</p>
+
+<p align="center">
+  ⭐ <b>If you find this model useful, please consider giving our <a href="https://github.com/ernie-research/NAVA">GitHub repo</a> a star!</b> ⭐
+</p>
+
+<p align="center">
+  📖 <a href="https://huggingface.co/baidu/NAVA/blob/main/README_zh.md"><b>中文版 README</b></a>
+</p>
+
+---
+
+## TL;DR
+
+NAVA is a **6.3 B-parameter joint audio-video generator** that synthesizes synchronized video **and** audio from a single prompt — including multi-speaker speech with reference-timbre control and image-conditioned continuations.
+
+Instead of post-hoc-aligned dual towers or fully unified tri-modal stacks, NAVA uses an **Align-then-Fuse MMDiT**: a dedicated alignment space first establishes audio-video correspondence, then context (text, speaker embeddings) is fused via cross-attention. On Verse-Bench it sets new SOTA on Sync-C / Sync-D / video quality / audio WER while using **2× to 5× fewer parameters** than open-source baselines.
+
+> **Highlights**
+> - **720p 1-min Fast Generation** — 720p synchronized audio-video in ~1 minute via 8-GPU Ulysses sequence parallel.
+> - **Dual-Channel Audio** — stereo audio (scene + speech) jointly denoised with video, no post-hoc vocoder alignment.
+> - **Precise Multi-Timbre Control** — reference WAVs bound to `<S>...<E>` speech spans for per-speaker voice identity.
+> - **Language-Described Camera Control** — shot composition, motion, and pacing directly from the prompt.
+> - **Multi-Resolution** — landscape / portrait / square aspect ratios from the same checkpoint.
+
+---
+
+## Model Details
+
+### Quick Facts
+
+| | |
+|---|---|
+| **Architecture** | Align-then-Fuse MMDiT (Wan2.2 backbone) |
+| **Parameters** | **6.3 B** (backbone, joint AV) |
+| **Modality** | Joint audio + video, text-conditioned |
+| **Resolution** | 1280×704 (recommended) · 960×960 also supported |
+| **Frames / FPS** | 37 frames @ 24 fps ≈ 6 s · 55–61 frames ≈ 9–10 s |
+| **Audio** | 25 latent tokens / sec, ≤ 10 s |
+| **Sampling** | Flow matching · UniPC scheduler · 50 default steps |
+| **Precision** | bf16 |
+| **Parallelism** | Single-GPU **or** Ulysses sequence parallel (up to 8 GPUs) |
+| **Base model** | [Wan-AI/Wan2.2-TI2V-5B](https://huggingface.co/Wan-AI/Wan2.2-TI2V-5B) |
+
+### Architecture
+
+<p align="center">
+  <img src="assets/arch.png" alt="NAVA Architecture" width="900">
+</p>
+
+NAVA instantiates *Native Audio-Visual Alignment* as an **Align-then-Fuse MMDiT** stack:
+
+- **Hierarchical Alignment Layers — 10 double-stream blocks.** Video and audio keep separate QKV projections and FFNs but share a joint self-attention over concatenated `[video_tokens; audio_tokens]`, plus dedicated cross-attention to text. This builds an alignment space where AV correspondence is learned without semantic context interference.
+- **Unified Fusion Layers — 20 single-stream blocks.** Video and audio share QKV/FFN; a unified joint attention treats all tokens as one stream, with a single text cross-attention path. This is where context-conditioned denoising happens.
+- **Backbone hyperparameters.** `dim=3072`, `ffn_dim=14336`, 24 attention heads, 30 layers (10 double + 20 single), `text_len=512`, patch size `(1, 2, 2)`. RMSNorm on QK; cross-attention norm; ε = 1e-6.
+- **Positional encoding.** 3D RoPE for video (temporal + height + width), 1D RoPE for audio, applied jointly inside the joint-attention path.
+- **Timbre-in-Context Conditioning.** Reference-WAV speaker embeddings (ReDimNet, 192-d) are injected through the context pathway and bound to `<S>...<E>` speech spans, enabling per-speaker timbre control in multi-speaker scenes.
+- **3D cross-modal CFG.** Independent classifier-free guidance scales for video, audio, and the cross-modal alignment direction (`video_align_guidance_scale`, `audio_align_guidance_scale`) keep AV synchronization tight at inference.
+
+### What's Different from Existing Open-Source AV Models
+
+| Design axis | Typical baselines | **NAVA** |
+|---|---|---|
+| Stream layout | Dual-tower (post-hoc align) **or** fully unified tri-modal | **Align-then-Fuse** — alignment space first, context fused after |
+| Speech control | Caption-only, no per-speaker timbre | **Timbre-in-Context** via reference WAVs |
+| Param budget | 10 B – 32 B | **6.3 B** |
+
+### Components Shipped Alongside the Backbone
+
+| Component | Description | Size |
+|---|---|---|
+| **WanAVModel** (backbone) | MMDiT, joint AV attention | 6.3 B |
+| **Wan2.2 Video VAE** | Causal 3D ConvNet · 16×16×4 spatial-temporal compression · 48 latent channels | 2.7 GB |
+| **LTX Audio VAE + Vocoder** | 128 latent channels · 25 tokens/sec · built-in waveform decoder | 348 MB |
+| **umt5-xxl Text Encoder** | T5 · 4096-d embeddings | 11 GB |
+| **ReDimNet** | Speaker embedding · 192-d | ~50 MB |
+
+---
+
+## Evaluation
+
+### Table 1 — VerseBench (general AV capability)
+
+NAVA achieves the **best** AV synchronization (Sync-C / Sync-D), video quality, and audio WER, with the smallest parameter budget.
+
+| Model | Params | Resolution | Sync-C ↑ | Sync-D ↓ | IB ↑ | Video Quality ↑ | WER ↓ | PQ ↑ | FD ↓ |
+|---|---|---|---|---|---|---|---|---|---|
+| Ovi 1.1 | 10 B | 720p | <u>7.4839</u> | 7.9791 | 0.199 | <u>0.636</u> | 0.102 | 5.8432 | 0.9418 |
+| MOVA | A18B (32 B) | 720p | 7.2888 | 7.808 | 0.269 | 0.603 | 0.126 | **7.2331** | 0.9222 |
+| Davinci | 15 B | 540p | 7.1487 | 7.8158 | 0.269 | 0.600 | 0.151 | 5.9559 | 0.9307 |
+| LTX 2.3 | 19 B | 512p | 7.2476 | <u>7.6902</u> | **0.337** | 0.576 | 0.106 | <u>6.9459</u> | **0.8287** |
+| **NAVA (ours)** | **6.3 B** | 720p | **7.7914** | **7.5655** | <u>0.313</u> | **0.659** | **0.099** | 6.8609 | <u>0.8328</u> |
+
+<sub>↑ higher is better · ↓ lower is better · **bold** = best · <u>underline</u> = 2nd best.</sub>
+
+### Table 2 — Seed-TTS-eval (speech quality)
+
+Among joint AV models, NAVA delivers speech quality close to dedicated audio-only systems. Audio-only rows are listed *for reference*; they are not directly comparable.
+
+| Category | Model | WER ↓ | Speaker Similarity ↑ |
+|---|---|---|---|
+| Audio-Only *(reference)* | CosyVoice | 4.29 | 60.9 |
+| Audio-Only *(reference)* | Qwen2.5-Omni | 2.72 | 63.2 |
+| Audio-Video Joint | DreamID-Omni | 33.44 | 34.1 |
+| Audio-Video Joint | **NAVA (ours)** | **5.81** | **62.4** |
+
+---
+
+## How to Use
+
+> **TL;DR command.** After §1 setup is complete:
+> ```bash
+> bash scripts/inference.sh           # General T2AV
+> bash scripts/inference_timbre.sh    # I2AV + timbre control
+> ```
+> Outputs land under `eval_results/`.
+
+### 1 · Setup (once)
+
+```bash
+git clone https://github.com/ernie-research/NAVA && cd NAVA
+
+# Python deps
+pip install torch torchvision torchaudio
+pip install diffusers transformers accelerate safetensors einops scipy PyYAML tqdm sentencepiece
+pip install flash-attn --no-build-isolation
+
+# All weights in one shot — main checkpoint + Wan2.2 VAE + T5 + LTX audio VAE
+huggingface-cli download <NAVA-repo-id> --local-dir .
+```
+
+<details>
+<summary><b>Expected on-disk layout</b></summary>
+
+```
+NAVA/
+├── NAVA.ckpt                                                    # main checkpoint (24 GB)
+├── Wan2.2-TI2V-5B/
+│   ├── Wan2.2_VAE.pth                                           # 2.7 GB
+│   ├── models_t5_umt5-xxl-enc-bf16.pth                          # 11 GB
+│   └── google/umt5-xxl/{spiece.model, tokenizer.json}
+├── params/
+│   └── LTX2/
+│       ├── ltx-2.3-22b-dev_audio_vae.safetensors                # 348 MB
+│       └── LICENSE                                              # LTX-2 Community License
+└── configs/                                                     # inference YAMLs
+```
+
+The LTX audio-VAE Python code is vendored under `nava_src/vendor/ltx_core/` (see its `NOTICE.md`), so no separate clone of the LTX-Video repo is needed. ReDimNet is fetched via `torch.hub` on first run.
+</details>
+
+### 2 · One-command inference (recommended, 8 GPU SP)
+
+The repo ships two end-to-end scripts that build a JSONL inline and launch SP=8 inference:
+
+```bash
+# General T2AV (text-only)
+bash scripts/inference.sh
+
+# I2AV + Timbre Control (first-frame image + reference voice)
+bash scripts/inference_timbre.sh
+```
+
+Override defaults via env vars:
+
+```bash
+CKPT=/path/to/NAVA.ckpt OUT_DIR=eval_results/run1 bash scripts/inference.sh
+TIMBRE_SCALE=3.0 SPK_WAV=/path/to/spk.wav    bash scripts/inference_timbre.sh
+```
+
+### 3 · Custom batches — write your own JSONL
+
+Each line is one prompt:
+
+```jsonl
+{"prompt": "一位男子在海边奔跑，镜头跟随。背景是海浪声和风声。"}
+{"prompt": "两人对话<S>Hello<E><S>Hi there<E>", "spk_wavs": ["spk1.wav", "spk2.wav"]}
+{"prompt": "镜头跟随主体...", "image_path": "/abs/path/first_frame.png"}
+```
+
+| Field | Required | Description |
+|---|---|---|
+| `prompt` | yes | Text caption (also accepts legacy `text` field name) |
+| `image_path` | no | Absolute path to first-frame image — auto-enables I2V for this sample |
+| `spk_wavs` | no | List of absolute paths to speaker reference WAVs (max 2) |
+
+Then launch:
+
+```bash
+SETUPTOOLS_USE_DISTUTILS=stdlib torchrun \
+    --nnodes=1 --nproc_per_node=8 \
+    --master_addr=127.0.0.1 --master_port=29507 \
+    inference_nava.py \
+    --config configs/baseline_t2av_demo_mmdit_no_split_ltx_control_unipc.yaml \
+    --ckpt NAVA.ckpt \
+    --out_dir ./outputs \
+    --data_format json --data_file my_prompts.jsonl \
+    --width 1280 --height 704 --frames 37 --fps 24 \
+    --steps 50 --save_sample --gen_turn 1 --use_sp
+```
+
+Outputs land at `outputs/{save_path}-{gen_turn}_av.mp4`. For timbre-controlled samples, also pass `--timbre_cfg --timbre_align_guidance_scale 3.0`.
+
+#### Mode cheatsheet
+
+| Goal | JSONL fields | Extra flags |
+|---|---|---|
+| Text → AV | `prompt` | — |
+| Image → AV | `prompt` + `image_path` | (auto-detected) |
+| Timbre-controlled speech | `prompt` + `spk_wavs` | `--timbre_cfg --timbre_align_guidance_scale 3.0` |
+| 9-second video | any | `--frames 55` |
+| Single-GPU (slower) | any | omit `--use_sp` |
+
+### 4 · Prompt rewriting (recommended for short / English inputs)
+
+NAVA is trained on Chinese dense captions; short or English prompts benefit substantially from rewriting before inference. Three pathways are provided, all sharing the same system prompt and sampling profile (so output style stays consistent), with `<S>...<E>` speech spans preserved verbatim.
+
+| Pathway | Backend | Speed | Best for |
+|---|---|---|---|
+| **vLLM batch server** (`pe_src/`) | Qwen3-4B-Thinking-2507 served via vLLM, async HTTP | **< 2 s** / prompt | Offline batches |
+| **Local transformers, single** (`gradio_demo/rewrite_single.py`) | Same model, in-process | 40–80 s / prompt | One-off CLI |
+| **Gradio "Rewrite" button** | Same as above, hosted in Gradio | 40–80 s / prompt | Interactive UI |
+
+```bash
+# Batch path: start vLLM server, then rewrite a txt of prompts
+bash pe_src/start_server.sh --gpu 0 --low-footprint
+python pe_src/rewrite.py -i prompts.txt -o prompts_rewritten.txt
+```
+
+### 5 · Gradio Web UI
+
+Interactive demo with click-to-rewrite (Qwen3-4B), image upload, and reference-WAV upload:
+
+```bash
+bash gradio_demo/start_gradio.sh \
+    --config configs/baseline_t2av_demo_mmdit_no_split_ltx_control_unipc.yaml \
+    --ckpt NAVA.ckpt \
+    --rewrite_model pe_src/Qwen3-4B-Thinking-2507 \
+    --port 8000 --nproc 8
+```
+
+<details>
+<summary><b>Debug mode (no models, UI only)</b></summary>
+
+```bash
+python gradio_demo/gradio_server.py --debug --port 8000
+```
+</details>
+
+---
+
+## Bias, Safety, and Misuse
+
+NAVA can synthesize video and speech conditioned on a reference image (`image_path`) and reference voice (`spk_wavs`). Using it to depict real persons without consent — including face-likeness or voice-likeness reproduction — is prohibited by the license and may also be illegal in your jurisdiction. We recommend:
+
+1. Only use **consent-approved** reference media.
+2. **Label generated content as synthetic.**
+3. Apply **provenance / watermarking** before redistribution.
+
+---
+
+## Citation
+
+```bibtex
+@article{nava2026,
+  title   = {NAVA: Native Audio-Visual Alignment for Joint Audio-Video Generation},
+  author  = {ERNIE Team},
+  journal = {arXiv preprint},
+  year    = {2026},
+}
+```
+
+## Acknowledgements
+
+NAVA builds on excellent upstream work: **Wan2.2-TI2V-5B** (video backbone & VAE), **LTX 2.3** (audio VAE + built-in vocoder), **umt5-xxl** (text encoder), and **ReDimNet** (speaker embedding). We also thank the open-source AV-generation community — Ovi, MOVA, Davinci, LTX — for releasing strong baselines that made fair benchmarking possible.
+
+## License & Contact
+
+Released under **Apache-2.0**. For research / commercial inquiries, contact the **ERNIE team at Baidu Inc.**

README_zh.md

@@ -0,0 +1,300 @@
+<p align="center">
+  <img src="assets/logo.png" alt="NAVA" width="160">
+</p>
+
+<h1 align="center">NAVA — 原生音视频对齐生成框架</h1>
+
+<p align="center">
+  <em>仅用 <b>6.3B</b> 参数即达到最先进的音视频同步效果。</em>
+</p>
+
+<p align="center">
+  <a href="https://arxiv.org/abs/2605.30073"><img alt="arXiv" src="https://img.shields.io/badge/Paper-arXiv-b31b1b.svg"></a>
+  <a href="https://github.com/ernie-research/NAVA"><img alt="Code" src="https://img.shields.io/badge/Code-GitHub-181717.svg"></a>
+  <a href="https://ernie-research.github.io/NAVA/"><img alt="Project Page" src="https://img.shields.io/badge/Project_Page-online-2c8ebb.svg"></a>
+  <img alt="License" src="https://img.shields.io/badge/license-Apache--2.0-green.svg">
+  <img alt="Params" src="https://img.shields.io/badge/params-6.3B-orange.svg">
+  <img alt="Base model" src="https://img.shields.io/badge/base-Wan2.2--TI2V--5B-7c5cff.svg">
+</p>
+
+<p align="center">
+  <b>ERNIE 团队</b> · 百度公司 · arXiv 2026
+</p>
+
+<p align="center">
+  ⭐ <b>如果本模型对你有帮助，欢迎给我们的 <a href="https://github.com/ernie-research/NAVA">GitHub 仓库</a> 点个 Star！</b> ⭐
+</p>
+
+<p align="center">
+  📖 <a href="https://huggingface.co/baidu/NAVA/blob/main/README.md"><b>English Version</b></a>
+</p>
+
+---
+
+## TL;DR
+
+NAVA 是一个 **6.3B 参数的联合音视频生成模型**，可以从一段提示词出发，端到端合成同步的视频**与**音频——包括带参考音色控制的多说话人语音，以及基于首帧图像的延续生成。
+
+不同于事后对齐的双塔结构、或完全统一的三模态大栈，NAVA 采用 **Align-then-Fuse MMDiT**：先在专用对齐空间中建立音视频对应关系，再通过 cross-attention 融入上下文（文本、说话人嵌入）。在 Verse-Bench 上，NAVA 在 Sync-C / Sync-D / 视频质量 / 音频 WER 等指标上均刷新 SOTA，同时参数量比开源基线**少 2× 到 5×**。
+
+> **亮点**
+> - **720p 1 分钟快速生成** —— 8 卡 Ulysses 序列并行下，约 1 分钟生成 720p 同步音视频。
+> - **双声道音频** —— 立体声音频（环境音 + 语音）与视频联合去噪，无需事后声码器对齐。
+> - **精准多音色控制** —— 参考 WAV 绑定到 `<S>...<E>` 语音片段，实现按说话人控制音色身份。
+> - **语言描述的镜头控制** —— 镜头构图、运动、节奏可直接由提示词驱动。
+> - **多分辨率** —— 同一权重支持横屏 / 竖屏 / 方形比例。
+
+---
+
+## 模型详情
+
+### 关键参数速览
+
+| | |
+|---|---|
+| **架构** | Align-then-Fuse MMDiT（Wan2.2 主干） |
+| **参数量** | **6.3B**（主干，联合 AV） |
+| **模态** | 联合音频 + 视频，文本条件 |
+| **分辨率** | 1280×704（推荐）· 同时支持 960×960 |
+| **帧数 / 帧率** | 37 帧 @ 24 fps ≈ 6 秒 · 55–61 帧 ≈ 9–10 秒 |
+| **音频** | 25 个 latent token / 秒，≤ 10 秒 |
+| **采样** | Flow matching · UniPC scheduler · 默认 50 步 |
+| **精度** | bf16 |
+| **并行** | 单卡 **或** Ulysses 序列并行（最多 8 卡） |
+| **基础模型** | [Wan-AI/Wan2.2-TI2V-5B](https://huggingface.co/Wan-AI/Wan2.2-TI2V-5B) |
+
+### 架构
+
+<p align="center">
+  <img src="assets/arch.png" alt="NAVA Architecture" width="900">
+</p>
+
+NAVA 把*原生音视频对齐*实例化为 **Align-then-Fuse MMDiT** 堆栈：
+
+- **层级对齐层 —— 10 个 double-stream 块。** 视频与音频保留各自独立的 QKV 投影与 FFN，但共享一个对 `[video_tokens; audio_tokens]` 拼接序列的联合 self-attention，并各自有专用的 cross-attention 接入文本。这样构建出一个对齐空间，让 AV 对应关系在不被语义上下文干扰的前提下被学到。
+- **统一融合层 —— 20 个 single-stream 块。** 视频与音频共享 QKV/FFN；统一的联合注意力把所有 token 视作一条序列，仅有一条文本 cross-attention 通路。上下文条件下的去噪在这里发生。
+- **主干超参。** `dim=3072`，`ffn_dim=14336`，24 个注意力头，30 层（10 double + 20 single），`text_len=512`，patch 大小 `(1, 2, 2)`。QK 上使用 RMSNorm；带 cross-attention 归一化；ε = 1e-6。
+- **位置编码。** 视频用 3D RoPE（时间 + 高 + 宽），音频用 1D RoPE，二者在联合注意力路径中联合应用。
+- **Timbre-in-Context 条件机制。** 参考 WAV 的说话人嵌入（ReDimNet，192 维）通过上下文通路注入，并绑定到 `<S>...<E>` 语音片段，使多说话人场景下的逐说话人音色控制成为可能。
+- **3D 跨模态 CFG。** 视频、音频以及跨模态对齐方向各自拥有独立的无分类器引导尺度（`video_align_guidance_scale`、`audio_align_guidance_scale`），在推理阶段把 AV 同步性卡得很紧。
+
+### 与现有开源 AV 模型的差异
+
+| 设计维度 | 典型基线 | **NAVA** |
+|---|---|---|
+| 数据流布局 | 双塔（事后对齐）**或** 完全统一三模态 | **Align-then-Fuse** —— 先对齐空间，再融合上下文 |
+| 语音控制 | 仅 caption，无逐说话人音色 | 通过参考 WAV 实现 **Timbre-in-Context** |
+| 参数预算 | 10B – 32B | **6.3B** |
+
+### 与主干一同发布的组件
+
+| 组件 | 说明 | 大小 |
+|---|---|---|
+| **WanAVModel**（主干） | MMDiT，联合 AV 注意力 | 6.3B |
+| **Wan2.2 Video VAE** | 因果 3D ConvNet · 16×16×4 时空压缩 · 48 latent 通道 | 2.7 GB |
+| **LTX Audio VAE + Vocoder** | 128 latent 通道 · 25 token/秒 · 内置波形解码器 | 348 MB |
+| **umt5-xxl 文本编码器** | T5 · 4096 维嵌入 | 11 GB |
+| **ReDimNet** | 说话人嵌入 · 192 维 | ~50 MB |
+
+---
+
+## 评测
+
+### 表 1 —— VerseBench（综合 AV 能力）
+
+NAVA 在 AV 同步（Sync-C / Sync-D）、视频质量与音频 WER 上均取得**最佳**，同时参数量最小。
+
+| 模型 | 参数量 | 分辨率 | Sync-C ↑ | Sync-D ↓ | IB ↑ | 视频质量 ↑ | WER ↓ | PQ ↑ | FD ↓ |
+|---|---|---|---|---|---|---|---|---|---|
+| Ovi 1.1 | 10B | 720p | <u>7.4839</u> | 7.9791 | 0.199 | <u>0.636</u> | 0.102 | 5.8432 | 0.9418 |
+| MOVA | A18B (32B) | 720p | 7.2888 | 7.808 | 0.269 | 0.603 | 0.126 | **7.2331** | 0.9222 |
+| Davinci | 15B | 540p | 7.1487 | 7.8158 | 0.269 | 0.600 | 0.151 | 5.9559 | 0.9307 |
+| LTX 2.3 | 19B | 512p | 7.2476 | <u>7.6902</u> | **0.337** | 0.576 | 0.106 | <u>6.9459</u> | **0.8287** |
+| **NAVA（本工作）** | **6.3B** | 720p | **7.7914** | **7.5655** | <u>0.313</u> | **0.659** | **0.099** | 6.8609 | <u>0.8328</u> |
+
+<sub>↑ 越高越好 · ↓ 越低越好 · **加粗** = 最佳 · <u>下划线</u> = 次佳。</sub>
+
+### 表 2 —— Seed-TTS-eval（语音质量）
+
+在联合 AV 模型中，NAVA 的语音质量已接近专用纯音频系统。纯音频行仅作*参考*，不可直接比较。
+
+| 类别 | 模型 | WER ↓ | 说话人相似度 ↑ |
+|---|---|---|---|
+| 纯音频（参考） | CosyVoice | 4.29 | 60.9 |
+| 纯音频（参考） | Qwen2.5-Omni | 2.72 | 63.2 |
+| 联合音视频 | DreamID-Omni | 33.44 | 34.1 |
+| 联合音视频 | **NAVA（本工作）** | **5.81** | **62.4** |
+
+---
+
+## 使用方法
+
+> **一行命令。** 完成第 §1 步环境准备后：
+> ```bash
+> bash scripts/inference.sh           # 通用 T2AV
+> bash scripts/inference_timbre.sh    # I2AV + 音色控制
+> ```
+> 输出落到 `eval_results/` 下。
+
+### 1 · 环境准备（一次性）
+
+```bash
+git clone https://github.com/ernie-research/NAVA && cd NAVA
+
+# Python 依赖
+pip install torch torchvision torchaudio
+pip install diffusers transformers accelerate safetensors einops scipy PyYAML tqdm sentencepiece
+pip install flash-attn --no-build-isolation
+
+# 一次性下载所有权重 —— 主 checkpoint + Wan2.2 VAE + T5 + LTX 音频 VAE
+huggingface-cli download <NAVA-repo-id> --local-dir .
+```
+
+<details>
+<summary><b>预期目录结构</b></summary>
+
+```
+NAVA/
+├── NAVA.ckpt                                                    # 主 checkpoint（24 GB）
+├── Wan2.2-TI2V-5B/
+│   ├── Wan2.2_VAE.pth                                           # 2.7 GB
+│   ├── models_t5_umt5-xxl-enc-bf16.pth                          # 11 GB
+│   └── google/umt5-xxl/{spiece.model, tokenizer.json}
+├── params/
+│   └── LTX2/
+│       ├── ltx-2.3-22b-dev_audio_vae.safetensors                # 348 MB
+│       └── LICENSE                                              # LTX-2 社区许可证
+└── configs/                                                     # 推理 YAML
+```
+
+LTX 音频 VAE 的 Python 代码已 vendor 到 `nava_src/vendor/ltx_core/`（见其 `NOTICE.md`），无需再单独 clone LTX-Video 仓库。ReDimNet 在首次运行时通过 `torch.hub` 下载。
+</details>
+
+### 2 · 一键推理（推荐，8 卡 SP）
+
+仓库提供了两个端到端脚本，会内联构造 JSONL 并启动 SP=8 推理：
+
+```bash
+# 通用 T2AV（纯文本）
+bash scripts/inference.sh
+
+# I2AV + 音色控制（首帧图 + 参考音频）
+bash scripts/inference_timbre.sh
+```
+
+通过环境变量覆盖默认值：
+
+```bash
+CKPT=/path/to/NAVA.ckpt OUT_DIR=eval_results/run1 bash scripts/inference.sh
+TIMBRE_SCALE=3.0 SPK_WAV=/path/to/spk.wav    bash scripts/inference_timbre.sh
+```
+
+### 3 · 自定义批次 —— 写自己的 JSONL
+
+每行一条提示词：
+
+```jsonl
+{"prompt": "一位男子在海边奔跑，镜头跟随。背景是海浪声和风声。"}
+{"prompt": "两人对话<S>Hello<E><S>Hi there<E>", "spk_wavs": ["spk1.wav", "spk2.wav"]}
+{"prompt": "镜头跟随主体...", "image_path": "/abs/path/first_frame.png"}
+```
+
+| 字段 | 必填 | 说明 |
+|---|---|---|
+| `prompt` | 是 | 文本 caption（也兼容旧字段名 `text`） |
+| `image_path` | 否 | 首帧图绝对路径 —— 自动为该样本启用 I2V |
+| `spk_wavs` | 否 | 说话人参考 WAV 的绝对路径列表（最多 2 条） |
+
+然后启动：
+
+```bash
+SETUPTOOLS_USE_DISTUTILS=stdlib torchrun \
+    --nnodes=1 --nproc_per_node=8 \
+    --master_addr=127.0.0.1 --master_port=29507 \
+    inference_nava.py \
+    --config configs/baseline_t2av_demo_mmdit_no_split_ltx_control_unipc.yaml \
+    --ckpt NAVA.ckpt \
+    --out_dir ./outputs \
+    --data_format json --data_file my_prompts.jsonl \
+    --width 1280 --height 704 --frames 37 --fps 24 \
+    --steps 50 --save_sample --gen_turn 1 --use_sp
+```
+
+输出落到 `outputs/{save_path}-{gen_turn}_av.mp4`。对带音色控制的样本，还需加上 `--timbre_cfg --timbre_align_guidance_scale 3.0`。
+
+#### 模式速查
+
+| 目标 | JSONL 字段 | 额外 flag |
+|---|---|---|
+| 文本 → AV | `prompt` | — |
+| 图像 → AV | `prompt` + `image_path` | （自动检测） |
+| 音色控制语音 | `prompt` + `spk_wavs` | `--timbre_cfg --timbre_align_guidance_scale 3.0` |
+| 9 秒视频 | 任意 | `--frames 55` |
+| 单卡（更慢） | 任意 | 去掉 `--use_sp` |
+
+### 4 · 提示词改写（短输入 / 英文输入推荐）
+
+NAVA 在中文密集 caption 上训练，短提示或英文提示在推理前先改写一遍会显著受益。我们提供三种通路，共用同一套 system prompt 与采样 profile（保证输出风格一致），且 `<S>...<E>` 语音片段会被原样保留。
+
+| 通路 | 后端 | 速度 | 适用场景 |
+|---|---|---|---|
+| **vLLM 批量服务**（`pe_src/`） | Qwen3-4B-Thinking-2507 由 vLLM 提供，异步 HTTP | **< 2 秒** / 条 | 离线批量 |
+| **本地 transformers，单条**（`gradio_demo/rewrite_single.py`） | 同模型，进程内 | 40–80 秒 / 条 | 单次 CLI |
+| **Gradio "Rewrite" 按钮** | 同上，托管在 Gradio 中 | 40–80 秒 / 条 | 交互式 UI |
+
+```bash
+# 批量路径：先起 vLLM 服务，再改写一个 prompt 列表
+bash pe_src/start_server.sh --gpu 0 --low-footprint
+python pe_src/rewrite.py -i prompts.txt -o prompts_rewritten.txt
+```
+
+### 5 · Gradio 网页 UI
+
+交互式 demo，支持点击改写（Qwen3-4B）、首帧图上传、参考 WAV 上传：
+
+```bash
+bash gradio_demo/start_gradio.sh \
+    --config configs/baseline_t2av_demo_mmdit_no_split_ltx_control_unipc.yaml \
+    --ckpt NAVA.ckpt \
+    --rewrite_model pe_src/Qwen3-4B-Thinking-2507 \
+    --port 8000 --nproc 8
+```
+
+<details>
+<summary><b>Debug 模式（不加载模型，仅 UI）</b></summary>
+
+```bash
+python gradio_demo/gradio_server.py --debug --port 8000
+```
+</details>
+
+---
+
+## 偏见、安全与滥用
+
+NAVA 可基于参考图（`image_path`）和参考音频（`spk_wavs`）合成视频与语音。**未经本人同意**用其再现真实人物的相貌或声音，是被许可证禁止的，并且在你所在司法辖区内可能也违法。我们建议：
+
+1. 仅使用**已获得授权**的参考素材。
+2. **将生成内容明确标注为合成**。
+3. 在再分发前应用**溯源 / 水印**机制。
+
+---
+
+## 引用
+
+```bibtex
+@article{nava2026,
+  title   = {NAVA: Native Audio-Visual Alignment for Joint Audio-Video Generation},
+  author  = {ERNIE Team},
+  journal = {arXiv preprint},
+  year    = {2026},
+}
+```
+
+## 致谢
+
+NAVA 站在优秀的上游工作之上：**Wan2.2-TI2V-5B**（视频主干与 VAE）、**LTX 2.3**（音频 VAE + 内置声码器）、**umt5-xxl**（文本编码器）、**ReDimNet**（说话人嵌入）。同时感谢开源 AV 生成社区 —— Ovi、MOVA、Davinci、LTX —— 公开了强力基线，让公平 benchmark 成为可能。
+
+## 许可证与联系方式
+
+以 **Apache-2.0** 协议发布。研究 / 商业合作请联系**百度公司 ERNIE 团队**。

Wan2.2-TI2V-5B/Wan2.2_VAE.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:20eb789667fa5e60e7516bf509512f6cb61f01b0aa0695eadaea930c13892b36
+size 2818839170

Wan2.2-TI2V-5B/google/umt5-xxl/spiece.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e3909a67b780650b35cf529ac782ad2b6b26e6d1f849d3fbb6a872905f452458
+size 4548313

Wan2.2-TI2V-5B/google/umt5-xxl/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e197b4d3dbd71da14b4eb255f4fa91c9c1f2068b20a2de2472967ca3d22602b
+size 16837417

Wan2.2-TI2V-5B/models_t5_umt5-xxl-enc-bf16.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7cace0da2b446bbbbc57d031ab6cf163a3d59b366da94e5afe36745b746fd81d
+size 11361920418

config.json

@@ -0,0 +1,21 @@
+{
+  "patch_size": [1, 2, 2],
+  "model_type": "ti2v",
+  "dim": 3072,
+  "ffn_dim": 14336,
+  "freq_dim": 256,
+  "num_heads": 24,
+  "num_layers": 30,
+  "num_double_layers": 10,
+  "num_single_layers": 20,
+  "vid_in_dim": 48,
+  "vid_out_dim": 48,
+  "audio_in_dim": 128, 
+  "audio_out_dim": 128, 
+  "text_len": 512, 
+  "window_size": [-1, -1],
+  "qk_norm": true,
+  "cross_attn_norm": true,
+  "eps": 1e-6,
+  "temporal_rope_scaling_factor": 0.24
+}

example_prompts.jsonl

@@ -0,0 +1,5 @@
+{"prompt": "一位男子在海边奔跑，镜头跟随其移动。写实电影感，中景跟随，自然光。背景是海浪声和风声。"}
+{"prompt": "画面呈现冷调写实电影质感，整体色调以冰蓝色、冷白色与浅灰色为主。一只北极熊正在冰面上行走，每一步都让爪子与冰面发生真实摩擦。背景伴有极地寒风与远处冰层轻微开裂声。"}
+{"prompt": "这段写实风格的视频记录了一个安静的公园棋局场景。左侧的男子看着棋盘，语气低沉地说道：<S>If you move the bishop now, you leave your king exposed.<E> 右侧的男子短暂停顿后抬起眼，语气冷静地回应道：<S>That is exactly why I want you to think the attack is obvious.<E>"}
+{"prompt": "这是一段具有史诗级写实电影质感的视频，蝙蝠侠站在城市高楼屋顶边缘，眼神深邃且坚定，他低沉沙哑且富有磁性地说道：<S>Small models can still do big things.<E>", "spk_wavs": ["/abs/path/to/batman_ref.wav"]}
+{"prompt": "一只猫从沙发上跳下，轻盈落地。柔和室内自然光，温暖居家氛围。", "image_path": "/abs/path/to/cat_first_frame.png"}

nava.yaml

@@ -0,0 +1,131 @@
+
+model_type: NAVA
+modality: audio_video # [audio, video, image, audio_video]
+
+# 模型
+model_id: /root/paddlejob/workspace/Qwen3-1.7B
+# model_id: GPT-B
+use_bf16: true
+audio_latent_ch: 128
+video_latent_ch: 48
+
+patch_size: 2
+image_size: 960
+log_width: 1280
+log_height: 704
+lambda_ddpm: 1.0
+
+pipeline: nava_src.pipeline_nava.AudioVideoPipeline
+scheduler_personalized: true
+scheduler_unipc: true
+use_loss_reweight: true
+attention_only: false
+
+use_mmdit_model: true
+audio_loss_coff: 0.2
+vision_loss_coff: 1
+
+# 推理
+video_guidance_scale: 3.0
+audio_guidance_scale: 2.0
+video_align_guidance_scale: 3.0
+audio_align_guidance_scale: 2.0
+masking_modality_prob: 0.2
+align_3d_cfg: true
+i2v_mode_prob: 0.5
+no_split_norm_ffn: true
+timbre_cfg: true
+timbre_align_guidance_scale: 3.0
+
+# 数据
+data:
+  # data_filelist: data/video_text/data_videogen_0630_256px_shijue.list
+  # data_weights: data/video_text/data_video_0630_shijue.weight
+  # data_filelist: data/audio_text/mixture_0207/data.list
+  # data_weights: data/audio_text/mixture_0207/data.weight
+  # data_filelist: av_data_v2.list
+  # data_weights: av_data.weight
+  data_filelist: av_data_v4_470w_filter10_with_audio.list
+  data_weights: av_data_v4_470w_filter10_with_audio.weight
+  # data_filelist: joint_train_mmdit.list
+  # data_weights: joint_train_mmdit.weight
+  use_length_buckets: true
+  num_length_buckets: 5
+  enable_ddp_bucket_sync: true
+  audio_tokens_per_sec: 25
+
+  queue_size: 16
+  io_workers: 4
+
+  modal_prob: 
+    text_to_audio: 1
+    text_to_video: 0.0
+    text_to_image: 0.0
+    text_to_av: 1
+
+  # server config
+  use_server: False
+  audio_scaling_factor: 1.0
+  audio_shift_factor: 0.0
+  video_scaling_factor: 1.0
+  video_shift_factor: 0.0
+  zlib_mode: false
+  zstd_mode: true
+
+  # video_fps: 16
+  # video_min_frames: 33
+  # video_max_frames: 300
+  # video_tgt_frames: 81
+  min_audio_duration: 0.0
+  max_audio_duration: 10.0
+
+  video_fps: 24
+  video_min_frames: 33
+  video_max_frames: 400
+  video_tgt_frames: 121
+  # video_tgt_frames: 241
+
+  add_spk_emb: true
+  spk_emb_prob: 0.9
+  use_speech_special_token: false
+
+model:
+  shift: 5
+  shift_audio: 5
+  num_train_timesteps: 1000
+  gradient_checkpointing: true
+  cross_gate: false
+  cross_1d_rope: false
+
+  # audio_config: nava_src/models/nava/configs/model/dit/audio.json
+  # video_config: nava_src/models/nava/configs/model/dit/video.json
+  audio_config: nava_src/models/nava/configs/model/dit/audio_tiny.json
+  video_config: nava_src/models/nava/configs/model/dit/video_tiny.json
+  joint_config: nava_src/models/nava/configs/model/dit/NAVA_6B.json
+  ckpt_dir: ./
+  audio_vae_ckpt_dir: ./huggingface_upload/params
+
+
+# 训练
+batch_size: 1
+lr: 1.0e-4
+weight_decay: 0
+warmup_steps: 0
+max_steps: 1000000
+save_every: 2500
+out_dir: outputs/t2av_baseline_demo_mmdit_test_no_separate_modulation
+seed: 42
+
+flow: true
+
+# 性能相关
+num_workers: 0
+prefetch_factor: 32
+grad_accum_steps: 1
+log_every: 1
+log_cases_every: 1000000        # >0 才会写图像
+log_sample_steps: 25
+log_cfg: 4
+max_grad_norm: 1.0
+amp_dtype: bf16            # bf16 / fp16 / null
+

params/LTX2/LICENSE

@@ -0,0 +1,381 @@
+                         LTX-2 Community License Agreement
+                           License date: January 5, 2026
+
+
+By using or distributing any portion or element of LTX-2, you agree
+to be bound by this Agreement.
+
+   1. Definitions.
+
+      "Agreement" means the terms and conditions for the license, use,
+      reproduction, and distribution of LTX-2 and the Complementary
+      Materials, as specified in this document.
+
+      "Control" means the direct or indirect ownership of more than
+      fifty percent (50%) of the voting securities or other ownership
+      interests, or the power to direct the management and policies of
+      such Entity through voting rights, contract, or otherwise.
+
+      "Data" means a collection of information and/or content extracted
+      from the dataset used with LTX-2, including to train, pretrain,
+      or otherwise evaluate LTX-2. The Data is not licensed under this
+      Agreement.
+
+      "Derivatives of LTX-2" means all modifications to LTX-2, works
+      based on LTX-2, or any other model which is created or initialized
+      by transfer of patterns of the weights, parameters, activations or
+      output of LTX-2, to the other model, in order to cause the other
+      model to perform similarly to LTX-2, including – but not limited
+      to - distillation methods entailing the use of intermediate data
+      representations or methods based on the generation of synthetic
+      data by LTX-2 for training the other model. For clarity, Derivatives
+      of LTX-2 include: (i) any fine-tuned or adapted weights, parameters,
+      or checkpoints derived from LTX-2; (ii) derivative model architectures
+      that incorporate or are based upon LTX-2's architecture; and
+      (iii) any modified or extended versions of the Complementary
+      Materials. All intellectual property rights in Derivatives of LTX-2
+      shall be subject to the terms of this Agreement, and you may not
+      claim exclusive ownership rights in any Derivatives of LTX-2 that
+      would restrict the rights granted herein.
+
+      "Entity" means any individual, corporation, partnership, limited
+      liability company, or other legal entity. For purposes of this
+      Agreement, an Entity shall be deemed to include, on an aggregative
+      basis, all subsidiaries, affiliates, and other companies under
+      common Control with such Entity. When determining whether an Entity
+      meets any threshold under this Agreement (including revenue
+      thresholds), all subsidiaries, affiliates, and companies under
+      common Control shall be considered collectively.
+
+      "Harm" includes but is not limited to physical, mental,
+      psychological, financial and reputational damage, pain, or loss.
+
+      "Licensor" or "Lightricks" means the owner that is granting the
+      license under this Agreement. For the purposes of this Agreement,
+      the Licensor is Lightricks Ltd.
+
+      "LTX-2" means the large language models, text/image/video/audio/3D
+      generation models, and multimodal large language models and their
+      software and algorithms, including trained model weights, parameters
+      (including optimizer states), machine-learning model code,
+      inference-enabling code, training-enabling code, fine-tuning
+      enabling code, accompanying source code, scripts, documentation,
+      tutorials, examples, and all other elements of the foregoing
+      distributed and made publicly available by Lightricks (including,
+      for example, at https://github.com/Lightricks/LTX-2) for the LTX-2
+      model released on January 5, 2026. This license is applicable to
+      all LTX-2 versions released since January 5, 2026, and all future
+      releases of LTX-2 under this license.
+
+      "Output" means the results of operating LTX-2 as embodied in
+      informational content resulting therefrom.
+
+      "you" (or "your") means an individual or legal Entity licensing
+      LTX-2 in accordance with this Agreement and/or making use of LTX-2
+      for whichever purpose and in any field of use, including usage of
+      LTX-2 in an end-use application - e.g. chatbot, translator, image
+      generator.
+
+   2. Grant of License. Subject to the terms and conditions of this
+      Agreement, you are granted a non-exclusive, worldwide,
+      non-transferable and royalty-free limited license under Licensor's
+      intellectual property or other rights owned by Licensor embodied
+      in LTX-2 to use, reproduce, prepare, distribute, publicly display,
+      publicly perform, sublicense, copy, create derivative works of,
+      and make modifications to LTX-2, for any purpose, subject to the
+      restrictions set forth in Attachment A; provided however, that
+      Entities with annual revenues of at least $10,000,000 (the
+      "Commercial Entities") are required to obtain a paid commercial
+      use license in order to use LTX-2 and Derivatives of LTX-2,
+      subject to the terms and provisions of a different license (the
+      "Commercial Use Agreement"), as will be provided by the Licensor.
+      Commercial Entities interested in such a commercial license are
+      required to [contact Licensor](https://ltx.io/model/licensing).
+      Any commercial use of LTX-2 or Derivatives of LTX-2 by the
+      Commercial Entities not in accordance with this Agreement and/or
+      the Commercial Use Agreement is strictly prohibited and shall be
+      deemed a material breach of this Agreement. Such material breach
+      will be subject, in addition to any license fees owed to Licensor
+      for the period such Commercial Entity used LTX-2 (as will be
+      determined by Licensor), to liquidated damages, which will be paid
+      to Licensor immediately upon demand, in an amount equal to double
+      the amount that would otherwise have been paid by you for the
+      relevant period of time. Such amount reflects a reasonable estimation
+      of the losses and administrative costs incurred due to such breach.
+      You agree and understand that this remedy does not limit the Licensor's
+      right to pursue other remedies available at law or equity.
+
+   3. Distribution and Redistribution. You may host for third parties
+      remote access purposes (e.g. software-as-a-service), reproduce
+      and distribute copies of LTX-2 or Derivatives of LTX-2 thereof in
+      any medium, with or without modifications, provided that you meet
+      the following conditions:
+
+      (a) Use-based restrictions as referenced in paragraph 4 and all
+          provisions of Attachment A MUST be included as an enforceable
+          provision by you in any type of legal agreement (e.g. a
+          license) governing the use and/or distribution of LTX-2 or
+          Derivatives of LTX-2, and you shall give notice to subsequent
+          users you distribute to, that LTX-2 or Derivatives of LTX-2
+          are subject to paragraph 4 and Attachment A in their entirety,
+          including all use restrictions and acceptable use policies;
+
+      (b) You must provide any third party recipients of LTX-2 or
+          Derivatives of LTX-2 a copy of this Agreement, including all
+          attachments and use policies. Any Derivative of LTX-2 (as
+          defined in Section 1, including but not limited to fine-tuned
+          weights, modified training code, models trained on Outputs, or
+          any other derivative) must be distributed exclusively under
+          the terms of this Agreement with a complete copy of this
+          license included;
+
+      (c) You must cause any modified files to carry prominent notices
+          stating that you changed the files;
+
+      (d) You must retain all copyright, patent, trademark, and
+          attribution notices excluding those notices that do not
+          pertain to any part of LTX-2, Derivatives of LTX-2.
+
+      You may add your own copyright statement to your modifications and
+      may provide additional or different license terms and conditions -
+      respecting paragraph 3(a) - for use, reproduction, or distribution
+      of your modifications, or for any such Derivatives of LTX-2 as a
+      whole, provided your use, reproduction, and distribution of LTX-2
+      otherwise complies with the conditions stated in this Agreement,
+      and you provide a complete copy of this Agreement with any such
+      use, reproduction and distribution of LTX-2 and any Derivatives
+      thereof.
+
+   4. Use-based restrictions. The restrictions set forth in Attachment A
+      are considered Use-based restrictions. Therefore, you cannot use
+      LTX-2 and the Derivatives of LTX-2 in violation of the specified
+      restricted uses. You may use LTX-2 subject to this Agreement,
+      including only for lawful purposes and in accordance with the
+      Agreement. "Use" may include creating any content with, fine-tuning,
+      updating, running, training, evaluating and/or re-parametrizing
+      LTX-2. You shall require all of your users who use LTX-2 or a
+      Derivative of LTX-2 to comply with the terms of this paragraph 4.
+
+   5. The Output You Generate. Except as set forth herein, Licensor
+      claims no rights in the Output you generate using LTX-2. You are
+      accountable for input you insert into LTX-2, the Output you
+      generate and its subsequent uses. No use of the Output can
+      contravene any provision as stated in the Agreement.
+
+   6. Updates and Runtime Restrictions. To the maximum extent permitted
+      by law, Licensor reserves the right to restrict (remotely or
+      otherwise) usage of LTX-2 in violation of this Agreement, update
+      LTX-2 through electronic means, or modify the Output of LTX-2
+      based on updates. You shall undertake reasonable efforts to use
+      the latest version of LTX-2. Any use of the non-current version
+      of LTX-2 is done solely at your risk.
+
+   7. Export Controls and Sanctions Compliance. You acknowledge that
+      LTX-2, Derivatives of LTX-2 may be subject to export control laws
+      and regulations, including but not limited to the U.S. Export
+      Administration Regulations and sanctions programs administered by
+      the Office of Foreign Assets Control (OFAC). You represent and
+      warrant that you and any users of LTX-2 are not (i) located in,
+      organized under the laws of, or ordinarily resident in any country
+      or territory subject to comprehensive sanctions; (ii) identified
+      on any U.S. government restricted party list, including the
+      Specially Designated Nationals and Blocked Persons List; or
+      (iii) otherwise prohibited from receiving LTX-2 under applicable
+      law. You shall not export, re-export, or transfer LTX-2, directly
+      or indirectly, in violation of any applicable export control or
+      sanctions laws or regulations. You agree to comply with all
+      applicable trade control laws and shall indemnify and hold
+      Licensor harmless from any claims arising from your failure to
+      comply with such laws.
+
+   8. Trademarks and related. Nothing in this Agreement permits you to
+      make use of Licensor's trademarks, trade names, logos or to
+      otherwise suggest endorsement or misrepresent the relationship
+      between the parties; and any rights not expressly granted herein
+      are reserved by the Licensor.
+
+   9. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides LTX-2 on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or
+      conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS
+      FOR A PARTICULAR PURPOSE. You are solely responsible for
+      determining the appropriateness of using or redistributing LTX-2
+      and Derivatives of LTX-2 and assume any risks associated with
+      your exercise of permissions under this Agreement.
+
+   10. Limitation of Liability. In no event and under no legal theory,
+       whether in tort (including negligence), contract, or otherwise,
+       unless required by applicable law (such as deliberate and grossly
+       negligent acts) or agreed to in writing, shall Licensor be liable
+       to you for damages, including any direct, indirect, special,
+       incidental, or consequential damages of any character arising as
+       a result of this Agreement or out of the use or inability to use
+       LTX-2 (including but not limited to damages for loss of goodwill,
+       work stoppage, computer failure or malfunction, or any and all
+       other commercial damages or losses), even if Licensor has been
+       advised of the possibility of such damages.
+
+   11. Accepting Warranty or Additional Liability. While redistributing
+       LTX-2 and Derivatives of LTX-2, you may, provided you do not
+       violate the terms of this Agreement, choose to offer and charge
+       a fee for, acceptance of support, warranty, indemnity, or other
+       liability obligations. However, in accepting such obligations,
+       you may act only on your own behalf and on your sole
+       responsibility, not on behalf of Licensor, and only if you agree
+       to indemnify, defend, and hold Licensor harmless for any liability
+       incurred by, or claims asserted against Licensor, by reason of
+       your accepting any such warranty or additional liability.
+
+   12. Governing Law. This Agreement and all relations, disputes, claims
+       and other matters arising hereunder (including non-contractual
+       disputes or claims) will be governed exclusively by, and construed
+       exclusively in accordance with, the laws of the State of New York.
+       To the extent permitted by law, choice of laws rules and the
+       United Nations Convention on Contracts for the International Sale
+       of Goods will not apply. For the purposes of adjudicating any
+       action or proceeding to enforce the terms of this Agreement, you
+       hereby irrevocably consent to the exclusive jurisdiction of, and
+       venue in, the federal and state courts located in the County of
+       New York within the State of New York. The prevailing party in
+       any claim or dispute between the parties under this Agreement
+       will be entitled to reimbursement of its reasonable attorneys'
+       fees and costs. You hereby waive the right to a trial by jury,
+       to participate in a class or representative action (including in
+       arbitration), or to combine individual proceedings in court or
+       in arbitration without the consent of all parties.
+
+   13. Term and Termination. This Agreement is effective upon your
+       acceptance and continues until terminated. Licensor may terminate
+       this Agreement immediately upon written notice to you if you
+       breach any provision of this Agreement, including but not limited
+       to violations of the use restrictions in Attachment A or
+       unauthorized commercial use. Upon termination: (a) all rights
+       granted to you under this Agreement will immediately cease;
+       (b) you must immediately cease all use of LTX-2 and Derivatives
+       of LTX-2; (c) you must delete or destroy all copies of LTX-2
+       and Derivatives of LTX-2 in your possession or control; and
+       (d) you must notify any third parties to whom you distributed
+       LTX-2 or Derivatives of LTX-2 of the termination. Sections 8-13,
+       and Section 15 shall survive termination of this Agreement.
+       Termination does not relieve you of any obligations incurred
+       prior to termination, including payment obligations under
+       Section 2. In addition, if You commence a lawsuit or other
+       proceedings (including a cross-claim or counterclaim in a lawsuit)
+       against Licensor or any person or entity alleging that LTX-2 or
+       any Output, or any portion of any of the foregoing, infringe any
+       intellectual property or other right owned or licensable by you,
+       then all licenses granted to you under this Agreement shall
+       terminate as of the date such lawsuit or other proceeding is filed.
+
+   14. Disputes and Arbitration. All disputes arising in connection with
+       this Agreement shall be finally settled by arbitration under the
+       Rules of Arbitration of the International Chamber of Commerce
+       ("ICC Rules"), by one (1) arbitrator appointed in accordance with
+       the ICC Rules. The seat of arbitration shall be New York, NY, USA,
+       and the proceedings shall be conducted in English. The arbitrator
+       shall be empowered to grant any relief that a court could grant.
+       Judgment on the arbitration award may be entered by any court
+       having jurisdiction thereof. Each party waives its right to a
+       trial by jury and to participate in any class or representative
+       action.
+
+   15. If any provision of this Agreement is held to be
+       invalid, illegal
+       or unenforceable, the remaining provisions shall be unaffected
+       thereby and remain valid as if such provision had not been set
+       forth herein.
+
+   END OF TERMS AND CONDITIONS
+
+   ATTACHMENT A: Use Restrictions
+
+      When using the Outputs, LTX-2 and any Derivatives thereof, you
+      will comply with the Acceptable Use Policy. In addition, you
+      agree not to use the Outputs, LTX-2 or its Derivatives in any
+      of the following ways:
+
+      1. In any way that violates any applicable national, federal,
+         state, local or international law or regulation;
+
+      2. For the purpose of exploiting, Harming or attempting to
+         exploit or Harm minors in any way;
+
+      3. To generate or disseminate false information and/or content
+         with the purpose of Harming others;
+
+      4. To generate or disseminate personal identifiable information
+         that can be used to Harm an individual;
+
+      5. To generate or disseminate information and/or content (e.g.
+         images, code, posts, articles), and place the information
+         and/or content in any context (e.g. bot generating tweets)
+         without expressly and intelligibly disclaiming that the
+         information and/or content is machine generated;
+
+      6. To defame, disparage or otherwise harass others;
+
+      7. To impersonate or attempt to impersonate (e.g. deepfakes)
+         others without their consent;
+
+      8. For fully automated decision making that adversely impacts an
+         individual's legal rights or otherwise creates or modifies a
+         binding, enforceable obligation;
+
+      9. For any use intended to or which has the effect of
+         discriminating against or Harming individuals or groups based
+         on online or offline social behavior or known or predicted
+         personal or personality characteristics;
+
+      10. To exploit any of the vulnerabilities of a specific group of
+          persons based on their age, social, physical or mental
+          characteristics, in order to materially distort the behavior
+          of a person pertaining to that group in a manner that causes
+          or is likely to cause that person or another person physical
+          or psychological Harm;
+
+      11. For any use intended to or which has the effect of
+          discriminating against individuals or groups based on legally
+          protected characteristics or categories;
+
+      12. To provide medical advice and medical results interpretation;
+
+      13. To generate or disseminate information for the purpose to be
+          used for administration of justice, law enforcement,
+          immigration or asylum processes, such as predicting an
+          individual will commit fraud/crime commitment (e.g. by text
+          profiling, drawing causal relationships between assertions
+          made in documents, indiscriminate and arbitrarily-targeted use);
+
+      14. To generate and/or disseminate malware (including – but not
+          limited to – ransomware) or any other content to be used for
+          the purpose of harming electronic systems;
+
+      15. To engage in, promote, incite, or facilitate discrimination
+          or other unlawful or harmful conduct in the provision of
+          employment, employment benefits, credit, housing, or other
+          essential goods and services;
+
+      16. To engage in, promote, incite, or facilitate the harassment,
+          abuse, threatening, or bullying of individuals or groups of
+          individuals;
+
+      17. For military, warfare, nuclear industries or applications,
+          weapons development, or any use in connection with activities
+          that may cause death, personal injury, or severe physical or
+          environmental damage;
+
+      18. For commercial use only: To train, improve, or fine-tune any
+          other machine learning model, artificial intelligence system,
+          or competing model, except for Derivatives of LTX-2 as
+          expressly permitted under this Agreement;
+
+      19. To circumvent, disable, or interfere with any technical
+          limitations, safety features, content filters, or use
+          restrictions implemented in LTX-2 by Licensor;
+
+      20. To use LTX-2 or Derivatives of LTX-2 in any product, service,
+          or application that directly competes with Licensor's
+          commercial products or services, or is designed to replace or
+          substitute Licensor's offerings in the market, without
+          obtaining a separate commercial license from Licensor.

params/LTX2/ltx-2.3-22b-dev_audio_vae.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7711812d9387ce940c2cd5d65a4f5a1e57bf6087cf618d89b56dd3c722c4dea
+size 364853140