Fix login bug

FL2VA/audio_vae/dac_alias_free_filter.py

@@ -49,11 +49,11 @@ def kaiser_sinc_filter1d(cutoff, half_width, kernel_size):  # return filter [1,1
         filter_ = torch.zeros_like(time)
     else:
         filter_ = 2 * cutoff * window * sinc(2 * cutoff * time)
-        """
-        Normalize filter to have sum = 1, otherwise we will have a small leakage of the constant component in the input signal.
-        """
+        # Normalize filter to have sum = 1, otherwise we will have a small
+        # leakage of the constant component in the input signal.
         filter_ /= filter_.sum()
-        filter = filter_.view(1, 1, kernel_size)
+    # Always reshape into [1, 1, kernel_size] so the return is never undefined.
+    filter = filter_.view(1, 1, kernel_size)
 
     return filter
 

FL2VA/audio_vae/dac_attn_proj.py

@@ -29,15 +29,19 @@ class CausalAttention(nn.Module):
     def __init__(self, in_dim, out_dim, num_heads):
         super().__init__()
         if in_dim > out_dim:
-            # assert in_dim // num_heads == out_dim
+            # Projection compresses in_dim → out_dim after attention.
+            # QKV operates in in_dim space so head_dim = in_dim // num_heads.
             self.head_dim = in_dim // num_heads
+            self.qkv_out_dim = in_dim
             self.qkv = nn.Linear(in_dim, in_dim * 3, bias=False)
             self.q_bias = nn.Parameter(torch.zeros(in_dim))
             self.v_bias = nn.Parameter(torch.zeros(in_dim))
             self.register_buffer("zero_k_bias", torch.zeros(in_dim))
         else:
-            # assert out_dim // num_heads == in_dim
+            # Projection expands (or keeps) in_dim → out_dim after attention.
+            # QKV operates in out_dim space so head_dim = out_dim // num_heads.
             self.head_dim = out_dim // num_heads
+            self.qkv_out_dim = out_dim
             self.qkv = nn.Linear(in_dim, out_dim * 3, bias=False)
             self.q_bias = nn.Parameter(torch.zeros(out_dim))
             self.v_bias = nn.Parameter(torch.zeros(out_dim))
@@ -47,21 +51,27 @@ def __init__(self, in_dim, out_dim, num_heads):
         self.out_dim = out_dim
         self.num_heads = num_heads
         self.scale = self.head_dim**-0.5
-        self.proj = nn.Linear(out_dim, out_dim)
+        # Final linear maps from the QKV output space to out_dim.
+        self.proj = nn.Linear(self.qkv_out_dim, out_dim)
 
     def forward(self, x: torch.Tensor) -> torch.Tensor:
         B, N, C = x.shape
-        qkv = F.linear(input=x, weight=self.qkv.weight, bias=torch.cat((self.q_bias, self.zero_k_bias, self.v_bias)))
-        q, k, v = qkv.reshape(B, N, 3, self.num_heads, self.head_dim).permute(2, 0, 3, 1, 4).unbind(0)
+        qkv = F.linear(
+            input=x,
+            weight=self.qkv.weight,
+            bias=torch.cat((self.q_bias, self.zero_k_bias, self.v_bias)),
+        )
+        q, k, v = (
+            qkv.reshape(B, N, 3, self.num_heads, self.head_dim)
+            .permute(2, 0, 3, 1, 4)
+            .unbind(0)
+        )
 
         x = scaled_dot_product_attention(q, k, v, attn_mask=None, dropout_p=0.0, is_causal=True)
 
-        if self.in_dim > self.out_dim:
-            x = torch.mean(x, dim=1)
-            if self.in_dim // self.num_heads != self.out_dim:
-                x = nn.functional.adaptive_avg_pool1d(x, self.out_dim)
-        else:
-            x = x.transpose(1, 2).reshape(B, N, -1)
+        # Always reshape to [B, N, qkv_out_dim] so self.proj receives the
+        # correct input dimension regardless of whether in_dim > out_dim or not.
+        x = x.transpose(1, 2).reshape(B, N, self.qkv_out_dim)
         x = self.proj(x)
         return x
 

FL2VA/audio_vae/dac_audio_vae.py

@@ -43,6 +43,17 @@ def forward(self, x):
 def init_weights(m):
     """Initialize Conv1d layers, including those wrapped with ``weight_norm``.
 
+    This is the canonical initializer for ``DacAudioVAE`` (encoder + decoder).
+    It uses ``trunc_normal_`` with ``std=0.02`` and explicitly zeroes biases,
+    matching the intended training configuration for this model family.
+
+    Note: ``dac_bigvgan.py`` / ``dac_utils.py`` also contain an ``init_weights``
+    helper (``std=0.01``, ``normal_``) used by BigVGAN's own module-level
+    ``apply()`` calls.  Because ``DacAudioVAE.__init__`` calls
+    ``self.apply(init_weights)`` *after* all sub-modules (including BigVGAN)
+    have been constructed, this function is the one that takes final effect on
+    every ``nn.Conv1d`` in the whole model graph.
+
     The encoder (``WNConv1d``) and decoder (BigVGAN) both wrap ``nn.Conv1d``
     with ``torch.nn.utils.parametrizations.weight_norm``. Under that
     parametrization ``m.weight`` is computed on access from
@@ -74,9 +85,15 @@ def __init__(self, dim: int = 16, dilation: int = 1):
 
     def forward(self, x):
         y = self.block(x)
-        pad = (x.shape[-1] - y.shape[-1]) // 2
-        if pad > 0:
-            x = x[..., pad:-pad]
+        diff = x.shape[-1] - y.shape[-1]
+        if diff > 0:
+            # Crop the residual path to match the convolved output length.
+            # Use explicit left/right amounts to handle both even and odd
+            # differences safely (avoids the x[..., pad:-pad] pattern which
+            # silently drops an extra sample when diff is odd).
+            pad_left = diff // 2
+            pad_right = diff - pad_left  # == pad_left + (diff % 2)
+            x = x[..., pad_left : x.shape[-1] - pad_right]
         return x + y
 
 
@@ -93,7 +110,11 @@ def __init__(self, dim: int = 16, stride: int = 1):
                 dim,
                 kernel_size=2 * stride,
                 stride=stride,
-                padding=math.ceil(stride / 2),
+                # For kernel_size = 2*stride the correct "same-length" padding
+                # is (kernel_size - stride) // 2 = stride // 2.
+                # The previous math.ceil(stride / 2) was off by 1 for odd
+                # strides (e.g. stride=5 gave padding=3 instead of 2).
+                padding=stride // 2,
             ),
         )
 
@@ -209,7 +230,9 @@ def __init__(
         if self.attn_proj:
             self.pre_block = AttnProjection(latent_dim, self.attn_proj_dim, num_heads=8)
 
-        self.sample_rate = sample_rate
+        # Apply canonical weight initialization to all Conv1d layers in the
+        # entire model graph (encoder + BigVGAN decoder). This single pass
+        # uses trunc_normal_(std=0.02) and is the intended final initializer.
         self.apply(init_weights)
 
     def preprocess(self, audio_data, sample_rate):

FL2VA/transformer/config.json

@@ -4,23 +4,31 @@
   "hidden_size": 5376,
   "num_layers": 50,
   "token_refiner_num_layers": 2,
+  "num_refiner_layers": 2,
   "num_attention_heads": 56,
   "attention_head_dim": 128,
   "ffn_hidden_size": 14336,
+  "ffn_dim": 14336,
   "latents_dim": 24,
+  "in_channels": 24,
   "audio_latents_dim": 32,
+  "audio_in_channels": 32,
   "patch_size": [
     1,
     2,
     2
   ],
   "text_dim": 5120,
   "timestep_input_dim": 256,
+  "freq_dim": 256,
   "time_embed_hidden_size": 5376,
+  "time_embed_hidden_dim": 5376,
   "time_embed_dim": 2688,
   "adaln_out_features": 96768,
   "final_adaln_out_features": 10752,
   "rope_inv_freq_len": 16,
+  "rope_freq_dim": 16,
+  "rope_theta": 10000.0,
   "norm_eps": 1e-05,
   "qk_norm_eps": 1e-05,
   "final_norm_eps": 1e-05

README.md

@@ -215,6 +215,68 @@ hf download MiniMaxAI/MiniMax-H3 --include "model_index.json" "FL2VA/*" "Ref2VA/
 hf download MiniMaxAI/MiniMax-H3 --include "model_index.json" "FL2VA/*" --local-dir MiniMax-H3
 ```
 
+#### Required Model Weights & Download Instructions
+
+> **Important:** This repository contains only code and configuration files.
+> The model weights (`.safetensors` files) are hosted separately on Hugging Face
+> and must be downloaded before running inference.
+
+**Step 1 — Install dependencies**
+
+```bash
+pip install -r requirements.txt
+# For SGLang / vLLM, also install the respective package as documented in the
+# framework sections below.
+```
+
+**Step 2 — Download the weights**
+
+```bash
+# Install the Hugging Face CLI if needed:
+pip install huggingface_hub
+
+# Download FL2VA checkpoint (T2VA + FL2VA tasks):
+hf download MiniMaxAI/MiniMax-H3 \
+  --include "model_index.json" "modular_model_index.json" \
+  --include "FL2VA/**" \
+  --include "vae/**" "audio_vae/**" "scheduler/**" "audio_scheduler/**" \
+  --local-dir ./MiniMax-H3
+
+# Download Ref2VA checkpoint additionally:
+hf download MiniMaxAI/MiniMax-H3 \
+  --include "Ref2VA/**" \
+  --local-dir ./MiniMax-H3
+```
+
+**Step 3 — Verify the download**
+
+After downloading, confirm the key weight files exist:
+
+```bash
+# FL2VA transformer weights (largest file — ~67 GB total for all shards)
+ls MiniMax-H3/FL2VA/transformer/model-*.safetensors
+
+# Audio VAE weights
+ls MiniMax-H3/FL2VA/audio_vae/*.safetensors
+
+# Visual VAE weights
+ls MiniMax-H3/FL2VA/vae/*.safetensors
+```
+
+If any files are missing, re-run the `hf download` command above. The `hf` CLI
+is resumable — it will skip files that are already complete.
+
+**Step 4 — Run inference**
+
+Once weights are downloaded, follow the SGLang, vLLM, or diffusers workflows
+documented in the sections below. For a quick smoke-test using the reproducible
+768p scripts, set your `SGLANG_DEPLOYMENT_URL` and run:
+
+```bash
+bash scripts/readme/reproducible-768p-t2va-request.sh
+```
+
+
 diffusers users do not need a manual download: `ModularPipeline.from_pretrained("MiniMaxAI/MiniMax-H3")` fetches exactly the components it needs. See the [diffusers documentation](https://github.com/huggingface/diffusers/blob/minimax-h3/docs/source/en/api/pipelines/minimax_h3.md) for loading recipes.
 
 We recommend the following inference frameworks to serve the model:

Ref2VA/audio_vae/dac_alias_free_filter.py

@@ -49,11 +49,11 @@ def kaiser_sinc_filter1d(cutoff, half_width, kernel_size):  # return filter [1,1
         filter_ = torch.zeros_like(time)
     else:
         filter_ = 2 * cutoff * window * sinc(2 * cutoff * time)
-        """
-        Normalize filter to have sum = 1, otherwise we will have a small leakage of the constant component in the input signal.
-        """
+        # Normalize filter to have sum = 1, otherwise we will have a small
+        # leakage of the constant component in the input signal.
         filter_ /= filter_.sum()
-        filter = filter_.view(1, 1, kernel_size)
+    # Always reshape into [1, 1, kernel_size] so the return is never undefined.
+    filter = filter_.view(1, 1, kernel_size)
 
     return filter
 

Ref2VA/audio_vae/dac_attn_proj.py

@@ -29,15 +29,19 @@ class CausalAttention(nn.Module):
     def __init__(self, in_dim, out_dim, num_heads):
         super().__init__()
         if in_dim > out_dim:
-            # assert in_dim // num_heads == out_dim
+            # Projection compresses in_dim → out_dim after attention.
+            # QKV operates in in_dim space so head_dim = in_dim // num_heads.
             self.head_dim = in_dim // num_heads
+            self.qkv_out_dim = in_dim
             self.qkv = nn.Linear(in_dim, in_dim * 3, bias=False)
             self.q_bias = nn.Parameter(torch.zeros(in_dim))
             self.v_bias = nn.Parameter(torch.zeros(in_dim))
             self.register_buffer("zero_k_bias", torch.zeros(in_dim))
         else:
-            # assert out_dim // num_heads == in_dim
+            # Projection expands (or keeps) in_dim → out_dim after attention.
+            # QKV operates in out_dim space so head_dim = out_dim // num_heads.
             self.head_dim = out_dim // num_heads
+            self.qkv_out_dim = out_dim
             self.qkv = nn.Linear(in_dim, out_dim * 3, bias=False)
             self.q_bias = nn.Parameter(torch.zeros(out_dim))
             self.v_bias = nn.Parameter(torch.zeros(out_dim))
@@ -47,21 +51,27 @@ def __init__(self, in_dim, out_dim, num_heads):
         self.out_dim = out_dim
         self.num_heads = num_heads
         self.scale = self.head_dim**-0.5
-        self.proj = nn.Linear(out_dim, out_dim)
+        # Final linear maps from the QKV output space to out_dim.
+        self.proj = nn.Linear(self.qkv_out_dim, out_dim)
 
     def forward(self, x: torch.Tensor) -> torch.Tensor:
         B, N, C = x.shape
-        qkv = F.linear(input=x, weight=self.qkv.weight, bias=torch.cat((self.q_bias, self.zero_k_bias, self.v_bias)))
-        q, k, v = qkv.reshape(B, N, 3, self.num_heads, self.head_dim).permute(2, 0, 3, 1, 4).unbind(0)
+        qkv = F.linear(
+            input=x,
+            weight=self.qkv.weight,
+            bias=torch.cat((self.q_bias, self.zero_k_bias, self.v_bias)),
+        )
+        q, k, v = (
+            qkv.reshape(B, N, 3, self.num_heads, self.head_dim)
+            .permute(2, 0, 3, 1, 4)
+            .unbind(0)
+        )
 
         x = scaled_dot_product_attention(q, k, v, attn_mask=None, dropout_p=0.0, is_causal=True)
 
-        if self.in_dim > self.out_dim:
-            x = torch.mean(x, dim=1)
-            if self.in_dim // self.num_heads != self.out_dim:
-                x = nn.functional.adaptive_avg_pool1d(x, self.out_dim)
-        else:
-            x = x.transpose(1, 2).reshape(B, N, -1)
+        # Always reshape to [B, N, qkv_out_dim] so self.proj receives the
+        # correct input dimension regardless of whether in_dim > out_dim or not.
+        x = x.transpose(1, 2).reshape(B, N, self.qkv_out_dim)
         x = self.proj(x)
         return x
 

Ref2VA/audio_vae/dac_audio_vae.py

@@ -43,6 +43,17 @@ def forward(self, x):
 def init_weights(m):
     """Initialize Conv1d layers, including those wrapped with ``weight_norm``.
 
+    This is the canonical initializer for ``DacAudioVAE`` (encoder + decoder).
+    It uses ``trunc_normal_`` with ``std=0.02`` and explicitly zeroes biases,
+    matching the intended training configuration for this model family.
+
+    Note: ``dac_bigvgan.py`` / ``dac_utils.py`` also contain an ``init_weights``
+    helper (``std=0.01``, ``normal_``) used by BigVGAN's own module-level
+    ``apply()`` calls.  Because ``DacAudioVAE.__init__`` calls
+    ``self.apply(init_weights)`` *after* all sub-modules (including BigVGAN)
+    have been constructed, this function is the one that takes final effect on
+    every ``nn.Conv1d`` in the whole model graph.
+
     The encoder (``WNConv1d``) and decoder (BigVGAN) both wrap ``nn.Conv1d``
     with ``torch.nn.utils.parametrizations.weight_norm``. Under that
     parametrization ``m.weight`` is computed on access from
@@ -74,9 +85,15 @@ def __init__(self, dim: int = 16, dilation: int = 1):
 
     def forward(self, x):
         y = self.block(x)
-        pad = (x.shape[-1] - y.shape[-1]) // 2
-        if pad > 0:
-            x = x[..., pad:-pad]
+        diff = x.shape[-1] - y.shape[-1]
+        if diff > 0:
+            # Crop the residual path to match the convolved output length.
+            # Use explicit left/right amounts to handle both even and odd
+            # differences safely (avoids the x[..., pad:-pad] pattern which
+            # silently drops an extra sample when diff is odd).
+            pad_left = diff // 2
+            pad_right = diff - pad_left  # == pad_left + (diff % 2)
+            x = x[..., pad_left : x.shape[-1] - pad_right]
         return x + y
 
 
@@ -93,7 +110,11 @@ def __init__(self, dim: int = 16, stride: int = 1):
                 dim,
                 kernel_size=2 * stride,
                 stride=stride,
-                padding=math.ceil(stride / 2),
+                # For kernel_size = 2*stride the correct "same-length" padding
+                # is (kernel_size - stride) // 2 = stride // 2.
+                # The previous math.ceil(stride / 2) was off by 1 for odd
+                # strides (e.g. stride=5 gave padding=3 instead of 2).
+                padding=stride // 2,
             ),
         )
 
@@ -209,7 +230,9 @@ def __init__(
         if self.attn_proj:
             self.pre_block = AttnProjection(latent_dim, self.attn_proj_dim, num_heads=8)
 
-        self.sample_rate = sample_rate
+        # Apply canonical weight initialization to all Conv1d layers in the
+        # entire model graph (encoder + BigVGAN decoder). This single pass
+        # uses trunc_normal_(std=0.02) and is the intended final initializer.
         self.apply(init_weights)
 
     def preprocess(self, audio_data, sample_rate):