Automated PR - 2026-01-29

packages/ltx-pipelines/src/ltx_pipelines/ic_lora.py

@@ -2,11 +2,12 @@ import logging
 from collections.abc import Iterator
 
 import torch
+from safetensors import safe_open
 
 from ltx_core.components.diffusion_steps import EulerDiffusionStep
 from ltx_core.components.noisers import GaussianNoiser
 from ltx_core.components.protocols import DiffusionStepProtocol
-from ltx_core.conditioning import ConditioningItem, VideoConditionByKeyframeIndex
+from ltx_core.conditioning import ConditioningItem, VideoConditionByReferenceLatent
 from ltx_core.loader import LoraPathStrengthAndSDOps
 from ltx_core.model.audio_vae import decode_audio as vae_decode_audio
 from ltx_core.model.upsampler import upsample_video
@@ -81,6 +82,21 @@ class ICLoraPipeline:
         )
         self.device = device
 
+        # Read reference downscale factor from LoRA metadata.
+        # IC-LoRAs trained with low-resolution reference videos store this factor
+        # so inference can resize reference videos to match training conditions.
+        self.reference_downscale_factor = 1
+        for lora in loras:
+            scale = _read_lora_reference_downscale_factor(lora.path)
+            if scale != 1:
+                if self.reference_downscale_factor not in (1, scale):
+                    raise ValueError(
+                        f"Conflicting reference_downscale_factor values in LoRAs: "
+                        f"already have {self.reference_downscale_factor}, but {lora.path} "
+                        f"specifies {scale}. Cannot combine LoRAs with different reference scales."
+                    )
+                self.reference_downscale_factor = scale
+
     @torch.inference_mode()
     def __call__(
         self,
@@ -249,17 +265,34 @@ class ICLoraPipeline:
             device=self.device,
         )
 
+        # Calculate scaled dimensions for reference video conditioning.
+        # IC-LoRAs trained with downscaled reference videos expect the same ratio at inference.
+        scale = self.reference_downscale_factor
+        if scale != 1 and (height % scale != 0 or width % scale != 0):
+            raise ValueError(
+                f"Output dimensions ({height}x{width}) must be divisible by reference_downscale_factor ({scale})"
+            )
+        ref_height = height // scale
+        ref_width = width // scale
+
         for video_path, strength in video_conditioning:
+            # Load video at scaled-down resolution (if scale > 1)
             video = load_video_conditioning(
                 video_path=video_path,
-                height=height,
-                width=width,
+                height=ref_height,
+                width=ref_width,
                 frame_cap=num_frames,
                 dtype=self.dtype,
                 device=self.device,
             )
             encoded_video = video_encoder(video)
-            conditionings.append(VideoConditionByKeyframeIndex(keyframes=encoded_video, frame_idx=0, strength=strength))
+            conditionings.append(
+                VideoConditionByReferenceLatent(
+                    latent=encoded_video,
+                    downscale_factor=scale,
+                    strength=strength,
+                )
+            )
 
         return conditionings
 
@@ -307,5 +340,26 @@ def main() -> None:
     )
 
 
+def _read_lora_reference_downscale_factor(lora_path: str) -> int:
+    """Read reference_downscale_factor from LoRA safetensors metadata.
+    Some IC-LoRA models are trained with reference videos at lower resolution than
+    the target output. This allows for more efficient training and can improve
+    generalization. The downscale factor indicates the ratio between target and
+    reference resolutions (e.g., factor=2 means reference is half the resolution).
+    Args:
+        lora_path: Path to the LoRA .safetensors file
+    Returns:
+        The reference downscale factor (1 if not specified in metadata, meaning
+        reference and target have the same resolution)
+    """
+    try:
+        with safe_open(lora_path, framework="pt") as f:
+            metadata = f.metadata() or {}
+            return int(metadata.get("reference_downscale_factor", 1))
+    except Exception as e:
+        logging.warning(f"Failed to read metadata from LoRA file '{lora_path}': {e}")
+        return 1
+
+
 if __name__ == "__main__":
     main()