indigo/LTX-2
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Automated PR - 2026-06-17

Browse Source
This commit is contained in:
github-actions[bot]
2026-06-17 14:21:07 +00:00
parent d6053703e0
commit f4b06fb977
103 changed files with 12887 additions and 3665 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/ltx-trainer/src/ltx_trainer/config.py
+310 -16
View File
@@ -1,10 +1,11 @@
from pathlib import Path
from typing import Annotated, Literal
from typing import Annotated, Literal, Union
from pydantic import BaseModel, ConfigDict, Discriminator, Field, Tag, ValidationInfo, field_validator, model_validator
from ltx_trainer.quantization import QuantizationOptions
from ltx_trainer.training_strategies.base_strategy import TrainingStrategyConfigBase
from ltx_trainer.training_strategies.flexible import FlexibleStrategyConfig
from ltx_trainer.training_strategies.text_to_video import TextToVideoConfig
from ltx_trainer.training_strategies.video_to_video import VideoToVideoConfig
@@ -13,6 +14,226 @@ class ConfigBaseModel(BaseModel):
model_config = ConfigDict(extra="forbid")
# =============================================================================
# Validation Condition Types
# =============================================================================
class FirstFrameConditionConfig(ConfigBaseModel):
"""First-frame conditioning (intrinsic, latent_idx=0). Always targets video.
If image_or_video points to a video file, the first frame is automatically extracted.
"""
type: Literal["first_frame"] = "first_frame"
image_or_video: str | Path
class PrefixConditionConfig(ConfigBaseModel):
"""Prefix conditioning for temporal extension (intrinsic). Exactly one of video/audio must be set."""
type: Literal["prefix"] = "prefix"
video: str | None = None
audio: str | None = None
num_frames: int | None = Field(
default=None,
ge=1,
description="Number of pixel frames for video prefix. Must satisfy num_frames %% 8 == 1.",
)
duration: float | None = Field(default=None, gt=0, description="Duration in seconds for audio prefix")
@model_validator(mode="after")
def validate_exactly_one_modality(self) -> "PrefixConditionConfig":
if (self.video is None) == (self.audio is None):
raise ValueError("Exactly one of 'video' or 'audio' must be set for prefix condition")
return self
@model_validator(mode="after")
def validate_num_frames_constraint(self) -> "PrefixConditionConfig":
if self.video is not None and self.num_frames is not None and self.num_frames % 8 != 1:
raise ValueError(
f"num_frames ({self.num_frames}) must satisfy num_frames % 8 == 1 "
f"for video prefix (e.g., 1, 9, 17, 25, ...)"
)
return self
class SuffixConditionConfig(ConfigBaseModel):
"""Suffix conditioning for temporal extension (intrinsic). Exactly one of video/audio must be set."""
type: Literal["suffix"] = "suffix"
video: str | None = None
audio: str | None = None
num_frames: int | None = Field(
default=None,
ge=1,
description="Number of pixel frames for video suffix. Must satisfy num_frames %% 8 == 0.",
)
duration: float | None = Field(default=None, gt=0, description="Duration in seconds for audio suffix")
@model_validator(mode="after")
def validate_exactly_one_modality(self) -> "SuffixConditionConfig":
if (self.video is None) == (self.audio is None):
raise ValueError("Exactly one of 'video' or 'audio' must be set for suffix condition")
return self
@model_validator(mode="after")
def validate_num_frames_constraint(self) -> "SuffixConditionConfig":
if self.video is not None and self.num_frames is not None and self.num_frames % 8 != 0:
raise ValueError(
f"num_frames ({self.num_frames}) must satisfy num_frames % 8 == 0 "
f"for video suffix (e.g., 8, 16, 24, 32, ...)"
)
return self
class SpatialCropConditionConfig(ConfigBaseModel):
"""Spatial crop conditioning for outpainting (intrinsic, video only)."""
type: Literal["spatial_crop"] = "spatial_crop"
video: str
spatial_region: tuple[int, int, int, int] = Field(
..., description="Spatial crop region as (y1, x1, y2, x2) in pixel coordinates"
)
class MaskConditionConfig(ConfigBaseModel):
"""Mask-based conditioning for inpainting (intrinsic). Exactly one of video/audio must be set."""
type: Literal["mask"] = "mask"
video: str | None = None
audio: str | None = None
mask: str
@model_validator(mode="after")
def validate_exactly_one_modality(self) -> "MaskConditionConfig":
if (self.video is None) == (self.audio is None):
raise ValueError("Exactly one of 'video' or 'audio' must be set for mask condition")
return self
class ReferenceConditionConfig(ConfigBaseModel):
"""Reference conditioning (IC-LoRA style concatenation). Exactly one of video/audio must be set."""
type: Literal["reference"] = "reference"
video: str | None = None
audio: str | None = None
downscale_factor: int = Field(default=1, ge=1)
temporal_scale_factor: int = Field(default=1, ge=1)
include_in_output: bool = False
@model_validator(mode="after")
def validate_exactly_one_modality(self) -> "ReferenceConditionConfig":
if (self.video is None) == (self.audio is None):
raise ValueError("Exactly one of 'video' or 'audio' must be set for reference condition")
return self
class VideoToAudioConditionConfig(ConfigBaseModel):
"""Video-to-audio — video is provided as frozen cross-modal conditioning.
The video is kept clean (sigma=0) and influences audio generation via cross-modal attention.
"""
type: Literal["video_to_audio"] = "video_to_audio"
video: str
class AudioToVideoConditionConfig(ConfigBaseModel):
"""Audio-to-video — audio is provided as frozen cross-modal conditioning.
The audio is kept clean (sigma=0) and influences video generation via cross-modal attention.
"""
type: Literal["audio_to_video"] = "audio_to_video"
audio: str
ValidationCondition = Annotated[
Union[
FirstFrameConditionConfig,
PrefixConditionConfig,
SuffixConditionConfig,
SpatialCropConditionConfig,
MaskConditionConfig,
ReferenceConditionConfig,
VideoToAudioConditionConfig,
AudioToVideoConditionConfig,
],
Field(discriminator="type"),
]
def _condition_targets_video(cond: ValidationCondition) -> bool:
"""Check if a validation condition targets the video modality."""
if cond.type in ("first_frame", "spatial_crop", "video_to_audio"):
return True
if cond.type in ("prefix", "suffix", "mask", "reference"):
return getattr(cond, "video", None) is not None
return False
def _condition_targets_audio(cond: ValidationCondition) -> bool:
"""Check if a validation condition targets the audio modality."""
if cond.type == "audio_to_video":
return True
if cond.type in ("prefix", "suffix", "mask", "reference"):
return getattr(cond, "audio", None) is not None
return False
class ValidationSample(ConfigBaseModel):
"""Configuration for a single validation sample — fully self-describing."""
prompt: str
conditions: list[ValidationCondition] = Field(default_factory=list)
video_dims: tuple[int, int, int] | None = Field(
default=None,
description="Per-sample override for (width, height, frames). None = inherit from ValidationConfig.",
)
seed: int | None = Field(
default=None,
description="Per-sample override for random seed. None = inherit from ValidationConfig.",
)
@field_validator("video_dims")
@classmethod
def validate_video_dims(cls, v: tuple[int, int, int] | None) -> tuple[int, int, int] | None:
if v is None:
return v
width, height, frames = v
if width % 32 != 0:
raise ValueError(f"Width ({width}) must be divisible by 32")
if height % 32 != 0:
raise ValueError(f"Height ({height}) must be divisible by 32")
if frames % 8 != 1:
raise ValueError(f"Frames ({frames}) must satisfy frames % 8 == 1 for LTX-2 (e.g., 1, 9, 17, 25, ...)")
return v
@model_validator(mode="after")
def validate_frozen_modality_conflicts(self) -> "ValidationSample":
frozen_types = {c.type for c in self.conditions if c.type in ("video_to_audio", "audio_to_video")}
if "video_to_audio" in frozen_types and "audio_to_video" in frozen_types:
raise ValueError(
"Cannot have both video_to_audio and audio_to_video conditions — nothing would be generated"
)
if "video_to_audio" in frozen_types:
for c in self.conditions:
if c.type != "video_to_audio" and _condition_targets_video(c):
raise ValueError(
f"Cannot use video-targeting '{c.type}' condition when video is frozen (video_to_audio)"
)
if "audio_to_video" in frozen_types:
for c in self.conditions:
if c.type != "audio_to_video" and _condition_targets_audio(c):
raise ValueError(
f"Cannot use audio-targeting '{c.type}' condition when audio is frozen (audio_to_video)"
)
return self
class ModelConfig(ConfigBaseModel):
"""Configuration for the base model and training mode"""
@@ -89,7 +310,9 @@ def _get_strategy_discriminator(v: dict | TrainingStrategyConfigBase) -> str:
# Union type for all strategy configs with discriminator
TrainingStrategyConfig = Annotated[
Annotated[TextToVideoConfig, Tag("text_to_video")] | Annotated[VideoToVideoConfig, Tag("video_to_video")],
Annotated[TextToVideoConfig, Tag("text_to_video")]
| Annotated[VideoToVideoConfig, Tag("video_to_video")]
| Annotated[FlexibleStrategyConfig, Tag("flexible")],
Discriminator(_get_strategy_discriminator),
]
@@ -191,13 +414,32 @@ class DataConfig(ConfigBaseModel):
ge=0,
)
@field_validator("preprocessed_data_root")
@classmethod
def validate_preprocessed_data_root(cls, v: str) -> str:
"""Validate that preprocessed_data_root exists."""
path = Path(v).expanduser().resolve()
if not path.exists():
raise ValueError(f"Dataset path does not exist: {v}")
if not path.is_dir():
raise ValueError(f"Dataset path is not a directory: {v}")
return str(path)
class ValidationConfig(ConfigBaseModel):
"""Configuration for validation during training"""
# Per-sample configuration (new format — preferred)
samples: list[ValidationSample] = Field(
default_factory=list,
description="List of validation samples. Each sample is fully self-describing with its own "
"prompt, conditions, and optional overrides. Replaces prompts/images/reference_videos.",
)
# Legacy fields (deprecated — converted to samples internally via convert_legacy_format)
prompts: list[str] = Field(
default_factory=list,
description="List of prompts to use for validation",
description="[DEPRECATED: use 'samples' instead] List of prompts to use for validation",
)
negative_prompt: str = Field(
@@ -207,19 +449,22 @@ class ValidationConfig(ConfigBaseModel):
images: list[str] | None = Field(
default=None,
description="List of image paths to use for validation. "
description="[DEPRECATED: use 'samples' with first_frame conditions] "
"List of image paths to use for validation. "
"One image path must be provided for each validation prompt",
)
reference_videos: list[str] | None = Field(
default=None,
description="List of reference video paths to use for validation. "
description="[DEPRECATED: use 'samples' with reference conditions] "
"List of reference video paths to use for validation. "
"One video path must be provided for each validation prompt",
)
reference_downscale_factor: int = Field(
default=1,
description="Downscale factor for reference videos in IC-LoRA validation. "
description="[DEPRECATED: use downscale_factor on ReferenceCondition] "
"Downscale factor for reference videos in IC-LoRA validation. "
"When > 1, reference videos are processed at 1/n resolution (e.g., 2 means half resolution). "
"Must match the factor used during dataset preprocessing.",
ge=1,
@@ -301,6 +546,13 @@ class ValidationConfig(ConfigBaseModel):
"in validation even when not training the audio branch.",
)
generate_video: bool = Field(
default=True,
description="Whether to generate video in validation samples. "
"Set to False for audio-only or v2a validation to save VRAM by skipping video VAE decoder loading. "
"When False, validation will only generate audio (requires generate_audio=True).",
)
skip_initial_validation: bool = Field(
default=False,
description="Skip validation video sampling at step 0 (beginning of training)",
@@ -308,7 +560,8 @@ class ValidationConfig(ConfigBaseModel):
include_reference_in_output: bool = Field(
default=False,
description="For video-to-video training: concatenate the original reference video side-by-side "
description="[DEPRECATED: use include_in_output on ReferenceCondition] "
"For video-to-video training: concatenate the original reference video side-by-side "
"with the generated output. The reference comes from the input video, not from the model's output.",
)
@@ -346,13 +599,33 @@ class ValidationConfig(ConfigBaseModel):
return v
@model_validator(mode="after")
def convert_legacy_format(self) -> "ValidationConfig":
"""Convert deprecated prompts/images/reference_videos to the new samples format."""
if self.prompts and not self.samples:
samples = []
for i, prompt in enumerate(self.prompts):
conditions: list[ValidationCondition] = []
if self.images and i < len(self.images):
conditions.append(FirstFrameConditionConfig(image_or_video=self.images[i]))
if self.reference_videos and i < len(self.reference_videos):
conditions.append(
ReferenceConditionConfig(
video=self.reference_videos[i],
downscale_factor=self.reference_downscale_factor,
include_in_output=self.include_reference_in_output,
)
)
samples.append(ValidationSample(prompt=prompt, conditions=conditions))
self.samples = samples
return self
@model_validator(mode="after")
def validate_scaled_reference_dimensions(self) -> "ValidationConfig":
"""Validate that scaled reference dimensions are valid when reference_downscale_factor > 1."""
if self.reference_downscale_factor > 1:
width, height, _frames = self.video_dims
# Validate that downscale factor evenly divides the target dimensions
if width % self.reference_downscale_factor != 0:
raise ValueError(
f"Width {width} is not evenly divisible by reference_downscale_factor "
@@ -367,7 +640,6 @@ class ValidationConfig(ConfigBaseModel):
scaled_width = width // self.reference_downscale_factor
scaled_height = height // self.reference_downscale_factor
# Validate scaled dimensions are divisible by 32
if scaled_width % 32 != 0:
raise ValueError(
f"Scaled reference width {scaled_width} (from {width} / {self.reference_downscale_factor}) "
@@ -381,6 +653,16 @@ class ValidationConfig(ConfigBaseModel):
return self
@model_validator(mode="after")
def validate_output_modality_requirements(self) -> "ValidationConfig":
"""Validate output modality settings when validation is configured."""
has_validation = bool(self.prompts) or bool(self.samples)
if has_validation and not self.generate_video and not self.generate_audio:
raise ValueError(
"At least one of generate_video or generate_audio must be True when validation is configured."
)
return self
class CheckpointsConfig(ConfigBaseModel):
"""Configuration for model checkpointing during training"""
@@ -514,19 +796,31 @@ class LtxTrainerConfig(ConfigBaseModel):
"""Expand user home directory in output path."""
return str(Path(v).expanduser().resolve())
def _validate_data_dirs_exist(self) -> None:
"""Verify that every directory declared by the training strategy exists under the data root."""
data_root = Path(self.data.preprocessed_data_root)
for dir_name in self.training_strategy.get_data_sources():
dir_path = data_root / dir_name
if not dir_path.is_dir():
raise ValueError(
f"Required data directory '{dir_name}' does not exist under preprocessed_data_root: {dir_path}"
)
@model_validator(mode="after")
def validate_strategy_compatibility(self) -> "LtxTrainerConfig":
"""Validate that training strategy and other configurations are compatible."""
self._validate_data_dirs_exist()
# Check that reference videos are provided when using video_to_video strategy
if (
self.training_strategy.name == "video_to_video"
and self.validation.interval
and not self.validation.reference_videos
):
raise ValueError(
"reference_videos must be provided in validation config when using video_to_video strategy"
if self.training_strategy.name == "video_to_video" and self.validation.interval:
has_reference = bool(self.validation.reference_videos) or any(
cond.type == "reference" for sample in self.validation.samples for cond in sample.conditions
)
if not has_reference:
raise ValueError(
"reference_videos or samples with reference conditions must be provided "
"in validation config when using video_to_video strategy"
)
# Check that LoRA config is provided when training mode is lora
if self.model.training_mode == "lora" and self.lora is None: