indigo/LTX-2
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Automated PR - 2026-01-29

Browse Source
This commit is contained in:
sync-bot
2026-01-29 18:42:17 +00:00
parent 727c43e998
commit ca1623ad2a
31 changed files with 1723 additions and 663 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/ltx-pipelines/src/ltx_pipelines/utils/args.py
+122 -12
View File
@@ -7,13 +7,14 @@ from ltx_pipelines.utils.constants import (
DEFAULT_1_STAGE_WIDTH,
DEFAULT_2_STAGE_HEIGHT,
DEFAULT_2_STAGE_WIDTH,
DEFAULT_CFG_GUIDANCE_SCALE,
DEFAULT_AUDIO_GUIDER_PARAMS,
DEFAULT_FRAME_RATE,
DEFAULT_LORA_STRENGTH,
DEFAULT_NEGATIVE_PROMPT,
DEFAULT_NUM_FRAMES,
DEFAULT_NUM_INFERENCE_STEPS,
DEFAULT_SEED,
DEFAULT_VIDEO_GUIDER_PARAMS,
)
@@ -185,16 +186,6 @@ def basic_arg_parser() -> argparse.ArgumentParser:
def default_1_stage_arg_parser() -> argparse.ArgumentParser:
parser = basic_arg_parser()
parser.add_argument(
"--cfg-guidance-scale",
type=float,
default=DEFAULT_CFG_GUIDANCE_SCALE,
help=(
f"Classifier-free guidance (CFG) scale controlling how strongly "
f"the model adheres to the prompt. Higher values increase prompt "
f"adherence but may reduce diversity (default: {DEFAULT_CFG_GUIDANCE_SCALE})."
),
)
parser.add_argument(
"--negative-prompt",
type=str,
@@ -205,7 +196,126 @@ def default_1_stage_arg_parser() -> argparse.ArgumentParser:
"Default: a comprehensive negative prompt covering common artifacts and quality issues."
),
)
parser.add_argument(
"--video-cfg-guidance-scale",
type=float,
default=DEFAULT_VIDEO_GUIDER_PARAMS.cfg_scale,
help=(
f"Classifier-free guidance (CFG) scale controlling how strongly "
f"the model adheres to the video prompt. Higher values increase prompt "
"adherence but may reduce diversity. 1.0 means no effect "
f"(default: {DEFAULT_VIDEO_GUIDER_PARAMS.cfg_scale})."
),
)
parser.add_argument(
"--video-stg-guidance-scale",
type=float,
default=DEFAULT_VIDEO_GUIDER_PARAMS.stg_scale,
help=(
f"STG (Spatio-Temporal Guidance) scale controlling how strongly "
f"the model reacts to the perturbation of the video modality. Higher values increase "
f"the effect but may reduce quality. 0.0 means no effect "
f"(default: {DEFAULT_VIDEO_GUIDER_PARAMS.stg_scale})."
),
)
parser.add_argument(
"--video-rescale-scale",
type=float,
default=DEFAULT_VIDEO_GUIDER_PARAMS.rescale_scale,
help=(
f"Rescale scale controlling how strongly "
f"the model rescales the video modality after applying other guidance. Higher values tend to decrease "
f"oversaturation effects. 0.0 means no effect (default: {DEFAULT_VIDEO_GUIDER_PARAMS.rescale_scale})."
),
)
parser.add_argument(
"--video-stg-blocks",
type=int,
nargs="*",
default=DEFAULT_VIDEO_GUIDER_PARAMS.stg_blocks,
help=(f"Which transformer blocks to perturb for STG. Default: {DEFAULT_VIDEO_GUIDER_PARAMS.stg_blocks}."),
)
parser.add_argument(
"--a2v-guidance-scale",
type=float,
default=DEFAULT_VIDEO_GUIDER_PARAMS.modality_scale,
help=(
f"A2V (Audio-to-Video) guidance scale controlling how strongly "
f"the model reacts to the perturbation of the audio-to-video cross-attention. Higher values may increase "
f"lipsync quality. 1.0 means no effect (default: {DEFAULT_VIDEO_GUIDER_PARAMS.modality_scale})."
),
)
parser.add_argument(
"--video-skip-step",
type=int,
default=DEFAULT_VIDEO_GUIDER_PARAMS.skip_step,
help=(
"Video skip step N controls periodic skipping during the video diffusion process: "
"only steps where step_index % (N + 1) == 0 are processed, all others are skipped "
f"(e.g., 0 = no skipping; 1 = skip every other step; 2 = skip 2 of every 3 steps; "
f"default: {DEFAULT_VIDEO_GUIDER_PARAMS.skip_step})."
),
)
parser.add_argument(
"--audio-cfg-guidance-scale",
type=float,
default=DEFAULT_AUDIO_GUIDER_PARAMS.cfg_scale,
help=(
f"Audio CFG (Classifier-free guidance) scale controlling how strongly "
f"the model adheres to the audio prompt. Higher values increase prompt "
f"adherence but may reduce diversity. 1.0 means no effect "
f"(default: {DEFAULT_AUDIO_GUIDER_PARAMS.cfg_scale})."
),
)
parser.add_argument(
"--audio-stg-guidance-scale",
type=float,
default=DEFAULT_AUDIO_GUIDER_PARAMS.stg_scale,
help=(
f"Audio STG (Spatio-Temporal Guidance) scale controlling how strongly "
f"the model reacts to the perturbation of the audio modality. Higher values increase "
f"the effect but may reduce quality. 0.0 means no effect "
f"(default: {DEFAULT_AUDIO_GUIDER_PARAMS.stg_scale})."
),
)
parser.add_argument(
"--audio-rescale-scale",
type=float,
default=DEFAULT_AUDIO_GUIDER_PARAMS.rescale_scale,
help=(
f"Audio rescale scale controlling how strongly "
f"the model rescales the audio modality after applying other guidance. "
f"Experimental. 0.0 means no effect (default: {DEFAULT_AUDIO_GUIDER_PARAMS.rescale_scale})."
),
)
parser.add_argument(
"--audio-stg-blocks",
type=int,
nargs="*",
default=DEFAULT_AUDIO_GUIDER_PARAMS.stg_blocks,
help=(f"Which transformer blocks to perturb for Audio STG. Default: {DEFAULT_AUDIO_GUIDER_PARAMS.stg_blocks}."),
)
parser.add_argument(
"--v2a-guidance-scale",
type=float,
default=DEFAULT_AUDIO_GUIDER_PARAMS.modality_scale,
help=(
f"V2A (Video-to-Audio) guidance scale controlling how strongly "
f"the model reacts to the perturbation of the video-to-audio cross-attention. Higher values may increase "
f"lipsync quality. 1.0 means no effect (default: {DEFAULT_AUDIO_GUIDER_PARAMS.modality_scale})."
),
)
parser.add_argument(
"--audio-skip-step",
type=int,
default=DEFAULT_AUDIO_GUIDER_PARAMS.skip_step,
help=(
"Audio skip step N controls periodic skipping during the audio diffusion process: "
"only steps where step_index % (N + 1) == 0 are processed, all others are skipped "
f"(e.g., 0 = no skipping; 1 = skip every other step; 2 = skip 2 of every 3 steps; "
f"default: {DEFAULT_AUDIO_GUIDER_PARAMS.skip_step})."
),
)
return parser
packages/ltx-pipelines/src/ltx_pipelines/utils/constants.py
+17 -1
View File
@@ -4,6 +4,7 @@
# Noise schedule for the distilled pipeline. These sigma values control noise
# levels at each denoising step and were tuned to match the distillation process.
from ltx_core.components.guiders import MultiModalGuiderParams
from ltx_core.types import SpatioTemporalScaleFactors
DISTILLED_SIGMA_VALUES = [1.0, 0.99375, 0.9875, 0.98125, 0.975, 0.909375, 0.725, 0.421875, 0.0]
@@ -24,13 +25,28 @@ DEFAULT_2_STAGE_WIDTH = DEFAULT_1_STAGE_WIDTH * 2
DEFAULT_NUM_FRAMES = 121
DEFAULT_FRAME_RATE = 24.0
DEFAULT_NUM_INFERENCE_STEPS = 40
DEFAULT_CFG_GUIDANCE_SCALE = 4.0
DEFAULT_VIDEO_GUIDER_PARAMS = MultiModalGuiderParams(
cfg_scale=3.0,
stg_scale=1.0,
rescale_scale=0.7,
modality_scale=3.0,
skip_step=0,
stg_blocks=[29],
)
# =============================================================================
# Audio
# =============================================================================
DEFAULT_AUDIO_GUIDER_PARAMS = MultiModalGuiderParams(
cfg_scale=7.0,
stg_scale=1.0,
rescale_scale=0.7,
modality_scale=3.0,
skip_step=0,
stg_blocks=[29],
)
AUDIO_SAMPLE_RATE = 24000
packages/ltx-pipelines/src/ltx_pipelines/utils/helpers.py
+114
View File
@@ -5,6 +5,7 @@ from dataclasses import replace
import torch
from tqdm import tqdm
from ltx_core.components.guiders import MultiModalGuider
from ltx_core.components.noisers import Noiser
from ltx_core.components.protocols import DiffusionStepProtocol, GuiderProtocol
from ltx_core.conditioning import (
@@ -12,6 +13,12 @@ from ltx_core.conditioning import (
VideoConditionByKeyframeIndex,
VideoConditionByLatentIndex,
)
from ltx_core.guidance.perturbations import (
BatchedPerturbationConfig,
Perturbation,
PerturbationConfig,
PerturbationType,
)
from ltx_core.model.transformer import Modality, X0Model
from ltx_core.model.video_vae import VideoEncoder
from ltx_core.text_encoders.gemma import GemmaTextEncoderModelBase
@@ -376,6 +383,113 @@ def guider_denoising_func(
return guider_denoising_step
def multi_modal_guider_denoising_func(
video_guider: MultiModalGuider,
audio_guider: MultiModalGuider,
v_context: torch.Tensor,
a_context: torch.Tensor,
transformer: X0Model,
) -> DenoisingFunc:
last_denoised_video = None
last_denoised_audio = None
def guider_denoising_step(
video_state: LatentState, audio_state: LatentState, sigmas: torch.Tensor, step_index: int
) -> tuple[torch.Tensor, torch.Tensor]:
nonlocal last_denoised_video, last_denoised_audio
if video_guider.should_skip_step(step_index) and audio_guider.should_skip_step(step_index):
return last_denoised_video, last_denoised_audio
sigma = sigmas[step_index]
pos_video_modality = modality_from_latent_state(
video_state, v_context, sigma, enabled=not video_guider.should_skip_step(step_index)
)
pos_audio_modality = modality_from_latent_state(
audio_state, a_context, sigma, enabled=not audio_guider.should_skip_step(step_index)
)
denoised_video, denoised_audio = transformer(
video=pos_video_modality, audio=pos_audio_modality, perturbations=None
)
neg_denoised_video, neg_denoised_audio = 0.0, 0.0
if video_guider.do_unconditional_generation() or audio_guider.do_unconditional_generation():
if video_guider.do_unconditional_generation() and video_guider.negative_context is None:
raise ValueError("Negative context is required for unconditioned denoising")
if audio_guider.do_unconditional_generation() and audio_guider.negative_context is None:
raise ValueError("Negative context is required for unconditioned denoising")
neg_video_modality = modality_from_latent_state(
video_state,
video_guider.negative_context
if video_guider.negative_context is not None
else pos_video_modality.context,
sigma,
)
neg_audio_modality = modality_from_latent_state(
audio_state,
audio_guider.negative_context
if audio_guider.negative_context is not None
else pos_audio_modality.context,
sigma,
)
neg_denoised_video, neg_denoised_audio = transformer(
video=neg_video_modality, audio=neg_audio_modality, perturbations=None
)
ptb_denoised_video, ptb_denoised_audio = 0.0, 0.0
if video_guider.do_perturbed_generation() or audio_guider.do_perturbed_generation():
perturbations = []
if video_guider.do_perturbed_generation():
perturbations.append(
Perturbation(type=PerturbationType.SKIP_VIDEO_SELF_ATTN, blocks=video_guider.params.stg_blocks)
)
if audio_guider.do_perturbed_generation():
perturbations.append(
Perturbation(type=PerturbationType.SKIP_AUDIO_SELF_ATTN, blocks=audio_guider.params.stg_blocks)
)
perturbation_config = PerturbationConfig(perturbations=perturbations)
ptb_denoised_video, ptb_denoised_audio = transformer(
video=pos_video_modality,
audio=pos_audio_modality,
perturbations=BatchedPerturbationConfig(perturbations=[perturbation_config]),
)
mod_denoised_video, mod_denoised_audio = 0.0, 0.0
if video_guider.do_isolated_modality_generation() or audio_guider.do_isolated_modality_generation():
perturbations = [
Perturbation(type=PerturbationType.SKIP_A2V_CROSS_ATTN, blocks=None),
Perturbation(type=PerturbationType.SKIP_V2A_CROSS_ATTN, blocks=None),
]
perturbation_config = PerturbationConfig(perturbations=perturbations)
mod_denoised_video, mod_denoised_audio = transformer(
video=pos_video_modality,
audio=pos_audio_modality,
perturbations=BatchedPerturbationConfig(perturbations=[perturbation_config]),
)
if video_guider.should_skip_step(step_index):
denoised_video = last_denoised_video
else:
denoised_video = video_guider.calculate(
denoised_video, neg_denoised_video, ptb_denoised_video, mod_denoised_video
)
if audio_guider.should_skip_step(step_index):
denoised_audio = last_denoised_audio
else:
denoised_audio = audio_guider.calculate(
denoised_audio, neg_denoised_audio, ptb_denoised_audio, mod_denoised_audio
)
last_denoised_video = denoised_video
last_denoised_audio = denoised_audio
return denoised_video, denoised_audio
return guider_denoising_step
def denoise_audio_video( # noqa: PLR0913
output_shape: VideoPixelShape,
conditionings: list[ConditioningItem],
packages/ltx-pipelines/src/ltx_pipelines/utils/model_ledger.py
+8 -2
View File
@@ -35,6 +35,8 @@ from ltx_core.text_encoders.gemma import (
AVGemmaTextEncoderModelConfigurator,
module_ops_from_gemma_root,
)
from ltx_core.text_encoders.gemma.encoders.av_encoder import GEMMA_MODEL_OPS
from ltx_core.utils import find_matching_file
class ModelLedger:
@@ -143,12 +145,16 @@ class ModelLedger:
)
if self.gemma_root_path is not None:
module_ops = module_ops_from_gemma_root(self.gemma_root_path)
model_folder = find_matching_file(self.gemma_root_path, "model*.safetensors").parent
weight_paths = [str(p) for p in model_folder.rglob("*.safetensors")]
self.text_encoder_builder = Builder(
model_path=self.checkpoint_path,
model_path=(str(self.checkpoint_path), *weight_paths),
model_class_configurator=AVGemmaTextEncoderModelConfigurator,
model_sd_ops=AV_GEMMA_TEXT_ENCODER_KEY_OPS,
registry=self.registry,
module_ops=module_ops_from_gemma_root(self.gemma_root_path),
module_ops=(GEMMA_MODEL_OPS, *module_ops),
)
if self.spatial_upsampler_path is not None: