indigo/LTX-2
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Automated PR - 2026-04-13

Browse Source
This commit is contained in:
github-actions[bot]
2026-04-13 14:29:35 +00:00
parent 59ca828d5a
commit d887bbd1e0
29 changed files with 463 additions and 184 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/ltx-pipelines/src/ltx_pipelines/a2vid_two_stage.py
+10 -5
View File
@@ -22,7 +22,7 @@ from ltx_pipelines.utils.blocks import (
VideoUpsampler,
)
from ltx_pipelines.utils.constants import (
STAGE_2_DISTILLED_SIGMA_VALUES,
STAGE_2_DISTILLED_SIGMAS,
)
from ltx_pipelines.utils.denoisers import GuidedDenoiser, SimpleDenoiser
from ltx_pipelines.utils.helpers import (
@@ -56,6 +56,7 @@ class A2VidPipelineTwoStage:
):
self.device = device or get_device()
self.dtype = torch.bfloat16
self._scheduler = LTX2Scheduler()
self.prompt_encoder = PromptEncoder(checkpoint_path, gemma_root, self.dtype, self.device, registry=registry)
self.image_conditioner = ImageConditioner(checkpoint_path, self.dtype, self.device, registry=registry)
@@ -103,6 +104,8 @@ class A2VidPipelineTwoStage:
enhance_prompt: bool = False,
streaming_prefetch_count: int | None = None,
max_batch_size: int = 1,
stage_1_sigmas: torch.Tensor | None = None,
stage_2_sigmas: torch.Tensor = STAGE_2_DISTILLED_SIGMAS,
) -> tuple[Iterator[torch.Tensor], Audio]:
assert_resolution(height=height, width=width, is_two_stage=True)
@@ -148,7 +151,9 @@ class A2VidPipelineTwoStage:
)
)
sigmas = LTX2Scheduler().execute(steps=num_inference_steps).to(dtype=torch.float32, device=self.device)
sigmas = (
stage_1_sigmas if stage_1_sigmas is not None else self._scheduler.execute(steps=num_inference_steps)
).to(dtype=torch.float32, device=self.device)
video_state, _ = self.stage_1(
denoiser=GuidedDenoiser(
@@ -185,7 +190,7 @@ class A2VidPipelineTwoStage:
# Stage 2: Upsample and refine the video at higher resolution with distilled LoRA.
upscaled_video_latent = self.upsampler(video_state.latent[:1])
distilled_sigmas = torch.Tensor(STAGE_2_DISTILLED_SIGMA_VALUES).to(self.device)
stage_2_sigmas = stage_2_sigmas.to(dtype=torch.float32, device=self.device)
stage_2_output_shape = VideoPixelShape(batch=1, frames=num_frames, width=width, height=height, fps=frame_rate)
stage_2_conditionings = self.image_conditioner(
lambda enc: combined_image_conditionings(
@@ -200,7 +205,7 @@ class A2VidPipelineTwoStage:
video_state, _ = self.stage_2(
denoiser=SimpleDenoiser(v_context_p, a_context_p),
sigmas=distilled_sigmas,
sigmas=stage_2_sigmas,
noiser=noiser,
width=width,
height=height,
@@ -209,7 +214,7 @@ class A2VidPipelineTwoStage:
video=ModalitySpec(
context=v_context_p,
conditionings=stage_2_conditionings,
noise_scale=distilled_sigmas[0].item(),
noise_scale=stage_2_sigmas[0].item(),
initial_latent=upscaled_video_latent,
),
audio=ModalitySpec(
packages/ltx-pipelines/src/ltx_pipelines/distilled.py
+7 -5
View File
@@ -23,8 +23,8 @@ from ltx_pipelines.utils.blocks import (
VideoUpsampler,
)
from ltx_pipelines.utils.constants import (
DISTILLED_SIGMA_VALUES,
STAGE_2_DISTILLED_SIGMA_VALUES,
DISTILLED_SIGMAS,
STAGE_2_DISTILLED_SIGMAS,
detect_params,
)
from ltx_pipelines.utils.denoisers import SimpleDenoiser
@@ -77,7 +77,7 @@ class DistilledPipeline:
self.video_decoder = VideoDecoder(distilled_checkpoint_path, self.dtype, self.device, registry=registry)
self.audio_decoder = AudioDecoder(distilled_checkpoint_path, self.dtype, self.device, registry=registry)
def __call__(
def __call__( # noqa: PLR0913
self,
prompt: str,
seed: int,
@@ -89,6 +89,8 @@ class DistilledPipeline:
tiling_config: TilingConfig | None = None,
enhance_prompt: bool = False,
streaming_prefetch_count: int | None = None,
stage_1_sigmas: torch.Tensor = DISTILLED_SIGMAS,
stage_2_sigmas: torch.Tensor = STAGE_2_DISTILLED_SIGMAS,
) -> tuple[Iterator[torch.Tensor], Audio]:
assert_resolution(height=height, width=width, is_two_stage=True)
@@ -105,7 +107,7 @@ class DistilledPipeline:
video_context, audio_context = ctx_p.video_encoding, ctx_p.audio_encoding
# Stage 1: Initial low resolution video generation.
stage_1_sigmas = torch.Tensor(DISTILLED_SIGMA_VALUES).to(self.device)
stage_1_sigmas = stage_1_sigmas.to(dtype=torch.float32, device=self.device)
stage_1_w, stage_1_h = width // 2, height // 2
stage_1_conditionings = self.image_conditioner(
lambda enc: combined_image_conditionings(
@@ -134,7 +136,7 @@ class DistilledPipeline:
# Stage 2: Upsample and refine the video at higher resolution with distilled LORA.
upscaled_video_latent = self.upsampler(video_state.latent[:1])
stage_2_sigmas = torch.Tensor(STAGE_2_DISTILLED_SIGMA_VALUES).to(self.device)
stage_2_sigmas = stage_2_sigmas.to(dtype=torch.float32, device=self.device)
stage_2_conditionings = self.image_conditioner(
lambda enc: combined_image_conditionings(
images=images,
packages/ltx-pipelines/src/ltx_pipelines/ic_lora.py
+9 -7
View File
@@ -32,8 +32,8 @@ from ltx_pipelines.utils.blocks import (
VideoUpsampler,
)
from ltx_pipelines.utils.constants import (
DISTILLED_SIGMA_VALUES,
STAGE_2_DISTILLED_SIGMA_VALUES,
DISTILLED_SIGMAS,
STAGE_2_DISTILLED_SIGMAS,
detect_params,
)
from ltx_pipelines.utils.denoisers import SimpleDenoiser
@@ -126,6 +126,8 @@ class ICLoraPipeline:
skip_stage_2: bool = False,
conditioning_attention_mask: torch.Tensor | None = None,
streaming_prefetch_count: int | None = None,
stage_1_sigmas: torch.Tensor = DISTILLED_SIGMAS,
stage_2_sigmas: torch.Tensor = STAGE_2_DISTILLED_SIGMAS,
) -> tuple[Iterator[torch.Tensor], Audio]:
"""
Generate video with IC-LoRA conditioning.
@@ -200,7 +202,7 @@ class ICLoraPipeline:
)
)
stage_1_sigmas = torch.Tensor(DISTILLED_SIGMA_VALUES).to(self.device)
stage_1_sigmas = stage_1_sigmas.to(dtype=torch.float32, device=self.device)
video_state, audio_state = self.stage_1(
denoiser=SimpleDenoiser(video_context, audio_context),
@@ -230,7 +232,7 @@ class ICLoraPipeline:
# Stage 2: Upsample and refine the video at higher resolution with distilled LORA.
upscaled_video_latent = self.upsampler(video_state.latent[:1])
distilled_sigmas = torch.Tensor(STAGE_2_DISTILLED_SIGMA_VALUES).to(self.device)
stage_2_sigmas = stage_2_sigmas.to(dtype=torch.float32, device=self.device)
stage_2_output_shape = VideoPixelShape(batch=1, frames=num_frames, width=width, height=height, fps=frame_rate)
stage_2_conditionings = self.image_conditioner(
lambda enc: combined_image_conditionings(
@@ -245,7 +247,7 @@ class ICLoraPipeline:
video_state, audio_state = self.stage_2(
denoiser=SimpleDenoiser(video_context, audio_context),
sigmas=distilled_sigmas,
sigmas=stage_2_sigmas,
noiser=noiser,
width=width,
height=height,
@@ -254,12 +256,12 @@ class ICLoraPipeline:
video=ModalitySpec(
context=video_context,
conditionings=stage_2_conditionings,
noise_scale=distilled_sigmas[0].item(),
noise_scale=stage_2_sigmas[0].item(),
initial_latent=upscaled_video_latent,
),
audio=ModalitySpec(
context=audio_context,
noise_scale=distilled_sigmas[0].item(),
noise_scale=stage_2_sigmas[0].item(),
initial_latent=audio_state.latent,
),
streaming_prefetch_count=streaming_prefetch_count,
packages/ltx-pipelines/src/ltx_pipelines/keyframe_interpolation.py
+11 -6
View File
@@ -25,7 +25,7 @@ from ltx_pipelines.utils.blocks import (
VideoUpsampler,
)
from ltx_pipelines.utils.constants import (
STAGE_2_DISTILLED_SIGMA_VALUES,
STAGE_2_DISTILLED_SIGMAS,
detect_params,
)
from ltx_pipelines.utils.denoisers import FactoryGuidedDenoiser, SimpleDenoiser
@@ -62,6 +62,7 @@ class KeyframeInterpolationPipeline:
):
self.device = device or get_device()
self.dtype = torch.bfloat16
self._scheduler = LTX2Scheduler()
self.prompt_encoder = PromptEncoder(checkpoint_path, gemma_root, self.dtype, self.device, registry=registry)
self.image_conditioner = ImageConditioner(checkpoint_path, self.dtype, self.device, registry=registry)
@@ -107,6 +108,8 @@ class KeyframeInterpolationPipeline:
enhance_prompt: bool = False,
streaming_prefetch_count: int | None = None,
max_batch_size: int = 1,
stage_1_sigmas: torch.Tensor | None = None,
stage_2_sigmas: torch.Tensor = STAGE_2_DISTILLED_SIGMAS,
) -> tuple[Iterator[torch.Tensor], Audio]:
assert_resolution(height=height, width=width, is_two_stage=True)
@@ -125,7 +128,9 @@ class KeyframeInterpolationPipeline:
v_context_n, a_context_n = ctx_n.video_encoding, ctx_n.audio_encoding
# Stage 1: Initial low resolution video generation.
sigmas = LTX2Scheduler().execute(steps=num_inference_steps).to(dtype=torch.float32, device=self.device)
sigmas = (
stage_1_sigmas if stage_1_sigmas is not None else self._scheduler.execute(steps=num_inference_steps)
).to(dtype=torch.float32, device=self.device)
stage_1_output_shape = VideoPixelShape(
batch=1,
@@ -181,7 +186,7 @@ class KeyframeInterpolationPipeline:
# Stage 2: Upsample and refine the video at higher resolution with distilled LORA.
upscaled_video_latent = self.upsampler(video_state.latent[:1])
distilled_sigmas = torch.Tensor(STAGE_2_DISTILLED_SIGMA_VALUES).to(self.device)
stage_2_sigmas = stage_2_sigmas.to(dtype=torch.float32, device=self.device)
stage_2_output_shape = VideoPixelShape(batch=1, frames=num_frames, width=width, height=height, fps=frame_rate)
stage_2_conditionings = self.image_conditioner(
lambda enc: image_conditionings_by_adding_guiding_latent(
@@ -196,7 +201,7 @@ class KeyframeInterpolationPipeline:
video_state, audio_state = self.stage_2(
denoiser=SimpleDenoiser(v_context_p, a_context_p),
sigmas=distilled_sigmas,
sigmas=stage_2_sigmas,
noiser=noiser,
width=width,
height=height,
@@ -205,12 +210,12 @@ class KeyframeInterpolationPipeline:
video=ModalitySpec(
context=v_context_p,
conditionings=stage_2_conditionings,
noise_scale=distilled_sigmas[0].item(),
noise_scale=stage_2_sigmas[0].item(),
initial_latent=upscaled_video_latent,
),
audio=ModalitySpec(
context=a_context_p,
noise_scale=distilled_sigmas[0].item(),
noise_scale=stage_2_sigmas[0].item(),
initial_latent=audio_state.latent,
),
streaming_prefetch_count=streaming_prefetch_count,
packages/ltx-pipelines/src/ltx_pipelines/retake.py
+10 -4
View File
@@ -25,7 +25,7 @@ from ltx_pipelines.utils.blocks import (
PromptEncoder,
VideoDecoder,
)
from ltx_pipelines.utils.constants import DISTILLED_SIGMA_VALUES, detect_params
from ltx_pipelines.utils.constants import DISTILLED_SIGMAS, detect_params
from ltx_pipelines.utils.denoisers import GuidedDenoiser, SimpleDenoiser
from ltx_pipelines.utils.helpers import (
audio_latent_from_file,
@@ -78,6 +78,8 @@ class RetakePipeline:
self.device = device or get_device()
self.dtype = torch.bfloat16
self.distilled = distilled
if not distilled:
self._scheduler = LTX2Scheduler()
self.prompt_encoder = PromptEncoder(
checkpoint_path=checkpoint_path,
gemma_root=gemma_root,
@@ -141,6 +143,7 @@ class RetakePipeline:
tiling_config: TilingConfig | None = None,
streaming_prefetch_count: int | None = None,
max_batch_size: int = 1,
sigmas: torch.Tensor | None = None,
) -> tuple[Iterator[torch.Tensor], torch.Tensor]:
"""Regenerate ``[start_time, end_time]`` of the source video (retake).
Parameters
@@ -227,15 +230,18 @@ class RetakePipeline:
initial_latent=initial_audio_latent,
frozen=initial_audio_latent is not None and not regenerate_audio,
)
# Build denoiser
# Build denoiser and resolve sigma schedule.
if sigmas is None:
sigmas = DISTILLED_SIGMAS if self.distilled else self._scheduler.execute(steps=num_inference_steps)
sigmas = sigmas.to(dtype=torch.float32, device=self.device)
if self.distilled:
sigmas = torch.tensor(DISTILLED_SIGMA_VALUES).to(dtype=torch.float32, device=self.device)
denoiser = SimpleDenoiser(
v_context=v_context_p,
a_context=a_context_p,
)
else:
sigmas = LTX2Scheduler().execute(steps=num_inference_steps).to(dtype=torch.float32, device=self.device)
v_context_n, a_context_n = contexts[1].video_encoding, contexts[1].audio_encoding
video_guider = MultiModalGuider(
params=video_guider_params,
packages/ltx-pipelines/src/ltx_pipelines/ti2vid_one_stage.py
+5 -1
View File
@@ -55,6 +55,7 @@ class TI2VidOneStagePipeline:
):
self.dtype = torch.bfloat16
self.device = device or get_device()
self._scheduler = LTX2Scheduler()
self.prompt_encoder = PromptEncoder(
checkpoint_path=checkpoint_path,
gemma_root=gemma_root,
@@ -107,6 +108,7 @@ class TI2VidOneStagePipeline:
streaming_prefetch_count: int | None = None,
tiling_config: TilingConfig | None = None,
max_batch_size: int = 1,
sigmas: torch.Tensor | None = None,
) -> tuple[Iterator[torch.Tensor], Audio]:
assert_resolution(height=height, width=width, is_two_stage=False)
@@ -135,7 +137,9 @@ class TI2VidOneStagePipeline:
)
)
sigmas = LTX2Scheduler().execute(steps=num_inference_steps).to(dtype=torch.float32, device=self.device)
sigmas = (sigmas if sigmas is not None else self._scheduler.execute(steps=num_inference_steps)).to(
dtype=torch.float32, device=self.device
)
video_guider_factory = create_multimodal_guider_factory(
params=video_guider_params,
packages/ltx-pipelines/src/ltx_pipelines/ti2vid_two_stages.py
+11 -6
View File
@@ -25,7 +25,7 @@ from ltx_pipelines.utils.blocks import (
VideoUpsampler,
)
from ltx_pipelines.utils.constants import (
STAGE_2_DISTILLED_SIGMA_VALUES,
STAGE_2_DISTILLED_SIGMAS,
detect_params,
)
from ltx_pipelines.utils.denoisers import FactoryGuidedDenoiser, SimpleDenoiser
@@ -61,6 +61,7 @@ class TI2VidTwoStagesPipeline:
):
self.device = device or get_device()
self.dtype = torch.bfloat16
self._scheduler = LTX2Scheduler()
self.prompt_encoder = PromptEncoder(checkpoint_path, gemma_root, self.dtype, self.device, registry=registry)
self.image_conditioner = ImageConditioner(checkpoint_path, self.dtype, self.device, registry=registry)
@@ -106,6 +107,8 @@ class TI2VidTwoStagesPipeline:
enhance_prompt: bool = False,
streaming_prefetch_count: int | None = None,
max_batch_size: int = 1,
stage_1_sigmas: torch.Tensor | None = None,
stage_2_sigmas: torch.Tensor = STAGE_2_DISTILLED_SIGMAS,
) -> tuple[Iterator[torch.Tensor], Audio]:
assert_resolution(height=height, width=width, is_two_stage=True)
@@ -142,7 +145,9 @@ class TI2VidTwoStagesPipeline:
)
)
sigmas = LTX2Scheduler().execute(steps=num_inference_steps).to(dtype=torch.float32, device=self.device)
sigmas = (
stage_1_sigmas if stage_1_sigmas is not None else self._scheduler.execute(steps=num_inference_steps)
).to(dtype=torch.float32, device=self.device)
video_state, audio_state = self.stage_1(
denoiser=FactoryGuidedDenoiser(
@@ -172,7 +177,7 @@ class TI2VidTwoStagesPipeline:
# Stage 2: Upsample and refine the video at higher resolution with distilled LoRA.
upscaled_video_latent = self.upsampler(video_state.latent[:1])
distilled_sigmas = torch.Tensor(STAGE_2_DISTILLED_SIGMA_VALUES).to(self.device)
stage_2_sigmas = stage_2_sigmas.to(dtype=torch.float32, device=self.device)
stage_2_conditionings = self.image_conditioner(
lambda enc: combined_image_conditionings(
images=images,
@@ -186,7 +191,7 @@ class TI2VidTwoStagesPipeline:
video_state, audio_state = self.stage_2(
denoiser=SimpleDenoiser(v_context=v_context_p, a_context=a_context_p),
sigmas=distilled_sigmas,
sigmas=stage_2_sigmas,
noiser=noiser,
width=width,
height=height,
@@ -195,12 +200,12 @@ class TI2VidTwoStagesPipeline:
video=ModalitySpec(
context=v_context_p,
conditionings=stage_2_conditionings,
noise_scale=distilled_sigmas[0].item(),
noise_scale=stage_2_sigmas[0].item(),
initial_latent=upscaled_video_latent,
),
audio=ModalitySpec(
context=a_context_p,
noise_scale=distilled_sigmas[0].item(),
noise_scale=stage_2_sigmas[0].item(),
initial_latent=audio_state.latent,
),
streaming_prefetch_count=streaming_prefetch_count,
packages/ltx-pipelines/src/ltx_pipelines/ti2vid_two_stages_hq.py
+13 -11
View File
@@ -23,7 +23,7 @@ from ltx_pipelines.utils.blocks import (
)
from ltx_pipelines.utils.constants import (
LTX_2_3_HQ_PARAMS,
STAGE_2_DISTILLED_SIGMA_VALUES,
STAGE_2_DISTILLED_SIGMAS,
)
from ltx_pipelines.utils.denoisers import GuidedDenoiser, SimpleDenoiser
from ltx_pipelines.utils.helpers import (
@@ -64,6 +64,7 @@ class TI2VidTwoStagesHQPipeline:
):
self.device = device or get_device()
self.dtype = torch.bfloat16
self._scheduler = LTX2Scheduler()
distilled_lora_stage_1 = LoraPathStrengthAndSDOps(
path=distilled_lora[0].path,
@@ -121,6 +122,8 @@ class TI2VidTwoStagesHQPipeline:
enhance_prompt: bool = False,
streaming_prefetch_count: int | None = None,
max_batch_size: int = 1,
stage_1_sigmas: torch.Tensor | None = None,
stage_2_sigmas: torch.Tensor = STAGE_2_DISTILLED_SIGMAS,
) -> tuple[Iterator[torch.Tensor], Audio]:
assert_resolution(height=height, width=width, is_two_stage=True)
@@ -157,13 +160,12 @@ class TI2VidTwoStagesHQPipeline:
)
)
empty_latent = torch.empty(VideoLatentShape.from_pixel_shape(stage_1_output_shape).to_torch_shape())
stepper = Res2sDiffusionStep()
sigmas = (
LTX2Scheduler()
.execute(latent=empty_latent, steps=num_inference_steps)
.to(dtype=torch.float32, device=self.device)
)
if stage_1_sigmas is None:
empty_latent = torch.empty(VideoLatentShape.from_pixel_shape(stage_1_output_shape).to_torch_shape())
stage_1_sigmas = self._scheduler.execute(latent=empty_latent, steps=num_inference_steps)
sigmas = stage_1_sigmas.to(dtype=torch.float32, device=self.device)
video_state, audio_state = self.stage_1(
denoiser=GuidedDenoiser(
@@ -195,7 +197,7 @@ class TI2VidTwoStagesHQPipeline:
# Stage 2: Upsample and refine the video at higher resolution with distilled LoRA.
upscaled_video_latent = self.upsampler(video_state.latent[:1])
distilled_sigmas = torch.tensor(STAGE_2_DISTILLED_SIGMA_VALUES, device=self.device)
stage_2_sigmas = stage_2_sigmas.to(dtype=torch.float32, device=self.device)
stage_2_output_shape = VideoPixelShape(batch=1, frames=num_frames, width=width, height=height, fps=frame_rate)
stage_2_conditionings = self.image_conditioner(
lambda enc: combined_image_conditionings(
@@ -210,7 +212,7 @@ class TI2VidTwoStagesHQPipeline:
video_state, audio_state = self.stage_2(
denoiser=SimpleDenoiser(v_context=v_context_p, a_context=a_context_p),
sigmas=distilled_sigmas,
sigmas=stage_2_sigmas,
noiser=noiser,
stepper=stepper,
width=width,
@@ -220,12 +222,12 @@ class TI2VidTwoStagesHQPipeline:
video=ModalitySpec(
context=v_context_p,
conditionings=stage_2_conditionings,
noise_scale=distilled_sigmas[0].item(),
noise_scale=stage_2_sigmas[0].item(),
initial_latent=upscaled_video_latent,
),
audio=ModalitySpec(
context=a_context_p,
noise_scale=distilled_sigmas[0].item(),
noise_scale=stage_2_sigmas[0].item(),
initial_latent=audio_state.latent,
),
loop=res2s_audio_video_denoising_loop,
packages/ltx-pipelines/src/ltx_pipelines/utils/constants.py
+4
View File
@@ -1,6 +1,7 @@
import logging
from dataclasses import dataclass, field, replace
import torch
from safetensors import safe_open
from ltx_core.components.guiders import MultiModalGuiderParams
@@ -17,6 +18,9 @@ DISTILLED_SIGMA_VALUES = [1.0, 0.99375, 0.9875, 0.98125, 0.975, 0.909375, 0.725,
# Reduced schedule for super-resolution stage 2 (subset of distilled values)
STAGE_2_DISTILLED_SIGMA_VALUES = [0.909375, 0.725, 0.421875, 0.0]
DISTILLED_SIGMAS = torch.tensor(DISTILLED_SIGMA_VALUES)
STAGE_2_DISTILLED_SIGMAS = torch.tensor(STAGE_2_DISTILLED_SIGMA_VALUES)
# =============================================================================
# Pipeline Parameters