indigo/LTX-2
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Automated PR - 2026-01-29

Browse Source
This commit is contained in:
sync-bot
2026-01-29 18:42:17 +00:00
parent 727c43e998
commit ca1623ad2a
31 changed files with 1723 additions and 663 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/ltx-pipelines/src/ltx_pipelines/ic_lora.py
+58 -4
View File
@@ -2,11 +2,12 @@ import logging
from collections.abc import Iterator
import torch
from safetensors import safe_open
from ltx_core.components.diffusion_steps import EulerDiffusionStep
from ltx_core.components.noisers import GaussianNoiser
from ltx_core.components.protocols import DiffusionStepProtocol
from ltx_core.conditioning import ConditioningItem, VideoConditionByKeyframeIndex
from ltx_core.conditioning import ConditioningItem, VideoConditionByReferenceLatent
from ltx_core.loader import LoraPathStrengthAndSDOps
from ltx_core.model.audio_vae import decode_audio as vae_decode_audio
from ltx_core.model.upsampler import upsample_video
@@ -81,6 +82,21 @@ class ICLoraPipeline:
)
self.device = device
# Read reference downscale factor from LoRA metadata.
# IC-LoRAs trained with low-resolution reference videos store this factor
# so inference can resize reference videos to match training conditions.
self.reference_downscale_factor = 1
for lora in loras:
scale = _read_lora_reference_downscale_factor(lora.path)
if scale != 1:
if self.reference_downscale_factor not in (1, scale):
raise ValueError(
f"Conflicting reference_downscale_factor values in LoRAs: "
f"already have {self.reference_downscale_factor}, but {lora.path} "
f"specifies {scale}. Cannot combine LoRAs with different reference scales."
)
self.reference_downscale_factor = scale
@torch.inference_mode()
def __call__(
self,
@@ -249,17 +265,34 @@ class ICLoraPipeline:
device=self.device,
)
# Calculate scaled dimensions for reference video conditioning.
# IC-LoRAs trained with downscaled reference videos expect the same ratio at inference.
scale = self.reference_downscale_factor
if scale != 1 and (height % scale != 0 or width % scale != 0):
raise ValueError(
f"Output dimensions ({height}x{width}) must be divisible by reference_downscale_factor ({scale})"
)
ref_height = height // scale
ref_width = width // scale
for video_path, strength in video_conditioning:
# Load video at scaled-down resolution (if scale > 1)
video = load_video_conditioning(
video_path=video_path,
height=height,
width=width,
height=ref_height,
width=ref_width,
frame_cap=num_frames,
dtype=self.dtype,
device=self.device,
)
encoded_video = video_encoder(video)
conditionings.append(VideoConditionByKeyframeIndex(keyframes=encoded_video, frame_idx=0, strength=strength))
conditionings.append(
VideoConditionByReferenceLatent(
latent=encoded_video,
downscale_factor=scale,
strength=strength,
)
)
return conditionings
@@ -307,5 +340,26 @@ def main() -> None:
)
def _read_lora_reference_downscale_factor(lora_path: str) -> int:
"""Read reference_downscale_factor from LoRA safetensors metadata.
Some IC-LoRA models are trained with reference videos at lower resolution than
the target output. This allows for more efficient training and can improve
generalization. The downscale factor indicates the ratio between target and
reference resolutions (e.g., factor=2 means reference is half the resolution).
Args:
lora_path: Path to the LoRA .safetensors file
Returns:
The reference downscale factor (1 if not specified in metadata, meaning
reference and target have the same resolution)
"""
try:
with safe_open(lora_path, framework="pt") as f:
metadata = f.metadata() or {}
return int(metadata.get("reference_downscale_factor", 1))
except Exception as e:
logging.warning(f"Failed to read metadata from LoRA file '{lora_path}': {e}")
return 1
if __name__ == "__main__":
main()
packages/ltx-pipelines/src/ltx_pipelines/keyframe_interpolation.py
+31 -11
View File
@@ -4,7 +4,7 @@ from collections.abc import Iterator
import torch
from ltx_core.components.diffusion_steps import EulerDiffusionStep
from ltx_core.components.guiders import CFGGuider
from ltx_core.components.guiders import MultiModalGuider, MultiModalGuiderParams
from ltx_core.components.noisers import GaussianNoiser
from ltx_core.components.protocols import DiffusionStepProtocol
from ltx_core.components.schedulers import LTX2Scheduler
@@ -28,8 +28,8 @@ from ltx_pipelines.utils.helpers import (
euler_denoising_loop,
generate_enhanced_prompt,
get_device,
guider_denoising_func,
image_conditionings_by_adding_guiding_latent,
multi_modal_guider_denoising_func,
simple_denoising_func,
)
from ltx_pipelines.utils.media_io import encode_video
@@ -86,7 +86,8 @@ class KeyframeInterpolationPipeline:
num_frames: int,
frame_rate: float,
num_inference_steps: int,
cfg_guidance_scale: float,
video_guider_params: MultiModalGuiderParams,
audio_guider_params: MultiModalGuiderParams,
images: list[tuple[str, int, float]],
tiling_config: TilingConfig | None = None,
enhance_prompt: bool = False,
@@ -96,7 +97,6 @@ class KeyframeInterpolationPipeline:
generator = torch.Generator(device=self.device).manual_seed(seed)
noiser = GaussianNoiser(generator=generator)
stepper = EulerDiffusionStep()
cfg_guider = CFGGuider(cfg_guidance_scale)
dtype = torch.bfloat16
text_encoder = self.stage_1_model_ledger.text_encoder()
@@ -125,12 +125,17 @@ class KeyframeInterpolationPipeline:
video_state=video_state,
audio_state=audio_state,
stepper=stepper,
denoise_fn=guider_denoising_func(
cfg_guider,
v_context_p,
v_context_n,
a_context_p,
a_context_n,
denoise_fn=multi_modal_guider_denoising_func(
video_guider=MultiModalGuider(
params=video_guider_params,
negative_context=v_context_n,
),
audio_guider=MultiModalGuider(
params=audio_guider_params,
negative_context=a_context_n,
),
v_context=v_context_p,
a_context=a_context_p,
transformer=transformer, # noqa: F821
),
)
@@ -256,7 +261,22 @@ def main() -> None:
num_frames=args.num_frames,
frame_rate=args.frame_rate,
num_inference_steps=args.num_inference_steps,
cfg_guidance_scale=args.cfg_guidance_scale,
video_guider_params=MultiModalGuiderParams(
cfg_scale=args.video_cfg_guidance_scale,
stg_scale=args.video_stg_guidance_scale,
rescale_scale=args.video_rescale_scale,
modality_scale=args.a2v_guidance_scale,
skip_step=args.video_skip_step,
stg_blocks=args.video_stg_blocks,
),
audio_guider_params=MultiModalGuiderParams(
cfg_scale=args.audio_cfg_guidance_scale,
stg_scale=args.audio_stg_guidance_scale,
rescale_scale=args.audio_rescale_scale,
modality_scale=args.v2a_guidance_scale,
skip_step=args.audio_skip_step,
stg_blocks=args.audio_stg_blocks,
),
images=args.images,
tiling_config=tiling_config,
)
packages/ltx-pipelines/src/ltx_pipelines/ti2vid_one_stage.py
+31 -11
View File
@@ -4,7 +4,7 @@ from collections.abc import Iterator
import torch
from ltx_core.components.diffusion_steps import EulerDiffusionStep
from ltx_core.components.guiders import CFGGuider
from ltx_core.components.guiders import MultiModalGuider, MultiModalGuiderParams
from ltx_core.components.noisers import GaussianNoiser
from ltx_core.components.protocols import DiffusionStepProtocol
from ltx_core.components.schedulers import LTX2Scheduler
@@ -23,8 +23,8 @@ from ltx_pipelines.utils.helpers import (
euler_denoising_loop,
generate_enhanced_prompt,
get_device,
guider_denoising_func,
image_conditionings_by_replacing_latent,
multi_modal_guider_denoising_func,
)
from ltx_pipelines.utils.media_io import encode_video
from ltx_pipelines.utils.types import PipelineComponents
@@ -73,7 +73,8 @@ class TI2VidOneStagePipeline:
num_frames: int,
frame_rate: float,
num_inference_steps: int,
cfg_guidance_scale: float,
video_guider_params: MultiModalGuiderParams,
audio_guider_params: MultiModalGuiderParams,
images: list[tuple[str, int, float]],
enhance_prompt: bool = False,
) -> tuple[Iterator[torch.Tensor], torch.Tensor]:
@@ -82,7 +83,6 @@ class TI2VidOneStagePipeline:
generator = torch.Generator(device=self.device).manual_seed(seed)
noiser = GaussianNoiser(generator=generator)
stepper = EulerDiffusionStep()
cfg_guider = CFGGuider(cfg_guidance_scale)
dtype = torch.bfloat16
text_encoder = self.model_ledger.text_encoder()
@@ -111,12 +111,17 @@ class TI2VidOneStagePipeline:
video_state=video_state,
audio_state=audio_state,
stepper=stepper,
denoise_fn=guider_denoising_func(
cfg_guider,
v_context_p,
v_context_n,
a_context_p,
a_context_n,
denoise_fn=multi_modal_guider_denoising_func(
video_guider=MultiModalGuider(
params=video_guider_params,
negative_context=v_context_n,
),
audio_guider=MultiModalGuider(
params=audio_guider_params,
negative_context=a_context_n,
),
v_context=v_context_p,
a_context=a_context_p,
transformer=transformer, # noqa: F821
),
)
@@ -175,7 +180,22 @@ def main() -> None:
num_frames=args.num_frames,
frame_rate=args.frame_rate,
num_inference_steps=args.num_inference_steps,
cfg_guidance_scale=args.cfg_guidance_scale,
video_guider_params=MultiModalGuiderParams(
cfg_scale=args.video_cfg_guidance_scale,
stg_scale=args.video_stg_guidance_scale,
rescale_scale=args.video_rescale_scale,
modality_scale=args.a2v_guidance_scale,
skip_step=args.video_skip_step,
stg_blocks=args.video_stg_blocks,
),
audio_guider_params=MultiModalGuiderParams(
cfg_scale=args.audio_cfg_guidance_scale,
stg_scale=args.audio_stg_guidance_scale,
rescale_scale=args.audio_rescale_scale,
modality_scale=args.v2a_guidance_scale,
skip_step=args.audio_skip_step,
stg_blocks=args.audio_stg_blocks,
),
images=args.images,
)
packages/ltx-pipelines/src/ltx_pipelines/ti2vid_two_stages.py
+31 -11
View File
@@ -4,7 +4,7 @@ from collections.abc import Iterator
import torch
from ltx_core.components.diffusion_steps import EulerDiffusionStep
from ltx_core.components.guiders import CFGGuider
from ltx_core.components.guiders import MultiModalGuider, MultiModalGuiderParams
from ltx_core.components.noisers import GaussianNoiser
from ltx_core.components.protocols import DiffusionStepProtocol
from ltx_core.components.schedulers import LTX2Scheduler
@@ -28,8 +28,8 @@ from ltx_pipelines.utils.helpers import (
euler_denoising_loop,
generate_enhanced_prompt,
get_device,
guider_denoising_func,
image_conditionings_by_replacing_latent,
multi_modal_guider_denoising_func,
simple_denoising_func,
)
from ltx_pipelines.utils.media_io import encode_video
@@ -88,7 +88,8 @@ class TI2VidTwoStagesPipeline:
num_frames: int,
frame_rate: float,
num_inference_steps: int,
cfg_guidance_scale: float,
video_guider_params: MultiModalGuiderParams,
audio_guider_params: MultiModalGuiderParams,
images: list[tuple[str, int, float]],
tiling_config: TilingConfig | None = None,
enhance_prompt: bool = False,
@@ -98,7 +99,6 @@ class TI2VidTwoStagesPipeline:
generator = torch.Generator(device=self.device).manual_seed(seed)
noiser = GaussianNoiser(generator=generator)
stepper = EulerDiffusionStep()
cfg_guider = CFGGuider(cfg_guidance_scale)
dtype = torch.bfloat16
text_encoder = self.stage_1_model_ledger.text_encoder()
@@ -127,12 +127,17 @@ class TI2VidTwoStagesPipeline:
video_state=video_state,
audio_state=audio_state,
stepper=stepper,
denoise_fn=guider_denoising_func(
cfg_guider,
v_context_p,
v_context_n,
a_context_p,
a_context_n,
denoise_fn=multi_modal_guider_denoising_func(
video_guider=MultiModalGuider(
params=video_guider_params,
negative_context=v_context_n,
),
audio_guider=MultiModalGuider(
params=audio_guider_params,
negative_context=a_context_n,
),
v_context=v_context_p,
a_context=a_context_p,
transformer=transformer, # noqa: F821
),
)
@@ -259,7 +264,22 @@ def main() -> None:
num_frames=args.num_frames,
frame_rate=args.frame_rate,
num_inference_steps=args.num_inference_steps,
cfg_guidance_scale=args.cfg_guidance_scale,
video_guider_params=MultiModalGuiderParams(
cfg_scale=args.video_cfg_guidance_scale,
stg_scale=args.video_stg_guidance_scale,
rescale_scale=args.video_rescale_scale,
modality_scale=args.a2v_guidance_scale,
skip_step=args.video_skip_step,
stg_blocks=args.video_stg_blocks,
),
audio_guider_params=MultiModalGuiderParams(
cfg_scale=args.audio_cfg_guidance_scale,
stg_scale=args.audio_stg_guidance_scale,
rescale_scale=args.audio_rescale_scale,
modality_scale=args.v2a_guidance_scale,
skip_step=args.audio_skip_step,
stg_blocks=args.audio_stg_blocks,
),
images=args.images,
tiling_config=tiling_config,
)
packages/ltx-pipelines/src/ltx_pipelines/utils/args.py
+122 -12
View File
@@ -7,13 +7,14 @@ from ltx_pipelines.utils.constants import (
DEFAULT_1_STAGE_WIDTH,
DEFAULT_2_STAGE_HEIGHT,
DEFAULT_2_STAGE_WIDTH,
DEFAULT_CFG_GUIDANCE_SCALE,
DEFAULT_AUDIO_GUIDER_PARAMS,
DEFAULT_FRAME_RATE,
DEFAULT_LORA_STRENGTH,
DEFAULT_NEGATIVE_PROMPT,
DEFAULT_NUM_FRAMES,
DEFAULT_NUM_INFERENCE_STEPS,
DEFAULT_SEED,
DEFAULT_VIDEO_GUIDER_PARAMS,
)
@@ -185,16 +186,6 @@ def basic_arg_parser() -> argparse.ArgumentParser:
def default_1_stage_arg_parser() -> argparse.ArgumentParser:
parser = basic_arg_parser()
parser.add_argument(
"--cfg-guidance-scale",
type=float,
default=DEFAULT_CFG_GUIDANCE_SCALE,
help=(
f"Classifier-free guidance (CFG) scale controlling how strongly "
f"the model adheres to the prompt. Higher values increase prompt "
f"adherence but may reduce diversity (default: {DEFAULT_CFG_GUIDANCE_SCALE})."
),
)
parser.add_argument(
"--negative-prompt",
type=str,
@@ -205,7 +196,126 @@ def default_1_stage_arg_parser() -> argparse.ArgumentParser:
"Default: a comprehensive negative prompt covering common artifacts and quality issues."
),
)
parser.add_argument(
"--video-cfg-guidance-scale",
type=float,
default=DEFAULT_VIDEO_GUIDER_PARAMS.cfg_scale,
help=(
f"Classifier-free guidance (CFG) scale controlling how strongly "
f"the model adheres to the video prompt. Higher values increase prompt "
"adherence but may reduce diversity. 1.0 means no effect "
f"(default: {DEFAULT_VIDEO_GUIDER_PARAMS.cfg_scale})."
),
)
parser.add_argument(
"--video-stg-guidance-scale",
type=float,
default=DEFAULT_VIDEO_GUIDER_PARAMS.stg_scale,
help=(
f"STG (Spatio-Temporal Guidance) scale controlling how strongly "
f"the model reacts to the perturbation of the video modality. Higher values increase "
f"the effect but may reduce quality. 0.0 means no effect "
f"(default: {DEFAULT_VIDEO_GUIDER_PARAMS.stg_scale})."
),
)
parser.add_argument(
"--video-rescale-scale",
type=float,
default=DEFAULT_VIDEO_GUIDER_PARAMS.rescale_scale,
help=(
f"Rescale scale controlling how strongly "
f"the model rescales the video modality after applying other guidance. Higher values tend to decrease "
f"oversaturation effects. 0.0 means no effect (default: {DEFAULT_VIDEO_GUIDER_PARAMS.rescale_scale})."
),
)
parser.add_argument(
"--video-stg-blocks",
type=int,
nargs="*",
default=DEFAULT_VIDEO_GUIDER_PARAMS.stg_blocks,
help=(f"Which transformer blocks to perturb for STG. Default: {DEFAULT_VIDEO_GUIDER_PARAMS.stg_blocks}."),
)
parser.add_argument(
"--a2v-guidance-scale",
type=float,
default=DEFAULT_VIDEO_GUIDER_PARAMS.modality_scale,
help=(
f"A2V (Audio-to-Video) guidance scale controlling how strongly "
f"the model reacts to the perturbation of the audio-to-video cross-attention. Higher values may increase "
f"lipsync quality. 1.0 means no effect (default: {DEFAULT_VIDEO_GUIDER_PARAMS.modality_scale})."
),
)
parser.add_argument(
"--video-skip-step",
type=int,
default=DEFAULT_VIDEO_GUIDER_PARAMS.skip_step,
help=(
"Video skip step N controls periodic skipping during the video diffusion process: "
"only steps where step_index % (N + 1) == 0 are processed, all others are skipped "
f"(e.g., 0 = no skipping; 1 = skip every other step; 2 = skip 2 of every 3 steps; "
f"default: {DEFAULT_VIDEO_GUIDER_PARAMS.skip_step})."
),
)
parser.add_argument(
"--audio-cfg-guidance-scale",
type=float,
default=DEFAULT_AUDIO_GUIDER_PARAMS.cfg_scale,
help=(
f"Audio CFG (Classifier-free guidance) scale controlling how strongly "
f"the model adheres to the audio prompt. Higher values increase prompt "
f"adherence but may reduce diversity. 1.0 means no effect "
f"(default: {DEFAULT_AUDIO_GUIDER_PARAMS.cfg_scale})."
),
)
parser.add_argument(
"--audio-stg-guidance-scale",
type=float,
default=DEFAULT_AUDIO_GUIDER_PARAMS.stg_scale,
help=(
f"Audio STG (Spatio-Temporal Guidance) scale controlling how strongly "
f"the model reacts to the perturbation of the audio modality. Higher values increase "
f"the effect but may reduce quality. 0.0 means no effect "
f"(default: {DEFAULT_AUDIO_GUIDER_PARAMS.stg_scale})."
),
)
parser.add_argument(
"--audio-rescale-scale",
type=float,
default=DEFAULT_AUDIO_GUIDER_PARAMS.rescale_scale,
help=(
f"Audio rescale scale controlling how strongly "
f"the model rescales the audio modality after applying other guidance. "
f"Experimental. 0.0 means no effect (default: {DEFAULT_AUDIO_GUIDER_PARAMS.rescale_scale})."
),
)
parser.add_argument(
"--audio-stg-blocks",
type=int,
nargs="*",
default=DEFAULT_AUDIO_GUIDER_PARAMS.stg_blocks,
help=(f"Which transformer blocks to perturb for Audio STG. Default: {DEFAULT_AUDIO_GUIDER_PARAMS.stg_blocks}."),
)
parser.add_argument(
"--v2a-guidance-scale",
type=float,
default=DEFAULT_AUDIO_GUIDER_PARAMS.modality_scale,
help=(
f"V2A (Video-to-Audio) guidance scale controlling how strongly "
f"the model reacts to the perturbation of the video-to-audio cross-attention. Higher values may increase "
f"lipsync quality. 1.0 means no effect (default: {DEFAULT_AUDIO_GUIDER_PARAMS.modality_scale})."
),
)
parser.add_argument(
"--audio-skip-step",
type=int,
default=DEFAULT_AUDIO_GUIDER_PARAMS.skip_step,
help=(
"Audio skip step N controls periodic skipping during the audio diffusion process: "
"only steps where step_index % (N + 1) == 0 are processed, all others are skipped "
f"(e.g., 0 = no skipping; 1 = skip every other step; 2 = skip 2 of every 3 steps; "
f"default: {DEFAULT_AUDIO_GUIDER_PARAMS.skip_step})."
),
)
return parser
packages/ltx-pipelines/src/ltx_pipelines/utils/constants.py
+17 -1
View File
@@ -4,6 +4,7 @@
# Noise schedule for the distilled pipeline. These sigma values control noise
# levels at each denoising step and were tuned to match the distillation process.
from ltx_core.components.guiders import MultiModalGuiderParams
from ltx_core.types import SpatioTemporalScaleFactors
DISTILLED_SIGMA_VALUES = [1.0, 0.99375, 0.9875, 0.98125, 0.975, 0.909375, 0.725, 0.421875, 0.0]
@@ -24,13 +25,28 @@ DEFAULT_2_STAGE_WIDTH = DEFAULT_1_STAGE_WIDTH * 2
DEFAULT_NUM_FRAMES = 121
DEFAULT_FRAME_RATE = 24.0
DEFAULT_NUM_INFERENCE_STEPS = 40
DEFAULT_CFG_GUIDANCE_SCALE = 4.0
DEFAULT_VIDEO_GUIDER_PARAMS = MultiModalGuiderParams(
cfg_scale=3.0,
stg_scale=1.0,
rescale_scale=0.7,
modality_scale=3.0,
skip_step=0,
stg_blocks=[29],
)
# =============================================================================
# Audio
# =============================================================================
DEFAULT_AUDIO_GUIDER_PARAMS = MultiModalGuiderParams(
cfg_scale=7.0,
stg_scale=1.0,
rescale_scale=0.7,
modality_scale=3.0,
skip_step=0,
stg_blocks=[29],
)
AUDIO_SAMPLE_RATE = 24000
packages/ltx-pipelines/src/ltx_pipelines/utils/helpers.py
+114
View File
@@ -5,6 +5,7 @@ from dataclasses import replace
import torch
from tqdm import tqdm
from ltx_core.components.guiders import MultiModalGuider
from ltx_core.components.noisers import Noiser
from ltx_core.components.protocols import DiffusionStepProtocol, GuiderProtocol
from ltx_core.conditioning import (
@@ -12,6 +13,12 @@ from ltx_core.conditioning import (
VideoConditionByKeyframeIndex,
VideoConditionByLatentIndex,
)
from ltx_core.guidance.perturbations import (
BatchedPerturbationConfig,
Perturbation,
PerturbationConfig,
PerturbationType,
)
from ltx_core.model.transformer import Modality, X0Model
from ltx_core.model.video_vae import VideoEncoder
from ltx_core.text_encoders.gemma import GemmaTextEncoderModelBase
@@ -376,6 +383,113 @@ def guider_denoising_func(
return guider_denoising_step
def multi_modal_guider_denoising_func(
video_guider: MultiModalGuider,
audio_guider: MultiModalGuider,
v_context: torch.Tensor,
a_context: torch.Tensor,
transformer: X0Model,
) -> DenoisingFunc:
last_denoised_video = None
last_denoised_audio = None
def guider_denoising_step(
video_state: LatentState, audio_state: LatentState, sigmas: torch.Tensor, step_index: int
) -> tuple[torch.Tensor, torch.Tensor]:
nonlocal last_denoised_video, last_denoised_audio
if video_guider.should_skip_step(step_index) and audio_guider.should_skip_step(step_index):
return last_denoised_video, last_denoised_audio
sigma = sigmas[step_index]
pos_video_modality = modality_from_latent_state(
video_state, v_context, sigma, enabled=not video_guider.should_skip_step(step_index)
)
pos_audio_modality = modality_from_latent_state(
audio_state, a_context, sigma, enabled=not audio_guider.should_skip_step(step_index)
)
denoised_video, denoised_audio = transformer(
video=pos_video_modality, audio=pos_audio_modality, perturbations=None
)
neg_denoised_video, neg_denoised_audio = 0.0, 0.0
if video_guider.do_unconditional_generation() or audio_guider.do_unconditional_generation():
if video_guider.do_unconditional_generation() and video_guider.negative_context is None:
raise ValueError("Negative context is required for unconditioned denoising")
if audio_guider.do_unconditional_generation() and audio_guider.negative_context is None:
raise ValueError("Negative context is required for unconditioned denoising")
neg_video_modality = modality_from_latent_state(
video_state,
video_guider.negative_context
if video_guider.negative_context is not None
else pos_video_modality.context,
sigma,
)
neg_audio_modality = modality_from_latent_state(
audio_state,
audio_guider.negative_context
if audio_guider.negative_context is not None
else pos_audio_modality.context,
sigma,
)
neg_denoised_video, neg_denoised_audio = transformer(
video=neg_video_modality, audio=neg_audio_modality, perturbations=None
)
ptb_denoised_video, ptb_denoised_audio = 0.0, 0.0
if video_guider.do_perturbed_generation() or audio_guider.do_perturbed_generation():
perturbations = []
if video_guider.do_perturbed_generation():
perturbations.append(
Perturbation(type=PerturbationType.SKIP_VIDEO_SELF_ATTN, blocks=video_guider.params.stg_blocks)
)
if audio_guider.do_perturbed_generation():
perturbations.append(
Perturbation(type=PerturbationType.SKIP_AUDIO_SELF_ATTN, blocks=audio_guider.params.stg_blocks)
)
perturbation_config = PerturbationConfig(perturbations=perturbations)
ptb_denoised_video, ptb_denoised_audio = transformer(
video=pos_video_modality,
audio=pos_audio_modality,
perturbations=BatchedPerturbationConfig(perturbations=[perturbation_config]),
)
mod_denoised_video, mod_denoised_audio = 0.0, 0.0
if video_guider.do_isolated_modality_generation() or audio_guider.do_isolated_modality_generation():
perturbations = [
Perturbation(type=PerturbationType.SKIP_A2V_CROSS_ATTN, blocks=None),
Perturbation(type=PerturbationType.SKIP_V2A_CROSS_ATTN, blocks=None),
]
perturbation_config = PerturbationConfig(perturbations=perturbations)
mod_denoised_video, mod_denoised_audio = transformer(
video=pos_video_modality,
audio=pos_audio_modality,
perturbations=BatchedPerturbationConfig(perturbations=[perturbation_config]),
)
if video_guider.should_skip_step(step_index):
denoised_video = last_denoised_video
else:
denoised_video = video_guider.calculate(
denoised_video, neg_denoised_video, ptb_denoised_video, mod_denoised_video
)
if audio_guider.should_skip_step(step_index):
denoised_audio = last_denoised_audio
else:
denoised_audio = audio_guider.calculate(
denoised_audio, neg_denoised_audio, ptb_denoised_audio, mod_denoised_audio
)
last_denoised_video = denoised_video
last_denoised_audio = denoised_audio
return denoised_video, denoised_audio
return guider_denoising_step
def denoise_audio_video( # noqa: PLR0913
output_shape: VideoPixelShape,
conditionings: list[ConditioningItem],
packages/ltx-pipelines/src/ltx_pipelines/utils/model_ledger.py
+8 -2
View File
@@ -35,6 +35,8 @@ from ltx_core.text_encoders.gemma import (
AVGemmaTextEncoderModelConfigurator,
module_ops_from_gemma_root,
)
from ltx_core.text_encoders.gemma.encoders.av_encoder import GEMMA_MODEL_OPS
from ltx_core.utils import find_matching_file
class ModelLedger:
@@ -143,12 +145,16 @@ class ModelLedger:
)
if self.gemma_root_path is not None:
module_ops = module_ops_from_gemma_root(self.gemma_root_path)
model_folder = find_matching_file(self.gemma_root_path, "model*.safetensors").parent
weight_paths = [str(p) for p in model_folder.rglob("*.safetensors")]
self.text_encoder_builder = Builder(
model_path=self.checkpoint_path,
model_path=(str(self.checkpoint_path), *weight_paths),
model_class_configurator=AVGemmaTextEncoderModelConfigurator,
model_sd_ops=AV_GEMMA_TEXT_ENCODER_KEY_OPS,
registry=self.registry,
module_ops=module_ops_from_gemma_root(self.gemma_root_path),
module_ops=(GEMMA_MODEL_OPS, *module_ops),
)
if self.spatial_upsampler_path is not None: