mirror of
https://github.com/Wan-Video/Wan2.1.git
synced 2025-07-14 03:30:10 +00:00
114 lines
3.4 KiB
Python
114 lines
3.4 KiB
Python
import math
|
|
|
|
import torch
|
|
import torch.cuda.amp as amp
|
|
import torch.nn as nn
|
|
from diffusers.configuration_utils import ConfigMixin, register_to_config
|
|
from diffusers.models.modeling_utils import ModelMixin
|
|
from wan.modules import WanModel
|
|
from wan.modules.model import sinusoidal_embedding_1d
|
|
from wan.taylorseer.cache_functions import cal_type
|
|
from wan.taylorseer.taylorseer_utils import taylor_formula
|
|
from .wan_cache_forward import wan_cache_forward
|
|
|
|
def wan_forward(
|
|
self:WanModel,
|
|
x,
|
|
t,
|
|
context,
|
|
seq_len,
|
|
current_step,
|
|
current_stream,
|
|
clip_fea=None,
|
|
y=None,
|
|
):
|
|
r"""
|
|
Forward pass through the diffusion model
|
|
|
|
Args:
|
|
x (List[Tensor]):
|
|
List of input video tensors, each with shape [C_in, F, H, W]
|
|
t (Tensor):
|
|
Diffusion timesteps tensor of shape [B]
|
|
context (List[Tensor]):
|
|
List of text embeddings each with shape [L, C]
|
|
seq_len (`int`):
|
|
Maximum sequence length for positional encoding
|
|
clip_fea (Tensor, *optional*):
|
|
CLIP image features for image-to-video mode
|
|
y (List[Tensor], *optional*):
|
|
Conditional video inputs for image-to-video mode, same shape as x
|
|
|
|
Returns:
|
|
List[Tensor]:
|
|
List of denoised video tensors with original input shapes [C_out, F, H / 8, W / 8]
|
|
"""
|
|
|
|
self.current['step'] = current_step
|
|
self.current['stream'] = current_stream
|
|
if current_stream == 'cond_stream':
|
|
cal_type(self.cache_dic, self.current)
|
|
|
|
if self.model_type == 'i2v':
|
|
assert clip_fea is not None and y is not None
|
|
# params
|
|
device = self.patch_embedding.weight.device
|
|
if self.freqs.device != device:
|
|
self.freqs = self.freqs.to(device)
|
|
|
|
if y is not None:
|
|
x = [torch.cat([u, v], dim=0) for u, v in zip(x, y)]
|
|
|
|
# embeddings
|
|
x = [self.patch_embedding(u.unsqueeze(0)) for u in x]
|
|
grid_sizes = torch.stack(
|
|
[torch.tensor(u.shape[2:], dtype=torch.long) for u in x])
|
|
x = [u.flatten(2).transpose(1, 2) for u in x]
|
|
seq_lens = torch.tensor([u.size(1) for u in x], dtype=torch.long)
|
|
assert seq_lens.max() <= seq_len
|
|
x = torch.cat([
|
|
torch.cat([u, u.new_zeros(1, seq_len - u.size(1), u.size(2))],
|
|
dim=1) for u in x
|
|
])
|
|
|
|
# time embeddings
|
|
with amp.autocast(dtype=torch.float32):
|
|
e = self.time_embedding(
|
|
sinusoidal_embedding_1d(self.freq_dim, t).float())
|
|
e0 = self.time_projection(e).unflatten(1, (6, self.dim))
|
|
assert e.dtype == torch.float32 and e0.dtype == torch.float32
|
|
|
|
# context
|
|
context_lens = None
|
|
context = self.text_embedding(
|
|
torch.stack([
|
|
torch.cat(
|
|
[u, u.new_zeros(self.text_len - u.size(0), u.size(1))])
|
|
for u in context
|
|
]))
|
|
|
|
if clip_fea is not None:
|
|
context_clip = self.img_emb(clip_fea) # bs x 257 x dim
|
|
context = torch.concat([context_clip, context], dim=1)
|
|
|
|
# arguments
|
|
kwargs = dict(
|
|
e=e0,
|
|
seq_lens=seq_lens,
|
|
grid_sizes=grid_sizes,
|
|
freqs=self.freqs,
|
|
context=context,
|
|
context_lens=context_lens,
|
|
cache_dic=self.cache_dic,
|
|
current=self.current)
|
|
|
|
for i, block in enumerate(self.blocks):
|
|
self.current['layer'] = i
|
|
x = block(x, **kwargs)
|
|
|
|
# head
|
|
x = self.head(x, e)
|
|
|
|
# unpatchify
|
|
x = self.unpatchify(x, grid_sizes)
|
|
return [u.float() for u in x] |