Diffusers代码学习：文本指导控制视频

InstructPix2Pix 允许您使用文本来描述要对视频进行的更改。

import os
os.environ["HF_ENDPOINT"] = "https://hf-mirror.com"
rom huggingface_hub import hf_hub_download
from PIL import Image
import imageio

# 以下代码加载文本指导原始视频

filename = "__assets__/pix2pix video/camel.mp4"

repo_id = "PAIR/Text2Video-Zero"
video_path = hf_hub_download(repo_type="space", repo_id=repo_id, filename=filename)

reader = imageio.get_reader(video_path, "ffmpeg")
frame_count = 8
video = [Image.fromarray(reader.get_data(i)) for i in range(frame_count)]

import torch
from diffusers import StableDiffusionInstructPix2PixPipeline
from diffusers.pipelines.text_to_video_synthesis.pipeline_text_to_video_zero import CrossFrameAttnProcessor

pipeline = StableDiffusionInstructPix2PixPipeline.from_pretrained("timbrooks/instruct-pix2pix", torch_dtype=torch.float16).to("cuda")
pipeline.unet.set_attn_processor(CrossFrameAttnProcessor(batch_size=3))

prompt = "make it Van Gogh Starry Night style"
result = pipeline(prompt=[prompt] * len(video), image=video).images
imageio.mimsave("edited_video.mp4", result, fps=4)

以下为文本指导使用的原始视频

文本指导原始视频

以下为生成的具有梵高风格的视频

Text2Video文本指导演示