飞桨PaddlePaddle中SDXL的常用方法

PaddlePaddle中使用SDXL的方法
controlnet、vae、unet及lora使用方法示例：

# controlnet、vae、unet及lora使用方法
# 安装develop的ppdiffusers
# pip install "ppdiffusers>=0.24.0"
import numpy as np
import cv2
from PIL import Image
import paddle
from ppdiffusers import (
    ControlNetModel, 
    StableDiffusionXLControlNetPipeline,
    AutoencoderKL, 
	UNet2DConditionModel, 
	EulerAncestralDiscreteScheduler
)
from ppdiffusers.utils import load_image


# load unet
unet = UNet2DConditionModel.from_pretrained("stabilityai/stable-diffusion-xl-base-1.0/unet", paddle_dtype=paddle.float16, variant="fp16")

# load controlnet
controlnet = ControlNetModel.from_pretrained("diffusers/controlnet-canny-sdxl-1.0")

# load vae
vae = AutoencoderKL.from_pretrained("madebyollin/sdxl-vae-fp16-fix", paddle_dtype=paddle.float16)

pipe = StableDiffusionXLControlNetPipeline.from_pretrained(
    "stabilityai/stable-diffusion-xl-base-1.0",
    # "SG161222/RealVisXL_V3.0", 
    paddle_dtype=paddle.float16, 
    safety_checker=None, 
    controlnet=controlnet, 
    variant="fp16",
    low_cpu_mem_usage=True, 
    vae=vae,
	unet=unet,
)

# denoise策略
pipe.scheduler = EulerAncestralDiscreteScheduler.from_config(pipe.scheduler.config)

# 开启 xformers
pipe.enable_xformers_memory_efficient_attention()

# Initialize LoRA model and weights
# 模型路径：./lora/anime-detailer-xl.safetensors
lora_model_id = "./lora"  # 模型所在目录
lora_filename = "anime-detailer-xl.safetensors"  # 模型名字
lora_scale_slider = 2  # -2 for less detailed result

# Load and fuse LoRA weights
pipe.load_lora_weights(
    lora_model_id, 
    weight_name=lora_filename, 
    from_diffusers=True, 
    from_hf_hub=True
)
pipe.fuse_lora(lora_scale=lora_scale_slider)

# 固定随机种子
generator = paddle.Generator().manual_seed(100)

# 定义prompt
prompt = "aerial view, a futuristic research complex in a bright foggy jungle, hard lighting"
negative_prompt = 'low quality, bad quality, sketches'

# controlnet参数设置
controlnet_conditioning_scale=0.5

# controlnet参考图
image = load_image(
    "https://hf-mirror.com/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/hf-logo.png"
)
# image = image.resize((1024, 1024)) # 修改尺寸(width, height),忽略
# 参考图生成线稿
image = np.array(image)
image = cv2.Canny(image, 100, 200)
image = image[:, :, None]
image = np.concatenate([image, image, image], axis=2)
image = Image.fromarray(image)

image = pipe(
    prompt,  
    negative_prompt=negative_prompt, 
    width=1024, 
    height=1024, 
    image=image,
    num_inference_steps=30,
    guidance_scale=3,
    num_images_per_prompt=1,
    generator=generator, 
    controlnet_conditioning_scale=controlnet_conditioning_scale, 
).images[0]

image.save("text_ctl_img.png")

更多详细使用方法可以参考官方文档以及hf-mirror.com中直接搜索比较官方的对应模型查看api使用方法，基本类似（提供了一种解决问题的思路）。