Datawhale X 魔搭 AI夏令营 第四期（进阶上分-实战优化）

使用ComfyUI生成图片

什么是ComfyUI

ComfyUI 是一个开源的用户界面框架，旨在提供一个简洁、直观的界面设计工具。它通常用于构建桌面应用程序或网页应用程序的用户界面，特别是在需要快速开发和高度可定制的情况下。ComfyUI 提供了丰富的组件和灵活的布局选项，帮助开发者轻松实现现代化的界面设计

主要核心模块

模型加载器：

功能：负责从磁盘或网络加载机器学习模型到内存中。模型加载器支持不同类型的模型格式和框架，如 PyTorch、TensorFlow 等，确保模型可以被正确解析和使用。

作用：使得界面可以通过与训练好的模型进行交互，进行各种推断和处理任务

提示词管理器：

功能：用于管理和处理提示词（Prompts）。提示词是指用于指导模型生成结果的输入文本。提示词管理器可以创建、存储、编辑和删除这些提示词。

作用：优化提示词的使用，确保模型生成的结果符合预期并能够快速调整提示词以改善输出效果。

采样器：

功能：控制模型生成文本或其他数据的过程。采样器通过不同的采样策略（如随机采样、贪婪采样或束搜索）来决定如何从模型中提取生成结果。

作用：影响生成内容的多样性和质量，可以根据需求调整采样策略，以获取最佳的生成结果。

解码器：

功能：将模型输出的原始数据转换为人类可读的格式。这通常涉及将生成的模型数据（如 token 序列）转换为自然语言文本或其他适用的格式。

作用：确保模型生成的结果能够正确地解码并呈现给用户，使得最终输出符合用户的阅读或使用需求。

安装ComfyUI

下载脚本代码文件

使用魔搭社区提供的Notebook，打开新的terminal，执行以下代码

git lfs install
git clone https://www.modelscope.cn/datasets/maochase/kolors_test_comfyui.git
mv kolors_test_comfyui/* ./
rm -rf kolors_test_comfyui/
mkdir -p /mnt/workspace/models/lightning_logs/version_0/checkpoints/
mv epoch=0-step=500.ckpt /mnt/workspace/models/lightning_logs/version_0/checkpoints/   

得到ComfyUI的安装文件并执行

运行程序

下载并安装 ComfyUI

from pathlib import Path

OPTIONS = {}
UPDATE_COMFY_UI = True  #@param {type:"boolean"}
INSTALL_COMFYUI_MANAGER = True  #@param {type:"boolean"}
INSTALL_KOLORS = True  #@param {type:"boolean"}
INSTALL_CUSTOM_NODES_DEPENDENCIES = True  #@param {type:"boolean"}
OPTIONS['UPDATE_COMFY_UI'] = UPDATE_COMFY_UI
OPTIONS['INSTALL_COMFYUI_MANAGER'] = INSTALL_COMFYUI_MANAGER
OPTIONS['INSTALL_KOLORS'] = INSTALL_KOLORS
OPTIONS['INSTALL_CUSTOM_NODES_DEPENDENCIES'] = INSTALL_CUSTOM_NODES_DEPENDENCIES

current_dir = !pwd
WORKSPACE = f"{current_dir[0]}/ComfyUI"



%cd /mnt/workspace/

![ ! -d $WORKSPACE ] && echo -= Initial setup ComfyUI =- && git clone https://github.com/comfyanonymous/ComfyUI
%cd $WORKSPACE

if OPTIONS['UPDATE_COMFY_UI']:
  !echo "-= Updating ComfyUI =-"
  !git pull


if OPTIONS['INSTALL_COMFYUI_MANAGER']:
  %cd custom_nodes
  ![ ! -d ComfyUI-Manager ] && echo -= Initial setup ComfyUI-Manager =- && git clone https://github.com/ltdrdata/ComfyUI-Manager
  %cd ComfyUI-Manager
  !git pull

if OPTIONS['INSTALL_KOLORS']:
  %cd ../
  ![ ! -d ComfyUI-KwaiKolorsWrapper ] && echo -= Initial setup KOLORS =- && git clone https://github.com/kijai/ComfyUI-KwaiKolorsWrapper.git
  %cd ComfyUI-KwaiKolorsWrapper
  !git pull

%cd $WORKSPACE

if OPTIONS['INSTALL_CUSTOM_NODES_DEPENDENCIES']:
  !pwd
  !echo "-= Install custom nodes dependencies =-"
  ![ -f "custom_nodes/ComfyUI-Manager/scripts/colab-dependencies.py" ] && python "custom_nodes/ComfyUI-Manager/scripts/colab-dependencies.py"

!wget "https://modelscope.oss-cn-beijing.aliyuncs.com/resource/cloudflared-linux-amd64.deb"
!dpkg -i cloudflared-linux-amd64.deb

下载模型

OPTIONS = {}

#@markdown **unet**

!wget -c "https://modelscope.cn/models/Kwai-Kolors/Kolors/resolve/master/unet/diffusion_pytorch_model.fp16.safetensors" -P ./models/diffusers/Kolors/unet/
!wget -c "https://modelscope.cn/models/Kwai-Kolors/Kolors/resolve/master/unet/config.json" -P ./models/diffusers/Kolors/unet/


#@markdown **encoder**

!modelscope download --model=ZhipuAI/chatglm3-6b-base --local_dir ./models/diffusers/Kolors/text_encoder/

#@markdown **vae**

!wget -c "https://modelscope.cn/models/AI-ModelScope/sdxl-vae-fp16-fix/resolve/master/sdxl.vae.safetensors" -P ./models/vae/ #sdxl-vae-fp16-fix.safetensors

#@markdown **scheduler**
!wget -c "https://modelscope.cn/models/Kwai-Kolors/Kolors/resolve/master/scheduler/scheduler_config.json" -P ./models/diffusers/Kolors/scheduler/

#@markdown **modelindex**
!wget -c "https://modelscope.cn/models/Kwai-Kolors/Kolors/resolve/master/model_index.json" -P ./models/diffusers/Kolors/

安装 LoRA 节点

lora_node = """
import torch
from peft import LoraConfig, inject_adapter_in_model


class LoadKolorsLoRA:
    @classmethod
    def INPUT_TYPES(s):
        return {
            "required": {
                "kolors_model": ("KOLORSMODEL", ),
                "lora_path": ("STRING", {"multiline": False, "default": "",}),
                "lora_alpha": ("FLOAT", {"default": 2.0, "min": 0.0, "max": 4.0, "step": 0.01}),
            },
        }

    RETURN_TYPES = ("KOLORSMODEL",)
    RETURN_NAMES = ("kolors_model",)
    FUNCTION = "add_lora"
    CATEGORY = "KwaiKolorsWrapper"

    def convert_state_dict(self, state_dict):
        prefix_rename_dict = {
            "blocks.7.transformer_blocks": "down_blocks.1.attentions.0.transformer_blocks",
            "blocks.10.transformer_blocks": "down_blocks.1.attentions.1.transformer_blocks",
            "blocks.15.transformer_blocks": "down_blocks.2.attentions.0.transformer_blocks",
            "blocks.18.transformer_blocks": "down_blocks.2.attentions.1.transformer_blocks",
            "blocks.21.transformer_blocks": "mid_block.attentions.0.transformer_blocks",
            "blocks.25.transformer_blocks": "up_blocks.0.attentions.0.transformer_blocks",
            "blocks.28.transformer_blocks": "up_blocks.0.attentions.1.transformer_blocks",
            "blocks.31.transformer_blocks": "up_blocks.0.attentions.2.transformer_blocks",
            "blocks.35.transformer_blocks": "up_blocks.1.attentions.0.transformer_blocks",
            "blocks.38.transformer_blocks": "up_blocks.1.attentions.1.transformer_blocks",
            "blocks.41.transformer_blocks": "up_blocks.1.attentions.2.transformer_blocks",
        }
        suffix_rename_dict = {
            ".to_out.lora_A.default.weight": ".to_out.0.lora_A.default.weight",
            ".to_out.lora_B.default.weight": ".to_out.0.lora_B.default.weight",
        }
        state_dict_ = {}
        for name, param in state_dict.items():
            for prefix in prefix_rename_dict:
                if name.startswith(prefix):
                    name = name.replace(prefix, prefix_rename_dict[prefix])
            for suffix in suffix_rename_dict:
                if name.endswith(suffix):
                    name = name.replace(suffix, suffix_rename_dict[suffix])
            state_dict_[name] = param
        lora_rank = state_dict_["up_blocks.1.attentions.2.transformer_blocks.1.attn2.to_q.lora_A.default.weight"].shape[0]
        return state_dict_, lora_rank

    def load_lora(self, model, lora_rank, lora_alpha, state_dict):
        lora_config = LoraConfig(
            r=lora_rank,
            lora_alpha=lora_alpha,
            init_lora_weights="gaussian",
            target_modules=["to_q", "to_k", "to_v", "to_out.0"],
        )
        model = inject_adapter_in_model(lora_config, model)
        model.load_state_dict(state_dict, strict=False)
        return model

    def add_lora(self, kolors_model, lora_path, lora_alpha):
        state_dict = torch.load(lora_path, map_location="cpu")
        state_dict, lora_rank = self.convert_state_dict(state_dict)
        kolors_model["pipeline"].unet = self.load_lora(kolors_model["pipeline"].unet, lora_rank, lora_alpha, state_dict)
        return (kolors_model,)


NODE_CLASS_MAPPINGS = {
    "LoadKolorsLoRA": LoadKolorsLoRA,
}
NODE_DISPLAY_NAME_MAPPINGS = {
    "LoadKolorsLoRA": "Load Kolors LoRA",
}
__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"]
""".strip()

import os

os.makedirs("/mnt/workspace/ComfyUI/custom_nodes/ComfyUI-LoRA", exist_ok=True)
with open("/mnt/workspace/ComfyUI/custom_nodes/ComfyUI-LoRA/__init__.py", "w", encoding="utf-8") as f:
    f.write(lora_node)

启动 ComfyUI

%cd /mnt/workspace/ComfyUI
import subprocess
import threading
import time
import socket
import urllib.request

def iframe_thread(port):
  while True:
      time.sleep(0.5)
      sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
      result = sock.connect_ex(('127.0.0.1', port))
      if result == 0:
        break
      sock.close()
  print("\nComfyUI finished loading, trying to launch cloudflared (if it gets stuck here cloudflared is having issues)\n")

  p = subprocess.Popen(["cloudflared", "tunnel", "--url", "http://127.0.0.1:{}".format(port)], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
  for line in p.stderr:
    l = line.decode()
    if "trycloudflare.com " in l:
      print("This is the URL to access ComfyUI:", l[l.find("http"):], end='')
    #print(l, end='')


threading.Thread(target=iframe_thread, daemon=True, args=(8188,)).start()

!python main.py --dont-print-server

得到ComfyUI链接

https://not-request-tex-voting.trycloudflare.com

ComfyUI

打开链接，进入ComfyUI

利用ComfyUI制图

不带Lora的工作流样例

点击右边Load，导入kolors_example.json，再点击右上方Queue Prompt，有

带Lora的工作流样例

导入kolors_with_lora_example.json，在中间上方的Load Kolors LoRA框中的第一行输入Lora的地址/mnt/workspace/models/lightning_logs/version_0/checkpoints/epoch=0-step=500.ckpt，再运行，有