寒武纪MLU370M8 stable_diffusion适配

1、起容器

下载MLU镜像

wget https://sdk.cambricon.com/static/PyTorch/MLU370_1.13_v1.17.0_X86_ubuntu20.04_python3.10_docker/pytorch-v1.17.0-torch1.13.1-ubuntu20.04-py310.tar.gz

起容器脚本

# 使用的镜像是pytorch:v1.17.0-torch1.13.1-ubuntu20.04-py310
export MY_CONTAINER="stable_diffusion"
num=`docker ps -a|grep "$MY_CONTAINER" | wc -l`
echo $num
echo $MY_CONTAINER
if [ 0 -eq $num ];then
docker run -it \
        --privileged \
        --pid=host \
        --net=host \
        --device /dev/cambricon_dev0 \
        --device /dev/cambricon_ctl \
        --device /dev/cambricon_ipcm0 \
        --name $MY_CONTAINER \
        -v /srv/workspace_llm/stable_diffusion:/workspace_stable_diffusion/ \
        -v /usr/bin/cnmon:/usr/bin/cnmon \
        yellow.hub.cambricon.com/pytorch/pytorch:v1.17.0-torch1.13.1-ubuntu20.04-py310
        /bin/bash
else
    docker start $MY_CONTAINER
    docker exec -ti $MY_CONTAINER /bin/bash
fi

进入容器

2、环境准备

• transformers

git clone -b v4.38.2 https://githubfast.com/huggingface/transformers.git
python /torch/src/catch/tools/torch_gpu2mlu/torch_gpu2mlu.py -i transformers/
pip install -e ./transformers_mlu/

• accelerate

git clone -b v0.22.0 https://githubfast.com/huggingface/accelerate.git
python /torch/src/catch/tools/torch_gpu2mlu/torch_gpu2mlu.py -i accelerate/
pip install -e ./accelerate_mlu/

• basicsr

git clone https://githubfast.com/XPixelGroup/BasicSR.git
python /torch/src/catch/tools/torch_gpu2mlu/torch_gpu2mlu.py -i BasicSR/
pip install -e ./BasicSR_mlu/

# 修改BasicSR_mlu/basicsr/utils/misc.py文件，添加以下几行代码
def get_device():
    if torch.mlu.is_available():
        return torch.device("mlu")
    else:
        return torch.device("cpu")
def gpu_is_available():
        return torch.mlu.is_available()

3、代码准备

git clone https://githubfast.com/AUTOMATIC1111/stable-diffusion-webui.git
# 代码转换
python /torch/src/catch/tools/torch_gpu2mlu/torch_gpu2mlu.py -i stable-diffusion-webui/

4、安装组件

cd stable-diffusion-webui_mlu && mkdir repositories
git clone https://githubfast.com/CompVis/stable-diffusion.git repositories/stable-diffusion-stability-ai
git clone https://githubfast.com/Stability-AI/generative-models.git repositories/generative-models
git clone https://githubfast.com/sczhou/CodeFormer.git repositories/CodeFormer
git clone https://githubfast.com/salesforce/BLIP.git repositories/BLIP
git clone https://githubfast.com/Stability-AI/stablediffusion.git
git clone -b v0.0.16 https://githubfast.com/crowsonkb/k-diffusion.git repositories/k-diffusion
git clone https://githubfast.com/CompVis/taming-transformers.git repositories/taming-transformers
cd repositories/
# ------------------------------------------------
python /torch/src/catch/tools/torch_gpu2mlu/torch_gpu2mlu.py -i BLIP/ && python /torch/src/catch/tools/torch_gpu2mlu/torch_gpu2mlu.py -i CodeFormer/ && python /torch/src/catch/tools/torch_gpu2mlu/torch_gpu2mlu.py  -i generative-models/ && python /torch/src/catch/tools/torch_gpu2mlu/torch_gpu2mlu.py -i k-diffusion/ && python /torch/src/catch/tools/torch_gpu2mlu/torch_gpu2mlu.py  -i stable-diffusion-stability-ai/ && python /torch/src/catch/tools/torch_gpu2mlu/torch_gpu2mlu.py -i taming-transformers/ && python /torch/src/catch/tools/torch_gpu2mlu/torch_gpu2mlu.py -i stablediffusion
# ------------------------------------------------
mkdir ../tmp && mv *_mlu ../tmp && rm -rf *
cd ../tmp
mv BLIP_mlu/ BLIP/ && mv CodeFormer_mlu/ CodeFormer/ && mv generative-models_mlu/ generative-models/ && mv k-diffusion_mlu/ k-diffusion/ && mv stable-diffusion-stability-ai_mlu/ stable-diffusion-stability-ai/ && mv taming-transformers_mlu/ taming-transformers
mv * ../repositories/
cd ../repositories/ && cp -r taming-transformers/taming/ stable-diffusion-stability-ai/
cd ../repositories/ && cp -r stable-diffusion-stability-ai/ldm/ ..

注释modules/devices.py 83行

# torch.mlu.ipc_collect()

注释modules/launch_utils.py 398-400行

# if (not is_installed("xformers") or args.reinstall_xformers) and args.xformers:
#     run_pip(f"install -U -I --no-deps {xformers_package}", "xformers")
#     startup_timer.record("install xformers")

注释modules/launch_utils.py 410-414 和 421-423行

# git_clone(assets_repo, repo_dir('stable-diffusion-webui-assets'), "assets", assets_commit_hash)
# git_clone(stable_diffusion_repo, repo_dir('stable-diffusion-stability-ai'), "Stable Diffusion", stable_diffusion_commit_hash)
# git_clone(stable_diffusion_xl_repo, repo_dir('generative-models'), "Stable Diffusion XL", stable_diffusion_xl_commit_hash)
# git_clone(k_diffusion_repo, repo_dir('k-diffusion'), "K-diffusion", k_diffusion_commit_hash)
# git_clone(blip_repo, repo_dir('BLIP'), "BLIP", blip_commit_hash)
-----------------------------------------------------------

# if not requirements_met(requirements_file):
#     run_pip(f"install -r \"{requirements_file}\"", "requirements")
#     startup_timer.record("install requirements")

5、下载模型

from modelscope import snapshot_download
model_dir = snapshot_download("AI-ModelScope/stable-diffusion-v1-5")
model_dir = snapshot_download("AI-ModelScope/clip-vit-large-patch14")

模型拷贝到自己项目路径

mv /root/.cache/modelscope/hub/AI-ModelScope/stable-diffusion-v1-5/v1-5-pruned-emaonly.safetensors /workspace_stable_diffusion/stable-diffusion-webui_mlu/models/Stable-diffusion/

mv /root/.cache/modelscope/hub/AI-ModelScope/stable-diffusion-v1-5/v1-5-pruned-emaonly.ckpt /workspace_stable_diffusion/stable-diffusion-webui_mlu/models/Stable-diffusion/pip

修改 ldm/moudles/encoders/moudles.py 100行 将version路径改成自己的模型路径

def __init__(self, version="/workspace_stable_diffusion/modle/clip-vit-large-patch14/", device="mlu", max_length=77,

修改modules/sd_vae_approx.py 57行

loaded_model.load_state_dict(torch.load(model_path, map_location='cpu' if devices.device.type != 'mlu' else None))
# =>改成
loaded_model.load_state_dict(torch.load(model_path, map_location='mlu'))

6、运行

./webui.sh -f --skip-torch-mlu-test

7、适配过程中遇到的Error

# AttributeError: 'NoneType' object has no attribute '_id'
# 修改/torch/venv3/pytorch/lib/python3.10/site-packages/gradio/blocks.py287行
"inputs": [block._id for block in inputs],
# =>修改成
"inputs": [block._id for block in inputs if block is not None],

# RuntimeError: Directory '/workspace_stable_diffusion/stable-diffusion-webui_mlu/repositories/stable-diffusion-webui-assets' does not exist
git clone https://github.com/AUTOMATIC1111/stable-diffusion-webui-assets

from pytorch_lightning.utilities.distributed import rank_zero_only
# 将所有存在上面的改为=>
from pytorch_lightning.utilities.rank_zero import rank_zero_only