LAVIS在Mac,M1PRO芯片下的安装实战

LAVIS在Mac,M1PRO芯片下的安装实战

契机

⚙ 本地想装个图片理解的大模型，看了下blip2感觉比较合适，macos安装的时候有点坑需要注意下，但是最终也无法使用mps加速，比较蛋疼。这里记录下安装步骤。

安装

LAVIS/projects/blip2 at main · salesforce/LAVIS

#直接安装
pip install salesforce-lavis

#报错
ERROR: Cannot install salesforce-lavis==1.0.0 and salesforce-lavis==1.0.2 because these package versions have conflicting dependencies.

The conflict is caused by:
salesforce-lavis 1.0.2 depends on decord
salesforce-lavis 1.0.0 depends on decord

To fix this you could try to:

loosen the range of package versions you've specified
remove package versions to allow pip attempt to solve the dependency conflict

#decord无法安装，解决方案如下

Build issues with decord for Mac with Python 3.9+ · Issue #15 · salesforce/LAVIS

#安装ffmeg

#install ffmpeg@4 using 
brew install ffmpeg@4
#update the link since I had ffmpeg5 using 
brew link --overwrite ffmpeg@4

#安装decode
#https://github.com/dmlc/decord#mac-os

brew install cmake ffmpeg
#克隆仓库
git clone --recursive https://github.com/dmlc/decord
#进入仓库编译
cd decord
mkdir build && cd build
cmake .. -DCMAKE_BUILD_TYPE=Release
make
#make出错如下
[ 3%] Building CXX object CMakeFiles/decord.dir/src/audio/audio_interface.cc.o
In file included from /Users/y/Desktop/decord/decord/src/audio/audio_interface.cc:5:
/Users/y/Desktop/decord/decord/src/audio/audio_reader.h:8:10: fatal error: 'vector' file not found
8 | #include <vector>
| ^~~~~~~~
1 error generated.
make[2]: *** [CMakeFiles/decord.dir/src/audio/audio_interface.cc.o] Error 1
make[1]: *** [CMakeFiles/decord.dir/all] Error 2
make: *** [all] Error 2

#decord/build/CMakeLists.txt
#加一行下面这个
set(CMAKE_CXX_COMPILER clang++)

#成功
 build git:(master) ✗ make
-- Could NOT find PkgConfig (missing: PKG_CONFIG_EXECUTABLE)
-- Found libavdevice, device input will be enabled
-- Found FFMPEG or Libav: /opt/homebrew/lib/libavformat.dylib;/opt/homebrew/lib/libavfilter.dylib;/opt/homebrew/lib/libavcodec.dylib;/opt/homebrew/lib/libavutil.dylib;/opt/homebrew/lib/libswresample.dylib;/opt/homebrew/lib/libavdevice.dylib, /opt/homebrew/include
FFMPEG_INCLUDE_DIR = /opt/homebrew/include
FFMPEG_LIBRARIES = /opt/homebrew/lib/libavformat.dylib;/opt/homebrew/lib/libavfilter.dylib;/opt/homebrew/lib/libavcodec.dylib;/opt/homebrew/lib/libavutil.dylib;/opt/homebrew/lib/libswresample.dylib;/opt/homebrew/lib/libavdevice.dylib
-- CUDA disabled, no nvdec capabilities will be enabled...
-- Configuring done (0.1s)
-- Generating done (0.0s)
-- Build files have been written to: /Users/y/Desktop/decord/decord/build
[  3%] Building CXX object CMakeFiles/decord.dir/src/audio/audio_interface.cc.o
[  7%] Building CXX object CMakeFiles/decord.dir/src/audio/audio_reader.cc.o
[ 11%] Building CXX object CMakeFiles/decord.dir/src/runtime/c_runtime_api.cc.o
[ 14%] Building CXX object CMakeFiles/decord.dir/src/runtime/cpu_device_api.cc.o
[ 18%] Building CXX object CMakeFiles/decord.dir/src/runtime/dso_module.cc.o
[ 22%] Building CXX object CMakeFiles/decord.dir/src/runtime/file_util.cc.o
[ 25%] Building CXX object CMakeFiles/decord.dir/src/runtime/module.cc.o
[ 29%] Building CXX object CMakeFiles/decord.dir/src/runtime/module_util.cc.o
[ 33%] Building CXX object CMakeFiles/decord.dir/src/runtime/ndarray.cc.o
[ 37%] Building CXX object CMakeFiles/decord.dir/src/runtime/registry.cc.o
[ 40%] Building CXX object CMakeFiles/decord.dir/src/runtime/str_util.cc.o
[ 44%] Building CXX object CMakeFiles/decord.dir/src/runtime/system_lib_module.cc.o
[ 48%] Building CXX object CMakeFiles/decord.dir/src/runtime/thread_pool.cc.o
[ 51%] Building CXX object CMakeFiles/decord.dir/src/runtime/threading_backend.cc.o
[ 55%] Building CXX object CMakeFiles/decord.dir/src/runtime/workspace_pool.cc.o
[ 59%] Building CXX object CMakeFiles/decord.dir/src/sampler/random_file_order_sampler.cc.o
[ 62%] Building CXX object CMakeFiles/decord.dir/src/sampler/random_sampler.cc.o
[ 66%] Building CXX object CMakeFiles/decord.dir/src/sampler/sequential_sampler.cc.o
[ 70%] Building CXX object CMakeFiles/decord.dir/src/sampler/smart_random_sampler.cc.o
[ 74%] Building CXX object CMakeFiles/decord.dir/src/video/logging.cc.o
[ 77%] Building CXX object CMakeFiles/decord.dir/src/video/storage_pool.cc.o
[ 81%] Building CXX object CMakeFiles/decord.dir/src/video/video_interface.cc.o
[ 85%] Building CXX object CMakeFiles/decord.dir/src/video/video_loader.cc.o
[ 88%] Building CXX object CMakeFiles/decord.dir/src/video/video_reader.cc.o
[ 92%] Building CXX object CMakeFiles/decord.dir/src/video/ffmpeg/filter_graph.cc.o
[ 96%] Building CXX object CMakeFiles/decord.dir/src/video/ffmpeg/threaded_decoder.cc.o
[100%] Linking CXX shared library libdecord.dylib
[100%] Built target decord

#安装到自己python库

#回到docord下的python目录
cd ../python
# option 1: add python path to $PYTHONPATH, you will need to install numpy separately
pwd=$PWD
echo "PYTHONPATH=$PYTHONPATH:$pwd" >> ~/.bash_profile
source ~/.bash_profile
# option 2: install with setuptools
python3 setup.py install --user

#再次安装lavis，安装成功！
pip install salesforce-lavis

测试代码

from PIL import Image
import requests
from transformers import Blip2Processor, Blip2ForConditionalGeneration
import torch

#强制指定mps
device = "mps"

processor = Blip2Processor.from_pretrained("Salesforce/blip2-opt-2.7b")
model = Blip2ForConditionalGeneration.from_pretrained(
    "Salesforce/blip2-opt-2.7b", torch_dtype=torch.float16
)
model.to(device)
url = "http://images.cocodataset.org/val2017/000000039769.jpg"
image = Image.open(requests.get(url, stream=True).raw)

inputs = processor(images=image, return_tensors="pt").to(device, torch.float16)

generated_ids = model.generate(**inputs)
generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True)[0].strip()
print(generated_text)

#第一次运行可能要等很久，因为要下载模型，最好网络加个魔法

#运行上面demo报错
Loading checkpoint shards: 100%|██████████| 2/2 [00:12<00:00,  6.36s/it]
Traceback (most recent call last):
    generated_ids = model.generate(**inputs)
    return func(*args, **kwargs)
    outputs = self.language_model.generate(
    return func(*args, **kwargs)
    result = self._greedy_search(
    unfinished_sequences = unfinished_sequences & ~stopping_criteria(input_ids, scores)
    is_done = is_done | criteria(input_ids, scores, **kwargs)
    is_done = torch.isin(input_ids[:, -1], self.eos_token_id.to(input_ids.device))
NotImplementedError: The operator 'aten::isin.Tensor_Tensor_out' is not currently implemented for the MPS device. If you want this op to be added in priority during the prototype phase of this feature, please comment on https://github.com/pytorch/pytorch/issues/77764. As a temporary fix, you can set the environment variable `PYTORCH_ENABLE_MPS_FALLBACK=1` to use the CPU as a fallback for this op. WARNING: this will be slower than running natively on MPS.

#强制指定环境变量如下
PYTORCH_ENABLE_MPS_FALLBACK=1

#最后输出
two cats laying on a couch

总结

• 其实效果一般，可能我用的float16
• 最主要就是安装decode
• decode的cmake有点麻烦
• 最后执行的时候，下载模型要卡很久，这里看网速就知道其实正在下载

写到最后