数字人生成项目geneface++ 安装文档

使用智星云AI平台，采用cuda11.7

#--------------Prepare the Environment

source /root/miniconda3/bin/activate

conda create -n geneface python=3.9

conda activate geneface

conda install conda-forge::ffmpeg # ffmpeg with libx264 codec to turn images to video

conda install pytorch==2.0.1 torchvision==0.15.2 torchaudio==2.0.2 pytorch-cuda=11.7 -c pytorch -c nvidia

pip install "git+https://github.com/facebookresearch/pytorch3d.git@stable"

pip install cython

pip install openmim==0.3.9

mim install mmcv==2.1.0

apt-get update

sudo apt-get install libasound2-dev portaudio19-dev

cd GeneFacePlusPlus/

pip install -r docs/prepare_env/requirements.txt -v

bash docs/prepare_env/install_ext.sh

export CUDA_VISIBLE_DEVICES=0

export VIDEO_ID=s1

echo $VIDEO_ID

#-----------------------------Prepare Data

#Step0. Crop videos to 512 * 512 and 25 FPS, ensure there is face in every frame.

ffmpeg -i data/raw/videos/${VIDEO_ID}.mp4 -vf fps=25,scale=w=512:h=512 -qmin 1 -q:v 1 data/raw/videos/${VIDEO_ID}_512.mp4

mv data/raw/videos/${VIDEO_ID}.mp4 data/raw/videos/${VIDEO_ID}_to_rm.mp4

mv data/raw/videos/${VIDEO_ID}_512.mp4 data/raw/videos/${VIDEO_ID}.mp4

#step1: Extract audio features, such as mel, f0, hubuert and esperanto

export CUDA_VISIBLE_DEVICES=0

export PYTHONPATH=./

mkdir -p data/processed/videos/${VIDEO_ID}

ffmpeg -i data/raw/videos/${VIDEO_ID}.mp4 -f wav -ar 16000 data/processed/videos/${VIDEO_ID}/aud.wav

export HF_ENDPOINT=https://hf-mirror.com
使用huggingface的镜像来下载huggingface的模型

python data_gen/utils/process_audio/extract_hubert.py --video_id=${VIDEO_ID}

python data_gen/utils/process_audio/extract_mel_f0.py --video_id=${VIDEO_ID}

#Step2. Extract images

mkdir -p data/processed/videos/${VIDEO_ID}/gt_imgs

ffmpeg -i data/raw/videos/${VIDEO_ID}.mp4 -vf fps=25,scale=w=512:h=512 -qmin 1 -q:v 1 -start_number 0 data/processed/videos/${VIDEO_ID}/gt_imgs/%08d.jpg

python data_gen/utils/process_video/extract_segment_imgs.py --ds_name=nerf --vid_dir=data/raw/videos/${VIDEO_ID}.mp4 --force_single_process # extract image, segmap, and background

#Step3. Extract lm2d_mediapipe

python data_gen/utils/process_video/extract_lm2d.py --ds_name=nerf --vid_dir=data/raw/videos/${VIDEO_ID}.mp4

Step3. Fit 3DMM

python data_gen/utils/process_video/fit_3dmm_landmark.py --ds_name=nerf --vid_dir=data/raw/videos/${VIDEO_ID}.mp4 --reset --debug --id_mode=global

Step4. Binarize

python data_gen/runs/binarizer_nerf.py --video_id=${VIDEO_ID}

#---------------------train_data

apt-get install rsync#安装rsync，不然安装时候报错

#————————训练headNerf模型

python tasks/run.py --config=egs/datasets/${VIDEO_ID}/lm3d_radnerf_sr.yaml --exp_name=motion2video_nerf/${VIDEO_ID}_head --reset

训练torse时候报错，需要按照https://github.com/yerfor/GeneFacePlusPlus/issues/122修改

CUDA_VISIBLE_DEVICES=0 python tasks/run.py --config=egs/datasets/${VIDEO_ID}/lm3d_radnerf_torso_sr.yaml --exp_name=motion2video_nerf/${VIDEO_ID}_torso --hparams=head_model_dir=checkpoints/motion2video_nerf/${VIDEO_ID}_head --reset

如果要修改训练步数，在egs/egs_bases/radnerf里面的base.xml里有

max_updates: 25_0000 # 40_0000 for training the whole head, 5_0000 for finetuning the mouth
finetune_lips: true
finetune_lips_start_iter: 20_0000

在egs/datasets/may里有 lm3d_radnerf_torso.yaml

lpips_start_iters: 20_0000 num_updates: 25_0000

lm3d_radnerf.yaml 里修改

lpips_start_iters: 20_0000 max_updates: 25_0000

#训练torseNerf模型

CUDA_VISIBLE_DEVICES=0 python tasks/run.py --config=egs/datasets/May/lm3d_radnerf_torso_sr.yaml --exp_name=motion2video_nerf/may_torso --hparams=head_model_dir=checkpoints/motion2video_nerf/may_head --reset