微调Grounding DINO

前言

    本文不生产技术，只做技术的搬运工！

环境配置

参考地址：https://github.com/longzw1997/Open-GroundingDino

权重下载

from transformers import BertConfig, BertModel
from transformers import AutoTokenizer

config = BertConfig.from_pretrained("bert-base-uncased")
model = BertModel.from_pretrained("bert-base-uncased", add_pooling_layer=False, config=config)
tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")

config.save_pretrained("/home/project_python/Open-GroundingDino/bert-base-uncased")
model.save_pretrained("/home/project_python/Open-GroundingDino/bert-base-uncased")
tokenizer.save_pretrained("/home/project_python/Open-GroundingDino/bert-base-uncased")

Swin-T：https://github.com/IDEA-Research/GroundingDINO/releases/download/v0.1.0-alpha/groundingdino_swint_ogc.pth

Swin-B：https://github.com/IDEA-Research/GroundingDINO/releases/download/v0.1.0-alpha2/groundingdino_swinb_cogcoor.pth

微调

创建以下文件，请注意修改每个文件中的路径，然后执行finetuning.sh即可

make_label_json.py

import json

# Define the content for the JSON file
content = {
    "0": "excavator",
    "1": "wagon"
}

# Define the file path
file_path = '/home/project_python/Open-GroundingDino/fine_tuning_input_param/singapore/label.json'

# Write the content to the JSON file
with open(file_path, 'w') as file:
    json.dump(content, file)

print(f"File '{file_path}' created successfully.")

rewirte_ovdg_json.py

#change the paths according to file locations
import json

# Define the data
data = {
    "train": [
        {
            "root": "/home/project_python/Open-GroundingDino/fine_tuning_data/singapore/images/train",#Train images
            "anno": "/home/project_python/Open-GroundingDino/fine_tuning_input_param/singapore/train.jsonl",#Odvg jsonl file
            "label_map": "/home/project_python/Open-GroundingDino/fine_tuning_input_param/singapore/label.json",# label.json file
            "dataset_mode": "odvg"
        }
    ],
    "val": [
        {
            "root": "/home/project_python/Open-GroundingDino/fine_tuning_data/singapore/images/val",#Test Images
            "anno": "/home/project_python/Open-GroundingDino/fine_tuning_data/singapore/val.json",#Test data Annotation file in COCO
            "label_map": None,
            "dataset_mode": "coco"
        }
    ]
}

file_path = '/home/project_python/Open-GroundingDino/config/datasets_singapore_odvg.json'

with open(file_path, 'w') as file:
    json.dump(data, file, indent=2)

print(f"Data has been written to {file_path}")

rewrite_cfg_coco_and_ovdg.py

import re

def modify_file(file_path):
    label_list_content = 'label_list = ["excavator","wagon"]\n'

    # Read the entire content of the file
    with open(file_path, 'r') as file:
        content = file.read()

    # Replace use_coco_eval =TRUE with use_coco_eval =FALSE using regex
    content = re.sub(r'use_coco_eval\s*=\s*True', 'use_coco_eval = False', content)

    # Insert label_list after use_coco_eval = FALSE using regex
    content = re.sub(r'use_coco_eval\s*=\s*False', r'use_coco_eval = False\n\n' + label_list_content, content, count=1, flags=re.MULTILINE)

    # Write the modified content back to the file
    with open(file_path, 'w') as file:
        file.write(content)

# Paths to the files
cfg_coco_path = '/home/project_python/Open-GroundingDino/config/cfg_coco.py'
cfg_odvg_path = '/home/project_python/Open-GroundingDino/config/cfg_odvg.py'

# Modify both files
modify_file(cfg_coco_path)
modify_file(cfg_odvg_path)

print("Updated use_coco_eval to FALSE and added label_list using regex in both files.")

rewrite_coco2odvg.py

import re

# Define the file path
file_path = 'tools/coco2odvg.py'

# Define the new values according to the dataset
new_id_map = '{0: 0, 1: 1}'# 7 classes of AquariumDataset
new_ori_map = '{"0": "excavator", "1": "wagon"}'

# Read the content of the file
with open(file_path, 'r') as file:
    content = file.read()

# Replace the id_map value using regex
content = re.sub(r'id_map\s*=\s*\{[^\}]*\}', f'id_map = {new_id_map}', content)

# Replace the ori_map value using regex
content = re.sub(r'ori_map\s*=\s*\{[^\}]*\}', f'ori_map = {new_ori_map}', content)

# Write the updated content back to the file
with open(file_path, 'w') as file:
    file.write(content)

print(f"Updated {file_path} successfully.")

rewrite_train_dist.py

def replace_file_content(file_path, new_content):
    try:
        # Open the file in write mode and replace its content
        with open(file_path, 'w') as file:
            file.write(new_content)
        print(f"Successfully replaced the content of {file_path}")
    except IOError as e:
        print(f"Error occurred while replacing content: {e}")

# Define the new content for the file
new_content = """\
CFG=$1
DATASETS=$2
OUTPUT_DIR=$3

# Set the environment variable for CUDA
export CUDA_VISIBLE_DEVICES=0

python main.py \\
    --config_file ${CFG} \\
    --datasets ${DATASETS} \\
    --output_dir ${OUTPUT_DIR} \\
    --pretrain_model_path /home/project_python/Open-GroundingDino/fine_tuning_weights/groundingdino_swint_ogc.pth \\
    --options text_encoder_type="/home/project_python/Open-GroundingDino/bert-base-uncased"
"""

# Specify the file path
file_path = '/home/project_python/Open-GroundingDino/train_dist.sh'

# Call the function to replace the content
replace_file_content(file_path, new_content)

train_dist.sh

CFG=$1
DATASETS=$2
OUTPUT_DIR=$3

# Set the environment variable for CUDA
export CUDA_VISIBLE_DEVICES=0

python main.py \
    --config_file ${CFG} \
    --datasets ${DATASETS} \
    --output_dir ${OUTPUT_DIR} \
    --pretrain_model_path /home/project_python/Open-GroundingDino/fine_tuning_weights/groundingdino_swint_ogc.pth \
    --options text_encoder_type="/home/project_python/Open-GroundingDino/bert-base-uncased"

finetuning.sh

source activate open-dino

python /home/project_python/Open-GroundingDino/fine_tuning_tools/rewrite_coco2odvg.py

python /home/project_python/Open-GroundingDino/tools/coco2odvg.py --input "/home/project_python/Open-GroundingDino/fine_tuning_data/singapore/train.json"  --output "/home/project_python/Open-GroundingDino/fine_tuning_input_param/singapore/train.jsonl"

python /home/project_python/Open-GroundingDino/fine_tuning_tools/make_label_json.py

python /home/project_python/Open-GroundingDino/fine_tuning_tools/rewirte_ovdg_json.py

python /home/project_python/Open-GroundingDino/fine_tuning_tools/rewrite_cfg_coco_and_ovdg.py

python /home/project_python/Open-GroundingDino/fine_tuning_tools/rewrite_train_dist.py

GPU_NUM=1
CGF="/home/project_python/Open-GroundingDino/config/cfg_odvg.py"
DATASETS="/home/project_python/Open-GroundingDino/config/datasets_singapore_odvg.json"
OUTPUT_DIR="/home/project_python/Open-GroundingDino/fine_tuning_output/singapore"

chmod +x train_dist.sh

sh train_dist.sh ${CGF} ${DATASETS} ${OUTPUT_DIR}

测试

infer_test_img.py

import os
import subprocess

# Directory containing the images
image_dir = "/home/project_python/Open-GroundingDino/fine_tuning_data/singapore/images/test"
# Get a list of all image files in the directory
image_files = [f for f in os.listdir(image_dir) if f.endswith('.png') or f.endswith('.jpg')]

# Define the other arguments for the inference script
config_path = "/home/project_python/Open-GroundingDino/tools/GroundingDINO_SwinT_OGC.py"
checkpoint_path = "/home/project_python/Open-GroundingDino/fine_tuning_output/singapore/checkpoint0099.pth"
text_prompts = "excavator"
output_dir = "/home/project_python/Open-GroundingDino/fine_tuning_test/pred_img"

# Loop over all image files and run the inference script on each one
for image_file in image_files:
    image_path = os.path.join(image_dir, image_file)
    command = [
        "python", "/home/project_python/Open-GroundingDino/tools/inference_on_a_image.py",
        "-c", config_path,
        "-p", checkpoint_path,
        "-i", image_path,
        "-t", text_prompts,
        "-o", output_dir+image_file
    ]
    subprocess.run(command)

finetuning_test.sh

source activate grounding

python /home/project_python/Open-GroundingDino/fine_tuning_tools/infer_test_img.py