Text2Image评价指标——CLIP Score（2）

最新推荐文章于 2024-09-11 22:42:53 发布

我是浮夸

最新推荐文章于 2024-09-11 22:42:53 发布

阅读量1.5k

点赞数 14

分类专栏：代码复现文章标签： python 图像处理 stable diffusion

本文链接：https://blog.csdn.net/qq_45791526/article/details/136326585

版权

代码复现专栏收录该内容

12 篇文章 9 订阅

订阅专栏

之前写的那篇是直接用了clip-vit-large-patch14，今天这里主要是使用CLIP模型来实现的，所以在运行代码之前请安装好CLIP。

代码主要的一些过程：

首先加载模型和预处理器，然后从文件夹中读取图片和文本，然后对图片和文本进行编码，然后来计算CLIP分数：计算图像和文本特征向量之间的点积，得到的结果就是两个特征向量之间的余弦相似度，也就是CLIP分数。

Text-Image：

import os

import numpy as np
import torch
import clip
from PIL import Image
from tqdm import tqdm

device = "cuda" if torch.cuda.is_available() else "cpu"
torch.cuda.set_device(7)
# 加载模型、数据处理器
model, preprocess = clip.load("ViT-B/32", device=device)
model.eval()
# model, preprocess = clip.load("ViT-L/14", device=device)



def get_all_folders(folder_path):
    # 获取文件夹中的所有文件和文件夹
    all_files = os.listdir(folder_path)
    # 过滤所有的文件夹
    folder_files = [file for file in all_files if os.path.isdir(os.path.join(folder_path, file))]
    # 将文件夹的路径添加到一个列表中
    folder_paths = [os.path.join(folder_path, folder_file) for folder_file in folder_files]
    # 返回列表
    return folder_paths

def get_all_images(folder_path):
    # 获取文件夹中的所有文件和文件夹
    all_files = os.listdir(folder_path)
    # 过滤所有的图片文件
    image_files = [file for file in all_files if file.endswith((".jpg", ".png", ".jpeg"))]
    # 将图片文件的路径添加到一个列表中
    image_paths = [os.path.join(folder_path, image_file) for image_file in image_files]
    # 返回列表
    return image_paths

def get_clip_score(image_path, text):
    # 打开图片并进行预处理
    image = preprocess(Image.open(image_path)).unsqueeze(0).to(device)
    # print(f"Image shape: {image.shape}")

    # 对文本进行编码
    text = clip.tokenize([text]).to(device)

    with torch.no_grad():
        # 对图片进行编码
        image_features = model.encode_image(image)
        # 对文本进行编码
        text_features = model.encode_text(text)
        # 对特征向量进行归一化处理
        image_features = image_features.cpu().numpy()
        text_features = text_features.cpu().numpy()
        # 对图片和文本的特征向量进行L2范数归一化。这样做的目的是使得每个特征向量的长度为1，

        image_features = image_features / np.sqrt(np.sum(image_features ** 2, axis=1, keepdims=True))
        text_features = text_features / np.sqrt(np.sum(text_features ** 2, axis=1, keepdims=True))

        # 计算图像和文本特征向量之间的点积，得到的结果就是两个特征向量之间的余弦相似度。
        similarity = image_features @ text_features.T
        # 计算相似度的平均值，得到CLIP分数。
        clip_score = np.mean(similarity)

    return clip_score  # 返回CLIP分数

def calculate_clip_scores_for_all_categories(images_folder_path, text_prompts):
    # 获取所有的类别文件夹
    category_folders = get_all_folders(images_folder_path)

    # 初始化一个字典来存储每个类别的 Clip Score
    category_clip_scores = {}
    # 初始化一个字典来存储每个类别的平均值
    category_mean_scores = {}

    # 遍历每个类别文件夹
    for category_folder, text_prompt in tqdm(zip(category_folders, text_prompts),total=len(category_folders),desc="Processing categories"):
        # 获取类别名称
        category_name = os.path.basename(category_folder)
        # 获取该类别下的所有图片
        images_path = get_all_images(category_folder)
        # 计算该类别的 Clip Score
        clip_scores = [get_clip_score(image_path, text_prompt) for image_path in images_path]
        # 将 Clip Score 存储在字典中
        category_clip_scores[category_name] = clip_scores
        # 计算平均值
        mean_score = np.mean(clip_scores)
        # 将平均值存储在字典中
        category_mean_scores[category_name] = mean_score

    return category_clip_scores,category_mean_scores


# 从文件中获取文本提示
def read_prompts_from_file(file_path):
    with open(file_path, 'r') as file:
        prompts = file.readlines()
    # 去除每行末尾的换行符
    prompts = [prompt.strip() for prompt in prompts]
    return prompts

# 使用函数读取文本提示
text_prompts = read_prompts_from_file('./texts/prompts.txt')


# 计算所有类别的 Clip Score 和平均值
category_clip_scores, category_mean_scores = calculate_clip_scores_for_all_categories("./samples_images", text_prompts)

# 打印结果
for category, clip_score in category_clip_scores.items():
    for score in clip_score:
        print(f"Category: {category}, Clip Score: {score:.4f}")



for category, mean_score in category_mean_scores.items():
    print(f"Category: {category}, Mean Score: {mean_score:.4f}")

Image-Image


import os

import numpy as np
import torch
import clip
from PIL import Image
from tqdm import tqdm

device = "cuda" if torch.cuda.is_available() else "cpu"
torch.cuda.set_device(7)
# 加载模型、数据处理器
model, preprocess = clip.load("ViT-B/32", device=device)
model.eval()
# model, preprocess = clip.load("ViT-L/14", device=device)


def get_all_folders(folder_path):
    # 获取文件夹中的所有文件和文件夹
    all_files = os.listdir(folder_path)
    # 过滤所有的文件夹
    folder_files = [file for file in all_files if os.path.isdir(os.path.join(folder_path, file))]
    # 将文件夹的路径添加到一个列表中
    folder_paths = [os.path.join(folder_path, folder_file) for folder_file in folder_files]
    # 返回列表
    return folder_paths

def get_all_images(folder_path):
    # 获取文件夹中的所有文件和文件夹
    all_files = os.listdir(folder_path)
    # 过滤所有的图片文件
    image_files = [file for file in all_files if file.endswith((".jpg", ".png", ".jpeg"))]
    # 将图片文件的路径添加到一个列表中
    image_paths = [os.path.join(folder_path, image_file) for image_file in image_files]
    # 返回列表
    return image_paths

def get_clip_score_between_images(image_path1, image_path2):
    # 打开第一张图片并进行预处理
    image1 = preprocess(Image.open(image_path1)).unsqueeze(0).to(device)
    # 打开第二张图片并进行预处理
    image2 = preprocess(Image.open(image_path2)).unsqueeze(0).to(device)

    with torch.no_grad():
        # 对第一张图片进行编码
        image_features1 = model.encode_image(image1)
        # 对第二张图片进行编码
        image_features2 = model.encode_image(image2)

        # 对特征向量进行归一化处理
        image_features1 = image_features1.cpu().numpy()
        image_features2 = image_features2.cpu().numpy()

        image_features1 = image_features1 / np.sqrt(np.sum(image_features1 ** 2, axis=1, keepdims=True))
        image_features2 = image_features2 / np.sqrt(np.sum(image_features2 ** 2, axis=1, keepdims=True))

        # 计算两个图像特征向量之间的点积，得到的结果就是两个特征向量之间的余弦相似度。
        similarity = image_features1 @ image_features2.T

        # 计算相似度的平均值，得到CLIP分数。
        clip_score = np.mean(similarity)

    return clip_score  # 返回CLIP分数

def calculate_clip_scores_for_all_categories(images_folder_path1, images_folder_path2):
    # 获取所有的类别文件夹
    category_folders1 = get_all_folders(images_folder_path1)
    category_folders2 = get_all_folders(images_folder_path2)

    # 初始化一个字典来存储每个类别的 Clip Score
    category_clip_scores = {}
    # 初始化一个字典来存储每个类别的平均值
    category_mean_scores = {}


    # 遍历每个类别文件夹
    for category_folder1 in tqdm(category_folders1, total=len(category_folders1), desc="Processing categories"):
        # 获取类别名称
        category_name = os.path.basename(category_folder1)
        # 获取该类别下的所有图片
        images_path1 = get_all_images(category_folder1)
        images_path2 = get_all_images(images_folder_path2)  # 获取文件夹2中的所有图片

        # 计算该类别的 Clip Score
        clip_scores = []
        for image_path1 in images_path1:
            image_scores = [get_clip_score_between_images(image_path1, image_path2) for image_path2 in images_path2]
            clip_scores.append(np.mean(image_scores))  # 对每张图片与文件夹2中所有图片的CLIP分数取平均值

        # 将 Clip Score 存储在字典中
        category_clip_scores[category_name] = clip_scores
        # 计算平均值
        mean_score = np.mean(clip_scores)
        # 将平均值存储在字典中
        category_mean_scores[category_name] = mean_score

    return category_clip_scores, category_mean_scores


# 计算所有类别的 Clip Score 和平均值
category_clip_scores, category_mean_scores = calculate_clip_scores_for_all_categories("./samples_images", "./regulation_images/bird")

# 打印结果
for category, clip_score in category_clip_scores.items():
    for score in clip_score:
        print(f"Category: {category}, Clip Score: {score:.4f}")
    # print(f"Category: {category}, Clip Score: {clip_score.item():.4f}")


for category, mean_score in category_mean_scores.items():
    print(f"Category: {category}, Mean Score: {mean_score:.4f}")
    # print(f"Category: {category}, Mean Score: {mean_score.item():.4f}")

为了方便理解代码，附上我的目录：