大语言模型评估指标BLEU,ROUGE,PPL

最新推荐文章于 2024-07-11 03:29:05 发布

凤凰AI

最新推荐文章于 2024-07-11 03:29:05 发布

阅读量95

点赞数 1

文章标签：语言模型人工智能自然语言处理

本文链接：https://blog.csdn.net/Daniel_Singularity/article/details/139998984

版权

# 第一步安装nltk的包-->pip install nltk
from nltk.translate.bleu_score import sentence_bleu

def cumulative_bleu(reference, candidate):

    bleu_1_gram = sentence_bleu(reference, candidate, weights=(1, 0, 0, 0))
    bleu_2_gram = sentence_bleu(reference, candidate, weights=(0.5, 0.5, 0, 0))
    bleu_3_gram = sentence_bleu(reference, candidate, weights=(0.33, 0.33, 0.33, 0))
    bleu_4_gram = sentence_bleu(reference, candidate, weights=(0.25, 0.25, 0.25, 0.25))

    # print('bleu 1-gram: %f' % bleu_1_gram)
    # print('bleu 2-gram: %f' % bleu_2_gram)
    # print('bleu 3-gram: %f' % bleu_3_gram)
    # print('bleu 4-gram: %f' % bleu_4_gram)

    return bleu_1_gram, bleu_2_gram, bleu_3_gram, bleu_4_gram
    # return bleu_1_gram, bleu_2_gram

# 生成文本
candidate_text = ["This", "is",  "some",  "generated", "text"]

# 参考文本列表
reference_texts = [["This", "is",  "a",  "reference", "text"],
                   ["This", "is",  "another", "reference", "text"]]

# 计算 Bleu 指标
c_bleu = cumulative_bleu(reference_texts, candidate_text)

# 打印结果

print("The Bleu score is:", c_bleu)

# 第一步：安装rouge-->pip install rouge
from rouge import Rouge

# 生成文本
generated_text = "This is some generated text."

# 参考文本列表
reference_texts = ["This is a reference text.", "This is another generated reference text."]

# 计算 ROUGE 指标
rouge = Rouge()
scores = rouge.get_scores(generated_text, reference_texts[1])
print(f'scores-->{scores}')

# 打印结果
print("ROUGE-1 precision:", scores[0]["rouge-1"]["p"])
print("ROUGE-1 recall:", scores[0]["rouge-1"]["r"])
print("ROUGE-1 F1 score:", scores[0]["rouge-1"]["f"])
# ROUGE-1 precision: 0.8
# ROUGE-1 recall: 0.6666666666666666
# ROUGE-1 F1 score: 0.7272727223140496

import math
# 定义语料库
sentences = [
['I', 'have', 'a', 'pen'],
['He', 'has', 'a', 'book'],
['She', 'has', 'a', 'cat']
]
# 定义语言模型
unigram = {
'I': 1/12,
'have': 1/12,
'a': 3/12,
'pen': 1/12,
'He': 1/12,
'has': 2/12,
'book': 1/12,
'She': 1/12,
'cat': 1/12
}
# 计算困惑度
perplexity = 0
for sentence in sentences:
    print(sentence)
    sentence_prob = 1
    for word in sentence:
        print(f'word-->{word}')
        sentence_prob *= unigram[word]
        print(sentence_prob)
        print('*'*80)
    print(sentence_prob)

    sentence_perplexity = 1/sentence_prob
    print(f'sentence_perplexity--》{sentence_perplexity}')
    perplexity += math.log(sentence_perplexity, 2) #以2为底
    print(f'perplexity-->{perplexity}')
    print('----' * 80)
perplexity = 2 ** (-perplexity/len(sentences))
print('困惑度为：', perplexity)
# 困惑度为： 0.000325