解决问题:读取xlsx文件内的中文文本,并测试参考答案和模型生成答案之间的Rouge-L指标。
import pandas as pd
import jieba
from rouge import Rouge
import sys
# 增加递归深度限制
sys.setrecursionlimit(10000)
def preprocess_text(text):
# 使用 jieba 进行中文分词
words = jieba.lcut(text)
return ' '.join(words)
def calculate_rouge_l(reference, candidate):
rouge = Rouge()
# 预处理文本
reference = preprocess_text(reference)
candidate = preprocess_text(candidate)
print("----------------")
print(reference)
print("----------------")
print(candidate)
# 计算ROUGE-L
scores = rouge.get_scores(candidate, reference)
return scores[0]['rouge-l']['r'] # 返回ROUGE-L的Recall得分
def main():
# 读取xlsx文件
file_path = '文件路径'
df = pd.read_excel(file_path)
# 打印列名以进行调试
print("Columns in the dataframe:", df.columns)
# 假设数据有一列 'reference' 和五列 'candidate1', 'candidate2', 'candidate3', 'candidate4', 'candidate5'
references = df['参考答案']
candidate_columns = ['one', 'two', 'three', 'four', 'five']
# 计算每个reference与每个candidate的ROUGE-L指标
for candidate in candidate_columns:
if candidate not in df.columns:
print(f"Column {candidate} not found in the dataframe.")
continue
rouge_scores = []
for ref, cand in zip(references, df[candidate]):
if pd.isna(ref) or pd.isna(cand):
rouge_scores.append(0)
continue
rouge_score = calculate_rouge_l(ref, cand)
rouge_scores.append(rouge_score)
# 将ROUGE-L指标添加到DataFrame中
df[f'ROUGE-L_{candidate}'] = rouge_scores
# 保存结果到新的xlsx文件
output_path = '文件路径'
df.to_excel(output_path, index=False)
print("Saved ROUGE scores to:", output_path)
if __name__ == "__main__":
main()