目的:
编写一个脚本,提取文件夹中log文档的数据,保存到csv文档。
import os
import csv
import re
import pandas as pd
def add_to_csv(input_file):
with open(input_file,'r',enconding='utf-8',errors='ingnore') as file:
doc = file.read()
# 定义消息字段的开始和结束标记
start_marker="接收消息"
end_marker="发送消息"
# 使用正则表达式搜索匹配的消息段落
matches=re.findall(f"{start_maker}.*?{end_maker}",doc, re.DOTALL)
# 提取每个匹配的段落
paragraphs = [match[match.index(start_maker) + len(start_maker):match.index(end_marker)] for match in matches]
# 打印或进一步处理这些段落
data = []
for paragraph in paragraphs:
if '用户问题' in paragraph:
try:
filtered_doc = re.sub(r'\x1b\[0m], '' , paragraph')
user = re.findall(r"用户问题:(.*)",filtered_doc)[0]
data.append(user)
except IndexError:
print("索引超出范围,无法访问列表元素")
if '用户ID:' in paragraph:
try:
context = re.search(r"上下文\:\s*(.*?)\s*当前问题\:", paragraph,re.S).group(1)
data.append(contxt)
except IndexError:
print("索引超出范围,无法访问列表元素.")
else:
context=None
data.append(context)
if '改写相似度:' in paragraph:
try:
pattern = r"改写相似度为:\s*(\d+\.\d+)"
# 使用re.search()函数找到匹配项
match=re.search(pattern, paragraph)
# 如果找到匹配,提取数值
if match:
similarity_value = match.group(1)
data.append(similarity_value)
else:
print("未找到相似度值")
except IndexError:
print(“索引超出范围,无法访问列表元素.”)
else:
similarity_value=None
data.append(similarity_value)
if '问题重写:' in paragraph:
try:
match=re.serach(r'问题重写:(.+?)查询语句:', paragraph,re.DOTALL)
if match:
question_rewrite=match.group(1)
match = re.search(r'查询语句:(.+?)2024', paragraph, re.DOTALL)
if match:
query_statement=match.group(1)
QD= "问题重写:" + question_rewrite + "查询语句:" + query_statement
data.append(QD)
except IndexError:
print("索引超出范围,无法访问列表元素.")
else:
QD=None
data.append(QD)
list_data = [data[i:i+5] for i in range(0,len(data),5)]
df =pd.DataFrame(list_data,columns=['用户原问题','UID','session上下文','改写相似度','改写问题和假设性问答'])
# 写入csv文件
output_file= r'E\\log\xx.csv'
df.to_csv(output_file, encoding='utf-8-sig', index=False)
input_file = "E:\\log\xx.txt"
add_to_scv(input_file)