查阅网上很多博客关于其他文本格式转化成utf-8,会出现中文乱码的情况,自己也遇到了,看到网上很多乱七八糟的博客,都没有什么实质性的解决方案。为此专门写了这篇博客,希望能对你们有帮助。
导入模块
# -*- coding: utf-8 -*-
# @Time : 2021/8/10 12:03
# @Author : xiehou
# @File : test.py
# @Software: PyCharm
import pandas as pd
from pandas import DataFrame
import os
from chardet import detect
对所有文本文件进行读取,并重写
def reformat_txt(txt_dir_path):
txt_names = os.listdir(txt_dir_path)
for txt_name in txt_names:
with open(txt_dir_path + '/' + txt_name, 'rb') as file:
content = file.read()
# 获取文件的编码格式
coding = detect(content)['encoding']
# 以该文件的格式进行读取
with open(txt_dir_path + '/' + txt_name, 'r', encoding=coding) as file:
content = file.read()
# 对之前读取的文件进行重写到result文件夹中,并以utf-8的格式进行写入
# 方便在对其文件内容进行读取
with open('./result/' + txt_name, 'w', encoding='utf-8') as f:
f.write(content)
对重写文件以utf-8的格式进行读取
def read_txt(txt_dir_path):
all_information = []
txt_names = os.listdir(txt_dir_path)
for txt_name in txt_names:
with open(txt_dir_path + '/' + txt_name, 'r', encoding='utf-8') as file:
content = file.readlines()
all_information.append((content, txt_name))
return all_information
将文本内容写入到csv文件中
def write_csv(all_information, tag, csv_path):
tags = []
titles = []
contents = []
for information in all_information:
title = information[1]
content = information[0]
titles.append(title)
contents.append(content)
tags.append(tag)
infor_dict = {
"标题": titles,
"内容": contents,
"标签": tags
}
data = DataFrame(infor_dict)
data.to_csv(csv_path)
主函数
if __name__ == '__main__':
# 原始文件夹,该文件夹包含utf-8格式和ansi格式等各种文本格式,
# 需要更改文件夹名称
txt_dir_path = './file'
reformat_txt(txt_dir_path)
# 中间文件夹,不需要改动(需要自行新建该文件夹,要不然会报错)
txt_dir_path = './result'
all_information = read_txt(txt_dir_path)
# 目标csv的标签
tag = '涉恐事件2.0'
csv_path = './' + tag + '.csv'
write_csv(all_information, tag, csv_path)
完整代码
# -*- coding: utf-8 -*-
# @Time : 2021/8/10 12:03
# @Author : xiehou
# @File : test.py
# @Software: PyCharm
import pandas as pd
from pandas import DataFrame
import os
from chardet import detect
def reformat_txt(txt_dir_path):
txt_names = os.listdir(txt_dir_path)
for txt_name in txt_names:
with open(txt_dir_path + '/' + txt_name, 'rb') as file:
content = file.read()
# 获取文件的编码格式
coding = detect(content)['encoding']
# 以该文件的格式进行读取
with open(txt_dir_path + '/' + txt_name, 'r', encoding=coding) as file:
content = file.read()
# 对之前读取的文件进行重写到result文件夹中,并以utf-8的格式进行写入
# 方便在对其文件内容进行读取
with open('./result/' + txt_name, 'w', encoding='utf-8') as f:
f.write(content)
def read_txt(txt_dir_path):
all_information = []
txt_names = os.listdir(txt_dir_path)
for txt_name in txt_names:
with open(txt_dir_path + '/' + txt_name, 'r', encoding='utf-8') as file:
content = file.readlines()
all_information.append((content, txt_name))
return all_information
def write_csv(all_information, tag, csv_path):
tags = []
titles = []
contents = []
for information in all_information:
title = information[1]
content = information[0]
titles.append(title)
contents.append(content)
tags.append(tag)
infor_dict = {
"标题": titles,
"内容": contents,
"标签": tags
}
data = DataFrame(infor_dict)
data.to_csv(csv_path)
if __name__ == '__main__':
# 原始文件夹,该文件夹包含utf-8格式和ansi格式等各种文本格式,
# 需要更改文件夹名称
txt_dir_path = './file'
reformat_txt(txt_dir_path)
# 中间文件夹,不需要改动(需要自行新建该文件夹,要不然会报错)
txt_dir_path = './result'
all_information = read_txt(txt_dir_path)
# 目标csv的标签
tag = '涉恐事件2.0'
csv_path = './' + tag + '.csv'
write_csv(all_information, tag, csv_path)
如果有问题可以在评论区私信我,如果有帮助请帮忙点个赞👍