将ansi文本格式以utf-8文本格式进行读取

查阅网上很多博客关于其他文本格式转化成utf-8，会出现中文乱码的情况，自己也遇到了，看到网上很多乱七八糟的博客，都没有什么实质性的解决方案。为此专门写了这篇博客，希望能对你们有帮助。

导入模块

# -*- coding: utf-8 -*-
# @Time    : 2021/8/10 12:03
# @Author  : xiehou
# @File    : test.py
# @Software: PyCharm
import pandas as pd
from pandas import DataFrame
import os
from chardet import detect

对所有文本文件进行读取，并重写

def reformat_txt(txt_dir_path):
    txt_names = os.listdir(txt_dir_path)
    for txt_name in txt_names:
        with open(txt_dir_path + '/' + txt_name, 'rb') as file:
            content = file.read()
		# 获取文件的编码格式
        coding = detect(content)['encoding']
		# 以该文件的格式进行读取
        with open(txt_dir_path + '/' + txt_name, 'r', encoding=coding) as file:
            content = file.read()
		# 对之前读取的文件进行重写到result文件夹中，并以utf-8的格式进行写入
		# 方便在对其文件内容进行读取
        with open('./result/' + txt_name, 'w', encoding='utf-8') as f:
            f.write(content)

对重写文件以utf-8的格式进行读取

def read_txt(txt_dir_path):
    all_information = []
    txt_names = os.listdir(txt_dir_path)
    for txt_name in txt_names:
        with open(txt_dir_path + '/' + txt_name, 'r', encoding='utf-8') as file:
            content = file.readlines()
            all_information.append((content, txt_name))
    return all_information

将文本内容写入到csv文件中

def write_csv(all_information, tag, csv_path):
    tags = []
    titles = []
    contents = []
    for information in all_information:
        title = information[1]
        content = information[0]
        titles.append(title)
        contents.append(content)
        tags.append(tag)
    infor_dict = {
        "标题": titles,
        "内容": contents,
        "标签": tags
    }
    data = DataFrame(infor_dict)
    data.to_csv(csv_path)

主函数

if __name__ == '__main__':
    # 原始文件夹,该文件夹包含utf-8格式和ansi格式等各种文本格式,
    # 需要更改文件夹名称
    txt_dir_path = './file'
    reformat_txt(txt_dir_path)

    # 中间文件夹,不需要改动(需要自行新建该文件夹，要不然会报错)
    txt_dir_path = './result'
    all_information = read_txt(txt_dir_path)

    # 目标csv的标签
    tag = '涉恐事件2.0'
    csv_path = './' + tag + '.csv'
    write_csv(all_information, tag, csv_path)

完整代码

# -*- coding: utf-8 -*-
# @Time    : 2021/8/10 12:03
# @Author  : xiehou
# @File    : test.py
# @Software: PyCharm
import pandas as pd
from pandas import DataFrame
import os
from chardet import detect
def reformat_txt(txt_dir_path):
    txt_names = os.listdir(txt_dir_path)
    for txt_name in txt_names:
        with open(txt_dir_path + '/' + txt_name, 'rb') as file:
            content = file.read()
		# 获取文件的编码格式
        coding = detect(content)['encoding']
		# 以该文件的格式进行读取
        with open(txt_dir_path + '/' + txt_name, 'r', encoding=coding) as file:
            content = file.read()
		# 对之前读取的文件进行重写到result文件夹中，并以utf-8的格式进行写入
		# 方便在对其文件内容进行读取
        with open('./result/' + txt_name, 'w', encoding='utf-8') as f:
            f.write(content)
def read_txt(txt_dir_path):
    all_information = []
    txt_names = os.listdir(txt_dir_path)
    for txt_name in txt_names:
        with open(txt_dir_path + '/' + txt_name, 'r', encoding='utf-8') as file:
            content = file.readlines()
            all_information.append((content, txt_name))
    return all_information
    def write_csv(all_information, tag, csv_path):
    tags = []
    titles = []
    contents = []
    for information in all_information:
        title = information[1]
        content = information[0]
        titles.append(title)
        contents.append(content)
        tags.append(tag)
    infor_dict = {
        "标题": titles,
        "内容": contents,
        "标签": tags
    }
    data = DataFrame(infor_dict)
    data.to_csv(csv_path)
    if __name__ == '__main__':
    # 原始文件夹,该文件夹包含utf-8格式和ansi格式等各种文本格式,
    # 需要更改文件夹名称
    txt_dir_path = './file'
    reformat_txt(txt_dir_path)

    # 中间文件夹,不需要改动(需要自行新建该文件夹，要不然会报错)
    txt_dir_path = './result'
    all_information = read_txt(txt_dir_path)

    # 目标csv的标签
    tag = '涉恐事件2.0'
    csv_path = './' + tag + '.csv'
    write_csv(all_information, tag, csv_path)

如果有问题可以在评论区私信我，如果有帮助请帮忙点个赞👍