全角文本变半角文本

在做自然语言处理的时候发现国内的好多语料库都是采用的全角编码的, 这会有些麻烦, 需要将其改编成半角编码

import os

def Q2B(uchar):
    """全角转半角"""
    inside_code = ord(uchar)
    if inside_code == 0x3000:
        inside_code = 0x0020
    else:
        inside_code -= 0xfee0
        
    # 转完之后不是半角字符返回原来的字符
    if inside_code < 0x0020 or inside_code > 0x7e:  
        return uchar
    return chr(inside_code)



if __name__ == '__main__':

    text = None
    path = "E:\\pycharm\\original\\"
    files = os.listdir(path)
    for file in files:
        file = path + file
        with open(file, "r", encoding="utf-8")as f:
            text = f.readlines()
        text_text = []
        for per_text in text:
            per_text = "".join([ Q2B(word) for word in per_text])
            text_text.append(per_text)

        fileNewname = file[:-4] + "2" + file[-4:]
        with open(fileNewname, "w", encoding="utf-8")as f2:
            for per_text in text_text:
                f2.write(per_text)

这样就能将目录下的所有文件的编码改成半角的