python 处理1998-01-2003版-带音.txt文本

今天处理了一下，转换gbk编码为utf-8，1998-01-2003版-带音.txt，我重新命令为1998-01-2003.txt

def convert_encoding(input_file, output_file,
                     input_encoding='gbk', output_encoding='utf8'):
    with open(input_file, 'rb') as input_fd, open(output_file, 'wb') as output_fd:
        file_content = input_fd.read()
        unicode_file_content = file_content.decode(input_encoding)
        output_fd.write(unicode_file_content.encode(output_encoding))

def create_if_dir_not_exists(path_dir):
    if os.path.exists(path_dir):
        if not os.path.isdir(path_dir):
            raise ValueError("path: {} exits, but is not a directory!".format(path_dir))
    else:
        os.makedirs(path_dir, exist_ok=True)

current_dir='.'
data_dir = os.path.join(current_dir, 'data')
create_if_dir_not_exists(data_dir)

input_file = os.path.join(data_dir, '1998-01-2003.txt')
output_file = os.path.join(data_dir, 'raw_data.txt')
convert_encoding(input_file, output_file)

with open(output_file) as f:
    list_data=f.readlines()

from tqdm import tqdm 
list_strip_data=[]
for line in tqdm(list_data):
    if(not line.strip()):
        continue
    line_arr=line.strip().split()
#     print(line_arr)
    key=line_arr[0].split('/')[0]
    values=line_arr[1:]
    list_strip_data.append([key,values])

processed_data=[]
for item in tqdm(list_strip_data):
    key=item[0].split('-')[0]
    values=item[1]
    words=[item.split('/')[0] for item in values]
    label=['B' if item.split('/')[1]=='nt' else 'O' for item in values]
    processed_data.append([key,' '.join(words),' '.join(label)])

train_data=[]
valid_data=[]
test_data=[]
for item in processed_data:
    if(item[0]<='19980120'):
        train_data.append(item)
    elif(item[0]>='19980120' and item[0]<='19980125'):
        valid_data.append(item)
    else:
        test_data.append(item)

import pandas as pd
data=pd.DataFrame(train_data,columns=['id','text','label'])
data.to_csv('train.csv',index=False,sep='\t')
data=pd.DataFrame(valid_data,columns=['id','text','label'])
data.to_csv('valid.csv',index=False,sep='\t')
data=pd.DataFrame(test_data,columns=['id','text','label'])
data.to_csv('test.csv',index=False,sep='\t')