linux分割csv文件路径,CSV文件拆分情况。。。

CSV文件拆分情况。。。

1.要注意读取的分隔符

2.保存时要注意保存的分隔符

分隔符坑很多，有的是“，”， 有的是空格，有的是tab，不一而足。。。。。。

import os

from sklearn.model_selection import train_test_split

import pandas as pd

import numpy as np

def rename():

for (root, dirs, files) in os.walk("tianchi_datasets", topdown=False):

for name in files:

new_name = os.path.join(root, root.split('/')[-1] + '_' + name)

print(os.path.join(root, "\t", name), new_name)

os.rename(os.path.join(root, name), new_name)

def break_data(target, rate=0.2):

for (root, dirs, files) in os.walk(target, topdown=False):

for name in files:

if "_" in name:

print(name)

origin_filename = os.path.join(root, name)

origin_dataset = pd.read_csv(origin_filename, delimiter="\t", header=None) # 加入参数

train_data, test_data = train_test_split(origin_dataset, test_size=rate)

train_filename = os.path.join(root, 'train.csv')

dev_filename = os.path.join(root, 'dev.csv')

train_data.to_csv(train_filename, index=False, sep="\t", header=None)

test_data.to_csv(dev_filename, index=False, sep="\t", header=None)

if __name__ == '__main__':

# rename()

break_data(target='datasets', rate=0.2)