【python处理数据】

读取文件并拆分为100万条/个（已知文件行数）

import os
import pandas as pd

# read DataFrame
data = pd.read_csv("youxiang202401.tsv",sep='\t',encoding='gb18030')

# number of csv files along with the row
k = 2
size = 1000000


for i in range(k):
    df = data[size*i:size*(i+1)]
    df.to_csv(f'zhuli{i+1}.csv', index=False)

file1 = pd.read_csv("zhuli1.csv")
print(file1)
print("\n")
file2 = pd.read_csv("zhuli2.csv")
print(file2)

读取文件并平均拆分

import pandas as pd

# 读取原始CSV文件
df = pd.read_csv('email202312.tsv',sep='\t',encoding='gb18030')

# 计算每个新文件应该包含的行数
chunk_size = len(df) // 2
print(len(df))
print(chunk_size)
# 拆分数据集并存储到新的CSV文件中
for i in range(2):
    start_row = chunk_size * i
    end_row = start_row + chunk_size
    chunk = df[start_row:end_row]
    chunk.to_csv(f'split_{i + 1}.csv', index=False)

添加文件表头&文件一列拆分多列

import pandas as pd

# 读取原始文件
data = pd.read_csv('202311.tsv',sep=',',header=None,low_memory=False)
#增加表头
data.columns=['结算账期','归属账期','结算关系标识','结算级别','省分','地市代码','文件标识','sp代码','结入账户','业务类','业务代码','产品代码','数据类型','费用类型','用户号码','订购量','重批价费率','计费费用','重批价后费用','结算比例','结算费用']
#print(data)

# 计算每个新文件应该包含的行数
chunk_size = len(data) // 6
print(len(data))
print(chunk_size)
# 拆分数据集并存储到新的CSV文件中
for i in range(6):
    start_row = chunk_size * i
    end_row = start_row + chunk_size
    chunk = data[start_row:end_row]
    chunk.to_csv(f'split_{i + 1}.csv', index=False)