海量数据按行数进行切分

最新推荐文章于 2022-10-01 16:24:24 发布

sxf_0123

最新推荐文章于 2022-10-01 16:24:24 发布

阅读量359

点赞数

分类专栏： python

本文链接：https://blog.csdn.net/sxf_123456/article/details/78178192

版权

python 专栏收录该内容

87 篇文章 1 订阅

订阅专栏

import os
import codecs

def split_file(read_file,write_file):
    readfile = open(read_file,"r")
    j = 0
    for line in readfile:
        line = line.strip()
        line_str = line.split('\t')
        if j%10000 == 0:  #按10000条数据切分
            num = int(j/10000) #设置文件的命名规则从0开始
            _write_file = write_file+str(num)+".txt"
            if os.path.exists(_write_file):
                os.remove(_write_file)
            writefile = open(_write_file,"w")
        new_line = "'"+line_str[1]+"'"+","
        writefile.write(new_line+ "\n")
        j += 1

    readfile.close()
    writefile.close()




if __name__=='__main__':
    read_file = "C:\\Users\\ziyht\\Desktop\\mysql_data_mac.txt"
    write_file = "C:\\Users\\ziyht\\Desktop\\test\\mac_"
    split_file(read_file,write_file)