python大文本文件处理_python – 逐行处理非常大(> 20GB)的文本文件

写这样的代码比较惯用

def ProcessLargeTextFile():

with open("filepath", "r") as r, open("outfilepath", "w") as w:

for line in r:

x, y, z = line.split(' ')[:3]

w.write(line.replace(x,x[:-3]).replace(y,y[:-3]).replace(z,z[:-3]))

这里的主要保存是只做一次拆分，但是如果CPU没有被征税，这很可能会有很大差异

它可能有助于一次节省几千行，并将其写入一个命中以减少您的硬盘驱动器的抖动。一百万行只有54MB的RAM！

def ProcessLargeTextFile():

bunchsize = 1000000 # Experiment with different sizes

bunch = []

with open("filepath", "r") as r, open("outfilepath", "w") as w:

for line in r:

x, y, z = line.split(' ')[:3]

bunch.append(line.replace(x,x[:-3]).replace(y,y[:-3]).replace(z,z[:-3]))

if len(bunch) == bunchsize:

w.writelines(bunch)

bunch = []

w.writelines(bunch)

建议@Janne，一种生成线条的替代方法

def ProcessLargeTextFile():

bunchsize = 1000000 # Experiment with different sizes

bunch = []

with open("filepath", "r") as r, open("outfilepath", "w") as w:

for line in r:

x, y, z, rest = line.split(' ', 3)

bunch.append(' '.join((x[:-3], y[:-3], z[:-3], rest)))

if len(bunch) == bunchsize:

w.writelines(bunch)

bunch = []

w.writelines(bunch)