python处理大文件_python对大文件的处理

多线程框架中采取queue来实现线程间资源的互斥。

在文件过大的情况下，如果都读入内存的话，占用内存就太多了。

这里手动实现了一个多线程调用文件迭代器来使用f.next()

# -*- coding:utf-8 -*-

import threading

class Geturl(object):

def __init__(self, open_file):

self.open_file = open_file

self.num = 0

self.__mutex = threading.RLock()

self.f = open(self.open_file, 'r')

self.kafka_mutex = threading.RLock()

def _line(self):

self.__mutex.acquire()

try:

line = self.f.__next__()

except StopIteration:

line = StopIteration

self.f.close()

'''

if self.num % 1000 == 0:

print(self.num)

self.num += 1

'''

self.__mutex.release()

return line

def get_line(self):

return self._line()

def _deal(deal_file):

while True:

try:

item = deal_file.get_line()

# 处理工作

if item == StopIteration:

raise ValueError()

except ValueError:

print("all task has done!")

break

except Exception as e:

print("error:", e)

if __name__ == "__main__":

filename = 'Bigfile'

geturl = Geturl(filename)

_deal(geturl)