一个简单的根据行数对大文件进行分片的python程序

项目中碰到了这么一个需求：
有一个record文件，每行一个item，整个文件大小在2G左右。根据要求，需要每天向其他系统提供100000个item，怎么处理比较好？
考虑之后觉得分片的主意不错，先根据每片的item数对这个大文件进行分片，然后每天投放一片即可。
具体python代码如下：

View Code

# -*- coding: utf-8 -*-

import os
import sys
import shutil
import time
# import linecache
import hashlib
import zlib
import binascii
import urllib2

import logging


datas_dir = "./datas/"
items_per_page = 10000
url_prefix = "http://172.16.1.110:80/download/"
check_result_dir  = "./results/"

logger = logging.getLogger(__name__)

def initialize():
"""
    @summary: initialize the working directory
"""
if os.path.exists(datas_dir) and os.path.isdir(datas_dir):
# clear datas dir
        print "begin to remove old datas directory"
        shutil.rmtree(datas_dir)
print "begin to make datas directory"
# to resove the conflict between rmtree and mkdir, so i will sleep 1 seconds
    time.sleep(1)
    os.mkdir(datas_dir)


def read_specific_lines(file, lines_to_read):
"""
    @summary: read specific lines from file
    file is any iterable; lines_to_read is an iterable
    containing int values
""" 
    lines = set(lines_to_read)
    last = max(lines)
for n, line in enumerate(file):
if n + 1 in lines:
yield line
if n + 1 > last:
return
def split_file(filename, lines_per_page):
"""
    @summary: split the file into n lines a page
"""
if lines_per_page <=0:
        lines_per_page = 1

    with open(filename, 'r') as fp:
        lines = []
for n, line in enumerate(fp):
            guard = n % lines_per_page
if guard == 0:
                lines = []
            lines.append(line)
if guard == lines_per_page - 1:
yield lines
yield lines

def write_to_file(lines, filename):
"""
    @summary: write lines to specified file
"""
    with open(filename, 'w') as fp:
for line in lines:
# construct content
            line_to_write = url_prefix + line
            fp.write(line_to_write)

def calculate_md5_crc32(msg):
"""
    @summary: calculate the md5 and crc32
"""
    m = hashlib.md5()
    m.update(msg)
    md5 = m.hexdigest().upper()
    crc32 = binascii.crc32(msg)
    crc32 = crc32 & 0xffffffff
    crc32_str = "%08x" % crc32
    crc32_str = crc32_str.upper()

return md5 + '.' + crc32_str

def check_file_integrity(download_url):
"""
    @summary: download file and check it's  integrity
    @return: True/False
"""
try:
        file_name = download_url.rsplit("/", 1)[1]
        response = urllib2.urlopen(download_url)
        md5_crc32 = calculate_md5_crc32(response.read())
print "file_name = %s, md5_crc32 = %s" %(file_name, md5_crc32)
if file_name == md5_crc32:
return True
else:
return False
except Exception, ex:
        logger.exception(ex)
return False

def do_check():
if os.path.exists(check_result_dir) and os.path.isdir(check_result_dir):
# clear datas dir
        print "begin to remove old result directory"
        shutil.rmtree(check_result_dir)
print "begin to make result directory"
# to resove the conflict between rmtree and mkdir, so i will sleep 1 seconds
    time.sleep(1)
    os.mkdir(check_result_dir)    
#    fp = open("not_integrity.list", 'w') 
    
for n, lines in enumerate(split_file("alive_sample.log", items_per_page)):
print "begin to check %d sample list" %( n+1)
if n >= 1:
break
        filename = os.path.join(check_result_dir, "notintergrity_" + str(n + 1) + ".list")
        fp = open(filename, 'w')
for line in lines:
try:
                download_url = url_prefix + line.strip()
                res = check_file_integrity(download_url)
if res == False:
                    fp.write(line)
                    fp.flush()
                    logger.error("check integrity error, download_url = %s", download_url)
else:
print "%s check OK" % line
except Exception, ex:
                logger.exception(ex)
        fp.close()
    fp.close()         
if __name__ == "__main__":
import myloggingconfig
#do_check()
    #assert False
    print check_file_integrity("http://172.16.1.110:80/download/B4D2EF861106F6812668D5163EA9CD58.4F38C168")
assert False
    initialize()
for n, lines in enumerate(split_file("20120106.rpt", items_per_page)):
print "begin construct %d sample list" %( n+1)
##        if n > 4:
##            break
        # construct file name
        filename = os.path.join(datas_dir, "samplelist_" + str(n + 1) + ".list")
        write_to_file(lines, filename)

上述代码中包含了计算md5和crc32的工具，整个分片功能包含在split_file函数中。

def split_file(filename, lines_per_page):
"""
    @summary: split the file into n lines a page
"""
if lines_per_page <=0:
        lines_per_page = 1

    with open(filename, 'r') as fp:
        lines = []
for n, line in enumerate(fp):
            guard = n % lines_per_page
if guard == 0:
                lines = []
            lines.append(line)
if guard == lines_per_page - 1:
yield lines
yield lines