ogr读写性能优化

ogr读写性能优化

在开发一个程序的时候，需要将一份mif格式的地理数据按照图幅进行切分，以便使用多进程并行的方式进行后续计算。在第一次实现该功能时，使用的是先将所有数据读取进内存，再将内存里的数据写入到文件中实现的。但是因为mif文件太大，直接读取进内存，在一些内存不足的机器上无法顺利运行，因此对该方法进行了改进。

封装的通用mif文件读写函数

def get_miffile_layer(ds_file):
    if ds_file is None:
        return None
    return ds_file.GetLayerByIndex(0)


def read_mif(file_path):
    try:
        mif_file = ogr.Open(file_path)
        lyr_file = mif_file.GetLayerByIndex(0)
    except:
        print "open file failure: %s" % file_path
        traceback.print_exc()
        return None

    return mif_file


def create_miffile(file_path, fieldDef_list) :
    ds_file = None;
    ds_driver = None;
    lyr_file = None;
    lyr_name = "";
    pos = file_path.rfind('.');
    if pos != -1 :
        lyr_name = file_path[0 : pos];
    pos = lyr_name.rfind(os.sep);
    if pos != -1 :
        lyr_name = lyr_name[pos + 1 : len(lyr_name)];
    ds_driver = ogr.GetDriverByName("MapInfo File");
    if ds_driver is None :
        return ds_file;
    ds_file = ds_driver.CreateDataSource(file_path);
    if ds_file is None :
        return ds_file;
    lyr_file = ds_file.CreateLayer(lyr_name);
    if lyr_file is None :
        ds_file = None;
        return ds_file;
    for fieldDef in fieldDef_list:
        lyr_file.CreateField(fieldDef);
    return ds_file;


def get_fieldDef_list(lyr_file):
    fieldDef_list = []
    feature = lyr_file.GetNextFeature()
    for i in range(feature.GetFieldCount()):
        fieldDef_list.append(feature.GetFieldDefnRef(i))
    return fieldDef_list


def write_mif(file_path, fieldDef_list, feat_list):
    ds_file = create_miffile(file_path, fieldDef_list)
    lyr_file = get_miffile_layer(ds_file)
    if lyr_file is None:
        print "write mif file failure! %s" % file_path
        return

    for feat in  feat_list:
        lyr_file.CreateFeature(feat)

在上述函数的基础上进行了再一次封装，使用一个对象来保留读取和写入数据的文件句柄，实现同时进行读取和写入。

class MifgReader(object):
    def __init__(self, file_path):
        self.file_path = file_path
        self.ds_file = read_mif(self.file_path)
        self.lyr_file = get_miffile_layer(self.ds_file)
        self.fieldDef_list = get_fieldDef_list(self.lyr_file)

    def get_fieldDef_list(self):
        return self.fieldDef_list

    def get_lyr_file(self):
        self.lyr_file.ResetReading()
        return self.lyr_file


class MifgWriter(object):
    def __init__(self, file_path):
        self.file_path = file_path
        self.ds_file = None
        self.lyr_file = None
        self.fieldDef_list = None

    def set_fieldDef_list(self, fieldDef_list):
        self.fieldDef_list = fieldDef_list
        self.ds_file = create_miffile(self.file_path, fieldDef_list)
        self.lyr_file = get_miffile_layer(self.ds_file)

    def is_valid(self):
        return self.fieldDef_list is not None and self.ds_file is not None and self.lyr_file is not None

    def write_feature(self, feature):
        if not self.is_valid():
            return False
        self.lyr_file.CreateFeature(feature)
        return True

上述类的使用方法如下：

def test(input_file_path, output_file_path):
    reader = MifgReader(input_file_path)
    writer = MifgWriter(output_file_path)
    fieldDef_list = reader.get_fieldDef_list()
    writer.set_fieldDef_list(fieldDef_list)
    
    for feature in reader.get_lyr_file():
        writer.write_feature(feature)
        feature.Destory()  # 释放内存

这样便可以实现不占用内存的情况下进行文件分幅或者其他处理