python 处理table

# -*- coding: utf-8 -*-
from collections import defaultdict


def table_to_list(table):
    dct = table_to_2d_dict(table)
    return list(iter_2d_dict(dct))


def table_to_2d_dict(table):
    result = defaultdict(lambda : defaultdict(str))
    for row_i, row in enumerate(table.xpath('./tr')):
        for col_i, col in enumerate(row.xpath('./td|./th')):
            colspan = int(col.get('colspan', 1))
            rowspan = int(col.get('rowspan', 1))
            col_data = "".join(col.xpath('.//text()')).replace(" ", "").replace("\n", "")
            while row_i in result and col_i in result[row_i]:
                col_i += 1
            for i in range(row_i, row_i + rowspan):
                for j in range(col_i, col_i + colspan):
                    result[i][j] = col_data
    return result


def iter_2d_dict(dct):
    for i, row in sorted(dct.items()):
        cols = []
        for j, col in sorted(row.items()):
            cols.append(col)
        print(len(cols), cols)
        yield cols


if __name__ == '__main__':

    test_html = """
    <div id="head"></div><!-- head finish --><!-- 中间部分 -->
    """
    from lxml import etree
    import pandas as pd

    doc = etree.HTML(test_html.replace("<tbody>", '').replace("</tbody>", '').replace("<thead>", '').replace("</thead>", '').replace("\xa0", ''))
    for table_el in doc.xpath('//table'):
        table = table_to_list(table_el)
        # 表格合并后，会有重复行，可对重复的行，进行去重
        new_table = [""]
        for _list in table:
            if _list != new_table[-1]:
                new_table.append(_list)
        new_table = new_table[1:]
        print(table)
        df = pd.DataFrame(new_table)
        for i in range(df.shape[1]):
            values = [_[0] for _ in df.iloc[:, i:i+1].values.tolist() if _ != ['']]
            if len(values) > 1 and (values[0] == '业主单位' or values[0] == '建设单位'):
                print("miss  list ", values[1:])