xml.sax处理DBLP数据集中特殊符号（外部实体）处理（python）

问题

DBLP数据集中存在“&”这样的特殊符号，一开始我在找各种处理特殊符号的方法，结果都跟我想的不一样。网上很多关于怎么用xml.sax处理dblp数据集的帖子，但都没讲怎么处理“&”。

思路

后来仔细看了一下，这些“&”都是在author标记下，后面都会跟几个字母，然后分号结束，例如

<author>G&uuml;nther Heinemann</author>

这里猜测ü可能是其他国家的字母。于是查到了entity。由于XML之前没学过，所以现学了一下里面的internal entity和external entity。

解决方法

dblp数据集中ü是作为外部实体（external entity）定义在“dblp.dtd”这个文件中，要想解析外部实体，就需要用到class xml.sax.handler.EntityResolver中提供的EntityResolver.resolveEntity(publicId, systemId)函数。
另外，Stack Overflow上的大佬还提到setFeature(feature_external_ges, True)这个设置是否有影响没有测试过。
总的来说，加上了EntityResolver的resolveEntity函数，顺利地帮我解析出了那些外部实体。像上面所举的Günther Heinemann，我顺利解析出Günther Heinemann。

致谢

在这里感谢shuaishuai3409，暴力的轮胎和arcsinM的帖子对我的代码的帮助，外部实体解析主要是看了IBM上的这个帖子。当然还有其他帖子，不过找不到了，一并感谢了！

代码

实现功能：对2015年前的作者构建一个索引（注释了作者协作关系图的代码，该部分尚未验证功能是否正确）。

# -*- coding: utf-8 -*-
"""
Created on Wed Oct 14 23:54:35 2020

@author: Administrator
"""

# use xml.sax parse dblp.xml. Give each author an ID and put them in a file.
# find collaboration relations of all authors and output them to a file. One relation's format is like (id1,id2)

import xml.sax
from xml.sax.handler import feature_external_ges

paper_tags = ('article', 'inproceedings', 'proceedings', 'book', 'incollection', 'phdthesis', 'mastersthesis', 'www')

class authorHandler(xml.sax.ContentHandler):  # extract all authors
    def __init__(self):
        self.CurrentData = ""  # tag's name
        self.dict = {}  # save all authors. The key is an author's name, the value is his id
        self.name = ""  # the name of an author
        self.id = 0  # the ID of an author
        self.contents = []
        self.author = []  # all authors for the same paper
        self.year = ""  # the year of publication


    def resolveEntity(self, publicID, systemID):
        print("TestHandler.resolveEntity(): %s %s" % (publicID, systemID))
        return systemID

    def startElement(self, tag, attributes):
        if tag != None and len(tag.strip()) > 0:
            self.CurrentData = tag

    def endElement(self, tag):
        if tag != None and len(tag.strip()) > 0:
            if tag in paper_tags:
                if len(self.author) > 0 and self.year != '2015':
                #if self.CurrentData == 'author':  # this tag is author, save it in the dict
                    for authorname in self.author:
                        exist = self.dict.get(authorname, -1)
                        if exist == -1:  # if this author have not been added into dict
                            self.dict[authorname] = self.id
                            self.id = self.id + 1

                self.author.clear()
            elif self.CurrentData == 'author':
                self.author.append(self.name)
                self.contents.clear()

    def characters(self, content):
        if content != '\n':
            if self.CurrentData == 'author':
                self.contents.append(content)
                self.name = ''.join(self.contents)
            # self.name += content.strip()
            elif self.CurrentData == "year":
                self.year = content.strip()

"""
class collabrationHandler(xml.sax.ContentHandler):  # extract all collaboration relations
    def __init__(self, dict, file):
        self.CurrentData = ""  # tag's name
        self.dict = dict  # the dict which is received ago
        self.name = ""  # the name of an author
        self.id = 0  # the ID of an author
        self.paper = False  # if the tag is article or inproceeding, paper = True
        self.author = []  # all authors' id in one <article> or <inproceeding>
        self.file = file  # Output collaboration relation to file
        self.edge = set()  # Edge's set

    def startElement(self, tag, attributes):
        self.CurrentData = tag
        if tag == 'article' or tag == 'inproceeding':
            self.author.clear()  # start processing a new paper, old collaboration neen to be deleted
            self.paper = True

    def endElement(self, tag):
        if (tag == 'article' or tag == 'inproceeding') and self.paper == True:  # One paper's tag close
            self.paper = False
            for i in self.author:
                for j in self.author:
                    if i < j and (i, j) not in self.edge:  # edge
                        self.file.write(str(i) + ' ' + str(j) + '\n')
                        self.edge.add((i, j))

    def characters(self, content):
        if self.paper == True:
            self.name = content
            isAuthor = self.dict.get(self.name, -1)  # isAuthor == -1 means that this content is not an author's name
            if isAuthor != -1:
                self.author.append(self.dict[self.name])  # add this author's id
"""

# set xml parser
parser = xml.sax.make_parser()
parser.setFeature(xml.sax.handler.feature_namespaces, 0)
parser.setFeature(feature_external_ges, True)
handler1 = authorHandler()
parser.setContentHandler(handler1)
parser.setEntityResolver(handler1)
parser.setDTDHandler(handler1)
parser.parse('dblp.xml')

with open('author.txt', 'w', encoding='utf-8') as f:
    for k, v in handler1.dict.items():
        f.write(str(v))
        f.write(' ' + k)
        f.write('\n')
f.close()

"""
with open('collaboration.txt', 'w') as f:
	handler2 = collabrationHandler(handler1.dict, f)
	parser.setContentHandler(handler2)
	parser.parse('dblp.xml')
f.close()
"""