问题
DBLP数据集中存在“&”这样的特殊符号,一开始我在找各种处理特殊符号的方法,结果都跟我想的不一样。网上很多关于怎么用xml.sax处理dblp数据集的帖子,但都没讲怎么处理“&”。
思路
后来仔细看了一下,这些“&”都是在author标记下,后面都会跟几个字母,然后分号结束,例如
<author>Günther Heinemann</author>
这里猜测ü
可能是其他国家的字母。于是查到了entity。由于XML之前没学过,所以现学了一下里面的internal entity和external entity。
解决方法
dblp数据集中ü
是作为外部实体(external entity)定义在“dblp.dtd”这个文件中,要想解析外部实体,就需要用到class xml.sax.handler.EntityResolver
中提供的EntityResolver.resolveEntity(publicId, systemId)
函数。
另外,Stack Overflow上的大佬还提到setFeature(feature_external_ges, True)
这个设置是否有影响没有测试过。
总的来说,加上了EntityResolver的resolveEntity函数,顺利地帮我解析出了那些外部实体。像上面所举的Günther Heinemann
,我顺利解析出Günther Heinemann。
致谢
在这里感谢shuaishuai3409,暴力的轮胎和arcsinM的帖子对我的代码的帮助,外部实体解析主要是看了IBM上的这个帖子。当然还有其他帖子,不过找不到了,一并感谢了!
代码
实现功能:对2015年前的作者构建一个索引(注释了作者协作关系图的代码,该部分尚未验证功能是否正确)。
# -*- coding: utf-8 -*-
"""
Created on Wed Oct 14 23:54:35 2020
@author: Administrator
"""
# use xml.sax parse dblp.xml. Give each author an ID and put them in a file.
# find collaboration relations of all authors and output them to a file. One relation's format is like (id1,id2)
import xml.sax
from xml.sax.handler import feature_external_ges
paper_tags = ('article', 'inproceedings', 'proceedings', 'book', 'incollection', 'phdthesis', 'mastersthesis', 'www')
class authorHandler(xml.sax.ContentHandler): # extract all authors
def __init__(self):
self.CurrentData = "" # tag's name
self.dict = {} # save all authors. The key is an author's name, the value is his id
self.name = "" # the name of an author
self.id = 0 # the ID of an author
self.contents = []
self.author = [] # all authors for the same paper
self.year = "" # the year of publication
def resolveEntity(self, publicID, systemID):
print("TestHandler.resolveEntity(): %s %s" % (publicID, systemID))
return systemID
def startElement(self, tag, attributes):
if tag != None and len(tag.strip()) > 0:
self.CurrentData = tag
def endElement(self, tag):
if tag != None and len(tag.strip()) > 0:
if tag in paper_tags:
if len(self.author) > 0 and self.year != '2015':
#if self.CurrentData == 'author': # this tag is author, save it in the dict
for authorname in self.author:
exist = self.dict.get(authorname, -1)
if exist == -1: # if this author have not been added into dict
self.dict[authorname] = self.id
self.id = self.id + 1
self.author.clear()
elif self.CurrentData == 'author':
self.author.append(self.name)
self.contents.clear()
def characters(self, content):
if content != '\n':
if self.CurrentData == 'author':
self.contents.append(content)
self.name = ''.join(self.contents)
# self.name += content.strip()
elif self.CurrentData == "year":
self.year = content.strip()
"""
class collabrationHandler(xml.sax.ContentHandler): # extract all collaboration relations
def __init__(self, dict, file):
self.CurrentData = "" # tag's name
self.dict = dict # the dict which is received ago
self.name = "" # the name of an author
self.id = 0 # the ID of an author
self.paper = False # if the tag is article or inproceeding, paper = True
self.author = [] # all authors' id in one <article> or <inproceeding>
self.file = file # Output collaboration relation to file
self.edge = set() # Edge's set
def startElement(self, tag, attributes):
self.CurrentData = tag
if tag == 'article' or tag == 'inproceeding':
self.author.clear() # start processing a new paper, old collaboration neen to be deleted
self.paper = True
def endElement(self, tag):
if (tag == 'article' or tag == 'inproceeding') and self.paper == True: # One paper's tag close
self.paper = False
for i in self.author:
for j in self.author:
if i < j and (i, j) not in self.edge: # edge
self.file.write(str(i) + ' ' + str(j) + '\n')
self.edge.add((i, j))
def characters(self, content):
if self.paper == True:
self.name = content
isAuthor = self.dict.get(self.name, -1) # isAuthor == -1 means that this content is not an author's name
if isAuthor != -1:
self.author.append(self.dict[self.name]) # add this author's id
"""
# set xml parser
parser = xml.sax.make_parser()
parser.setFeature(xml.sax.handler.feature_namespaces, 0)
parser.setFeature(feature_external_ges, True)
handler1 = authorHandler()
parser.setContentHandler(handler1)
parser.setEntityResolver(handler1)
parser.setDTDHandler(handler1)
parser.parse('dblp.xml')
with open('author.txt', 'w', encoding='utf-8') as f:
for k, v in handler1.dict.items():
f.write(str(v))
f.write(' ' + k)
f.write('\n')
f.close()
"""
with open('collaboration.txt', 'w') as f:
handler2 = collabrationHandler(handler1.dict, f)
parser.setContentHandler(handler2)
parser.parse('dblp.xml')
f.close()
"""