从enwiki.xml中筛选类别
为了方便工作,我需要从enwiki-pages-articles.xml中过滤类别,并将它们存储相同的XML格式的类别。因此我选用了SAX解析器,在PyPy和CPython中都适用的包装器解析。对外的原生编译包(同事在PyPy和CPython 中) 。
代码非常简单:
class WikiCategoryHandler(handler.ContentHandler):
"""Class which detecs category pages and stores them separately
"""
ignored = set(('contributor', 'comment', 'meta'))
def __init__(self, f_out):
handler.ContentHandler.__init__(self)
self.f_out = f_out
self.curr_page = None
self.curr_tag = ''
self.curr_elem = Element('root', {})
self.root = self.curr_elem
self.stack = Stack()
self.stack.push(self.curr_elem)
self.skip = 0
def startElement(self, name, attrs):
if self.skip>0 or name in self.ignored:
self.skip += 1
return
self.curr_tag = name
elem = Element(name, attrs)
if name == 'page':
elem.ns = -1
self.curr_page = elem
else: # we don't want to keep old pages in memory
self.curr_elem.append(elem)
self.stack.push(elem)
self.curr_elem = elem
def endElement(self, name):
if self.skip>0:
self.skip -= 1
return
if name == 'page':
self.task()
self.curr_page = None
self.stack.pop()
self.curr_elem = self.stack.top()
self.curr_tag = self.curr_elem.tag
def characters(self, content):
if content.isspace(): return
if self.skip == 0:
self.curr_elem.append(TextElement(content))
if self.curr_tag == 'ns':
self.curr_page.ns = int(content)
def startDocument(self):
self.f_out.write("\n")
def endDocument(self):
self.f_out.write("\n")
print("FINISH PROCESSING WIKIPEDIA")
def task(self):
if self.curr_page.ns == 14:
self.f_out.write(self.curr_page.render())
class Element(object):
def __init__(self, tag, attrs):
self.tag = tag
self.attrs = attrs
self.childrens = []
self.append = self.childrens.append
def __repr__(self):
return "Element {}".format(self.tag)
def render(self, margin=0):
if not self.childrens:
return u"{0}".format(
" "*margin,
self.tag,
"".join([' {}="{}"'.format(k,v) for k,v in {}.iteritems()]))
if isinstance(self.childrens[0], TextElement) and len(self.childrens)==1:
return u"{0}{3}{1}>".format(
" "*margin,
self.tag,
"".join([u' {}="{}"'.format(k,v) for k,v in {}.iteritems()]),
self.childrens[0].render())
return u"{0}\n{3}\n{0}{1}>".format(
" "*margin,
self.tag,
"".join([u' {}="{}"'.format(k,v) for k,v in {}.iteritems()]),
"\n".join((c.render(margin+2) for c in self.childrens)))
class TextElement(object):
def __init__(self, content):
self.content = content
def __repr__(self):
return "TextElement" def render(self, margin=0):
return self.content
Element和TextElement元素包换tag和body信息,同时提供了一个方法来渲染它。
下面是我想要的PyPy和CPython比较结果。
/
time
PyPy
2169.90s
CPython
4494.69s
我很对PyPy的结果很吃惊。