import requests import urllib.parse import datetime from lxml import etree fhout = open("result.txt", 'a') baseurl = 'https://zh.wikipedia.org/wiki/' begin_date = datetime.datetime.strptime('2016-01-01', "%Y-%m-%d") contents=[] for i in range(196,366): content = [] mid_date = begin_date + datetime.timedelta(days=i) thedate = str(mid_date.month) + '月' + str(mid_date.day) + '日' print(thedate) urlthedate = urllib.parse.quote(thedate) url = baseurl + urlthedate print(url) html = requests.get(url).text.encode("utf-8") tree = etree.HTML(html) ul = tree.xpath('//li[@class="toclevel-1 tocsection-1"]/ul/li/a/span[@class="toctext"]/text()') num = len(ul) fhout.write("data_"+str(mid_date.month)+"_"+str(mid_date.day)+"=[") for i in range(num, 0, -1): records = tree.xpath('//div[@id="mw-content-text"]/div[@class="mw-parser-output"]/ul['+ str(i) +']/li[descendant-or-self::text()]') ulen = len(records) for j in range(ulen-1,-1,-1): content.append(records[j].xpath('string(.)')) fhout.write("'"+records[j].xpath('string(.)')+"'\n") fhout.write("]\n") print(content) contents.append(content) fhout.close()