多线程爬取糗事百科
import urllib.request
import re
import urllib.error
import sys
import threading
headers=("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3947.100 Safari/537.36")
opener=urllib.request.build_opener()
opener.addheaders=[headers]
urllib.request.install_opener(opener)
class One(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
def run(self):
for i in range(1,5,2):
url="https://www.qiushibaike.com/8hr/page/"+str(i)
pagedata=urllib.request.urlopen(url).read().decode("utf-8","ignore")
pat=".*?id='qiushi_tag_(.*?)'>"
datalist=re.compile(pat,re.S).findall(pagedata)
for j in range(0,len(datalist)):
thispagedata=datalist[j]
thisurl="https://www.qiushibaike.com/article/"+thispagedata
trueurl=urllib.request.urlopen(thisurl).read().decode("utf-8","ignore")
non_bmp_map = dict.fromkeys(range(0x10000, sys.maxunicode + 1), 0xfffd)
trueurl=trueurl.translate(non_bmp_map)
pat2='<meta name="description" content="(.*?)" />'
data=re.compile(pat2,re.S).findall(trueurl)
print("第"+str(i)+"页第"+str(j)+"个段子的内容是:")
print(data)
class Two(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
def run(self):
for i in range(0,5,2):
url="https://www.qiushibaike.com/8hr/page/"+str(i)
pagedata=urllib.request.urlopen(url).read().decode("utf-8","ignore")
pat=".*?id='qiushi_tag_(.*?)'>"
datalist=re.compile(pat,re.S).findall(pagedata)
for j in range(0,len(datalist)):
thispagedata=datalist[j]
thisurl="https://www.qiushibaike.com/article/"+thispagedata
trueurl=urllib.request.urlopen(thisurl).read().decode("utf-8","ignore")
non_bmp_map = dict.fromkeys(range(0x10000, sys.maxunicode + 1), 0xfffd)
trueurl=trueurl.translate(non_bmp_map)
pat2='<meta name="description" content="(.*?)" />'
data=re.compile(pat2,re.S).findall(trueurl)
print("第"+str(i)+"页第"+str(j)+"个段子的内容是:")
print(data)
one=One()
one.start()
two=Two()
two.start()