爬虫学习笔记

爬取糗事百科的段子
def getcontent(url,pat):
 headers=(‘User-Agent’, ‘Mozilla/6.0 (iPhone; CPU iPhone OS 8_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/8.0 Mobile/10A5376e Safari/8536.25’)
 
 opener=urllib.request.build_opener()
 opener.addheaders=[headers]
 urllib.request.install_opener(opener)
 data=urllib.request.urlopen(url).read().decode(‘utf-8’)
 #pat=’

.+?

’#正则
 contentlist=re.compile(pat).findall(data)
 return contentlist
for page in range(1,3):
 url=initurl+‘page/’+str(page)+’/’
 contentlist=getcontent(url,pat)
 print(contentlist)
 #for content in contentlist:
  #print(content)