实战爬虫抓取糗事百科段子(抓段子详情页)
- 先抓取详情页链接,拼接成为正确地址
- 抓取详情页数据,处理掉不需要的字符
- 当正则写的结果不唯一时,通过切片获取需要的数据
# !/usr/bin/python
# Filename: 实战 糗事百科(抓详情页).py
# Data : 2020/06/15
# Author : --king--
# ctrl+alt+L自动加空格格式化
import requests
import re
import time
# 获取详情页面url
def detail_pages(url):
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36'}
html = requests.get(url, headers=headers)
text = html.text
detail_pages = re.findall(r'''.+?<div.+?untagged.+?<a.+?(/article/\d+)".+?</a>''', text, re.VERBOSE | re.DOTALL)
urls = []
for detail_page in detail_pages:
detail_page = 'https://www.qiushibaike.com' + detail_page
urls.append(detail_page)
return urls
# 解析详情页面获取joke
def parse_page(urls):
for url in urls:
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36'}
html = requests.get(url, headers=headers)
text = html.text
# 获取的有三个值,切片取第一个
joke = re.findall(r'''.+?<div.+?word.+?<div.+?>(.+?)</div>''', text, re.VERBOSE | re.DOTALL)
# 法案先输出结果有表情链接<img src="xxx" align="yyy">,替换掉
joke = re.sub(r'<br/>|<img.+?>| ', '', joke[0])
# jokes = jokes[0].replace('<br/>','')
print(joke)
time.sleep(2)
def main():
base_url = 'https://www.qiushibaike.com/text/page/{}/'
for i in range(1, 11):
page_url = base_url.format(i)
detai_page_list = detail_pages(page_url)
parse_page(detai_page_list)
time.sleep(2)
print(i)
break
if __name__ == '__main__':
main()