python3 爬取糗事百科

1.准备：

python 3.6

需要用到的包：re  request BeautiflSoup  urllib

2.代码如下：

---

# -*- coding: utf-8 -*-
import urllib.request
import re
from urllib import request
from bs4 import BeautifulSoup

articleUrl = "https://www.qiushibaike.com/textnew/page/%d"
#段子地址
commentUrl = "https://www.qiushibaike.com/article/%s"
#评论
page = 0

Url = articleUrl % page
#1.获取url源码

def getContentOrComment(Url):
	user_agent = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36'
	headers = {'User-agent': user_agent} #浏览器信息
	req = request.Request(url=Url, headers=headers)
	response = urllib.request.urlopen(req) #打开网址
	content = response.read().decode('utf-8') #读取所有源代码
	#print(content)
	return content

articlePage = getContentOrComment(Url)

#2.获取话题内容
soup = BeautifulSoup(articlePage, 'html.parser')

#print(soup) #格式化输出
floor = 1
#attrs属性
for string in soup.find_all(attrs="article block untagged mb15"):
	#print(string)
	#切片
	commentId = str(string.get('id')).strip()[11:]
	#print(commentId) #获取内容链接的后面id 9位数
	print('\n')
	#获取内容
	print(floor, '.', string.find(attrs="content").get_text().strip()) 
	floor += 1

#3.获取评论
commentPage = getContentOrComment(commentUrl%commentId)
soup = BeautifulSoup(commentPage, 'html.parser')
Cfloor = 1
for comment in soup.find_all(attrs="body"):
 	print("\n  ", Cfloor, " 楼回复：", comment.get_text().strip())
 	Cfloor += 1