用urllib、BeautifulSoup抓取糗事百科段子

python版本为：

chao@chao-machine:~/python_study$ python
Python 3.4.3 (default, May 31 2015, 17:07:22) 
[GCC 4.9.1] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> 

代码

#_*_ coding:utf-8 _*_
from bs4 import BeautifulSoup
import urllib
import urllib.request
import urllib.error
import urllib.parse


def print_qiushi(item):
	#过滤掉有图片的段子
	if item.find('div',class_='thumb'):
		return
	#过滤掉有视频的段子
	if item.find(name="div",class_='video_holder'):
		return
	#获取发表这条段子的用户名
	author = item.find("div",class_='author')
	if author != None:
		author = author.get_text().strip()
	else:
		author = 'anonymous'
	#获取段子的发表时间
	times = item.find("div",class_='content').contents[-2]
	if times == None:
		times = ''
	else:
		times = str(times)
		times.strip()
	#获取段子内容
	content = item.find("div",class_='content').get_text().strip()


	print('-_-:',author,"  ",times,'\n')
	print(content)
	print("\n\n")


url="http://www.qiushibaike.com/text"     换成"http://www.qiushibaike.com/"也可以的，反正有过滤机制 
user_agent = 'Mozellb/4.0 (compatible;MSIE 5.5;Windows NT)'
heads = {'User-Agent':user_agent}
try:
	request = urllib.request.Request(url,headers=heads)
	response = urllib.request.urlopen(request)
	soup = BeautifulSoup(response.read())
	items = soup.find_all(name='div',class_='article block untagged mb15')
	#循环处理没一条状态，包括用户名，内容，发布的时间
	for item in items:
		print_qiushi(item)
except urllib.error.URLError as e:
	if hasattr(e,'code'):
		print(e.code)
	if hasattr(e,'reason'):
		print(e.reason)

输出结果：

这次发的有点匆忙，下次再分析过程