python爬取糗事百科

转载：静觅 »Python爬虫实战一之爬取糗事百科段子

#!/usr/bin/env python
# _*_coding:utf-8 _*_
# @Time     :2017/8/21 23:32
# @Author   :luoyu_bie
# @File     :QsBaike.py
# @Software :PyCharm Community Edition
import urllib2
import re

page = 1
url = "http://www.qiushibaike.com/hot/page/"+str(page)
agent = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:54.0) Gecko/20100101 Firefox/54.0"
headers = {"User-Agent":agent}
req = urllib2.Request(url,headers=headers)
response = urllib2.urlopen(req)
html = response.read()
#编译正则表达式
pattern = re.compile('<div class="author clearfix">.*?<h2>(.*?)</h2>.*?<div class="content">.*?<span>(.*?)</span>.*?<div class="stats">.*?<i class="number">(\d)</i>',re.S)
items = re.findall(pattern,html)
for item in items:
    print "发布者："+item[0].strip()+"\n","段子："+"\n"+item[1].strip().replace("<br/>","\n")+"\n"+"点赞数："+item[2]+"\n"+"*"*10