【爬虫】爬取百度搜索结果页面

今日看了一下爬虫，写了一个爬取百度搜索页面的小程序。可以在代码中改动搜索词，代码如下：

#coding=utf-8
#python version：2.7
#author:sharpdeep

import urllib
import urllib2
import re
from bs4 import BeautifulSoup as BS

baseUrl = 'http://www.baidu.com/s'
page = 1 #第几页
word = '穿戴设备'  #搜索关键词

data = {'wd':word,'pn':str(page-1)+'0','tn':'baidurt','ie':'utf-8','bsst':'1'}
data = urllib.urlencode(data)
url = baseUrl+'?'+data

try:
    request = urllib2.Request(url)
    response = urllib2.urlopen(request)
except urllib2.HttpError,e:
    print e.code
    exit(0)
except urllib2.URLError,e:
    print e.reason
    exit(0)

html = response.read()
soup = BS(html)
td = soup.find_all(class_='f')

for t in td:
    print t.h3.a.get_text()
    print t.h3.a['href']

    font_str = t.find_all('font',attrs={'size':'-1'})[0].get_text()
    start = 0 #起始
    realtime = t.find_all('div',attrs={'class':'realtime'})
    if realtime:
        realtime_str = realtime[0].get_text()
        start = len(realtime_str)
        print realtime_str
    end = font_str.find('...')
    print font_str[start:end+3],'\n'

https://github.com/sharpdeep/CrawlerBaidu

转载于:https://www.cnblogs.com/sharpdeep/p/10629551.html