# -*- coding: UTF-8 -*-
import sys
import time
import urllib
import requests
import numpy as np
from bs4 import BeautifulSoup
from openpyxl import Workbook
# Some User Agents
hds = [{'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6'}, \
{
'User-Agent': 'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.12 Safari/535.11'}, \
{'User-Agent': 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Trident/6.0)'}]
def book_spider(book_tag):
page_num = 0;
book_list = []
try_times = 0
while (1):
# url='http://www.douban.com/tag/%E5%B0%8F%E8%AF%B4/book?start=0' # For Test
url = 'http://www.douban.com/tag/' + urllib.request.quote(book_tag) + '/book?start=' + str(page_num * 15)
time.sleep(np.random.rand() * 5)
# Last Version
try:
req = urllib.request.Request(url, headers=hds[page_num % len(hds)])
plain_text = urllib.request.urlopen(req).read()
#plain_text = str(source_code)
except (urllib.request.HTTPError, urllib.request.URLError) as e:
pri
【python爬虫实例】爬取豆瓣图书及信息
本文通过Python爬虫技术详细讲解如何抓取豆瓣网站上的图书信息,包括书名、作者、评分等关键数据,展示了从网页数据解析到数据存储的完整流程。
摘要由CSDN通过智能技术生成