python爬取豆瓣短评_Python爬取豆瓣指定书籍的短评

Python爬取豆瓣指定书籍的短评

#!/usr/bin/python

# coding=utf-8

import re

import sys

import time

import random

import urllib

import urllib2

import MySQLdb

# 爬取豆瓣评论

class Douban:

# 构造函数

def __init__(self, url, name):

# 采集的地址

#self.url = 'https://book.douban.com/subject/26356948/comments/hot'

self.url = url

# 存储的文件名

self.filename = str(name) + '.txt'

# 数据库配置

self.database = {

'host': '127.0.0.1',

'username': 'root',

'password': 'root',

'database': 'douban',

'charset': 'utf8',

'table': str(name),

}

# HTTP请求超时

self.http_timeout = 10

# 请求计数器

self.request_counter = 0

# 错误代码计数器

self.error_counter = 0

# 数据库第一次错误

self.db_error_first = True

# 替换换行符

def replaceSpace(self, string):

string = string.replace('\r\n', ' ')

string = string.replace('\n', ' ')

string = string.replace('\r', ' ')

return string

# 清洗单页评论，返回为单条评论的列表

def getCommentsPage(self, url):

user_agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.119 Safari/537.36'

headers = {'User-Agent': user_agent}

request = urllib2.Request(url, headers=headers)

try:

response = urllib2.urlopen(request, timeout=self.http_timeout)

comments_page_raw = response.read()

reg = r'.*?

(.*?).*?'

comments_page = re.findall(reg, comments_page_raw, re.I | re.M | re.S)

return comments_page

except urllib2.HTTPError, e:

errmsg = 'HTTP Error: ' + e.code + ',' + e.reason

print unicode(errmsg, 'utf-8')

if e.code == 403 or e.code == '403':

# 服务器拒绝服务，等等...等等... 等120秒

print unicode('服务器拒绝服务，等等...等等... 等120秒', 'utf-8')

time.sleep(120)

response = urllib2.urlopen(request, timeout=self.http_timeout)

comments_page_raw = response.read()

reg = r'.*?

(.*?).*?'

comments_page = re.findall(reg, comments_page_raw, re.I | re.M | re.S)

return comments_page

elif e.code == 404 or e.code == '404':

# not found... 已爬取完毕，结束程序

print unicode('not found... 已爬取完毕，结束程序', 'utf-8')

# 结束程序

sys.exit(0)

else:

print unicode('HTTP 未知错误', 'utf-8')

# 错误次数累加

self.error_counter += 1

# 判断是否进行中断

if self.error_counter == 20:

info = '错误次数已达%d次，结束程序' % self.error_counter

print unicode(info, 'utf-8')

sys.exit(1)

# 清洗单条评论，返回单条评论的昵称、头像、内容...的列表

def getCommentsItem(self, comment_raw):

# 昵称， 豆瓣主页，头像， 点赞， 星级评价， 日期， 评价内容

reg = r'

(.*?).*?.*? (.*?).*?

(.*?)

.*?

'