python爬取豆瓣电影评论_Python爬取豆瓣电影

Python爬取豆瓣电影

需要导入的模块：

from urllib import request

import urllib

from html.parser import HTMLParser

关于

HtmlParser，顾名思义，是解析Html的一个工具。python自带的。

一、常用属性和方法介绍

HtmlParser是一个类，在使用时一般继承它然后重载它的方法，来达到解析出需要的数据的目的。

1.常用属性：

lasttag，保存上一个解析的标签名，是字符串。

2.常用方法：

handle_starttag(tag, attrs) ，处理开始标签，比如

；这里的attrs获取到的是属性列表，属性以元组的方式展示

handle_endtag(tag) ，处理结束标签,比如

handle_startendtag(tag, attrs) ，处理自己结束的标签，如

handle_data(data) ，处理数据，标签之间的文本

handle_comment(data) ，处理注释，之间的文本

from urllib import request

import urllib

from html.parser import HTMLParser

class MovieParser(HTMLParser):

def __init__(self):

HTMLParser.__init__(self)

self.movies = []

def handle_starttag(self, tag, attrs):

def _attr(attrlist, attrname):

for attr in attrlist:

if attr[0] == attrname:

return attr[1]

return None

if tag == 'li' and _attr(attrs, 'data-title'):

movie= {}

movie['title'] = _attr(attrs, 'data-title')

movie['rate'] = _attr(attrs, 'data-rate')

movie['director'] = _attr(attrs, 'data-director')

movie['actors'] = _attr(attrs, 'data-actors')

self.movies.append(movie)

print('%(title)s|%(rate)s|%(director)s|%(actors)s' % movie)

def print_movies(url):

header = {

'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36'}

req = urllib.request.Request(url, headers=header)

s = urllib.request.urlopen(req)

parser = MovieParser()

parser.feed((s.read()).decode('utf-8'))

s.close()

if __name__ == '__main__':

url = 'https://movie.douban.com/'

print_movies(url)

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

fromurllibimportrequest

importurllib

fromhtml.parserimportHTMLParser

classMovieParser(HTMLParser):

def__init__(self):

HTMLParser.__init__(self)

self.movies=[]

defhandle_starttag(self,tag,attrs):

def_attr(attrlist,attrname):

forattrinattrlist:

ifattr[0]==attrname:

returnattr[1]

returnNone

iftag=='li'and_attr(attrs,'data-title'):

movie={}

movie['title']=_attr(attrs,'data-title')

movie['rate']=_attr(attrs,'data-rate')

movie['director']=_attr(attrs,'data-director')

movie['actors']=_attr(attrs,'data-actors')

self.movies.append(movie)

print('%(title)s|%(rate)s|%(director)s|%(actors)s'%movie)

defprint_movies(url):

header={

'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36'}

req=urllib.request.Request(url,headers=header)

s=urllib.request.urlopen(req)

parser=MovieParser()

parser.feed((s.read()).decode('utf-8'))

s.close()

if__name__=='__main__':

url='https://movie.douban.com/'

print_movies(url)