python2.7.x+request+xpath爬取豆瓣电影信息

先记录一下代码，最近开始学习python爬虫，照着视频教程进行练习，再对相关知识点进行扩充。

#!/usr/bin/env python 
# -*- coding:utf-8 -*-

import requests
from lxml import etree

#1.将目标网站上的页面抓取下来
headers = {
    'User-Agent': "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) "
                  "Chrome/63.0.3239.26 Safari/537.36 Core/1.63.6788.400 QQBrowser/10.3.2767.400",
    'Referer': 'https://movie.douban.com/'
}

url = 'https://movie.douban.com/cinema/nowplaying/changsha/'
response = requests.get(url, headers=headers)
text = response.text

#print(response.text)
#print(response.content)

html = etree.HTML(text)
ul = html.xpath("//ul[@class='lists']")[0]
#print(etree.tostring(ul, encoding='gbk'))
lis = ul.xpath("./li")
movies = []
for li in lis:
    #print(etree.tostring(li, encoding='utf-8').decode('utf-8'))
    title = li.xpath("@data-title")[0]
    #print(title)
    score = li.xpath("@data-score")[0]
    duration = li.xpath("@data-duration")[0]
    region = li.xpath("@data-region")[0]
    director = li.xpath("@data-director")[0]
    actors = li.xpath("@data-actors")[0]
    thumbnail = li.xpath(".//img/@src")[0]
    movie = {
        'title': title,
        'score': score,
        'duration': duration,
        'region': region,
        'director':director,
        'actors': actors,
        'thumbnail': thumbnail
    }
    movies.append(movie)

for movie in movies:
    print(movie['title'])
    print(movie['score'])
    print(movie['duration'])
    print(movie['region'])
    print(movie['director'])
    print(movie['actors'])
    print(movie['thumbnail'])