# coding = utf-8
# Author:'Alvin'
# Date: '2018/8/10'
import requests
from lxml import html
import csv
#step1 确定url,设置请求头user-agent,利用requests库请求并得到网页源码
def get_content():
url = 'https://movie.douban.com/top250'
user_agent = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36'
headers = {
"User_Agent": user_agent
}
response = requests.get(url, headers=headers)
get_movie_list(response.content)
#setp2 分析返回的源码,利用xpath解析源码得到想要获取的内容并以字典的方式存到列表中
def get_movie_list(source):
selector = html.document_fromstring(source)
movieList = selector.xpath('//div[@class="info"]')
movies = []
for movie in movieList:
t
python利用requests和lxml库爬取豆瓣top250的电影信息并保存到csv
最新推荐文章于 2022-11-08 14:35:55 发布