使用python爬虫爬取豆瓣top250电影名

import requests
import re
url = 'https://movie.douban.com/top250'
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36'}
resp = requests.get(url,headers = headers)
page_content = resp.text
//re.S 参数可以匹配包括换行在内的所有字符
//?P<name>
ob1 = re.compile(r'<li>.*?<span class="title">(?P<name>.*?)</span>',re.S) //电影名
ob2 = re.compile(r'</span>.*?<br>(?P<year>.*?)&nbsp',re.S)	//导演
ob3 = re.compile(r'<span class="rating_num" property="v:average">.*?(?P<score>.*?)</span>',re.S) //评分
ob4 = re.compile(r'<div class="star">.*?<span>(?P<number>.*?)人评价',re.S) //评价人数
result1 = ob1.finditer(page_content)
result2 = ob2.finditer(page_content)
result3 = ob3.finditer(page_content)
result4 = ob4.finditer(page_content)
for i in result1:
    print(i.group('name'))
for i in result2:
    print(i.group('year').strip())  //strip()函数可以去掉空格
for i in result3:
    print(i.group('score'))
for i in result4:
    print(i.group('number'))

上面这种方式只能依次输出相同项，所以进行一些改进,并将文件保存在csv中。

import requests
import re
import csv

url = 'https://movie.douban.com/top250'
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36'}
resp = requests.get(url, headers=headers)
page_content = resp.text
//这种方式的正则表达式要表示的内容是要连续的
ob1 = re.compile(r'<li>.*?<span class="title">(?P<name>.*?)</span>.*?'
                 r'<br>(?P<year>.*?)&n'
                 r'bsp.*?<span class="rating_num" property="v:average">(?P<score>.*?)</span>.*?'
                 r'<span>(?P<number>.*?)</span>', re.S)

result1 = ob1.finditer(page_content)
f = open('250.csv',mode = 'w')
csvwriter = csv.writer(f)
for i in result1:
    print(i.group('name'))
    print(i.group('year').strip())
    print(i.group('score'))
    print(i.group('number'))
    dic = i.groupdict()
    dic['year'] = dic['year'].strip()
    csvwriter.writerow(dic.values())
f.close()
print('over')