刚开始学爬虫,对照着崔庆才的“猫眼TOP100”修改而来,有不对的地方希望大佬指正
第一天,利用requerts库和正则表达式抓取练习
import requests
import re
import json
import time
def get_one_page(url):
headers = {'User-Agent' : 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36'}
response = requests.get(url,headers=headers)
if response.status_code == 200:
return response.text
return None
def parse_one_page(html):
pattern = re.compile('<li>.*?<div.*?<div.*?<em.*?"">(.*?)</em>.*?<img.*?alt="(.*?)".*?class="info">.*?<a.*?<span.*?</a>.*?<div.*?class="bd".*?<p.*? / (.*?) .*?</p>.*?<div.*?<span.*?<span.*?average">(.*?)</span.*?<span.*?<span>(.*?)人评价</span>.*?',re.S)
items = re.findall(pattern, html)
for item in items:
yield {
'index': item[0],
'title': item[1],
'county' : item[2],
'score' : item[3],
'number of people' : item[4],
}
def write_to_file(content):
with open('豆瓣电影排行250.txt', 'a', encoding='utf-8') as f:
f.write(json.dumps(content, ensure_ascii=False) + '\n')
def main(offset):
url = 'https://movie.douban.com/top250?start='+str(offset)+'&filter='
html = get_one_page(url)
for item in parse_one_page(html):
print(item)
write_to_file(item)
if __name__ == '__main__':
for i in range(10):
main(i * 25)
time.sleep(1)
运行结果:
保存的txt文件: