爬取猫眼电影榜单
import json
import os
import requests
from bs4 import BeautifulSoup
def get_html(url):
try:
d = {'User Agent':"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 SE 2.X MetaSr 1.0"}
r = requests.get(url, params=d)
if r.status_code == 200:
return r.text
return None
except:
return None
def parse_html(html):
soup = BeautifulSoup(html, "html.parser")
# 创建BeautifulSoup对象,解析HTML网页
data = soup.body.find_all('div', {'class': 'movie-item-info'})
# 获取soup的body部分,找到class为movie-item-info的div
name = []
# 建立一个空列表,储存爬取的数据
for i in data:
lis = i.text.replace('\n', " ")
# 取标签文本部分,并用空格代替换行
name.append(lis)
# 存入列表
t = 1
for i in name:
print(t, i)
t += 1
return name
def write_down(lis):
with open('C:/Users/root/Desktop/py/猫眼.txt','a') as f:
for i in lis:
f.write(json.dumps(i,ensure_ascii=False) + '\n')
# 注意编码false,
f.close()
def main():
url = 'https://maoyan.com/board/4'
html = get_html(url)
list = parse_html(html)
write_down(list)
# __main__python的主函数
if __name__ == '__main__':
main()