首先测试一下网页是否有用
import requests,json
from requests.exceptions import RequestException
from bs4 import BeautifulSoup
def get_one_page(url):
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36'
}
response=requests.get(url,headers=headers)
try:
if response.status_code==200: #如果状态码为200,则返回抓取的网页文本
return response.text
return None
except RequestException:
return print("访问页面错误(●'◡'●)")
def main():
url = 'https://maoyan.com/board/4?offset=0'
html=get_one_page(url)
print(html)
if __name__ == '__main__':
main()
运行成功,返回如下text。这一段包含我们想要爬去的所有的内容。
<dd>
<i class="board-index board-index-9">9</i>
<a href="/films/1212" title="千与千寻" class="image-link" data-act="boarditem-click" data-val="{movieId:1212}">
<img src="//ms0.meituan.net/mywww/image/loading_2.e3d934bf.png" alt="" class="poster-default" />
<img data-src="http://p0.meituan.net/movi