ajax_web_parser.py
'''
数据为动态加载(ajax,使用js控制数据)
鼠标滚动(下拉)数据自动显示在页面
通过观察在加载是url的变化,构造出url
url:ttps://knewone.com/discover?page=7
'''
import requests
from bs4 import BeautifulSoup
import time
# 通过url发起请求,获取返回的返回的response信息
def getResponseMsg(url):
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) '
'Chrome/68.0.3440.106 Safari/537.36'}
try:
r = requests.get(url, headers=headers)
r.raise_for_status()
r.encoding = r.apparent_encoding
return r.text
except:
return "请求失败"
def parseHtml(html):
soup = BeautifulSoup(html, 'lxml')
# 提取出所在的a标签
imgs = soup.select('a.cover-inner > img')
titles = soup.select('h4.title > a')
links = soup.select('h4.title > a')
for img,title,link in zip(imgs,titles,links):
data = {
'img':img.get('src'),
'title':title.get('title'),
'link':link.get('href')
}
print(data)
#观察发送请求是的url变化,每刷新出内容/discover?page=7&page=4,page+1
def get_more_page(start, end, url): #设置爬取的页面数
for i in range(start, end):
html = getResponseMsg(url+str(i))
parseHtml(html)
time.sleep(2)
if __name__=='__main__':
url = 'https://knewone.com/discover?page=7&page='
#爬取10页数据
get_more_page(1, 10, url)