编译环境:pycharm 4.5.3
python版本:3.5.1
首先从pycharm库中下载并安装BeautifulSoup4,lxml,requests,time插件
以KnewOne为例:
from bs4 import BeautifulSoup import requests import time url='https://knewone.com/things' web_data=requests.get(url) #利用requests访问网页 soup=BeautifulSoup(web_data.text,'lxml') #解析网页 titles=soup.select(' section.content > h4 > a ') #网页元素定位 imgs=soup.select('a.cover-inner > img') for title,img in zip(titles,imgs): #少部分数据可以先用字典封装,大数据再考虑用数据库 data={ 'title':title.get_text(), 'img':img.get('src'), } print(data)
此外,发现KnewOne的“产品”页面为一个异步加载网页,那么,如何爬取一页所有的信息呢?
def get_info(url,data=None): web_data=requests.get(url) soup=BeautifulSoup(web_data.text,'lxml') titles=soup.select(' section.content > h4 > a ') imgs=soup.select('a.cover-inner > img') if(data==None): for title,img in zip(titles,imgs): data={ 'title':title.get_text(), 'img':img.get('src'), } print(data) def get_mor_pages(start,end): for one in range(start,end): get_info(url+str(one))