import json
from urllib.parse import quote
from selenium.common.exceptions import TimeoutException
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait
from bs4 import BeautifulSoup
browser = webdriver.Chrome()
wait = WebDriverWait(browser,15)
KEYWORD = 'iPad'
def index_page(page):
'''
抓取所搜页
:param page: 页码
:return:
'''
print('正在抓取第',page,'页')
try:
url = 'https://s.taobao.com/search?q=' + quote(KEYWORD)
browser.get(url)
if page > 1:
input = wait.until(EC.presence_of_element_located((By.CSS_SELECTOR,'#mainsrp-pager div.form > input')))
submit = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR,'#mainsrp-pager div.form > span.btn.J_Submit')))
input.clear()
input.send_keys(page)
submit.click()
wait.until(EC.text_to_be_present_in_element((By.CSS_SELECTOR,'#mainsrp-pager li.item.active > span'),str(page)))
wait.until(EC.presence_of_element_located((By.CSS_SELECTOR,'.m-itemlist .items .item')))
get_products()
except TimeoutException:
index_page(page)
def get_products():
'''
提取商品
:return:
'''
html = browser.page_source
soup = BeautifulSoup(html,'lxml')
items = soup.find('div',{'class':'m-itemlist'}).find_all('div',{'data-category':'auctions'})
#print(items)
product_list = []
for item in items:
#print(item)
product = {
'image':item.find('div',{'class':'pic'}).a.img['src'],
'price':item.find('div',{'class':'price g_price g_price-highlight'}).get_text().strip(),
'deal':item.find('div',{'class':'deal-cnt'}).get_text(),
'title':item.find('div',{'class':'row row-2 title'}).get_text().strip(),
'shop':item.find('div',{'class':'shop'}).get_text().strip(),
'location':item.find('div',{'class':'location'}).get_text().strip()
}
#print(product)
product_list.append(product)
with open('product.json','a',encoding='utf-8') as f:
f.write(json.dumps(product_list,indent=2,ensure_ascii=False))
print('ok')
for i in range(1,101):
index_page(i)
部分结果为:
{
"image": "//g-search1.alicdn.com/img/bao/uploaded/i4/imgextra/i2/13022581/O1CN01x8TLzf1Uw8ijNtCZa_!!0-saturn_solar.jpg_230x230.jpg",
"price": "¥2398.00",
"deal": "1734人付款",
"title": "12期免息【送电动牙刷+1年延保】Apple/苹果 iPad 2018款 9.7英寸平板电脑 wifi平板电脑 吃鸡玩家",
"shop": "绿森数码官方旗舰店",
"location": "浙江 杭州"
},
{
"image": "//g-search1.alicdn.com/img/bao/uploaded/i4/i2/1669409267/O1CN012IKKoeqk1Nwld2s_!!0-item_pic.jpg_230x230.jpg",
"price": "¥2398.00",
"deal": "5244人付款",
"title": "[12期分期][两年保修]Apple/苹果 iPad 2018款 9.7英寸wifi新款平板电脑32G/128G 正品国行新品授权旗舰店",
"shop": "卓辰数码旗舰店",
"location": "浙江 杭州"
},
{
"image": "//g-search1.alicdn.com/img/bao/uploaded/i4/i4/268451883/O1CN01fxAXtR1PmSGc2xFVa_!!0-item_pic.jpg_230x230.jpg",
"price": "¥2328.00",
"deal": "1825人付款",
"title": "2018款ipad Apple/苹果 iPad 2018款 9.7英寸32/128Gwifi 平板电脑 3/6/12期分期 正品国行 全国联保",
"shop": "三际数码官方旗舰店",
"location": "山东 济南"
},
{
"image": "//g-search3.alicdn.com/img/bao/uploaded/i4/i2/2616970884/O1CN011IOub5IKnvUp9nh_!!0-item_pic.jpg_230x230.jpg",
"price": "¥2688.00",
"deal": "8681人付款",
"title": "【官网价优惠高达971元】Apple/苹果 iPad mini4 7.9英寸平板电脑 128G WiFi版 迷你智能平板电脑 金属轻薄",
"shop": "苏宁易购官方旗舰店",
"location": "江苏 南京"
},
{
"image": "//g-search3.alicdn.com/img/bao/uploaded/i4/i1/2616970884/O1CN01Ld0NTf1IOub5txwnH_!!0-item_pic.jpg_230x230.jpg",
"price": "¥2565.00",
"deal": "5521人付款",
"title": "【官网价优惠至高511元】2018新款 Apple/苹果 9.7英寸iPad 智能平板电脑 正品国行32G 128G",
"shop": "苏宁易购官方旗舰店",
"location": "江苏 南京"
},
{
"image": "//g-search2.alicdn.com/img/bao/uploaded/i4/i3/1669409267/O1CN019lgc8W2IKKoe8pxMJ_!!0-item_pic.jpg_230x230.jpg",
"price": "¥2158.00",
"deal": "534人付款",
"title": "Apple/苹果 Ipad 2017款 9.7英寸 平板电脑 32G/128G Wifi版",
"shop": "卓辰数码旗舰店",
"location": "浙江 杭州"
},
{
"image": "//g-search3.alicdn.com/img/bao/uploaded/i4/i4/1115488308/O1CN012BF6yiFNCxMurOm_!!0-item_pic.jpg_230x230.jpg",
"price": "¥2328.00",
"deal": "33人付款",
"title": "2328起!送钢化膜/现货速发/12期分期 Apple/苹果 iPad 2018款9.7英寸平板电脑WiFi新款平板电脑",
"shop": "君问数码官方旗舰店",
"location": "浙江 杭州"
},
{
"image": "//g-search2.alicdn.com/img/bao/uploaded/i4/i3/1917047079/O1CN0122AEDnF6rAZNIFG_!!0-item_pic.jpg_230x230.jpg",
"price": "¥6499.00",
"deal": "",
"title": "Apple/苹果 11 英寸 iPad Pro",
"shop": "applestore官方旗舰店",
"location": "上海"
},