京东商品信息爬取
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
from bs4 import BeautifulSoup
class Spider():
def __init__(self):
self.url = 'https://www.jd.com/'
self.opt = webdriver.ChromeOptions()
self.opt.add_experimental_option('excludeSwitches', ['enable-automation'])
self.browser = webdriver.Chrome(options=self.opt)
def get_jd(self):
self.browser.get(self.url)
wait = WebDriverWait(self.browser,280)
wait.until(EC.presence_of_element_located((By.ID,'key')))
text = self.browser.find_element(By.ID,'key')
text.send_keys('美食')
text.send_keys(Keys.ENTER)
time.sleep(20)
def get_data(self):
data = self.browser.page_source
print(self.browser.current_url)
self.parse_data(data=data)
def parse_data(self,data):
soup = BeautifulSoup(data,'lxml')
title = soup.select('.gl-i-wrap a em')
price = soup.select('.gl-i-wrap .p-price strong i')
dianming = soup.select('.gl-i-wrap .J_im_icon a')
for titles,prices,dianmings in zip(title,price,dianming):
n = titles.get_text()
p = prices.get_text()
d = dianmings.get_text()
print(n)
print(p)
print(d)
print('==============')
if __name__ == '__main__':
s = Spider()
s.get_jd()
s.get_data()