代码如下
from selenium import webdriver
from bs4 import BeautifulSoup
import time
import pymongo
client = pymongo.MongoClient('localhost',27017)
ceshi = client['ceshi']
lagou_python = ceshi['lagou_web']
url = 'https://www.lagou.com/'
def lagou(city,job):
driver = webdriver.Chrome()
driver.get(url)
time.sleep(1)
driver.find_element_by_id("cboxClose").click()
time.sleep(1)
driver.find_element_by_id("search_input").send_keys(city)
driver.find_element_by_id("search_button").click()
driver.find_element_by_id("keyword").send_keys(job)
driver.find_element_by_id("submit").click()
time.sleep(1)
t = 1
while True:
print('抓取第{}页数据'.format(str(t)))
html = driver.page_source
soup = BeautifulSoup(html, 'lxml')
work_list = soup.select('li.con_li