爬虫入门
- requests+bs4
- selenium+geckodriver
- scrapy框架
Mac安装geckodriver
brew install geckodriver
设置配置文件~/.bash_profile文件
export path
import requests
from bs4 import BeautifulSoup
url = "https://b2c.csair.com/B2C40/newTrips/static/main/page/booking/index.html?t=S&c1=BJS&c2=SHA&d1=2021-04-20&at=1&ct=0&it=0&b1=PEK-PKX&b2=SHA-PVG"
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36'}
r = requests.get(url, headers=headers)
r.encoding = 'UTF-8'
soup = BeautifulSoup(r.text, "html.parser")
title = soup.find("title").text
print(title)
from selenium import webdriver
driver = webdriver.Firefox(executable_path=r'geckodriver')
url = "https://b2c.csair.com/B2C40/newTrips/static/main/page/booking/index.html?t=S&c1=BJS&c2=SHA&d1=2021-04-20&at=1&ct=0&it=0&b1=PEK-PKX&b2=SHA-PVG"
driver.get(url)
print(driver.page_source)
comment = driver.find_element_by_css_selector('div.zls-flplace')
content = comment.find_element_by_class_name('zls-flplace')
print(content.text)