一、爬取网页数据(urllib和requests)
》 urllib
import urllib.request
#调用库中的urlopen()方法,并传入一个url
response=urllib.request.urlopen('http://www.baidu.com')
#使用read()方法读取获取到的网页内容
html=response.read().decode('UTF-8')
print(html)
》requests
下载(如果下载速度慢的话可以使用国内源)
#进入python文件进行下载
python.exe -m pip install requests
二、数据解析(Xpath、lxml、beautifulsoup、jsonpath)
》Xpath
#抓取网页源代码
import requests #使用XPath爬虫
ua = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0'
html = requests.get('http://gra.hnu.edu.cn/index/zsxx.htm',headers={'User-Agent':ua})
html.encoding = 'utf-8'
#使用XPath筛选数据
#print(html.text)
xpath = "//a[@target='_blank']"
from lxml import etree
page = etree.HTML(html.text)
result = page.xpath(xpath)
#输出所有符合XPath的结果
print(result)
for item in result:
print(item.text)
#对感兴趣的数据进行提纯(2024年湖南大学研究生院招生简章)
print()
print("符合条件的结果有")
for item in result:
if "2024" in item.text and '招生简章' in item.text:
print(item.text)
》lxml
下载
pip install lxml
使用
》beautifulsoup
下载
pip install beautifulsoup4
使用
import requests
from bs4 import BeautifulSoup #使用BeautifulSoup爬虫
ua = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0'
#构造headers
headers = {
'User-Agent':ua
}
#发起请求
html = requests.get('http://gra.hnu.edu.cn/index/zsxx.htm')
html.encoding = 'utf-8'
#print(html.text)
#解析数据
soup = BeautifulSoup(html.text,'lxml')
result = soup.find_all('a',target='_blank')
for item in result:
if "2024" in item.text and '招生简章' in item.text:
print(item.text)
三、爬取动态内容(selenuim、phantomJS)
》selenuim
操作鼠标键盘
from selenium import webdriver
#设置不关闭浏览器
options = webdriver.EdgeOptions()
options.add_experimental_option('detach',True)
driver = webdriver.ChromiumEdge()#新版edge浏览器
driver.get('https://www.baidu.com')
import time
time.sleep(3)
from selenium.webdriver.common.by import By
driver.find_element(By.ID,'kw').send_keys('湘潭理工学院')
import time
time.sleep(3)
driver.find_element(By.ID,'su').click()
#或者使用键盘点击
#from selenium.webdriver.common.keys import Keys
#driver.find_element(By.ID,'kw').send_keys(Keys.ENTER)
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.action_chains import ActionChains
#设置不关闭浏览器
options = webdriver.EdgeOptions()
options.add_experimental_option('detach',True)
driver = webdriver.ChromiumEdge(options=options)#新版edge浏览器
driver.get('https://yjsy.hunnu.edu.cn/')
import time
time.sleep(3)
el = driver.find_element(By.XPATH,"//ul[@class='menu']/li[4]/a")
ActionChains(driver).move_to_element(el).perform() # 鼠标悬停
time.sleep(3)
ek = driver.find_element(By.XPATH,"//ul[@class='menu']/li[4]/ul/li[2]/a")
ActionChains(driver).move_to_element(ek).perform() # 鼠标悬停
time.sleep(3)
ActionChains(driver).move_to_element(ek).click().perform()
time.sleep(3)
动态抓取
from selenium import webdriver
from selenium.webdriver.common.by import By
options = webdriver.EdgeOptions()
options .add_experimental_option('detach',True)
driver = webdriver.ChromiumEdge(options=options)
#加载网页获取源代码
url = 'https://www.bilibili.com/v/popular/all/'
driver.get(url)
#导入BeautifulSoup,筛选数据
from bs4 import BeautifulSoup
soup = BeautifulSoup(driver.page_source,'lxml')
result = soup.find_all('div',class_='video-card')
for item in result:
title = item.find('p',class_='video-name')
up = item.find('span',class_='up-name__text')
count = item.find('span',class_='play-text')
print(f'视频:{title.text},up:{up.text},播放量:{count.text.strip()}')
动态抓取(升级版)
from selenium import webdriver
from selenium.webdriver.common.by import By
options = webdriver.EdgeOptions()
options.add_experimental_option('detach',True)
driver = webdriver.ChromiumEdge(options=options)
url='https://www.bilibili.com/video/BV1SC4y1F7Vj/'
driver.get(url)
import time
time.sleep(5)
from bs4 import BeautifulSoup
soup = BeautifulSoup(driver.page_source,'lxml')
result = soup.find_all('div',class_='app-v1')
for item in result:
title = item.find('h1',class_='video-title')
count = item.find('span',class_='view item')
abd = item.find('span',class_='dm item')
datetime = item.find('span',class_='pubdate-text')
contents = soup.find_all('div',class_='content-warp')
contents_text = []
for content in contents:
ad = content.find('div',class_='user-info').text
ac = content.find('span',class_='reply-content').text
contents_text.append({
'name':ad,
'text':ac
})
print(f'视频:{title.text},播放量:{count.text.strip()},弹幕数:{abd.text.strip()},时间:{datetime.text}')
for content in contents_text:
print(f"评论:\nID:{content['name']},内容:{content['text']}")
driver.close()
正则表达式
import re
#待匹配字符串
title="你好, hello, 世界"
#创建正则表达式,用于只匹配中文
pattern=re.complie(r"[\u4e00-\u9fa5]+")
#检索整个字符串,将匹配的中文放到列表中
result=pattern.findall(title)
print(result)
天气查询
# -*- coding: utf-8 -*-
from __future__ import print_function
import ssl, hmac, base64, hashlib
from datetime import datetime as pydatetime
try:
from urllib import urlencode
from urllib.request import Request, urlopen
except ImportError:
from urllib.parse import urlencode
from urllib.request import Request, urlopen
# 云市场分配的密钥Id
secretId = "AKID1c6k2R0Duhg3eCnZP3ljQ8asmIe25sYnfUot"
# 云市场分配的密钥Key
secretKey = "Grb2a7e3UCjuHqLk03ub9nLAalmmc2bStAGI0dq3"
source = "market"
# 签名
datetime = pydatetime.utcnow().strftime('%a, %d %b %Y %H:%M:%S GMT')
signStr = "x-date: %s\nx-source: %s" % (datetime, source)
sign = base64.b64encode(hmac.new(secretKey.encode('utf-8'), signStr.encode('utf-8'), hashlib.sha1).digest())
auth = 'hmac id="%s", algorithm="hmac-sha1", headers="x-date x-source", signature="%s"' % (secretId, sign.decode('utf-8'))
# 请求方法
method = 'GET'
# 请求头
headers = {
'X-Source': source,
'X-Date': datetime,
'Authorization': auth,
}
# 查询参数
queryParams = {
"areaCn": "湘潭",
}
# body参数(POST方法下存在)
bodyParams = {
}
# url参数拼接
url = 'https://service-6drgk6su-1258850945.gz.apigw.tencentcs.com/release/lundear/weather1d'
if len(queryParams.keys()) > 0:
url = url + '?' + urlencode(queryParams)
request = Request(url, headers=headers)
request.get_method = lambda: method
if method in ('POST', 'PUT', 'PATCH'):
request.data = urlencode(bodyParams).encode('utf-8')
request.add_header('Content-Type', 'application/x-www-form-urlencoded')
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
response = urlopen(request, context=ctx)
content = response.read()
if content:
#print(content.decode('utf-8'))
import json
weather =json.loads(content.decode('utf-8'))
print(f"湘潭的天气:{weather['data']['now']['weather']},温度:{weather['data']['now']['temp']}")