题目:使用正则表达式、BeautifulSoup、Xpath三种方法爬取糗图百科的热门图片。
正则表达式:
import requests
import re
import os
if not os.path.exists("qiutu"):
os.mkdir("qiutu")
url = "https://www.qiushibaike.com/imgrank/"
headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit\
/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36 Edg/\
87.0.664.75'}
page_text = requests.get(url = url, headers = headers).text
s = '<div class="thumb">.*?<img src="(.*?)" alt.*?</div>'
img_src = re.findall(s,page_text,re.S)
for src in img_src:
src = "https:"+src
img = requests.get(url = src, headers = headers).content
img_name = src.split('/')[-1]
img_path = "qiutu/"+img_name
with open(img_path,'wb') as f:
f.write(img)
print(img_name,"下载完成")
BeautifulSoup:
import requests
from bs4 import BeautifulSoup
import os
if not os.path.exists("qiutu"):
os.mkdir("qiutu")
url = "https://www.qiushibaike.com/imgrank/"
headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit\
/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36 Edg/\
87.0.664.75'}
page_text = requests.get(url = url, headers = headers).text
soup = BeautifulSoup(page_text,'lxml')
img_data = soup.select('.thumb > a > img')
srcs = []
for each_data in img_data:
srcs.append(each_data['src'])
for src in srcs:
src = "https:"+src
img = requests.get(url = src, headers = headers).content
img_name = src.split('/')[-1]
img_path = "qiutu/"+img_name
with open(img_path,'wb') as f:
f.write(img)
print(img_name,"下载完成")
Xpath
import requests
from lxml import etree
import os
if not os.path.exists("qiutu"):
os.mkdir("qiutu")
url = "https://www.qiushibaike.com/imgrank/"
headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit\
/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36 Edg/\
87.0.664.75'}
page_text = requests.get(url = url, headers = headers).text
tree = etree.HTML(page_text)
srcs = tree.xpath('/html/body//div[@class="thumb"]/a/img/@src')
for src in srcs:
src = "https:"+src
img = requests.get(url = src, headers = headers).content
img_name = src.split('/')[-1]
img_path = "qiutu/"+img_name
with open(img_path,'wb') as f:
f.write(img)
print(img_name,"下载完成")