实例1 白杨网爬取图片:
// An highlighted block
import requests
import os
url = "http://www.cuc.edu.cn/_upload/article/images/92/67/6e6b9b08471eb4ecfeca0e7091cf/c01c28b9-d48a-43a9-9727-899ba4b6067b.jpg"
root = "D://pics//"#根目录
path = root + url.split('/')[-1]#网络文件名称最后一部分截取
try:
if not os.path.exists(root):
os.mkdir(root)#根目录在与否
if not os.path.exists(path):
r = requests.get(url)#文件在与否
with open(path,'wb') as f:
f.write(r.content)#二进制
f.close()
print("文件保存成功")
else:
print("文件已存在")
except:
print("error")
实例2 demo网页爬取文字:
// An highlighted block
import requests
from bs4 import BeautifulSoup
keyword = "python"
try:
kv = {'wd':keyword}
demo = requests.get("https://python123.io/ws/demo.html",params=kv)
print(demo.request.url)#Response对象属性 request
demo.raise_for_status()#??
#print(demo.text)
except:
print("error!")
soup= BeautifulSoup(demo.content,"html.parser")#soup表示解析后的demo页面。
# print(soup.head)
# print(soup.head.contents)
# print(soup.body.contents)#tag类型的和字符串类型的结点都是叶子节点
# print(len(soup.body.contents))
# print(soup.body.contents[1])
# print(soup.title.parent)
# print(soup.html.parent)
#soup = BeautifulSoup("<p>中文</p>,html.parser")
#print(soup.p.string)
print(soup.find_all(string = "Basic Python"))
import re
print(soup.find_all(string = re.compile("python")))
实例3 bs4及requests合用:
// An highlighted block
######为啥不输出?????????????????????
import requests
from bs4 import BeautifulSoup
import bs4
#第一步:搭框架:用的函数,主函数
#第二步:写函数
#
def gethtmltext(url):#获取url信息
try:
r = requests.get(url,timeout =30 )
r.raise_for_status()
r.encoding = r.apparent_encoding
return r.text
except:
return ""
def filluniverlist(ulist,html):#把html的内容放到ulist里
soup = BeautifulSoup(html,"html.parser")
for tr in soup.find('tbody').children:
if isinstance(tr,bs4.element.Tag):#对类型做判断,如果不是bs4库的tag类型,则过滤掉。
tds = tr('td')# tr已经都是tag类,将所有的td标签存进一个列表里
ulist.append([tds[0].string,tds[1].string,tds[2].string])#在列表末尾添加新的对象
#大学排名 名称 分数
def printlist(ulist,num):#要打印的参数num
tplt = "{0:^10}\t{1:{3}^10}\t{2:^10}"
print(tplt.format("排名","学校名称","总分",chr(12288)))
for i in range(num):#各学校的信息循环打印
u = ulist[i]
print(tplt.format(u[0],u[1],u[2],chr(1228)))
def main():
uninfo = []#大学信息方法到列表中
url = "https://www.shanghairanking.cn/rankings/bcur/2020"
html = gethtmltext(url)
filluniverlist(uninfo,html)
printlist(uninfo,20)
main()
实例4 爬取《斗破苍穹》小说:
// An highlighted block
import requests
import re
import time
#伪装成浏览器
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36'}
#
f = open('D:/pics/doupo.txt','+a')
#
def get_info(url):
res = requests.get(url,headers=headers)#获取网页html
if res.status_code==200:
contents = re.findall('<p>(.*?)</p>',res.content.decode('utf-8'),re.S)
for content in contents:
f.write(content+'\n')#正则获取数据写入txt文件中
else:
pass
if __name__=='__main__':
urls = ['http://www.doupoxs.com/doupocangqiong/{}.html'.format(str(i))for i in range(2,1665)]#构造多页的url
for url in urls:
get_info(url)
time.sleep(1)
f.close()
实例5 淘宝爬取商品信息:
// An highlighted block
import requests
import re
def gethtmltext(url):
try:
r = requests.get(url,timeout = 30)
r.raise_for_status()
r.encoding = r.apparent_encoding
return r.text
except:
return ""
def parsepage(ilt,html):#解析网页,ilt结果列表
try:
plt = re.findall(r'\"view_price\"\:\"[\d\.]*\"',html)#\是转意字符,价格
tlt = re.findall(r'\"title\"\:\".*?\"',html)#名称
for i in range(len(plt)):
price = eval(plt[i].split(':')[1])#切掉不需要的字符
title = eval(tlt[i].split(':')[1])#以:为断点,断句 eval是干啥的函数
ilt.append([price,title])#把提取出来的价格,名称往列表里放.0位置是商品价格,1位置是商品的名称
except:
print("")
def printlist(ilt):
tplt = "{:4}\t{:8}\t{:16}"#\t 缩进4个字符相当于Tab键,:之前是输出的起始位置.python是从0开始输出的:之后是位宽
print(tplt.format("序号","价格","商品名称"))#表头
count = 0#序号
for g in ilt:
count = count+1
print(tplt.format(count,g[0],g[1]))
print("")
def main():#主函数
goods = '书包'
depth = 3# 当前页与第二页
start_url = 'https://www.taobao.com/search?q='+goods
infolist = []#输出结果
for i in range(depth):
try:
url = start_url+'&s='+str(44*i)
html = gethtmltext(url)#获取
parsepage(infolist,html)#解析
except:
continue
printlist(infolist)
main()# 调用整个程序