储存为json
关键步骤:
1、requests.get()get请求网页
2、BeautifulSoup()提取数据
3、find_all(class_=。。。)找到需要的数据
4、json.dump()编码存入j.txt
5、json.load(fp)读取文本内容
import requests
from bs4 import BeautifulSoup
import json
# 获取html内容
u = "Mozilla/4.0(compatable; MSTE 5.5; Windoes NT)"
h = {'User-Agent':u}
r = requests.get('http://seputu.com/',headers = h)
s = BeautifulSoup(r.text,'html.parser',from_encoding='utf-8')
txt = []
for m in s.find_all(class_='mulu'):
c = m.find('center')
h2 = m.find('h2') #标题
if h2 != None:
h2 = h2.string
l = []
for a in m.find(class_='box').find_all('a'):
hr = a.get('href')
b = a.get('title')
l.append({b:hr})
txt.append(h2)
txt.append(l)
#编码
print(txt)
with open("j.txt","w") as fp:
json.dump(txt,fp=fp,ensure_ascii=False)
#解码
with open('j.txt','r') as fp:
print(json.load(fp))
储存为csv
关键步骤:
1、requests.get()get请求网页
2、BeautifulSoup()提取数据
3、find_all(class_=。。。)找到需要的数据
4、f=csv.writer(fp)
f.writerow(csv_h)
f.writerows(csv_r)存入数据
5、f=csv.reader(fp)
fh=next(f)
Row = namedtuple(‘Row’,fh) #使用命名元组
import requests
from bs4 import BeautifulSoup
import csv
from collections import namedtuple
# 获取html内容
u = "Mozilla/4.0(compatable; MSTE 5.5; Windoes NT)"
h = {'User-Agent':u}
r = requests.get('http://seputu.com/',headers = h)
s = BeautifulSoup(r.text,'html.parser',from_encoding='utf-8')
csv_h = ['UserName','Id1','Id2']
csv_r = []
for m in s.find_all(class_='mulu'):
c = m.find('center')
h2 = m.find('h2') #标题
if h2 != None:
h2 = h2.string
for a in m.find(class_='box').find_all('a'):
hr = a.get('href')
b = a.get('title')
csv_r.append([h2,b,hr])
# print(csv_r)
with open("c.csv","w") as fp:
f=csv.writer(fp)
f.writerow(csv_h)
f.writerows(csv_r)
with open('c.csv','r') as fp:
f=csv.reader(fp)
fh=next(f)
Row = namedtuple('Row',fh) #使用命名元组
for r in f:
if r != []:
# print(r)
row =Row(*r)
print(row.UserName)
lxml解析,xpath、re
关键步骤:
1、requests.get()
2、etree.HTML()下载网页
3、html.xpath()找到需要匹配的子节点
4、re.compile()正则匹配数据里需要的对象
from lxml import etree
import requests
import re
r = requests.get('http://seputu.com/',headers = {'User-Agent':"Mozilla/4.0(compatable; MSTE 5.5; Windoes NT)"})
html = etree.HTML(r.text)
d = html.xpath('.//*[@class="mulu"]') #找到所有class="mulu"的标记
for dm in d:
h2 = dm.xpath('./div[@class="mulu-title"]/center/h2/text()') #找到div h2
if len(h2)>0:
h2_title = h2[0]
a_s = dm.xpath('./div[@class="box"]/ul/li/a')
for a in a_s:
href = a.xpath('./@href')[0]
title = a.xpath('./@title')[0].encode('utf-8')
title = title.decode()
tre = re.compile(r'\s*\[(.*)\]\s+(.*)')
title2= tre.search(title).group(2)
print(title2,href)
多媒体文件提取,获取下载进度
关键步骤:
1、requests.get()
2、etree.HTML()下载网页
3、html.xpath()找到需要匹配的子节点
4、urllib.request.urlretrieve()存入并获取当前下载进度
import urllib.request
from lxml import etree
import requests
import re
import os
def Sc(bnum,bsize,tsize):
'''
:param bnum:当前传输的块数
:param bsize:数据块大小
:param tsize:远程文件大小
:return:
'''
per = 100.0 *bnum*bsize/tsize
if per>100:
per = 100
print('当前下载进度:%d' % per)
u = "Mozilla/4.0(compatable; MSTE 5.5; Windoes NT)"
h = {'User-Agent':u}
r = requests.get('https://www.rouding.com/roudingtuku/shishangzhaopian/',headers = h)
html = etree.HTML(r.text)
d = html.xpath('.//img/@src') #选取所有img节点的src属性,也就是图片网址
i=0
for img in d:
img = 'https:'+img
file_path = '.\img00002'
if not os.path.exists(file_path):
os.mkdir(file_path)
urllib.request.urlretrieve(img,'.\img00002\img'+str(i)+'.jpg',Sc)
i += 10
通过stmp发送邮件
from email.header import Header
from email.mime.text import MIMEText
from email.utils import parseaddr,formataddr
import smtplib
def _format_addr(s):
name,addr = parseaddr(s)
return formataddr((Header(name,'utf-8').encode(),addr))
# #发件人地址
from_addr = '*****@163.com'
# #邮箱密码
password = '********'
# #收件人地址
to_addr = '*****@qq.com'
# #163网易邮箱服务器地址
smtp_server = 'smtp.163.com'
#设置邮箱信息
# msg = MIMEText(html,'html','utf-8')#发送网页
msg = MIMEText('python运行异常,异常信息遇到http403','plain','utf-8')
msg['From'] = _format_addr('一号爬虫<%s>'%from_addr)
msg['To'] = _format_addr('管理员<%s>'%to_addr)
msg['Subject'] = Header('一号爬虫运行状态','utf-8').encode()
#发送邮件
server = smtplib.SMTP('smtp.163.com',25)
server.login(from_addr,password)
server.sendmail(from_addr,[to_addr],msg.as_string())
server.quit()