学习使用xpath代替正则表达式,首先得了解xpath语法
xpath语法详解
1.使用xpath遇到的几个问题,一个是不会提取a标签的内容,后来查资料发现需要在后面加/text()
2.提取倒数标签数据使用[last()-xx]的方式
今天还顺便了解了requests模块,发现这个模块蛮好用的
注释写中文出错后再开头加上# -- coding:gbk --即可
附上今天使用xpath改造的项目源码:
test2.py
import sys
import requests
import re
def download(url):
print("downloading:"+url)
headers={'User-agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'}
try:
content=requests.get(url,headers=headers)
except urllib.error.URLError as e:
print("download error:"+e.reason)
content=None
return content
def save(file_name, file_content):
print("saving.......")
with open(file_name + ".html", "wb") as f:
f.write(file_content.content)
f.close()
murl="http://blog.csdn.net/Joliph/article/list/1"
html = download(murl)
save(re.split('/',murl)[-1], html)
budejie2.py
# -*- coding:gbk -*-
import sys
import requests
import re
from lxml import etree
from multiprocessing.dummy import Pool as threadpool
import urllib.request
import time
def wp():
print("plz input 1~50")
page=input("which budejie page you want to download?(1~50):")
page=int(page)
while page>50:
print("plz input 1~50")
page=input("which budejie page you want to download?(1~50):")
page=int(page)
return page
pg=wp()
headers={'User-agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'}
url="http://www.budejie.com/"+str(pg)
html=requests.get(url,headers=headers).text
selector=etree.HTML(html)
mp4url=[]
start_time=time.time()
mp4list=selector.xpath('//*[@class="j-r-list-tool-l-down f-tar j-down-video j-down-hide ipad-hide"]/a')
for i in mp4list:
mp4url.append(str(i.xpath('@href')[0]))
end_time=time.time()
print("匹配耗时:"+str(end_time-start_time)+"Seconds")
for i in mp4url:
print("downloading:"+i)
filename=re.split('/',i)[-1]
urllib.request.urlretrieve(i,filename)
csdn2.py:
# -*- coding:gbk -*-
import sys
import requests
import re
from lxml import etree
def download(url):
headers = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36"}
html=requests.get(url,headers=headers).text
return html
html2=download("http://blog.csdn.net/Joliph")
selector2=etree.HTML(html2)
pagelist=selector2.xpath('//*[@id="papelist"]/a[last()-2]/text()')[0]
#这里有有个潜在的问题,在我博客写到5页以上时出现...后无法判断页数
pagelist=int(pagelist)
for page in range(1,pagelist+1):
url="http://blog.csdn.net/Joliph/article/list/"+str(page)
html=download(url)
selector=etree.HTML(html)
titlelist=selector.xpath('//*[@class="list_c_t"]/a/text()')
yearlist=selector.xpath('//*[@class="date_t"]/span/text()')
monthlist=selector.xpath('//*[@class="date_t"]/em/text()')
daylist=selector.xpath('//*[@class="date_b"]/text()')
#/text()!!!!!!!!!!!!!!!!!!!!!!!
number=len(titlelist)
for i in range(1,number+1):
print(yearlist[i-1]+"."+monthlist[i-1]+"."+daylist[i-1]+"----"+titlelist[i-1])