利用request&xpath爬取网站图片以及图片信息

coding=utf-8

import urllib.request
import re
import os
import requests
from lxml import html

‘’’
Urllib 模块提供了读取web页面数据的接口，我们可以像读取本地文件一样读取www和ftp上的数据
urlopen 方法用来打开一个url
read方法 用于读取Url上的数据
‘’’

def getHtml(url):
page = urllib.request.urlopen(url);
html = page.read();
return html;

def getImg(html):
#findall(pattern, string, flags=0) 返回string中所有与pattern相匹配的全部字串，返回形式为数组
#imglist = re.findall(‘img src="(http.?)"’, html)
imglist = re.findall('img src="(/uploads.?)"’, html)
#result = html.xpath(’//div[@class=“catalog04 catalog04a”]/dl/dt/a/img/@src’) # 需要获取的数据

reg =r’url"(/templets/.*?)"’

imgre = re.compile(reg);

#imglist = re.findall(imgre, html)
return imglist

def getWenzi(url):
page = requests.Session().get(url)
tree = html.fromstring(page.text)
result = tree.xpath(’//div[@class=“catalog04 catalog04a”]//dd/ol/text()’) # 需要获取的职称数据
result1 = tree.xpath(’//div[@class=“catalog04 catalog04a”]//ul//a/text()’) # 需要获取的姓名数据
# result2 = tree.xpath(’//tr[last()]//a/@href’)
name = result1
zhicheng = result
print(name)
print(len(name))
print(zhicheng)
for i in range(0, len(name)):
fo = open(“wenzi.txt”, “a”)
fo.write(‘姓名：’ + str(name[i]) + ’ ’ + ‘职称：’ + str(zhicheng[i]) + ’ ’ + ‘照片名称：’ + str(i))
fo.write(’\n’)
fo.write(’\n’)
fo.close()

url=“http://www.dongfangyy.com.cn/departments/article/44.html”
getWenzi(url)
html = getHtml(url).decode(“utf-8”);
imagesUrl = getImg(html);

if os.path.exists(“E:/imags”) == False:
os.mkdir(“E:/imags”);

if os.path.exists(“E:/wenzi”) == False:
os.mkdir(“E:/wenzi”);

count=0;
for url in imagesUrl :
url=‘http://www.dongfangyy.com.cn’+url
print(url)
if (url.find(’.’) != -1):
name = url[url.find(’.’, len(url) - 5):];
bytes = urllib.request.urlopen(url);
f = open(“E:/imags/” + str(count) + name, ‘wb’);
f.write(bytes.read());
f.flush();
f.close();
count += 1;