coding=utf-8
import urllib.request
import re
import os
import requests
from lxml import html
‘’’
Urllib 模块提供了读取web页面数据的接口,我们可以像读取本地文件一样读取www和ftp上的数据
urlopen 方法用来打开一个url
read方法 用于读取Url上的数据
‘’’
def getHtml(url):
page = urllib.request.urlopen(url);
html = page.read();
return html;
def getImg(html):
#findall(pattern, string, flags=0) 返回string中所有与pattern相匹配的全部字串,返回形式为数组
#imglist = re.findall(‘img src="(http.?)"’, html)
imglist = re.findall('img src="(/uploads.?)"’, html)
#result = html.xpath(’//div[@class=“catalog04 catalog04a”]/dl/dt/a/img/@src’) # 需要获取的数据
reg =r’url"(/templets/.*?)"’
imgre = re.compile(reg);
#imglist = re.findall(imgre, html)
return imglist
def getWenzi(url):
page = requests.Session().get(url)
tree = html.fromstring(page.text)
result = tree.xpath(’//div[@class=“catalog04 catalog04a”]//dd/ol/text()’) # 需要获取的职称数据
result1 = tree.xpath(’//div[@class=“catalog04 catalog04a”]//ul//a/text()’) # 需要获取的姓名数据
# result2 = tree.xpath(’//tr[last()]//a/@href’)
name = result1
zhicheng = result
print(name)
print(len(name))
print(zhicheng)
for i in range(0, len(name)):
fo = open(“wenzi.txt”, “a”)
fo.write(‘姓名:’ + str(name[i]) + ’ ’ + ‘职称:’ + str(zhicheng[i]) + ’ ’ + ‘照片名称:’ + str(i))
fo.write(’\n’)
fo.write(’\n’)
fo.close()
url=“http://www.dongfangyy.com.cn/departments/article/44.html”
getWenzi(url)
html = getHtml(url).decode(“utf-8”);
imagesUrl = getImg(html);
if os.path.exists(“E:/imags”) == False:
os.mkdir(“E:/imags”);
if os.path.exists(“E:/wenzi”) == False:
os.mkdir(“E:/wenzi”);
count=0;
for url in imagesUrl :
url=‘http://www.dongfangyy.com.cn’+url
print(url)
if (url.find(’.’) != -1):
name = url[url.find(’.’, len(url) - 5):];
bytes = urllib.request.urlopen(url);
f = open(“E:/imags/” + str(count) + name, ‘wb’);
f.write(bytes.read());
f.flush();
f.close();
count += 1;