最近几天在学习python,写了个简单的爬虫程序:
成功运行代码的前提是安装了Python需要的第三方库,以下是代码:
# -*- coding:utf8 -*-
import os
import re
import requests as rq
import urllib
from bs4 import BeautifulSoup
#下载图片
def getHtml2():
html = rq.get(urls).text
soup = BeautifulSoup(html,"html.parser")#文档对象
for k in soup.find_all('a',class_="tit"):#获取class为tit的a标签
print(k)
addr = re.compile('<a class="tit" href="(.*?)"')
url = re.findall(addr,repr(k))[0]
#创建路径文件夹名称
cate_name = re.findall('<a .*?>(.*?)</a>',repr(k),re.S|re.M)[0]
path = 'C:/data/images\\%s' % cate_name
# 创建文件目录
os.mkdir(path)
# 切换到该目录
os.chdir(path)
#print(url)
for x in range(1,20):
url1 = url.split(".html")[0]+"_"+str(x)+".html"
html1 = rq.get(url1).text
soup1 = BeautifulSoup(html1,"html.parser")#文档对象
image = soup1.find_all('img')[0]
print(image)
res_url = r"(?<=src=\").+?(?=\")|(?<=src=\').+?(?=\')"
img_url = re.findall(res_url ,repr(image), re.I|re.S|re.M)[0]
urllib.request.urlretrieve(img_url, '%s.jpg' % x) #prython3.6写法
def main():
print("开始解析网页===>")
global urls
urls = "http://www.58gc.cn/brjr/"
getHtml2()
print("网页解析结束===>")
main()