通过python实现斗图啦网站上的表情的获取和下载
#encoding=utf-8
#导入模块
import random
import requests
import re
from bs4 import BeautifulSoup
import bs4
import os
#创建请求头列表,帮助我们在进行数据爬取的时候伪装成浏览器
my_headers = [
"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36",
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:30.0) Gecko/20100101 Firefox/30.0",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/537.75.14",
"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Win64; x64; Trident/6.0)",
]
kv = {"User-Agent": "Mozilla/5.0"}
def getHTMLText(url, headers):
try:
#随机从headers列表中选择一个header使用
random_header = random.choice(headers)
r = requests.get(url, headers={"User-Agent": random_header}, timeout = 30)
#校验是否爬取成功,如果获取失败,输出“爬取失败”
r.raise_for_status()
r.encoding = r.apparent_encoding
# print r.text
return r.text
except:
print "爬取失败"
def getImgList(Ilist, html):
#使用python自带的html解析器,html.parser进行返回的html数据的解析工作
soup = BeautifulSoup(html, "html.parser")
# print html
#分析解析后的html代码,通过正则表达式获取每一个图片对应的url地址,然后组成获取url的正则表达式
pattern_img = re.compile(r'data-original="(.+?)"')
#获取图片对应的标题
pattern_title = re.compile(r'alt="(.+?)"')
#找到所有的图片url值
imgList = re.findall(pattern_img, html)
# print imgList
#获取所有的图片对应的标题信息
titleList = re.findall(pattern_title, html)
# print titleList[0].encode('utf-8')
#将每一对urli地址和title组成一个列表项,放入到另外一个列表项中可以通过下表进行调用
for i in range(len(imgList)):
# print i,
titleList[i] = titleList[i].encode('utf-8')
# print titleList[i]
Ilist.append([imgList[i], titleList[i]])
return Ilist
#判断是否存在指定的文件夹,然后创建文件夹
def mkdir(i):
if not os.path.exists('img-0%d' %i):
os.mkdir('img-0%d' %i)
def printImg(Ilist,page):
# print len(Ilist)
for i in range(len(Ilist)):
# print Ilist[i][0], Ilist[i][1].encode('utf-8')
# print Ilist[i][0][-4:] == '.jpg'
# print type(Ilist[i][1].decode())
# print type(Ilist[0][1].decode('utf-8'))
img_content = requests.get(Ilist[i][0]).content
if (Ilist[i][0][-4:] == '.jpg'):
with open('img-0%d/%s.jpg' % ( page, Ilist[i][1].decode('utf-8')), 'wb') as f:
f.write(img_content)
f.close()
elif (Ilist[i][0][-4:] == '.gif'):
with open('img-0%d/%s.gif' % ( page, Ilist[i][1].decode('utf-8')), 'wb') as f:
f.write(img_content)
f.close()
def main(page):
Ilist = []
url = "https://www.doutula.com/photo/list/?page=%d" %page
html = getHTMLText(url, my_headers)
Ilist = getImgList(Ilist, html)
mkdir(page)
printImg(Ilist, page)
# page = 1
if __name__ == "__main__":
print "开始获取表情包图片..."
for page in range(16):
print "开始获取第%d页中的内容..." % page
main(page)
print "第%d页的内容获取完毕..."%page
page += 1
print "恭喜你,你想要的内容获取完毕!!"
# i = 1
# for i in range(12):
# print i,