爬取腾讯漫画的一个例子
# encoding: utf-8
from __future__ import print_function
from __future__ import unicode_literals
import requests
import base64
import re
import json
import os
requestSession = requests.session()
def getId(url):
numRE = re.compile(r'\d+')#r的意思是不转义,即\表示原样的\。否则有可能被视图按\d为一个字符解析转义。
id = numRE.findall(url)[0]
return id
def getContent(id):
getComicInfoUrl = 'http://m.ac.qq.com/GetData/getComicInfo?id={}'.format(id)
getComicInfo = requestSession.get(getComicInfoUrl)
comicInfoJson = getComicInfo.text
comicInfo = json.loads(comicInfoJson)
comicName = comicInfo['title']#解析出漫画名
getChapterListUrl = 'http://m.ac.qq.com/GetData/getChapterList?id={}'.format(id)
getChapterList = requestSession.get(getChapterListUrl)
contentJson = json.loads(getChapterList.text)
count = contentJson['length']#获得章节数
sortedContentList = []
#按顺序整理章节
for i in range(count + 1):
for item in contentJson:
if isinstance(contentJson[item], dict) and contentJson[item]['seq'] == i:
sortedContentList.append({item: contentJson[item]})
break
return (comicName, count, sortedContentList)
def getImgList(contentJson, id):
#cid = contentJson.keys()[0] #原作者使用的是Python2
#在python2.x中,dict.keys()返回一个列表,在python3.x中,dict.keys()返回一个dict_keys对象,比起列表,这个对象的行为更像是set,所以不支持索引的。解决方案:list(dict.keys())[index]
cid = list(contentJson.keys())[0]
#getPicHashURL = 'http://m.ac.qq.com/View/mGetPicHash?id={}&cid={}'.format(id, cid) #原作者请求失效
#getPicHashURL='http://m.ac.qq.com/chapter/index/id/{}/cid/{}'.format(id,cid) #异步加载,修改后仍是错误的
picJsonPage = requestSession.get(getPicHashURL).text
picJson = json.loads(picJsonPage)
count = picJson['pCount'] #统计图片数量
pHash = picJson['pHash']
sortedImgDictList = []
for i in range(1, count + 1):
for item in pHash:
if pHash[item]['seq'] == i:
sortedImgDictList.append(pHash[item])
break
imgList = []
for imgDict in sortedImgDictList:
k = imgDict['cid']
m = imgDict['pid']
j = int(id)
uin = max(j + k + m, 10001)
l = [j % 1000 / 100, j % 100, j, k]
n = '/mif800/' + '/'.join(str(j) for j in l) + '/'
h = str(m) + '.mif2'
g="http://ac.tc.qq.com/store_file_download?buid=15017&uin="+str(uin)+"&dir_path="+n+"&name="+h
imgList.append(g)
return imgList
def downloadImg(imgUrlList, contentPath):
count = len(imgUrlList)
print('该集漫画共计{}张图片'.format(count))
i = 1
for imgUrl in imgUrlList:
print('\r正在下载第{}张图片...'.format(i), end='')
imgPath = os.path.join(contentPath, '{0:0>3}.jpg'.format(i))
downloadRequest = requestSession.get(imgUrl, stream=True)
with open(imgPath, 'wb') as f:
for chunk in downloadRequest.iter_content(chunk_size=1024):
if chunk: # filter out keep-alive new chunks
f.write(chunk)
f.flush()
i += 1
print('完毕!\n')
def main():
url = 'http://m.ac.qq.com/comic/index/id/623622' #要爬取的漫画首页
path = 'E:\\WorkSpace\\test' #下载图片存放路径
if not os.path.isdir(path):
os.mkdir(path)
id = getId(url)
comicName,count,contentList = getContent(id)
contentNameList = []
for item in contentList:
for k in item:
contentNameList.append(item[k]['t'])
print('漫画名: {}'.format(comicName))
print('章节数: {}'.format(count))
print('章节列表:')
print('\n'.join(contentNameList))
comicPath = os.path.join(path, comicName)
if not os.path.isdir(comicPath):
os.mkdir(comicPath)
print()
i = 0
for content in contentList:
print('正在下载第{0:0>4}话: {1}'.format(i + 1, contentNameList[i]))
contentPath = os.path.join(comicPath, '第{0:0>4}话-{1}'.format(i + 1, contentNameList[i]))
if not os.path.isdir(contentPath):
os.mkdir(contentPath)
imgList = getImgList(content, id)
downloadImg(imgList, contentPath)
i += 1
if __name__ == '__main__':
main()
#coding=utf-8
#破解采集图片数据
"""
http://api.yyhao.com/app_api/v3/getcomicinfo/?comic_id=27284 #获得目录信息
"""
import requests
import base64
import re
import json
import os
import urllib
#获取网页源代码
def getHtmlText(url):
try:
head={}
head['User-Agent'] ="Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.94 Safari/537.36"
r=requests.get(url,headers=head)
r.raise_for_status()#如果状态不是200,引发HTTPError异常
r.encoding=r.apparent_encoding
return r.text
except:
print("获取网页产生异常")
#获取漫画信息
def getComicInfo(comicId):
url='http://api.yyhao.com/app_api/v3/getcomicinfo/?comic_id={}'.format(comicId)
info=json.loads(getHtmlText(url))
comicName=info['comic_name']
comicAuthor=info['comic_author']
comicDesc=info['comic_desc']
chapterList=info['comic_chapter'][1]['chapter_list']
return(comicName,comicAuthor,comicDesc,chapterList)
#下载某个url对应的图片
def downloadImag(url,picPath,num):
head={}
head['User-Agent'] ="Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.94 Safari/537.36"
image = requests.get(url,headers=head)
if image.status_code == 200:
picPath=picPath+str(num)+'.jpg'
f=open(picPath, 'wb')
f.write(image.content)
f.close()
else:
print("下载图片出错!")
#下载一部漫画所有的图片
def downloadAll(path,chapterList):
for chapter in chapterList:
chapterPath=path+chapter['chapter_name']+'/'
if not os.path.exists(chapterPath):
os.mkdir(chapterPath)
start=chapter['chapter_source'][0]['start_num']
end=chapter['chapter_source'][0]['end_num']
ofDomain=chapter['chapter_source'][0]['chapter_domain']
if ofDomain=='':
domain='mhpic.samanlehua.com'
else:
domain='mhpic.'+ofDomain
rule=urllib.parse.quote(chapter['chapter_source'][0]['rule'])
imageUrl='http://'+domain+rule
imageUrl=imageUrl.replace("%24%24",'{}')
#imageUrl=urllib.parse.quote(imageUrl)
while(start<=end):
imageUrlAll=imageUrl.format(start)#+'-kmw.middle'
downloadImag(imageUrlAll,chapterPath,start)
start+=1
print(chapter['chapter_name']+" has downloaded")
if __name__ == '__main__':
comicId=3130
comicName,coimcAuthor,comicDesc,chapterList=getComicInfo(comicId)
print(comicName+' downloading...')
#存放路径
path='E:/WorkSpace/comic/'+comicName+'/'
if not os.path.exists(path):
os.mkdir(path)
downloadAll(path,chapterList)
print("All finished!")
其他资料
www.manhuagui.com的网站是动态加载的,用静态提取的方法失败了
参考博客
[1]http://blog.sina.com.cn/s/blog_700376570102x80k.html #scrapy静态抓取
[2]http://blog.sina.com.cn/s/blog_700376570102x8er.html
[3]http://blog.sina.com.cn/s/blog_700376570102x96k.html #动态抓取
对于数据来说,我们通常的做法就是进行Ajax处理,即异步请求数据,然后局部渲染生成整个网页。因此,我们设想,它是不是有ajax请求呢,
让我们来看看Network tab页下面查不查得到相关信息
使用PantomJS
#coding=utf-8
from selenium import webdriver
import time
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
#基于PhantomJS创建一个浏览器,并且设置一下用户代理,否则可能出现界面不兼容的情况
dcap = dict(DesiredCapabilities.PHANTOMJS)
dcap["phantomjs.page.settings.userAgent"] = ("Mozilla/4.0 (compatible; MSIE 5.5; windows NT)" )
browser = webdriver.PhantomJS(desired_capabilities=dcap)
#打开动漫的第一页
browser.get("http://ac.qq.com/ComicView/index/id/539443/cid/1")
#通过该循环,我们可以依次进行自动滑动,模拟滑动后自然会触发后续的图片资源。
for i in range(10):
js='window.scrollTo('+str(i*1280)+','+str((i+1)*1280)+')'
browser.execute_script(js)
time.sleep(1)
#将打开的界面截图保存,方便观察
a=browser.get_screenshot_as_file("E:/WorkSpace/Python/test.jpg")
#获取当前页面所有源码(此时包含触发出来的异步加载的资源)
data=browser.page_source
#将相关网页源码写入本地文件中,方便分析
fh=open("E:/WorkSpace/Python/dongman.html","w",encoding="utf-8")
fh.write(data)
fh.close()
#结束了PhantomJS的使用之后,我们需要关闭一下浏览器,所以,我们在代码后添加如下一行代码
browser.quit()
#我们可以通过正则表达式'<img src="(http:..ac.tc.qq.com.store_file_download.buid=.*?name=.*?).jpg"'将所有动漫资源图片网址提取出来,提取出来之后,通过urllib对这些图片进行爬取,爬到本地。
import re
import urllib
#构造正则表达式提取动漫图片资源网址
pat='<img src="(http:..ac.tc.qq.com.store_file_download.buid=.*?name=.*?).jpg"'
#获取所有动漫图片资源网址
allid=re.compile(pat).findall(data)
for i in range(0,len(allid)):
#得到当前网址
thisurl=allid[i]
#去除网址中的多余元素amp;
thisurl2=thisurl.replace("amp;","")+".jpg"
#输出当前爬取的网址
print(thisurl2)
#设置将动漫存储到本地的本地目录
localpath="E:/WorkSpace/dongman/"+str(i)+".jpg"
#通过urllib对动漫图片资源进行爬取
urllib.request.urlretrieve(thisurl2,filename=localpath)
爬取小说的例子
#coding=utf-8
#BeautifulSoup处理HTML标签包括搜索,遍历相应的标签,但是将HTML文档转化为可读文档,属于浏览器渲染的行为,bs一般是不提供的,具体的解决办法有用正则表达式,用nltk,用js的innnerHTML
#import nltk
#content=nltk.clean_html(contentHTML) #nltk后来的版本不支持clear_html()和clear_url()
#[s.extract() for s in contentHTML.find_all('p',text="")] #去除了所有的p标签
import requests
from bs4 import BeautifulSoup
#from HTMLParser import HTMLParser #python2
from html.parser import HTMLParser #python3
from re import sub
from sys import stderr
from traceback import print_exc
import os
class MyHTMLParser(HTMLParser):
def __init__(self):
HTMLParser.__init__(self)
self.texts=""#存放处理的结果
def handle_data(self, data):
data=data.strip() #去除首尾空格
data=' '+data #段落首部空格
self.texts+=data
#处理标签
#def handle_starttag(self, tag, attrs): #在遇到开始标签时发生的行为
#def handle_startendtag(self, tag, attrs): #在遇到开始或者结束标签时发生的行为
def handle_endtag(self,tag): #在遇到结束标签时发生的行为
if tag == 'p':
self.texts+='\n\n'
def text(self):
return self.texts
#解析HTML为Text
def dehtml(text):
try:
parser = MyHTMLParser()
parser.feed(text)
parser.close()
return parser.text()
except:
print_exc(file=stderr)
return text
#获取网页源代码
def getHTMLText(url):
try:
head={}
head['User-Agent'] ="Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.94 Safari/537.36"
r=requests.get(url,headers=head)
r.raise_for_status()#如果状态不是200,引发HTTPError异常
r.encoding=r.apparent_encoding
return r.text
except:
return "获取网页产生异常"
if __name__=="__main__":
print('starting...')
bookUrl="http://www.shuhuanghuang.com/book/44674.html"#小说首页
path="E:/WorkSpace/novel/"#下载小说位置
soup=BeautifulSoup(getHTMLText(bookUrl),'lxml')
infoSoup=soup.find('div',class_='info-box')
bookTitle=infoSoup.find('p',class_='title').text
bookDescription=infoSoup.get_text()
bookPath=path+bookTitle;
if not os.path.exists(bookPath): #不能用!来取非
os.makedirs(bookPath)
descriptionFile=bookPath+'/'+'Description.txt'
if not os.path.exists(descriptionFile):
open(descriptionFile,'w+').close()
f=open(descriptionFile,'w')
f.write(bookDescription)
f.close()
print('downloading...')
chapterUrl=soup.find('a',class_='read',text='阅读')['href'] #小说第一章url
hasNext=True;
while(hasNext):
chapterSoup = BeautifulSoup(getHTMLText(chapterUrl),'lxml')
title=chapterSoup.find('div',class_='title').string #小说标题
title=title.replace('\r\n','').replace('\t','')
contentHTML=chapterSoup.find('div',class_='content')
contentStr=contentHTML.prettify().replace('\n','').replace('<p> </p>','')
content=dehtml(contentStr)
chapterPath=bookPath+'/'+title+'.txt'
if not os.path.exists(chapterPath):
open(chapterPath,'w+').close()
f=open(chapterPath,'w')
f.write(content)
f.close()
print(title+' has downloaded!')
nextChapter=chapterSoup.find('a',class_='next')
if not nextChapter:
hasNext=False
break
chapterUrl=nextChapter['href']
print('All finished!')