python爬虫百度文库_Python百度文库爬虫之pdf文件

Python百度文库爬虫之ppt文件

对于文件的所有类型，我都会用一篇文章进行说明，链接：

一.网页分析

说明一下：分析doc文件和pdf文件非常相似，从爬取过程来说基本类似。因此我不会花费太多的语言来说明，直接使用代码

from IPython.display import Image

Image("./Images/pdf_0.png",width="600px",height="400px")

Image("./Images/pdf_1.png",width="600px",height="400px")

Image("./Images/pdf_2.png",width="600px",height="400px")

二.数据URL获取

import requests

import json

import re

import json

session=requests.session()

url=input("请输入下载的文件URL地址：")

content=session.get(url).content.decode('gbk')

doc_id=re.findall('view/(.*?).html',url)[0]

types=re.findall(r"docType.*?:.*?'(.*?)'",content)[0]

title=re.findall(r"title.*?:.*?'(.*?)'",content)[0]

请输入下载的文件URL地址： https://wenku.baidu.com/view/5fd3a55b3d1ec5da50e2524de518964bcf84d2b6.html?fr=search

title

'网络爬虫技术研究'

url_list=re.findall(r'(https.*?0.json.*?)\\x22}',content)

url_list=[addr.replace("\\\\\\/","/") for addr in url_list]

三.程序调试

result=""

for url in set(url_list):

content=session.get(url).content.decode('gbk')

y=0

txtlists=re.findall(r'"c":"(.*?)".*?"y":(.*?),',content)

for item in txtlists:

# 当item[1]的值与前面不同时，代表要换行了

if not y==item[1]:

y=item[1]

n='\n'

else:

n=''

result+=n

result+=item[0].encode('utf-8').decode('unicode_escape','ignore')

E:\Anaconda\envs\mypython\lib\site-packages\ipykernel_launcher.py:16: DeprecationWarning: invalid escape sequence '\/'

app.launch_new_instance()

filename="./Files/"+title+'.txt'

with open(filename,'w',encoding="utf-8") as f:

f.write(result)

f.close()

四.函数编程

1.功能完善

提供下载[‘txt’,‘doc’,‘pdf’,‘ppt’]文件

import requests

import json

import re

import os

session=requests.session()

path="F:\\桌面\\Files"

if not os.path.exists(path):

os.mkdir(path)

def parse_txt1(code,doc_id):

content_url='https://wenku.baidu.com/api/doc/getdocinfo?callback=cb&doc_id='+doc_id

content=session.get(content_url).content.decode(code)

md5sum=re.findall('"md5sum":"(.*?)",',content)[0]

rsign=re.findall('"rsign":"(.*?)"',content)[0]

pn=re.findall('"totalPageNum":"(.*?)"',content)[0]

content_url='https://wkretype.bdimg.com/retype/text/'+doc_id+'?rn='+pn+'&type=txt'+md5sum+'&rsign='+rsign

content=json.loads(session.get(content_url).content.decode('gbk'))

result=''

for item in content:

for i in item['parags']:

result+=i['c']

return result

def parse_txt2(content,code,doc_id):

md5sum=re.findall('"md5sum":"(.*?)",',content)[0]

rsign=re.findall('"rsign":"(.*?)"',content)[0]

pn=re.findall('"show_page":"(.*?)"',content)[0]

content_url='https://wkretype.bdimg.com/retype/text/'+doc_id+'?rn='+pn+'&type=txt'+md5sum+'&rsign='+rsign

content=json.loads(session.get(content_url).content.decode('utf-8'))

result=''

for item in content:

for i in item['parags']:

result+=i['c']

return result

def parse_doc(content):

url_list=re.findall(r'(https.*?0.json.*?)\\x22}',content)

url_list=[addr.replace("\\\\\\/","/") for addr in url_list]

result=""

for url in set(url_list):

content=session.get(url).content.decode('gbk')

y=0

txtlists=re.findall(r'"c":"(.*?)".*?"y":(.*?),',content)

for item in txtlists:

# 当item[1]的值与前面不同时，代表要换行了

if not y==item[1]:

y=item[1]

n='\n'

else:

n=''

result+=n

result+=item[0].encode('utf-8').decode('unicode_escape','ignore')

return result

def parse_pdf(content):

url_list=re.findall(r'(https.*?0.json.*?)\\x22}',content)

url_list=[addr.replace("\\\\\\/","/") for addr in url_list]

result=""

for url in set(url_list):

content=session.get(url).content.decode('gbk')

y=0

txtlists=re.findall(r'"c":"(.*?)".*?"y":(.*?),',content)

for item in txtlists:

# 当item[1]的值与前面不同时，代表要换行了

if not y==item[1]:

y=item[1]

n='\n'

else:

n=''

result+=n

result+=item[0].encode('utf-8').decode('unicode_escape','ignore')

return result

def save_file(title,filename,content):

with open(filename,'w',encoding='utf-8') as f:

f.write(content)

print("文件"+title+"保存成功")

f.close()

def main():

print("欢迎来到百度文库文件下载：")

print("-----------------------\r\n")

while True:

try:

print("1.doc \n 2.txt \n 3.ppt \n 4.xls\n 5.ppt\n")

types=input("请输入需要下载文件的格式(0退出)：")

if types=="0":

break

if types not in ['txt','doc','pdf']:

print("抱歉功能尚未开发")

continue

url=input("请输入下载的文库URL地址：")

# 网页内容

response=session.get(url)

code=re.findall('charset=(.*?)"',response.text)[0]

if code.lower()!='utf-8':

code='gbk'

content=response.content.decode(code)

# 文件id

doc_id=re.findall('view/(.*?).html',url)[0]

# 文件类型

#types=re.findall(r"docType.*?:.*?'(.*?)'",content)[0]

# 文件主题

#title=re.findall(r"title.*?:.*?'(.*?)'",content)[0]

if types=='txt':

md5sum=re.findall('"md5sum":"(.*?)",',content)

if md5sum!=[]:

result=parse_txt2(content,code,doc_id)

title=re.findall(r'

(.*?). ',content)[0]

#filename=os.getcwd()+"\\Files\\"+title+'.txt'

filename=path+"\\"+title+".txt"

save_file(title,filename,result)

else:

result=parse_txt1(code,doc_id)

title=re.findall(r"title.*?:.*?'(.*?)'",content)[0]

#filename=os.getcwd()+"\\Files\\"+title+'.txt'

filename=path+"\\"+title+".txt"

save_file(title,filename,result)

elif types=='doc':

title=re.findall(r"title.*?:.*?'(.*?)'",content)[0]

result=parse_doc(content)

filename=path+"\\"+title+".doc"

save_file(title,filename,result)

elif types=='pdf':

title=re.findall(r"title.*?:.*?'(.*?)'",content)[0]

result=parse_pdf(content)

filename=path+"\\"+title+".txt"

save_file(title,filename,result)

except Exception as e:

print(e)

if __name__=='__main__':

main()

欢迎来到百度文库文件下载：

-----------------------

1.doc

2.txt

3.ppt

4.xls

5.ppt

请输入需要下载文件的格式(0退出)： pdf

请输入下载的文库URL地址： https://wenku.baidu.com/view/5fd3a55b3d1ec5da50e2524de518964bcf84d2b6.html?fr=search

E:\Anaconda\envs\mypython\lib\site-packages\ipykernel_launcher.py:93: DeprecationWarning: invalid escape sequence '\/'

文件网络爬虫技术研究保存成功

1.doc

2.txt

3.ppt

4.xls

5.ppt

请输入需要下载文件的格式(0退出)： 0