自己写的爬取代码,还需进一步优化,仅供参考
import requests
import re
import pandas as pd
class spider():
def numbers(link):
number=('正在爬取:'+link+'\n\n')
return(number)
def page(url,totalpage):
changepage=[]
for i in range(1,totalpage+1):
page_number=re.sub('/page/\d+','/page/%s'%i,url,re.S)
changepage.append(page_number)
return(changepage)
def htmls(url): #网页
html=requests.get(url)
txt=html.text
return(txt)
def offer(home):
whole=re.findall('<div class="article block untagged mb15"(.*?)<div class="single-clear">',txt,re.S)
return(whole)
def author(source):
name=re.findall('<h2>(.*?)</h2>',source,re.S)
return('发布人:'+name[0]+'\n\n')
def contents(cont):
content=re.findall('<div class="content">(.*?)</div>',cont,re.S)
return('发布内容:'+content[0])
if __name__=='__main__':
url='http://www.qiushibaike.com/8hr/page/1/?s=4897603'
links=spider.page(url,3)
file=open('jiusihi.txt','w')
info=[]
for link in links:
prin=spider.numbers(link)
info.append(prin)
txt=spider.htmls(link)
datas=spider.offer(txt)
for data in datas:
print1=spider.author(data)
info.append(print1)
print2=spider.contents(data)
info.append(print2)
for each in info:
file.write(each)
file.close()