百度搜索关键词
输入搜索的词,输入页数
返回标题、页数、url。
import requests
from lxml import etree
def gethtml(url):
text=requests.get(url).text
text=text.replace('<em>','')
html=etree.HTML(text)
return html
def gettitle(html):
title=html.xpath('//h3[@class="t"]//a[@data-click]/text()')
tit=[]
for each in title:
tit.append(each)
return(tit)
def getcontent(html):
content=html.xpath('//div[@class="c-abstract"]/text()')
con=[]
for each in content:
con.append(each)
return(con)
def geturlid(html):
urlid=html.xpath('//div[@class="f13"]//a[@class="c-showurl"]/text()')
url=[]
for each in urlid:
url.append(each)
return(url)
def search(name,page):
urlname=[]
for each in range(page):
url='http://www.baidu.com/s?wd='+name+'&pn=%s&rsv_bp=0&rsv_spt=3&rsv_n=2&inputT=6391'%(each*10)
urlname.append(url)
return(urlname)
file=open('output.txt','w+',encoding='utf-8')
name=input('搜索内容:')
page=int(input('搜索页数:'))
urlname=search(name,page)
i=1
search=[]
for url in urlname:
html=gethtml(url)
title_len=len(gettitle(html))
for each in range(title_len):
#print(i)
#print(gettitle(html)[each])
#print(getcontent(html)[each])
#print(geturlid(html)[each])
search.append('第%s条'%i)
search.append(gettitle(html)[each])
search.append(getcontent(html)[each])
search.append(geturlid(html)[each])
i+=1
for each in search:
file.write(each+'\n')
file.close()
代码还在完善,目前可以保存在文本里,接下来要保存到数据库中