xpath 百度爬虫

百度搜索关键词
输入搜索的词，输入页数
返回标题、页数、url。

import requests
from lxml import etree



def gethtml(url):
    text=requests.get(url).text
    text=text.replace('<em>','')
    html=etree.HTML(text)
    return html

def gettitle(html):
    title=html.xpath('//h3[@class="t"]//a[@data-click]/text()')
    tit=[]
    for each in title:
        tit.append(each)
    return(tit) 


def getcontent(html):
    content=html.xpath('//div[@class="c-abstract"]/text()')
    con=[]
    for each in content:
        con.append(each)
    return(con)



def geturlid(html):
    urlid=html.xpath('//div[@class="f13"]//a[@class="c-showurl"]/text()')
    url=[]
    for each in urlid:
        url.append(each)
    return(url)


def search(name,page):
    urlname=[]
    for each in range(page):
        url='http://www.baidu.com/s?wd='+name+'&pn=%s&rsv_bp=0&rsv_spt=3&rsv_n=2&inputT=6391'%(each*10)
        urlname.append(url)
    return(urlname)


file=open('output.txt','w+',encoding='utf-8')
name=input('搜索内容：')
page=int(input('搜索页数：'))
urlname=search(name,page)
i=1
search=[]
for url in urlname:
    html=gethtml(url)
    title_len=len(gettitle(html))
    for each in range(title_len):
        #print(i)
        #print(gettitle(html)[each])
        #print(getcontent(html)[each])
        #print(geturlid(html)[each])

        search.append('第%s条'%i)
        search.append(gettitle(html)[each])
        search.append(getcontent(html)[each])
        search.append(geturlid(html)[each])
        i+=1
for each in search:
    file.write(each+'\n')
file.close()

代码还在完善，目前可以保存在文本里，接下来要保存到数据库中