百度贴吧爬虫

import urllib

def loadPage(url,filename):
    """"
    作用：根据url发送请求，获取服务器响应文件
    url：需要爬取的url地址
    filename：处理的文件名
    """
    print("正在下载"+filename)
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36"
    }
    request = urllib.request.Request(url,headers = headers) #py3与py2不同
    return urllib.request.urlopen(request).read()




def writePage(html, filename):
    """

    作用：将html内容写入到本地
    html：服务器相应的文件内容
    """
    print("正在保存"+filename)
    with open(filename,"wb") as f: #这里读取二进制数据，故用wb
        f.write(html)

def tiebaSpider(url, beginPage, endPage):
    """
    作用：贴吧爬虫调度器，负责组合处理每个页面的url
    url:贴吧url的前部分
    beginPage：起始页
    endPage：结束页
    """
    for page in range(beginPage,endPage+1):
        pn = (page-1)*50
        filename = "第"+str(page)+"页.html"
        fullurl = url +"&pn" +str(pn)
        html = loadPage(fullurl, filename)
        writePage(html,filename)
        #print(html)
        #print (fullurl)





if __name__ =="__main__":
    kw = input("请输入需要爬取的贴吧名：")
    beginPage = int(input("请输入起始页："))
    endPage = int(input("请输入结束页："))

    url = "http://tieba.baidu.com/f?"
    key = urllib.parse.urlencode({"kw":kw})#py3与py2不同
    fullurl = url +key
    tiebaSpider(fullurl, beginPage, endPage)