python之爬取百度贴吧

有道翻译

import urllib.request
import urllib.parse
url='http://fanyi.youdao.com/translate?smartresult=dict&smartresult=rule'   # 有道翻译错误代码50时，去掉translate_o?的_o
headers={"User-Agent":"Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:69.0) Gecko/20100101 Firefox/69.0"}   # 编码错误时，检查用户代理是否有省略号
formdata={
    "i":"你",        # 要翻译的内容 
    "from":"en",     # 翻译成英文
    "to":"zh-CHS",   # 从中文
    "smartresult":"dict",
    "client":"fanyideskweb",
    "doctype":"json",
    "version":"2.1",
    "keyfrom":"fanyi.web",
    "action":"FY_BY_CLICKBUTTION"
}
data=urllib.parse.urlencode(formdata).encode('utf-8')
request=urllib.request.Request(url,data=data,headers=headers)
response=urllib.request.urlopen(request)
print(response.read().decode('utf-8'))

requests库的使用：

import requests
url="https://tieba.baidu.com/f?"
param={"kw":"囚徒健身"}
headers={"User-Agent":"Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:69.0) Gecko/20100101 Firefox/69.0"}
res=requests.get(url,params=param,headers=headers)
print(res.text)

爬取百度贴吧的网页

import requests
def tieba_spider(url,begin_page,end_page):
    for page in range(begin_page,end_page):
        pn=(page-1)*50
        file_name="第"+str(page)+"页.html"
        full_url=url+"&pn="+str(pn)
        html=load_page(full_url,file_name)
        write_page(html,file_name)
def load_page(url,filename):
    headers={"User-Agent":"Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:69.0) Gecko/20100101 Firefox/69.0"}
    return requests.get(url,params=param,headers=headers).text
def write_page(html,filename):
    print("正在保存"+filename)
    with open(filename,'w',encoding='utf-8') as file:
        file.write(html)
if __name__=="__main__":
    kw=input("请输入需要爬取的贴吧名：")
    begin_page=int(input("请输入起始页："))
    end_page=int(input("请输入结束页："))
    url="https://tieba.baidu.com/f?"
    param={"kw":kw}
    tieba_spider(url,begin_page,end_page)