python爬虫第三讲-爬虫网络模块下

案例练习

爬取贴吧数据

学习目标：练习urllib post的实现方式
需求：
1 输入要爬取贴吧的名称(例如 海贼王)
2 要进行翻页的爬取(起始页 终止页)
3 把每一页的内容保存到本地 1.html 2.html（第一页.html 1.txt）

import urllib.parse
import urllib.request

name = input('请输入要贴吧的名称')
begin = int(input('请输入起始页'))
end = int(input('请输入终止页'))
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36'
}
url = 'https://tieba.baidu.com/f?'
kw = {'kw': name}
result = urllib.parse.urlencode(kw)
for i in range(begin, end + 1):
    pn = (i - 1) * 50
    url_real = url + result + '&pn=' + str(pn)

    req = urllib.request.Request(url_real, headers=headers)
    res = urllib.request.urlopen(req)
    html = res.read().decode('utf-8')

    file_name = '第{}页.html'.format(i)
    with open(file_name, 'w', encoding='utf-8') as f:
        print('正在爬取第{}页'.format(i))
        f.write(html)

改写成函数形式

# 读取页面
def read_page(url_real):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36'
    }
    req = urllib.request.Request(url_real, headers=headers)
    res = urllib.request.urlopen(req)
    html = res.read().decode('utf-8')
    return html


# 写入文件
def write_page(file_name, html, i):
    with open(file_name, 'w', encoding='utf-8') as f:
        print('正在爬取第{}页'.format(i))
        f.write(html)


# 主函数
def main():
    name = input('请输入要贴吧的名称')
    begin = int(input('请输入起始页'))
    end = int(input('请输入终止页'))
    url = 'https://tieba.baidu.com/f?'
    kw = {'kw': name}
    result = urllib.parse.urlencode(kw)
    for i in range(begin, end + 1):
        pn = (i - 1) * 50
        url_real = url + result + '&pn=' + str(pn)
        html = read_page(url_real)
        file_name = '第{}页.html'.format(i)
        write_page(file_name, html, i)


if __name__ == main():
    main()

改成面向对象

import urllib.parse
import urllib.request


class TeibaSpider(object):
    def __init__(self):
        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36'
        }
        self.baseurl = 'https://tieba.baidu.com/f?'

    def readpage(self, url_real):
        req = urllib.request.Request(url_real, headers=self.headers)
        res = urllib.request.urlopen(req)
        html = res.read().decode('utf-8')
        return html

    def writepage(self, file_name, html, i):
        with open(file_name, 'w', encoding='utf-8') as f:
            print('正在爬取第{}页'.format(i))
            f.write(html)

    def main(self):
        name = input('请输入要贴吧的名称')
        begin = int(input('请输入起始页'))
        end = int(input('请输入终止页'))
        kw = {'kw': name}
        result = urllib.parse.urlencode(kw)
        for i in range(begin, end + 1):
            pn = (i - 1) * 50
            url_real = self.baseurl + result + '&pn=' + str(pn)
            html = self.readpage(url_real)
            file_name = '第{}页.html'.format(i)
            self.writepage(file_name, html, i)


if __name__ == '__main__':
    TeibaSpider().main()

有道翻译

需求: 通过python制作一个小翻译软件

from urllib import request, parse
import json

key = input('请输入你要翻译的内容')

url = 'https://fanyi.youdao.com/translate?smartresult=dict&smartresult=rule'
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36'
}
data = {
    'i': key,
    'from': 'AUTO',
    'to': 'AUTO',
    'smartresult': 'dict',
    'client': 'fanyideskweb',
    'salt': '16226278326643',
    'sign': '3e5e1551836159cd404ae34a612b6eb5',
    'lts': '1622627832664',
    'bv': '24ecb70ba6203e4453baed50aa26b78e',
    'doctype': 'json',
    'version': '2.1',
    'keyfrom': 'fanyi.web',
    'action': 'FY_BY_REALTlME'
}
data = parse.urlencode(data)
data = bytes(data, 'utf-8')

req = request.Request(url, data=data, headers=headers, method='POST')
res = request.urlopen(req)
html = res.read().decode('utf-8')

r = json.loads(html)
result = r['translateResult'][0][0]['tgt']
print(f'翻译的结果是{result}')

requests快速入门

第三方的http请求模块
pip install requests -i https://pypi.douban.com/simple

import requests

#  https://tieba.baidu.com/f?kw=海贼王&pn=0
'''
requests.get(url=url,params=params,headers=headers)
url 为基准的url 不包含参数
params 以键值对的形式来添加参数

'''
# 没毛病
# url = 'https://tieba.baidu.com/f?'
# params = {'kw':'海贼王','pn':'0'}
# headers = {
#     'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36'
# }
# # 开始请求
# html = requests.get(url=url,params=params,headers=headers)
# print(html.text)
# ---------------------------------------------------

# url = 'https://tieba.baidu.com/f?kw=海贼王&pn=0'
#
# headers = {
#     'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36'
# }
#
# html = requests.get(url,headers=headers)
# print(html.text)

# -------------------------------------------------

url = 'https://qq.yh31.com/zjbq/2920180.html'

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36'
}

response = requests.get(url,headers=headers)
# print(type(response.text)) # 返回的是str类型的数据 出现乱码
# print(response.content) # 返回的是二进制数据(字节流 bytes) 数据看不懂
# print(type(response.content.decode('utf-8')),response.content.decode('utf-8')) # str
# response.encoding = 'utf-8'
# print(response.text) # 有可能会产生乱码的

# 你爬取的是 图片 文字
'''
和乱码有什么关系？怎么解决呢？原理
response.content 是直接从网站上抓取数据没有做任何处理 没有做任何的解码
response.content.decode('utf-8') 手动的指定 并且这种方式返回的是字符串

response.text 是requests模块将response.content解码后得到的字符串 requests库会猜一个解码的方式 手动指定一个编码 encoding

'''