Python爬虫入门（一）requests库

最新推荐文章于 2022-10-22 10:37:53 发布

咸鱼hao

最新推荐文章于 2022-10-22 10:37:53 发布

阅读量160

点赞数

分类专栏： Python 文章标签： Python 爬虫 requests

本文链接：https://blog.csdn.net/weixin_43868436/article/details/89706357

版权

Python 专栏收录该内容

13 篇文章 0 订阅

订阅专栏

Python爬虫入门（一）requests库

import requests
import os
url1 = "http://www.baidu.com"
# get方法
# r = requests.get(url1)
# print(r.status_code)
# print(r.encoding)
# print(r.apparent_encoding)
# r.encoding = 'utf-8'
# print(r.text)
# requests库异常


def find_html(url):
    # 简单的访问
    try:
        r = requests.get(url, timeout=30)
        print(r.request.headers)  # 'User-Agent': 'python-requests/2.20.0' 说明来访问的是python
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        return r.text
    except:
        return "产生异常"


# print((url1))


def view_in_browser(url):
    # 模拟浏览器对网页访问
    try:
        kv = {'User-Agent': 'Mozilla/5.0'}  # 标准浏览器标识
        r = requests.get(url, headers=kv)
        print(r.request.headers)
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        return r.text
    except:
        return "产生异常"


# print(view_in_browser(url1))


def search_in_baidu():
    # 百度的搜索URL是 http://www.baidu.com/s  key为wd
    kv = {'wd': 'python'}
    try:
        r = requests.get("http://www.baidu.com/s", params=kv)
        print(r.request.url)
        r.raise_for_status()
        print(len(r.text))  # 内容太长，仅打印长度
    except:
        print("爬取失败")


# search_in_baidu()


def store_picture(url):
    # 爬取图片，其他格式同理
    root = "pictures\\"
    path = root + url.split('/')[-1]  # 路径+文件名
    try:
        if not os.path.exists(root):
            os.mkdir(root)
        if not os.path.exists(path):
            r = requests.get(url)
            # 参数w 写入，会覆盖源文件内容
            # 参数b binary mode即二进制模式
            with open(path, 'wb') as f:
                f.write(r.content)
                f.close()
                print("文件保存成功")
        else:
            print("文件已存在")
    except:
        print("爬取失败")


def search_IP():
    kv = {'ip': '202.204.80.112'}
    ip_url = "http://www.ip138.com/ips138.asp?"
    try:
        r = requests.get(ip_url, params=kv)
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        print(r.text)
    except:
        print("爬取失败")


search_IP()