Python练习-爬取并下载漫画

利用python做的一个小练习

1.导入相应所需要的包。

import os
import requests
from bs4 import BeautifulSoup
import re
import random

2.定义多个头文段和ip地址，访问时随机生成，防止ip被封。

headers = [
    "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36",
    "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:30.0) Gecko/20100101 Firefox/30.0",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/537.75.14",
    "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Win64; x64; Trident/6.0)",
    'Mozilla/5.0 (Windows; U; Windows NT 5.1; it; rv:1.8.1.11) Gecko/20071127 Firefox/2.0.0.11',
    'Opera/9.25 (Windows NT 5.1; U; en)',
    'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)',
    'Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.5 (like Gecko) (Kubuntu)',
    'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.0.12) Gecko/20070731 Ubuntu/dapper-security Firefox/1.5.0.12',
    'Lynx/2.8.5rel.1 libwww-FM/2.14 SSL-MM/1.4.1 GNUTLS/1.2.9',
    "Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.7 (KHTML, like Gecko) Ubuntu/11.04 Chromium/16.0.912.77 Chrome/16.0.912.77 Safari/535.7",
    "Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:10.0) Gecko/20100101 Firefox/10.0 "
]

# 代理ip
# https://ip.jiangxianli.com/?page=1
proxy_list = [
    'http://1.117.111.162:7890',
    'http://59.60.128.61:4216',
    'http://120.43.116.50:4216',
    'http://120.40.180.208:4216',
    'http://120.40.185.28:4216',
    'http://221.4.245.242:9091'
]

3.利用url获取网页的源代码的方法

# 获取网页
def askUrl(url):
    head = {
        'user-agent': random.choice(headers)
    }
    proxy = {
        'http': random.choice(proxy_list)
    }
    try:
        response = requests.get(url, timeout=30, headers=head, proxies=proxy)
        response.raise_for_status()  # 如果状态码不是200, 引发异常
        response.encoding = 'utf-8'  # 防止中文乱码
        return response.text
    except:
        return ''

4.要爬取的首页面，对页面排行进行筛选，这里没有加密的，比较简单。

# 查询排行榜
def searchTops(home_url):
    html = askUrl(home_url)
    ranking = int(input('请输入您要进行的操作(查询总排行 0 / 查询少年漫画排行 1 / 查询少女漫画排行 2):\n'))
    weekOrMonth = int(input('请输入您需要的排行(查询周排行 0 / 查询月排行 1):\n')