爬虫代理获取、使用、保存

import requests
import re
import csv
from bs4 import BeautifulSoup
import time

#爬取西刺代理第一页的代理IP   一定要加headers请求头
def get_ip(url,headers):
    r = requests.get(url, headers=headers)
    html = r.text
    # print(html)
    soup = BeautifulSoup(html, "lxml")
    # datas1 = soup.find_all(name = 'tr',class_='odd')
    # for data1 in datas1:
    # result1 = data1.find_all('td')[1]  #result 是td节点 的list型
    # print(result1.string)
    # datas2 = soup.find_all(name = 'tr',class_='')  #51个
    # for data2 in datas2:
    # result2 = data2.find_all('td')  #result 是td节点 的list型  这里第一个没有数据，所以按照上面的方法不行
    # print(result2)
    # 重写
    proxy_ip_list = []
    parent = soup.find(id="ip_list")
    lis = parent.find_all('tr')
    lis.pop(0)  # 因为第一条没有数据，要删掉
    for i in lis:
        ip = i.find_all('td')[1].string  # ip
        dk = i.find_all('td')[2].string  # port
        ty = i.find_all('td')[5].string  # portocol
        if ty in ("HTTP", "HTTPS", "http", "https"):  # 挑选出合格的IP
            proxy_ip_list.append(f'{ty}://{ip}:{dk}')
    return proxy_ip_list
def open_url_by_proxy(headers,proxy_ip_list):
#print("共爬到",len(datalist),'条IP数据') #一页100条数据
#print(proxy_ip_list)  #一页100条数据
##使用代理
#proxies 是一个字典型，('http':'http://IP:port','https':'https://IP:port')直接传入requests的get中
#web_data = requests.get(url,headers = hearders,proxies= proxies)
    url = 'http://www.baidu.com'
    proxies = {}  #将HTTPS 与HTTP分开
    for proxy in proxy_ip_list:
        if proxy.startswith('HTTPS'):
            proxies['https'] = proxy

        else:
            proxies["http"] = proxy
        try:
            r = requests.get(url,headers,proxies =proxies)
            r.raise_for_status()
            r.encoding = r.apparent_encoding
            return(proxy,r.status_code)  #代理有效返回网页代码以及状态码
        except:
            return False

def check_proxy(proxy,status_code):  #检查返回的状态码
    if status_code == 200:
        print('有效代理IP: ' + proxy)
    else:
        print('无效代理IP: ' + proxy)


def main(url,headers):
    proxy_ip_list = get_ip(url,headers)
    proxy,status_code = open_url_by_proxy(headers,proxy_ip_list)
    check_proxy( proxy,status_code)

if __name__ =="__main__":
    url = "https://www.xicidaili.com/nn"
    headers = {
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
        "Accept-Encoding": "gzip, deflate, br",
        "Accept-Language": "zh-CN,zh;q=0.9",
        "Cache-Control": "max-age=0",
        "Connection": "keep-alive",
        "Cookie": "_free_proxy_session=BAh7B0kiD3Nlc3Npb25faWQGOgZFVEkiJTZjMDU0Nzc5MjNhMDFjNDYyMWViYjM2MGFlYTg" +
                  "2NmUyBjsAVEkiEF9jc3JmX3Rva2VuBjsARkkiMWUzdUtabE9HREd0RjJ0c0hQOVcweWp5ODdzcHMzT2FQcnNsYW9SZXV" +
                  "oV2c9BjsARg%3D%3D--" +
                  "a31243018cd2aeb2d317f5888017bf6711ec9bb1; Hm_lvt_0cf76c77469e965d2957f0553e6ecf59=1554887960; " +
                  "Hm_lpvt_0cf76c77469e965d2957f0553e6ecf59=1554888022",
        "User-Agent": "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36"
    }
    main(url,headers)
结果如下：
D:\pycharm安装包\test\doubandata.py\Scripts\python.exe D:/pycharm安装包/douban/xici.py
有效代理IP: HTTPS://171.41.82.37:9999

Process finished with exit code 0

更加详细学习可以参见