urllib爬虫封装（可设置代理，记录日志）方法

最新推荐文章于 2021-10-14 09:02:45 发布

紫色的梅花

最新推荐文章于 2021-10-14 09:02:45 发布

阅读量1.1w

点赞数

分类专栏：爬虫文章标签：爬虫 urllib

本文链接：https://blog.csdn.net/qq_41805981/article/details/100964596

版权

urllib爬虫封装（具体其他功能根据需求完善）

1.支持设置HTTP Request Headers ,能设置UA。
2.支持代理服务器的设置。
3.它支持timeout超时机制。
4.它支持网页的编码指定。
5.它支持服务器返回的错误处理如果>400 and <500 直接记录日志，
如果错误码是500-600 则重新发起请求。

# -*- coding: utf-8 -*-
"""
Created on Tue Sep 17 19:46:32 2019

@author: Administrator
"""

import logging
from logging import handlers
import random
from urllib import request
from urllib import error
import time
import socket


# 日志
class Logger(object):
    level_relations = {
        'debug': logging.DEBUG,
        'info': logging.INFO,
        'warning': logging.WARNING,
        'error': logging.ERROR,
        'crit': logging.CRITICAL
    }  # 日志级别关系映射
    #='info'
    def __init__(self, filename, level, when='D', backCount=3,
                 fmt='%(asctime)s - %(pathname)s[line:%(lineno)d] - %(levelname)s: %(message)s'):
        pass
        # 创建日志的方法

    def create_log(self,filename):
        self.logger = logging.getLogger(filename)
        return self.logger

    def set_log_format(self, fmt):
        format_str = logging.Formatter(fmt)  # 设置日志格式
        return format_str

    def write_log(self, filename,level, when, backCount, fmt, encoding='utf-8'):
        self.logger = Logger.create_log(self,filename)
        self.logger.setLevel(self.level_relations.get(level))  # 设置日志级别
        sh = logging.StreamHandler()  # 往屏幕上输出
        sh.setFormatter(Logger.set_log_format(self, fmt))  # 设置屏幕上显示的格式
        # 实例化TimedRotatingFileHandler
        # interval是时间间隔，backCount是备份文件的个数，如果超过这个个数，就会自动删除，when是间隔的时间单位，单位有以下几种：
        # S 秒
        # M 分
        # H 小时、
        # D 天、
        # W 每星期（interval==0时代表星期一）
        # midnight 每天凌晨
        th = handlers.TimedRotatingFileHandler(filename=filename, when=when, backupCount=backCount,
                                               encoding='utf-8')  # 往文件里写入#指定间隔时间自动生成文件的处理器
        th.setFormatter(Logger.set_log_format(self, fmt))
        # 设置文件里写入的格式
        self.logger.addHandler(sh)  # 把对象加到logger里
        self.logger.addHandler(th)
        return  sh,th,self.logger
    def remove_hanler(self,filename, level, when, backCount, fmt, encoding='utf-8'):
        sh,th,self.logger=Logger.write_log(self, filename, level, when, backCount, fmt, encoding='utf-8')
        self.logger.removeHandler(sh)
        self.logger.removeHandler(th)

#
# if __name__ == '__main__':
#     filename="i://all.log"
#     level='debug'
#     when='D'
#     backCount=3
#     fmt='%(asctime)s - %(pathname)s[line:%(lineno)d] - %(levelname)s: %(message)s'
#
#     log = Logger(filename,level='debug')
#     log.create_log(filename)
#     log.write_log(filename, level, when, backCount, fmt, encoding='utf-8')
#     log.logger.debug('debug')
#     log.logger.info('info')
#     log.logger.warning('警告')
#     log.logger.error('报错')
#     log.logger.critical('严重')
#     # Logger('i://error.log', level='error').logger.error('error')
#     log.remove_hanler( filename, 'error', 'D', 3, '%(asctime)s - %(pathname)s[line:%(lineno)d] - %(levelname)s: %(message)s', encoding='utf-8')
#

minRangeForProxy=1
maxRangeForProxy=10
minsleepTime=1
maxsleepTime=3
filename='i://all.log'
level = 'debug'
when = 'D'
backCount = 3
fmt = '%(asctime)s - %(pathname)s[line:%(lineno)d] - %(levelname)s: %(message)s'

log = Logger(filename, level='debug')


def downloadHtml(log,url,headers=[], useProxyRate=0,\
  proxy={},timeout=socket._GLOBAL_DEFAULT_TIMEOUT,\
  decodeInfo='utf-8',num_retries=5):

   """
   这是一个爬取网页数据的函数
   headers     它支持设置HTTP  Request  Headers ,能设置UA
   proxy       支持代理服务器的设置
   timeout     它支持timeout超时机制
   decodeInfo  它支持网页 的编码指定
   它支持服务器返回的错误处理  如果>400 and <500  直接记录日志
                          如果500-600        重新发起请求
   """
   if  random.randint(minRangeForProxy,maxRangeForProxy
                      )  >useProxyRate:
       proxy=None  #不使用代理服务器
   #创建proxy  Handler
   proxy_handler=request.ProxyHandler(proxy)
   #创建opener
   opener=request.build_opener(proxy_handler)
   #设置http   request 的headers
   opener.addheaders=headers
   #把proxy handler安装到urllib库中去
   request.install_opener(opener)
   #初始值将html的返回值设置成None
   #如果成功的返回，则得到想要的html信息
   #否则，None值不会发生改变
   html=None
   #发起请求
   try:
       #在这里考虑超时的问题urlopen有内部的timeout参数
       res=request.urlopen(url,timeout=timeout)
       html=res.read().decode(decodeInfo)
   except UnicodeDecodeError:
       log.logger.error("UnicodeDecodeError")
   except error.URLError or error.HTTPError as e:
       log.logger.error("Download Error")
       #4xx,5xx
       #出现4xx错误怎么办，直接记录日志
       if  hasattr(e,'code') and 400<=e.code <500:
           log.create_log(filename)
           log.write_log(filename, level, when, backCount, fmt, encoding='utf-8')
           log.logger.debug('debug')
           log.logger.info('info')
           log.logger.warning('警告')
           log.logger.error('报错')
           log.logger.critical('严重')

           return  html
       #如果出现5xx错误怎么办,服务器的问题
       if num_retries>0:
           #如果状态码为500-600之间，则继续抓取，直到抓取的次数
           #超出最大上限
           if   hasattr(e,"code") and 500 <=e.code<600:
               #如果请求返回服务器失败随机休息1到3秒钟,然后继续获取当前.
               time.sleep(random.randint(minsleepTime,maxsleepTime))
               html=downloadHtml(url,
                      headers,proxy,useProxyRate,
                      timeout,num_retries-1)


   return html

headers=[("User-Agent", "Mozilla/5.0 (Windows NT 6.1;\
   Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)\
   Chrome/76.0.3809.132 Safari/537.36")]

url="http://www.baidu.com"
#http://121.199.42.16/VAD/GetIp.aspx?act=get&num=200&time=60&plat=1&re=0&type=0&ow=1&prov=浙江省
proxy_addr={"http","ip:端口号"}
print(downloadHtml(log,url=url,
                  headers=headers,proxy=proxy_addr))
log.remove_hanler( filename, 'error', 'D', 3, '%(asctime)s - %(pathname)s[line:%(lineno)d] - %(levelname)s: %(message)s', encoding='utf-8')

运行结果:

返回百度首页。

H:\ANACONDA\python.exe I:/ainlp/spiderBasic.py
<!DOCTYPE html>
<!--STATUS OK-->






<html>
<head>
    
    <meta http-equiv="content-type" content="text/html;charset=utf-8">
    <meta http-equiv="X-UA-Compatible" content="IE=Edge">
	<meta content="always" name="referrer">
    <meta name="theme-color" content="#2932e1">
    <link rel="shortcut icon" href="/favicon.ico" type="image/x-icon" />
    <link rel="search" type="application/opensearchdescription+xml" href="/content-search.xml" title="百度搜索" />
    <link rel="icon" sizes="any" mask href="//www.baidu.com/img/baidu_85beaf5496f291521eb75ba38eacbd87.svg">
	
	
	<link rel="dns-prefetch" href="//s1.bdstatic.com"/>
	<link rel="dns-prefetch" href="//t1.baidu.com"/>
	<link rel="dns-prefetch" href="//t2.baidu.com"/>
	<link rel="dns-prefetch" href="//t3.baidu.com"/>
	<link rel="dns-prefetch" href="//t10.baidu.com"/>
	<link rel="dns-prefetch" href="//t11.baidu.com"/>
	<link rel="dns-prefetch" href="//t12.baidu.com"/>
	<link rel="dns-prefetch" href="//b1.bdstatic.com"/>
    
    <title>百度一下，你就知道</title>
    

<style id="css_index" index="index" type="text/css">html,body{height:100%}
html{overflow-y:auto}
body{font:12px arial;text-align:;background:#fff}
body,p,form,ul,li{margin:0;padding:0;list-style:none}
body,form,#fm{position:relative}
td{text-align:left}
img{border:0}
a{color:#00c}
a:active{color:#f60}
input{border:0;padding:0}
#wrapper{position:relative;_position:;min-height:100%}
#head{padding-bottom:100px;text-align:center;*z-index:1}
#ftCon{height:50px;position:absolute;bottom:47px;text-align:left;width:100%;margin:0 auto;z-index:0;overflow:hidden}
.ftCon-Wrapper{overflow:hidden;margin:0 auto;text-align:center;*width:640px}
.qrcodeCon{text-align:center;position:absolute;bottom:132px;width:100%}
#qrcode{display:inline-block;*float:left;*margin-top:4px}
#qrcode .qrcode-item{float:left}
#qrcode .qrcode-item-2{margin-left:33px}
#qrcode .qrcode-img{width:106px;height:106px;margin:0 auto;padding:5px;border:1px solid #f3f3f3}
#qrcode .qrcode-item-1 .qrcode-img{background:url(https://ss1.bdstatic.com/5eN1bjq8AAUYm2zgoY3K/r/www/cache/static/protocol/https/home/img/qrcode/zbios_09b6296.png) 0 0 no-repeat;background-position:5px 5px}
#qrcode .qrcode-item-2 .qrcode-img{background:url(https://ss1.bdstatic.com/5eN1bjq8AAUYm2zgoY3K/r/www/cache/static/protocol/https/home/img/qrcode/nuomi_365eabd.png) 0 0 no-repeat;background-position:5px 5px}
@media only screen and (-webkit-min-device-pixel-ratio:2){#qrcode .qrcode-item-1 .qrcode-img{background-image:url(https://ss1.bdstatic.com/5eN1bjq8AAUYm2zgoY3K/r/www/cache/static/protocol/https/home/img/qrcode/zbios_x2_5869f49.png);background-size:106px 106px}
#qrcode .qrcode-item-2 .qrcode-img{background-image:url(https://ss1.bdstatic.com/5eN1bjq8AAUYm2zgoY3K/r/www/cache/static/protocol/https/home/img/qrcode/nuomi_x2_55dc5b7.png);background-size:106px 106px}}
#qrcode .qrcode-text{color:#999;line-height:23px}
#qrcode .qrcode-text a{color:#999;text-decoration:none}
#qrcode .qrcode-text p{text-align:center}
#qrcode .qrcode-text b{color:#666;font-weight:700}
#qrcode .qrcode-text span{letter-spacing:1px}
#qrcode .qrcode-text .title{color:#333;font-size:16px;line-height:32px;letter-spacing:3px}
#qrcode .qrcode-text .sub-title{color:#999;font-size:14px;line-height:14px;font-weight:300}
#ftConw{display:inline-block;text-align:left;margin-left:33px;line-height:22px;position:relative;top:-2px;*float:right;*margin-left:0;*position:static}
#ftConw,#ftConw a{color:#999}
#ftConw{text-align:center;margin-left:0}
.bg{background-image:url(https://ss1.bdstatic.com/5eN1bjq8AAUYm2zgoY3K/r/www/cache/static/protocol/https/global/img/icons_441e82f.png);background-repeat:no-repeat;_background-image:url(https://ss1.bdstatic.com/5eN1bjq8AAUYm2zgoY3K/r/www/cache/static/protocol/https/global/img/icons_d5b04cc.gif)}
.c-icon{display:inline-block;width:14px;height:14px;vertical-align:text-bottom;font-style:normal;overflow:hidden;background:url(https://ss1.bdstatic.com/5eN1bjq8AAUYm2zgoY3K/r/www/cache/static/protocol/https/global/img/icons_441e82f.png) no-repeat 0 0;_background-image:url(https://ss1.bdstatic.com/5eN1bjq8AAUYm2zgoY3K/r/www/cache/static/protocol/https/global/img/icons_d5b04cc.gif)}
.c-icon-triangle-down-blue{background-position:-480px -168px}
.c-icon-chevron-unfold2{background-position:-504px -168px}
#m{width:720px;margin:0 auto}
#nv a,#nv b,.btn,#lk{font-size:14px}
#nv{height:19px;font-size:16px;margin:0 0 4px;text-align:left;text-indent:137px}
.s_btn{width:95px;height:32px;padding-top:2px\9;font-size:14px;background-color:#ddd;background-position:0 -48px;cursor:pointer}
.s_btn_h{background-position:-240px -48px}
.s_btn_wr{width:97px;height:34px;display:inline-block;background-position:-120px -48px;*position:relative;z-index:0;vertical-align:top}
#jgwab{margin-left:19px}
#cp .c-icon-icrlogo,.c-icon-jgwablogo{width:14px;height:17px;display:inline-block;overflow:hidden;background:url(https://ss1.bdstatic.com/5eN1bjq8AAUYm2zgoY3K/r/www/cache/static/protocol/https/global/img/icons_441e82f.png) no-repeat;_background-image:url(https://ss1.bdstatic.com/5eN1bjq8AAUYm2zgoY3K/r/www/cache/static/protocol/https/global/img/icons_d5b04cc.gif)}
#cp .c-icon-icrlogo{background-position:-600px -96px;position:relative;top:3px}
.c-icon-jgwablogo{background-position:-623px -96px;position:relative;top:3px}
#shouji{margin-right:14px}
#u{display:none}
#c-tips-container{display:none}
#wrapper{min-width:810px;height:100%;min-height:600px}
#head{position:relative;padding-bottom:0;height:100%;min-height:600px}
#head .head_wrapper{height:100%}
#m{position:relative}
#fm{padding-left:40px;top:-37px}
#lh a{margin-left:62px}
#lh #seth,#lh #setf{margin-left:0}
#lk{position:absolute;display:none;top:0;right:0;margin:33px 0}
#lk span{font:14px "宋体"}
#nv{position:absolute;display:none;top:0;right:0}
#lm{color:#666;width:100%;height:60px;margin-top:60px;line-height:15px;font-size:13px;position:absolute;top:0;left:0}
#lm a{color:#666}
#pad-version{line-height:40px}
.s_ipt_wr.bg,.s_btn_wr.bg,#su.bg{background-image:none}
.s_btn_wr{width:auto;height:auto;border-bottom:1px solid transparent;*border-bottom:0}
.s_btn{width:100px;height:36px;color:#fff;font-size:15px;letter-spacing:1px;background:#3385ff;border-bottom:1px solid #2d78f4;outline:medium;*border-bottom:0;-webkit-appearance:none;-webkit-border-radius:0}
.s_btn.btnhover{background:#317ef3;border-bottom:1px solid #2868c8;*border-bottom:0;box-shadow:1px 1px 1px #ccc}
.s_btn_h{background:#3075dc;box-shadow:inset 1px 1px 5px #2964bb;-webkit-box-shadow:inset 1px 1px 5px #2964bb;-moz-box-shadow:inset 1px 1px 5px #2964bb;-o-box-shadow:inset 1px 1px 5px #2964bb}
#result_logo{display:none}
#index_logo img{display:inline-block;width:270px;height:129px}
#s_tab{display:none}
.s_form{position:relative;top:38.2%}
.s_form_wrapper{position:relative;top:-191px}
.s_ipt_wr{height:34px}
#head .c-icon-bear-round{display:none}
#form{margin:22px auto 0;width:641px;text-align:left;z-index:100}
#form .bdsug,#fm .bdsug{top:35px}
.bdsug{display:none;position:absolute;width:538px;background:#fff;border:1px solid #ccc;_overflow:hidden;box-shadow:1px 1px 3px #ededed;-webkit-box-shadow:1px 1px 3px #ededed;-moz-box-shadow:1px 1px 3px #ededed;-o-box-shadow:1px 1px 3px #ededed}
.bdsug.bdsugbg ul{background:url(https://ss1.bdstatic.com/5eN1bjq8AAUYm2zgoY3K/r/www/cache/static/protocol/https/home/img/sugbg_1762fe7.png) 100% 100% no-repeat;background-size:100px 110px;background-image:url(https://ss1.bdstatic.com/5eN1bjq8AAUYm2zgoY3K/r/www/cache/static/protocol/https/home/img/sugbg_90fc9cf.gif)\9}
.bdsug li{width:522px;color:#000;font:14px arial;line-height:25px;padding:0 8px;position:relative;cursor:default}
.bdsug li.bdsug-s{background:#f0f0f0}
.bdsug-store span,.bdsug-store b{color:#7A77C8}
.bdsug-store-del{font-size:12px;color:#666;text-decoration:underline;position:absolute;right:8px;top:0;cursor:pointer;display:none}
.bdsug-s .bdsug-store-del{display:inline-block}
.bdsug-ala{display:inline-block;border-bottom:1px solid #e6e6e6}
.bdsug-ala h3{line-height:14px;background:url(//www.baidu.com/img/sug_bd.png) no-repeat left center;margin:8px 0 5px;font-size:12px;font-weight:400;color:#7B7B7B;padding-left:20px}
.bdsug-ala p{font-size:14px;font-weight:700;padding-left:20px}
.bdsug .bdsug-direct{width:auto;padding:0;border-bottom:1px solid #f1f1f1}
.bdsug .bdsug-direct p{color:#00c;font-weight:700;line-height:34px;padding:0 8px;cursor:pointer;white-space:nowrap;overflow:hidden}
.bdsug .bdsug-direct p img{width:16px;height:16px;margin:7px 6px 9px 0;vertical-align:middle}
.bdsug .bdsug-direct p span{margin-left:8px}
.bdsug .bdsug-direct p i{font-size:12px;line-height:100%;font-style:normal;font-weight:400;color:#fff;background-color:#2b99ff;display:inline;text-align:center;padding:1px 5px;*padding:2px 5px 0;margin-left:8px;overflow:hidden}
.bdsug .bdsug-pcDirect{color:#000;font-size:14px;line-height:30px;height:30px;background-color:#f8f8f8}
.bdsug .bdsug-pc-direct-tip{position:absolute;right:15px;top:8px;width:55px;height:15px;display:block;background:url(https://ss1.bdstatic.com/5eN1bjq8AAUYm2zgoY3K/r/www/cache/static/protocol/https/global/img/pc_direct_42d6311.png) no-repeat 0 0}
.bdsug li.bdsug-pcDirect-s{background-color:#f0f0f0}
.bdsug .bdsug-pcDirect-is{color:#000;font-size:14px;line-height:22px;background-color:#f8f8f8}
.bdsug .bdsug-pc-direct-tip-is{position:absolute;right:15px;top:3px;width:55px;height:15px;display:block;background:url(https://ss1.bdstatic.com/5eN1bjq8AAUYm2zgoY3K/r/www/cache/static/protocol/https/global/img/pc_direct_42d6311.png) no-repeat 0 0}
.bdsug li.bdsug-pcDirect-is-s{background-color:#f0f0f0}
.bdsug .bdsug-pcDirect-s .bdsug-pc-direct-tip,.bdsug .bdsug-pcDirect-is-s .bdsug-pc-direct-tip-is{background-position:0 -15px}
.bdsug .bdsug-newicon{color:#929292;opacity:.7;font-size:12px;display:inline-block;line-height:22px;letter-spacing:2px}
.bdsug .bdsug-s .bdsug-newicon{opacity:1}
.bdsug .bdsug-newicon i{letter-spacing:0;font-style:normal}
.bdsug .bdsug-feedback-wrap{text-align:right;background:#fafafa;color:#666;height:25px;line-height:25px;display:none}
.bdsug .bdsug-feedback{margin-right:10px;text-decoration:underline;color:#666}
.toggle-underline{text-decoration:none}
.toggle-underline:hover{text-decoration:underline}
.tools{position:absolute;right:-75px}
#mHolder{width:62px;position:relative;z-index:296;display:none}
#mCon{height:18px;line-height:18px;position:absolute;cursor:pointer}
#mCon span{color:#00c;display:block;width:24px}
#mCon .hw{text-decoration:underline;cursor:pointer;display:inline-block}
#mCon .pinyin{display:inline-block}
#mCon .c-icon-chevron-unfold2{margin-left:5px}
#mMenu a{width:100%;height:100%;display:block;line-height:22px;text-indent:6px;text-decoration:none;filter:none\9}
#mMenu,#user ul{box-shadow:1px 1px 2px #ccc;-moz-box-shadow:1px 1px 2px #ccc;-webkit-box-shadow:1px 1px 2px #ccc;filter:progid:DXImageTransform.Microsoft.Shadow(Strength=2, Direction=135, Color=#cccccc)\9}
#mMenu{width:56px;border:1px solid #9b9b9b;list-style:none;position:absolute;right:27px;top:28px;display:none;background:#fff}
#mMenu a:hover{background:#ebebeb}
#mMenu .ln{height:1px;background:#ebebeb;overflow:hidden;font-size:1px;line-height:1px;margin-top:-1px}
#u1 a:link,#u1 a:visited{color:#666;text-decoration:none}
#u1 a:hover,#u1 a:active{text-decoration:underline}
#u1 a:active{color:#00c}
#u1{z-index:2;color:#fff;position:absolute;right:0;top:0;margin:19px 0 5px;padding:0 96px 0 0}
#u1 .reg{display:none}
#u1 a.pf,#u1 a.pf:visited{display:inline-block;float:left;color:#333;line-height:24px;font-size:13px;margin-left:20px;overflow:hidden;text-decoration:underline}
#u1 a.lb,#u1 a.lb:visited,#u1 a.username{display:inline-block;float:left;color:#333;font-size:13px;line-height:24px;margin-left:20px;text-decoration:underline}
#u1 a.bri,#u1 a.bri:visited{display:inline-block;position:absolute;right:10px;width:60px;height:23px;float:left;color:#fff;background:#38f;line-height:24px;font-size:13px;text-align:center;overflow:hidden;border-bottom:1px solid #38f;margin-left:19px;margin-right:2px}
#u1 a.bri.brihover{display:none;text-decoration:none;color:#333;background:0 0;border-bottom:1px solid transparent;margin-left:19px}
#u1 #lm a{color:#00c;text-decoration:underline}
#u1 a.mnav,#u1 a.mnav:visited{float:left;color:#333;font-weight:700;line-height:24px;margin-left:20px;font-size:13px;text-decoration:underline}
#u1 a.pf:hover,#u1 a.lb:hover,#u1 a.mnav:hover{color:#00c}
.briiconsbg{background-repeat:no-repeat;background-size:300px 18px;background-image:url(https://ss1.bdstatic.com/5eN1bjq8AAUYm2zgoY3K/r/www/cache/static/protocol/https/home/img/icons_0c37e9b.png);background-image:url(https://ss1.bdstatic.com/5eN1bjq8AAUYm2zgoY3K/r/www/cache/static/protocol/https/home/img/icons_809ae65.gif)\9}
.bdpfmenu{background-color:#fff;border:1px solid #d1d1d1;position:absolute;right:160px;width:68px;top:36px;margin-top:-1px;_margin-top:-3px;z-index:2;box-shadow:1px 1px 5px #d1d1d1;-webkit-box-shadow:1px 1px 5px #d1d1d1;-moz-box-shadow:1px 1px 5px #d1d1d1;-o-box-shadow:1px 1px 5px #d1d1d1}
.bdpfmenu a{display:block;text-align:left;margin:0!important;padding:0 9px;line-height:26px;text-decoration:none}
#wrapper .bdpfmenu a:link,#wrapper .bdpfmenu a:visited{background:#fff;color:#333}
#wrapper .bdpfmenu a:hover,#wrapper .bdpfmenu a:active{background:#38f;text-decoration:none;color:#fff}
#wrapper .bdnuarrow{width:0;height:0;font-size:0;line-height:0;display:block;position:absolute;top:-10px;left:50%;margin-left:-5px}
#wrapper .bdnuarrow em,#wrapper .bdnuarrow i{width:0;height:0;font-size:0;line-height:0;display:block;position:absolute;border:5px solid transparent;border-style:dashed dashed solid}
#wrapper .bdnuarrow em{border-bottom-color:#d8d8d8;top:-1px}
#wrapper .bdnuarrow i{border-bottom-color:#fff;top:0}
.s-isindex-wrap #wrapper .bdnuarrow{height:13px;background:url(https://ss1.bdstatic.com/5eN1bjq8AAUYm2zgoY3K/r/www/cache/static/protocol/https/home/img/icons_0c37e9b.png) no-repeat -90px -1px}
#wrapper .bdnuarrow.bdbriarrow{right:104px;display:none!important}
#wrapper .bdbri{width:85px;min-height:100px;border-left:1px solid #e7e7e7;position:absolute;background-color:#f9f9f9;overflow:hidden;z-index:2;right:0;top:0}
#prefpanel{background:#fafafa;display:none;opacity:0;position:fixed;_position:absolute;top:-359px;z-index:1000;width:100%;min-width:960px;border-bottom:1px solid #ebebeb}
#prefpanel form{_width:850px}
#wrapper .bdbriimgtitle{color:#333;text-align:center;width:66px;height:43px;line-height:43px;padding-top:9px;margin:0 auto;border-bottom:#f0f0f0 1px solid;font-size:13px;cursor:default}
#wrapper .briscrollwrapper{overflow:hidden}
#wrapper .briscrollwrapperContainer{position:relative}
#wrapper .bdbri.bdbriimg .bdmainlink a,#wrapper .bdbri.bdbriimg .bdothlink a{display:block;text-align:center;width:66px;height:76px;margin:0 auto;border-bottom:#f0f0f0 1px solid;color:#666;text-decoration:none;overflow:hidden}
#wrapper .bdbri.bdbriimg .bdmainlink a:visited,#wrapper .bdbri.bdbriimg .bdothlink a:visited{color:#666}
#wrapper .bdbri.bdbriimg .bdmainlink a:hover,#wrapper .bdbri.bdbriimg .bdothlink a:hover{color:#666;text-decoration:underline}
#wrapper .bdbri.bdbriimg .bdmainlink a:active,#wrapper .bdbri.bdbriimg .bdothlink a:active{color:#00c;text-decoration:underline}
#wrapper .bdbri.bdbriimg span{width:36px;height:36px;display:block;margin:10px auto 5px;background:url(https://ss1.bdstatic.com/5eN1bjq8AAUYm2zgoY3K/r/www/cache/static/protocol/https/home/img/logos/bdbri_icons_194ae06.png) no-repeat;cursor:pointer}
#wrapper .bdbri.bdbriimg .bdbrimore,#wrapper .bdbri.bdbriimg .bdbrievenmore{clear:both;text-align:center}
#wrapper .bdbri.bdbriimg .bdbrievenmore{margin-top:15px;height:30px;width:85px;overflow:hidden}
#wrapper .bdbri.bdbriimg span.bdbriimgitem_1{background:url(https://ss1.bdstatic.com/5eN1bjq8AAUYm2zgoY3K/r/www/cache/static/protocol/https/home/img/logos/nuomi_7f5cd35.png) no-repeat;background-size:cover}
#wrapper .bdbri.bdbriimg span.bdbriimgitem_2{background:url(https://ss1.bdstatic.com/5eN1bjq8AAUYm2zgoY3K/r/www/cache/static/protocol/https/home/img/logos/zhidao_cbf2aff.png) no-repeat;background-size:cover}
#wrapper .bdbri.bdbriimg span.bdbriimgitem_3{width:36px;background:url(https://ss1.bdstatic.com/5eN1bjq8AAUYm2zgoY3K/r/www/cache/static/protocol/https/home/img/logos/qqjt_9809ca8.png) no-repeat;background-size:cover}
#wrapper .bdbri.bdbriimg span.bdbriimgitem_4{background:url(https://ss1.bdstatic.com/5eN1bjq8AAUYm2zgoY3K/r/www/cache/static/protocol/https/home/img/logos/image_55b5909.png) no-repeat;background-size:cover}
#wrapper .bdbri.bdbriimg span.bdbriimgitem_5{background:url(https://ss1.bdstatic.com/5eN1bjq8AAUYm2zgoY3K/r/www/cache/static/protocol/https/home/img/logos/wenku_aaf198d.png) no-repeat;background-size:cover}
#wrapper .bdbri.bdbriimg span.bdbriimgitem_6{background:url(https://ss1.bdstatic.com/5eN1bjq8AAUYm2zgoY3K/r/www/cache/static/protocol/https/home/img/logos/fengyunbang_1986a40.png) no-repeat;background-size:cover}
#wrapper .bdbri.bdbriimg span.bdbriimgitem_7{background-position:-220px 0}
#wrapper .bdbri.bdbriimg .bdbrievenmore a:link,#wrapper .bdbri.bdbriimg .bdbrievenmore a:visited{color:#666;text-decoration:underline}
#wrapper .bdbri.bdbriimg .bdbrievenmore a:hover{color:#666;text-decoration:underline}
#wrapper .bdbri.bdbriimg .bdbrievenmore a:active{color:#00c}
.bdbriscroll-ctrl-scroll{position:absolute;top:10px;right:1px;width:8px;border-top:1px solid #e4e4e4;border-left:1px solid #e4e4e4;cursor:default;-webkit-user-select:none;-moz-user-select:none}
.bdbriscroll-ctrl-scroll .bdbriscroll-axis{width:8px;left:0;z-index:0;position:absolute;background:#f2f2f2}
.bdbriscroll-ctrl-scroll-touch .bdbriscroll-axis{width:7px;background:#f2f2f2}
.bdbriscroll-ctrl-scroll-hover .bdbriscroll-axis{background:#f2f2f2}
.bdbriscroll-ctrl-scroll .bdbriscroll-slider{overflow:hidden;width:7px;height:14px;position:absolute;left:0;z-index:10;display:none;background:#d9d9d9;margin-top:-1px;margin-left:-1px;border-right:1px solid #cecece;border-bottom:1px solid #cecece;cursor:default}
.bdbriscroll-ctrl-scroll-touch .bdbriscroll-slider,.bdbriscroll-ctrl-scroll-hover .bdbriscroll-slider{background:#b8b8b8;border-right:1px solid #afafaf;border-bottom:1px solid #afafaf}
.s_ipt::-webkit-input-placeholder{padding-left:3px;color:#aaa;font-size:13px}
.s_ipt::-moz-placeholder{padding-left:3px;color:#aaa;font-size:13px}
.s_ipt:-ms-input-placeholder{padding-left:3px;color:#aaa;font-size:13px}
.s_ipt::placeholder{padding-left:3px;color:#aaa;font-size:13px}
.kw-placeholder{position:absolute;top:0;left:0;color:#aaa;font-size:13px;height:35px;line-height:35px;padding-left:10px;max-width:360px;z-index:99;pointer-events:none}
.kw-placeholder.placeholders-hidden{visibility:hidden}
.qrcode-chunwan{width:530px;height:90px;position:absolute;left:50%;margin-left:-265px;z-index:1;bottom:115px;background-size:100%;background-repeat:no-repeat;background-position:center}
.qrcode-chunwan .close-chunwan{width:23px;height:23px;position:absolute;top:0;right:0;display:none;_display:block;background-size:100%}
.qrcode-chunwan:hover .close-chunwan{display:block}</style>

<!--[if lte IE 8]>
<style index="index" data-compress="strip">
.s_form{top:260px}
</style>
<![endif]-->
<!--[if IE 8]>
<style index="index" data-compress="strip">
#u1 a.mnav,#u1 a.mnav:visited,#u1 a.lb,#u1 a.lb:visited,#u1 a.pf,#u1 a.pf:visited,#u1 a.bri,#u1 a.bri:visited{font-family:simsun;}
</style>
<![endif]-->
<style data-for="debug">
#debug{display:none!important;}
</style>
<style data-for="result" id="css_index_result" type="text/css">#seth{display:inline;behavior:url(#default#homepage)}
#setf{display:inline}
#sekj{margin-left:14px}
#st,#sekj{display:none}
.s_ipt_wr{border:1px solid #b6b6b6;border-color:#7b7b7b #b6b6b6 #b6b6b6 #7b7b7b;background:#fff;display:inline-block;vertical-align:top;width:539px;margin-right:0;border-right-width:0;border-color:#b8b8b8 transparent #ccc #b8b8b8;overflow:hidden}
.wrapper_s .s_ipt_wr{width:439px}
.wrapper_s .s_ipt{width:434px}
.wrapper_s .s_ipt_tip{width:434px}
.s_ipt_wr:hover,.s_ipt_wr.ipthover{border-color:#999 transparent #b3b3b3 #999}
.s_ipt_wr.iptfocus{border-color:#4791ff transparent #4791ff #4791ff}
.s_ipt_tip{color:#aaa;position:absolute;z-index:-10;font:16px/22px arial;height:32px;line-height:32px;padding-left:7px;overflow:hidden;width:526px}
.s_ipt{width:526px;height:22px;font:16px/18px arial;line-height:22px;margin:6px 0 0 7px;padding:0;background:transparent;border:0;outline:0;-webkit-appearance:none}
#kw{position:relative}
#u .username i{background-position:-408px -144px}
.bdpfmenu,.usermenu{border:1px solid #d1d1d1;position:absolute;width:105px;top:36px;z-index:302;box-shadow:1px 1px 5px #d1d1d1;-webkit-box-shadow:1px 1px 5px #d1d1d1;-moz-box-shadow:1px 1px 5px #d1d1d1;-o-box-shadow:1px 1px 5px #d1d1d1}
.bdpfmenu{font-size:12px;background-color:#fff}
.bdpfmenu a,.usermenu a{display:block;text-align:left;margin:0!important;padding:0 9px;line-height:26px;text-decoration:none}
.briiconsbg{background-repeat:no-repeat;background-size:300px 18px;background-image:url(https://ss1.bdstatic.com/5eN1bjq8AAUYm2zgoY3K/r/www/cache/static/protocol/https/home/img/icons_0c37e9b.png);background-image:url(https://ss1.bdstatic.com/5eN1bjq8AAUYm2zgoY3K/r/www/cache/static/protocol/https/home/img/icons_809ae65.gif)\9}
#u{z-index:301;position:absolute;right:0;top:0;margin:21px 9px 5px 0;padding:0}
.wrapper_s #u{margin-right:3px}
#u a{text-decoration:underline;color:#333;margin:0 7px}
.wrapper_s #u a{margin-right:0 6px}
#u div a{text-decoration:none}
#u a:hover{text-decoration:underline}
#u .back_org{color:#666;float:left;display:inline-block;height:24px;line-height:24px}
#u .bri{display:inline-block;width:24px;height:24px;float:left;line-height:24px;color:transparent;background:url(https://ss1.bdstatic.com/5eN1bjq8AAUYm2zgoY3K/r/www/cache/static/protocol/https/home/img/icons_0c37e9b.png) no-repeat 4px 3px;background-size:300px 18px;background-image:url(https://ss1.bdstatic.com/5eN1bjq8AAUYm2zgoY3K/r/www/cache/static/protocol/https/home/img/icons_809ae65.gif)\9;overflow:hidden}
#u .bri:hover,#u .bri.brihover{background-position:-18px 3px}
#mCon #imeSIcon{background-position:-408px -144px;margin-left:0}
#mCon span{color:#333}
.bdpfmenu a:link,.bdpfmenu a:visited,#u .usermenu a:link,#u .usermenu a:visited{background:#fff;color:#333}
.bdpfmenu a:hover,.bdpfmenu a:active,#u .usermenu a:hover,#u .usermenu a:active{background:#38f;text-decoration:none;color:#fff}
.bdpfmenu{width:70px}
.usermenu{width:68px;right:8px}
#wrapper .bdnuarrow{width:0;height:0;font-size:0;line-height:0;display:block;position:absolute;top:-10px;left:50%;margin-left:-5px}
#wrapper .bdnuarrow em,#wrapper .bdnuarrow i{width:0;height:0;font-size:0;line-height:0;display:block;position:absolute;border:5px solid transparent;border-style:dashed dashed solid}
#wrapper .bdnuarrow em{border-bottom-color:#d8d8d8;top:-1px}
#wrapper .bdnuarrow i{border-bottom-color:#fff;top:0}
#prefpanel{background:#fafafa;display:none;opacity:0;position:fixed;_position:absolute;top:-359px;z-index:500;width:100%;min-width:960px;border-bottom:1px solid #ebebeb}
#prefpanel form{_width:850px}
#kw_tip{cursor:default;display:none;margin-top:1px}
#bds-message-wrapper{top:43px}
.quickdelete-wrap{position:relative}
.quickdelete-wrap input{width:500px}
.wrapper_l .quickdelete-wrap input{width:500px}
.wrapper_s .quickdelete-wrap input{width:402px}
input::-ms-clear{display:none}
.quickdelete{width:32px;height:32px;background:url(https://ss1.bdstatic.com/5eN1bjq8AAUYm2zgoY3K/r/www/cache/static/protocol/https/global/img/quickdelete_33e3eb8.png) no-repeat;background-position:10px 10px;position:absolute;display:block}
.quickdelete:hover{background-position:10px -24px}
#lh a{margin-left:25px}
.bdbriwrapper-tuiguang{display:none!important}</style>
<!--[if IE 8]>
<style index="index" data-compress="strip">
.s_ipt{background-color:#FFF;}
</style>
<![endif]-->

    
<script data-compress="strip">
function h(obj){
    obj.style.behavior='url(#default#homepage)';
	var a = obj.setHomePage('//www.baidu.com/');
}
</script>

<noscript>
    <meta http-equiv="refresh" content="0; url=/baidu.html?from=noscript"/>
</noscript>

    <script>window._ASYNC_START=new Date().getTime();</script>
</head>

<body link="#0000cc">
	<script>
	if (/Chrome\/37.0.2062.94/i.test(navigator.userAgent) && (/(windows 7)|(windows nt 6.1)/i.test(navigator.userAgent))) {
		var _chrome_37_fix = document.createElement("style");
		_chrome_37_fix.type="text/css";
		_chrome_37_fix.setAttribute("data-for","result");
		_chrome_37_fix.innerHTML = ".t,.f16,#kw,.s_ipt,.c-title,.c-title-size,.to_zhidao,.to_tieba,.to_zhidao_bottom{font-size:15px;} .ec-hospital-info-main h2,.ad-widget-gx_sck-ylzx-doctor-info h2,.ec-card-main h2,.ad-widget-h1 h2,.ad-widget-title h2,.ad-widget-small-head h2,.ad-widget-small-head h2 a,.ad-widget-header .ec-figcaption h2{font-size: 15px !important;}";
		document.getElementsByTagName("head")[0].appendChild(_chrome_37_fix);
	}
	</script>
    <div id="wrapper" style="display:none;">
        


        


<script>if(window.bds&&bds.util&&bds.util.setContainerWidth){bds.util.setContainerWidth();}</script><div id="head"><div class="head_wrapper"><div class="s_form"><div class="s_form_wrapper"><style>.index-logo-srcnew {display: none;}@media (-webkit-min-device-pixel-ratio: 2),(min--moz-device-pixel-ratio: 2),(-o-min-device-pixel-ratio: 2),(min-device-pixel-ratio: 2){.index-logo-src {display: none;}.index-logo-srcnew {display: inline;}}</style><div id="lg"><img hidefocus="true" class='index-logo-src' src="//www.baidu.com/img/bd_logo1.png" width="270" height="129" usemap="#mp"><img hidefocus="true" class='index-logo-srcnew' src="//www.baidu.com/img/bd_logo1.png?qua=high" width="270" height="129" usemap="#mp"><map name="mp"><area style="outline:none;" hidefocus="true" shape="rect" coords="0,0,270,129" href="//www.baidu.com/s?wd=%E4%BB%8A%E6%97%A5%E6%96%B0%E9%B2%9C%E4%BA%8B&tn=SE_PclogoS_8whnvm25&sa=ire_dl_gh_logo&rsv_dl=igh_logo_pcs" onmousedown="return ns_c({fm: 'tab', tab: 'felogo', rsv_platform: 'wwwhome' })" target

最低0.47元/天解锁文章

紫色的梅花

关注

0
点赞
踩
0

收藏

觉得还不错? 一键收藏
0
评论
urllib爬虫封装（可设置代理，记录日志）方法

urllib爬虫封装（具体其他功能根据需求完善）1.支持设置HTTP Request Headers ,能设置UA。2.支持代理服务器的设置。3.它支持timeout超时机制。4.它支持网页的编码指定。5.它支持服务器返回的错误处理如果>400 and <500 直接记录日志，如果错误码是500-600 则重新发起请求。# -*- coding: ...
复制链接

扫一扫

专栏目录