urllib2 gzipped gzip压缩

urllib2 gzipped gzip压缩

    #rr=httppool[getdomain(url)[0]].urlopen("get",getdomain(url)[1],retries=1,headers=headers).data

    def do(url):

        try:

            return httppool[getdomain(url)[0]].get_url(getdomain(url)[1],retries=1,headers=headers).data

        except urllib3.HTTPError,e:

            print "在获取url=%s的源码时出现了异常:%s"%(url,e)

            return ""

    

    rr = with_timeout(5, do, url, timeout_value="")

    ee=datetime.now()

    print "cost time--------------------------:%s"%(ee-ss).seconds

    #rr=httppool[getdomain(url)[0]].get_url(getdomain(url)[1],retries=1,headers=headers)

    pdata = StringIO.StringIO(rr)#下面6行是实现解压缩

    gzipper = gzip.GzipFile(fileobj = pdata)

    isUseGzip=True

    try:

        data = gzipper.read()

    except(IOError):

        isUseGzip=False

        print 'unused gzip'

        data = rr#当有的服务器不支持gzip格

    #print data

    

    print "isUseGzip:%s"%isUseGzip

    return data

http://man.lupaworld.com/content/develop/diveintopython-html-5.4/diveintopython-5.4/http_web_services/gzip_compression.html

posted on 2010-10-16 08:48  lexus 阅读( ...) 评论( ...) 编辑 收藏

转载于:https://www.cnblogs.com/lexus/archive/2010/10/16/1852885.html