python 下载文件-Python实现批量下载文件

Python实现批量下载文件

#!/usr/bin/env python

# -*- coding:utf-8 -*-

from gevent import monkey

monkey.patch_all()

from gevent.pool import Pool

import requests

import sys

import os

def download(url):

chrome = 'Mozilla/5.0 (X11; Linux i86_64) AppleWebKit/537.36 ' +

'(KHTML, like Gecko) Chrome/41.0.2272.101 Safari/537.36'

headers = {'User-Agent': chrome}

filename = url.split('/')[-1].strip()

r = requests.get(url.strip(), headers=headers, stream=True)

with open(filename, 'wb') as f:

for chunk in r.iter_content(chunk_size=1024):

if chunk:

f.write(chunk)

f.flush()

print filename,"is ok"

def removeLine(key, filename):

os.system('sed -i /%s/d %s' % (key, filename))

if __name__ =="__main__":

if len(sys.argv) &