python批量_python批量下载

以下载图片为例：

from gevent import monkey

monkey.patch_all()

from gevent.pool import Pool

from urllib.parse import urlparse

import requests

import sys

import os

def download(url):

chrome = 'Mozilla/5.0 (X11; Linux i86_64) AppleWebKit/537.36 ' + '(KHTML, like Gecko) Chrome/41.0.2272.101 Safari/537.36'

headers = {'User-Agent': chrome}

r = requests.get(url.strip(), headers=headers, stream=True)

output_dir = "pic" # 图片存放文件夹

# 判断输出文件夹是否存在。不存在就创建

if not os.path.isdir(output_dir):

os.makedirs(output_dir)

url_parse = urlparse(url) # 图片来源url

fname = os.path.split(url_parse.path)[1] # 图片名

filename = output_dir + os.sep + fname # 图片存放位置

print("开始下载：%s" % fname)

with open(filename, 'wb')as f:

f.write(r.content) # 保存图片

print("下载完成：%s" % fname)

if __name__ == "__main__":

if len(sys.argv) == 2:

filename = sys.argv[1]

f = open(filename, "r")

p = Pool(100)

for line in f.readlines():

if line:

p.spawn(download, line.strip())

f.close()

p.join()

else:

print('Usage: python %s urls.txt' % sys.argv[0])

python  downloader.py test.txt