多线程下载图片

import sys
import requests
import threading
import time


def download_image(image_url):
    try:
        r = requests.get(image_url)
        image_name = "/Users/Desktop/pictures/" + image_url.split("/")[-1]
        with open(image_name,'wb') as f:
            f.write(r.content)
    except Exception as e:
        print(e)


image_url_list = []

for line in sys.stdin:
    image_url = line.strip()
    while threading.active_count()>200:
        time.sleep(0.01)
    thread = threading.Thread(target=download_image, args=(image_url,))
    thread.start()

while threading.active_count()>1:
    time.sleep(0.01)



另外的写法

#coding:utf-8
import sys
import csv
import threading
import time
import requests
import os
import glob

lock = threading.Lock()


def download(line, save_dir, i, writer):
    url=line[2]
    name = os.path.join(save_dir, "{0:02d}/{1:08d}.jpg".format(i//1000000, i))
    if not os.path.isfile(name):
        res = requests.get(url,timeout=3)
        if 200 != res.status_code:
            return
        with open(name, "wb") as f:
            f.write(res.content)
    with lock:
        obj = line+[name]
        writer.writerow(obj)

for i in range(13):
    csv_name ="csv/train-{0:05d}-of-00013.csv".format(i)
    if not os.path.isfile(csv_name):
        print("{0} not exists!".format(csv_name))
        continue
    print(csv_name)
    save_dir = os.path.join("data", csv_name.split("/")[-1].split(".")[0])
    print (save_dir)
    if not os.path.isdir(save_dir):
        os.mkdir(save_dir)
    out_csv = csv_name.replace(".csv", "_out.csv")
    print (out_csv)
    if os.path.isfile(out_csv):
        print ("out_csv exists!")
        continue
        #exit(0)
    for k in range(20):
        p = "{0}/{1:02d}".format(save_dir, k)
        if not os.path.isdir(p):
            os.makedirs(p)
    csvfile = open(out_csv,"w")
    writer = csv.writer(csvfile)

    lines = csv.reader(open(csv_name))

    for j,line in enumerate(lines):
        while threading.active_count()>200:
            time.sleep(0.01)

        thread = threading.Thread(target=download, args=(line,save_dir, j, writer))
        thread.start()
while threading.active_count()>1:
    time.sleep(0.01)