# -*- coding: UTF-8 -*-
import requests
import re
import os
import cv2
import argparse
ap = argparse.ArgumentParser()
ap.add_argument("-w", "--word", required=False, default="皮卡丘",
help="image search word")
ap.add_argument("-o", "--output", required=False,
help="path to output directory of images")
ap.add_argument("-s", "--size", required=False, type=int, default=300,
help="search image size")
args = vars(ap.parse_args())
def request_get(url):
r = requests.get(url, headers=get_headers(), timeout=30)
r.encoding = 'utf-8'
return r
def get_headers():
headers = {
}
return headers
def get_url(word):
return ""
def get_url_limit(word):
return ""
def read_txt(file_path):
fo = open(file_path, "r+", encoding='utf-8')
data = fo.read()
fo.close()
return data
def get_urls(txt):
urls = []
# print(urls)
return urls
def get_txt(r):
return r.text
def append(srcs, lists):
for l in lists:
srcs.append(l)
return srcs
def get_image_urls(word, size):
page_size = 30;
urls = []
if size <= page_size:
urls = get_urls(get_txt(request_get(get_url(word))))
else :
us = get_urls(get_txt(request_get(get_url(word))))
urls = append(urls, us)
# print(int(((size - page_size) / page_size)))
for n in range(int((size / page_size))) :
us = get_urls(get_txt(request_get(get_url_limit(word, n * page_size, page_size))))
urls = append(urls, us)
return list(set(urls))
def save_images(path, urls):
print(path, len(urls))
if os.path.exists(path) == False:
os.makedirs(path)
for n in range(len(urls)):
try:
# print(n)
save_image(path, n, urls[n])
except Exception as e :
print(e, urls[n])
def save_image(path, n, url):
r = requests.get(url, timeout=30)
ext = ".jpg"
p = os.path.sep.join([path, "{}{}".format(
str(n).zfill(8), ext)])
f = open(p, "wb")
f.write(r.content)
f.close()
# check_image(p)
def check_image(p):
image = cv2.imread(p)
if image is None:
print("deleting: {}".format(p))
os.remove(p)
def get_images(word, size):
return get_images_by_path(word, os.getcwd() + os.sep + word, size)
def get_images_by_path(word, path, size):
urls = get_image_urls(word, size)
save_images(path, urls)
if args["output"] is None :
get_images(args["word"], args["size"])
else :
get_images_by_path(args["word"], args["output"], args["size"])
需要完整代码(抓取网站),请私信