我抓取的是美女模特这一栏
执行程序
import requests
import time
import os
from multiprocessing import Pool,cpu_count,current_process,Process
from lxml import etree
import parsel
import re
from PIL import Image
from io import BytesIO
timeout = 10
# 下载图片保存路径
DIR_PATH = r"d:\meizi\明星模特"
header = \
{
"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36"
}
# 获取每一页的地址
def get_page_num():
r = requests.get(url=base_url+'/bz/nxxz/nxmt/',timeout=10,headers=header)
html = parsel.Selector(r.text)
list=html.xpath('//span[@class="ea"]/select[@name="sldd"]/option/@value').extract()
return list
# 组装每一页上每个item的地址
def get_page_detail(list):
page_list=[]
for i in list:
url =base_url+'/bz/nxxz/nxmt/'+i
r = requests.get(url=url,timeout=10,headers=header)
html = parsel.Selector(r.text)
list=html.xpath('//ul[@class="picbz"]/li/a[@target="_blank"]/@href').extract()
nums=html.xpath('//ul[@class="picbz"]/li/text()').extract()
total=[]
for d in nums:
if re.findall(r"\d+\.?\d*",d):
total.append(re.findall(r"\d+\.?\d*",d)[0])
for ind, i in enumerate(list):
id = re.findall(r"\d+",i)[0]
for i3 in range(int(total[ind])):
if i3 !=0:
s = f'{id}_{i3+1}'
page_list.append(i.replace(id,s))
else:
page_list.append(i)
return page_list
# 获取原图地址
def get_img_orangin_url(url,index):
r = requests.get(url=url,timeout=10,headers=header)
r.encoding=r.apparent_encoding
html = parsel.Selector(r.text)
file_name=html.xpath('/html/body/div[3]/h1/span/text()').get()
img_url = html.xpath('/html/body/script[1]').get()
pattern = re.compile("'(.*)'")
img_url=pattern.findall(img_url)[0]
img_url= 'http://pic.jj20.com'+img_url
file_name = re.sub('\(.*\)','',file_name)
print("url",url)
print("img_url",img_url)
if '_' in url:
file_name=file_name+img_url[img_url.index('-')+1:]
else:
file_name=f'{file_name}-1.jpg'
# pool.apply_async(save_img,(img_url,file_name))
print("file_name",file_name)
save_img(img_url,file_name)
# 保存图片
def save_img(img_url,file_name):
try:
# r = requests.get(img_url, headers=header, timeout=timeout).content
if not os.path.exists(DIR_PATH):
os.makedirs(DIR_PATH)
os.chdir(DIR_PATH)
# with open(file_name, 'wb') as f:
# f.write(r)
img_header={
"Accept":"image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8",
"Cache-Control":"no-cache",
"Cookie":"__yjs_duid=1_6ab2ea97fb9890674b30afac7438c78d1623378472065; UM_distinctid=179f8e5766edd-01960ab8d9a8cc-68141f7b-1fa400-179f8e5766f714; Hm_lvt_d9f1c8630a7aa5c7ce2a72d4b564c044=1623378459; Hm_lpvt_d9f1c8630a7aa5c7ce2a72d4b564c044=1623734936",
"Host": "pic.jj20.com",
"Pragma": "no-cache",
"Referer": "http://cj.jj20.com/",
"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36"
}
response = requests.get(img_url, headers=img_header, timeout=timeout)
image = Image.open(BytesIO(response.content))
image.save(file_name)
print("保存完毕")
except Exception as e:
print(e)
def get_img(page_list):
for index, i in enumerate(page_list):
time.sleep(0.1)
# pool.apply_async(get_img_orangin_url,(base_url+i,))
get_img_orangin_url(base_url+i,index+1)
if __name__ == '__main__':
base_url='http://www.jj20.com'
pool=Pool(6)
list=get_page_num()
# list=["list_57_1.html"]
page_list = get_page_detail(list)
get_img(page_list)
print(len(page_list))
pool.close()
pool.join()