线程
import requests
from lxml import etree
import threading
from queue import Queue
pageUrlQueue = Queue()
downloadQueue = Queue()
def parsePageUrl(pages):
for page in range(1,pages+1):
url = "https://bizhi.ijinshan.com/dtag_109/index_"+str(page)+".shtml"
print(url)
headers = {
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36 Edg/92.0.902.55",
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
}
res = requests.get(url=url,headers=headers)
res.encoding = res.apparent_encoding
html = etree.HTML(res.text)
divs = html.xpath("//div[@class='wallpaper-wrapper