综合案例
import requests
from lxml import etree
from urllib import request
import os
import re
def parse_page(url):
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36"
}
response = requests.get(url, headers=headers)
text = response.text
html = etree.HTML(text)
imgs = html.xpath("//div[@class='page-content text-center']//img")
for img in imgs:
img_url = img.get("data-original")
alt = img.get("alt")
alt = re.sub(r'[??!!@ ε=(´ο`*)))]', '', alt)
after_name = os.path.splitext(img_url)[1]
full_name = alt + after_name
request.urlretrieve(img_url, 'download/'+full_name)
def main():
for x in range(1, 101):
url = "https://www.doutula.com/photo/list/?page=%d" % x
parse_page(url)
break
if __name__ == '__main__':
main()