python爬虫下一页_python 爬虫 循环分页

import os

from time import sleep

import faker

import requests

from lxml import etree

fake = faker.Faker()

base_url = "http://angelimg.spbeen.com"

def get_next_link(url):

content = downloadHtml(url)

html = etree.HTML(content)

next_url = html.xpath("//a[@class='ch next']/@href")

if next_url:

return base_url + next_url[0]

else:

return False

def downloadHtml(ur):

user_agent = fake.user_agent()

headers = {'User-Agent': user_agent,"Referer":"http://angelimg.spbeen.com/"}

response = requests.get(url, headers=headers,timeout=20)

if response.status_code != 200:

return None

else:

return response.text

def getImgUrl(content):

html = etree.HTML(content)

img_url = html.xpath('//*[@id="content"]/a/img/@src')

title = html.xpath(".//div['@class=article']/h2/text()")

return img_url[0],title[0]

def saveImg(title,img_url):

if img_url is not None and title is not None:

title = title.split('【')[0]

file_path = 'isssss/{}/'.format(title)

if not os.path.exists(file_path):

os.makedirs(file_path)

file_name = img_url.split('/')[-1]

with open(file_path+file_name+".jpg",'wb') as f:

user_agent = fake.user_agent()

headers = {'User-Agent': user_agent,"Referer":"http://angelimg.spbeen.com/"}

content = requests.get(img_url, headers=headers,timeout=20)

#request_view(content)

f.write(content.content)

print("save img "+ img_url)

f.close()

def request_view(response):

import webbrowser

request_url = response.url

base_url = '

' %(request_url)

base_url = base_url.encode()

content = response.content.replace(b"

",base_url)

tem_html = open('tmp.html','wb')

tem_html.write(content)

tem_html.close()

webbrowser.open_new_tab('tmp.html')

def optimizeContent(res):

res = res.replace('b\'', '')

res = res.replace('\\n', '')

res = res.replace('\'', '')

res = res.replace('style', 'nouse')

res = res.replace('\.', '')

return res

def crawl_img(url):

content = downloadHtml(url)

if content is not None:

res = getImgUrl(content)

title = res[1]

img_url = res[0]

title = optimizeContent(title)

title = title.replace('.', '')

print(title)

saveImg(title,img_url)

return True

else:

return None

if __name__ == "__main__":

try:

root_url = "http://angelimg.spbeen.com/ang/{}"

for i in range(37,10000):

url = root_url.format(i)

try:

while url:

res = crawl_img(url)

if res is None:

print(url + ' 无数据')

next = i + 1

url = root_url.format(next)

break

else:

url = get_next_link(url)

print("爬取页面：" + url)

i = i + 1

except Exception as e:

print(str(e))

except Exception as e:

print(str(e))

结果