python爬取多个网页,使用BeautifulSoup在python中抓取多个页面

I have managed to write code to scrape data from the first page and now the I am stuck with writing a loop in this code to scrape the next 'n' pages. Below is the code

I would appreciate if someone could guide/help me to write the code that would scrape the data from remaining pages.

Thanks!

from bs4 import BeautifulSoup

import requests

import csv

url = requests.get('https://wsc.nmbe.ch/search?sFamily=Salticidae&fMt=begin&sGenus=&gMt=begin&sSpecies=&sMt=begin&multiPurpose=slsid&sMulti=&mMt=contain&searchSpec=s').text

soup = BeautifulSoup(url, 'lxml')

elements = soup.find_all('div', style="border-bottom: 1px solid #C0C0C0; padding: 10px 0;")

#print(elements)

csv_file = open('wsc_scrape.csv', 'w')

csv_writer = csv.writer(csv_file)

csv_writer.writerow(['sp_name', 'species_author', 'status', 'family'])

for element in elements:

sp_name = element.i.text.strip()

print(sp_name)

status = element.find('span', class_ = ['success label', 'error label']).text.strip()

print(status)

author_family = element.i.next_sibling.strip().split('|')

species_author = author_family[0].strip()

family = author_family[1].strip()

print(species_author)

print(family)

print()

csv_writer.writerow([sp_name, species_author, status, family])

csv_file.close()

解决方案

You have to pass page= parameter in URL and iterate over all pages:

from bs4 import BeautifulSoup

import requests

import csv

csv_file = open('wsc_scrape.csv', 'w', encoding='utf-8')

csv_writer = csv.writer(csv_file)

csv_writer.writerow(['sp_name', 'species_author', 'status', 'family'])

for i in range(151):

url = requests.get('https://wsc.nmbe.ch/search?page={}&sFamily=Salticidae&fMt=begin&sGenus=&gMt=begin&sSpecies=&sMt=begin&multiPurpose=slsid&sMulti=&mMt=contain&searchSpec=s'.format(i+1)).text

soup = BeautifulSoup(url, 'lxml')

elements = soup.find_all('div', style="border-bottom: 1px solid #C0C0C0; padding: 10px 0;")

for element in elements:

sp_name = element.i.text.strip()

print(sp_name)

status = element.find('span', class_ = ['success label', 'error label']).text.strip()

print(status)

author_family = element.i.next_sibling.strip().split('|')

species_author = author_family[0].strip()

family = author_family[1].strip()

print(species_author)

print(family)

print()

csv_writer.writerow([sp_name, species_author, status, family])

csv_file.close()