python爬取下一页数据_抓取网页及下一页

def get_text():

driver = webdriver.Firefox()

urls = ["url1",

"url2",

"url3"

]

ref = ['url1','url2','url3']

for i in range(len(urls)):

driver.get(urls[i])

count=0

while True:

count +=1

input = driver.find_element_by_class_name("search-list")

with codecs.open(ref[i]+'_page_'+str(count)+'.txt','w',encoding='utf-8') as f:  #保存网页源代码

f.write(input.text)

try:

detail_url = driver.find_element_by_link_text("下一页").get_attribute('href')

driver.get(detail_url)

except:

break