今天seo的同事需要一个简单的爬虫工具, 根据一个url地址,抓取改页面的a连接,然后进入a连接里面的页面再次抓取a连接
1.需要一个全局的set([])集合来保存抓取的url地址
2.由于现在单页面也来越多,所以我们借用selenium来抓取页面内容, 由于页面内容比较多, 我们程序需要将滚动条滚到最下面,如:driver.execute_script("return document.body.scrollHeight;")
3.需要查找页面2的超链接 driver.find_elements_by_xpath("//a[@href]")
4.为了便于查看数据记录,每抓取一个地址就记录到日志中去(曾经尝试过爬网完毕后再记录,但是爬网时间太长,一旦出现异常就一条记录都没有了)
整个代码如下:
from selenium import webdriver
from selenium.webdriver.firefox.options import Options
from selenium.common.exceptions import TimeoutException
import time
import datetime
from urllib import parse
import os
urls = set([])
def getUrl(url,host):
driver = webdriver.Ie()
try:
#driver = webdriver.Firefox()
driver.set_page_load_timeout(10)
driver.get(url)
#time.sleep(2)
all_window_height = []
all_window_height.append(driver.execute_script("return document.body.scrollHeight;"))
while True:
driver.execute_script("scroll(0,100000)")
time.sleep(1)
check_height = driver.execute_script("return document.body.scrollHeight;")
if check_height == all_window_height[-1]:
print("我已下拉完毕")
break
else:
all_window_height.append(check_height)
print("我正在下拉")
#for link in driver.find_elements_by_xpath("//*[@href]"):
#for link in driver.find_elements_by_tag_name("a"):
for link in driver.find_elements_by_xpath("//a[@href]"):
try:
tempurl1=link.get_attribute('href')
if tempurl1.startswith("http"):
if tempurl1 not in urls:
urls.add(tempurl1)
log(host,url+','+tempurl1)
print(tempurl1)
except:
print(link)
except Exception as e:
print(e)
finally:
driver.quit()
def log(name,msg):
filename='D://'+name+'.csv'
if not os.path.exists(filename):
with open(filename,'w') as f:
print('create file:'+filename)
f.write('parentUrl,currenturl'+'\n')
f.close()
with open(filename,'a') as f:
f.write(msg+'\n')
f.close()
url= input("Enter a url")
try:
urls.clear()
url= url.strip()
if len(url)>0:
host =parse.urlparse(url).netloc
print(url+"下面的连接:")
t1=datetime.datetime.now()
getUrl(url,host)
l=list(urls)
for item in l:
print(item+"下面的连接:")
getUrl(item,host)
t2=datetime.datetime.now()
tt =(t2-t1).seconds
minutes=tt//60
seconds=tt%60
print("total cost %d minutes %d seconds" % (minutes,seconds))
except Exception as e:
print(e)
然后运行pyinstaller -F a.py 打包
关于selenium 的IE 可以参考https://blog.csdn.net/ma_jiang/article/details/96022775