链接爬虫,通过跟踪页面内的链接,通过正则表达式来确定需要下载的页面的url,通过set来去除重复的链接
# coding=utf-8
import urllib2
import re
import urlparse
def download(url, user_agent='wswp', num_retries=2):
print 'Downloading:', url
headers = {'User-agent': user_agent}
request = urllib2.Request(url, headers=headers)
try:
html = urllib2.urlopen(request).read()
except urllib2.URLError as e:
print 'Download error:', e.reason
html = None
if num_retries > 0:
if hasattr(e, 'code') and 500 <= e.code < 600:
return download(url, num_retries - 1)
return html
def link_crawler(seed_url, link_regex):
"""Crawl from the given seed URL follwoing links matched by link_regex"""
crawl_queue = [seed_url]
#防止抓取重复的url
seen = set(crawl_queue)
while crawl_queue:
url = crawl_queue.pop()
html = download(url)
for link in get_links(html):
if re.match(link_regex, link):
link = urlparse.urljoin(seed_url, link)
if link not in seen:
seen.add(link)
crawl_queue.append(link)
#测试一下,打印出抓取到的url
print link
def get_links(html):
"""Return a list of links from html"""
webpage_regex = re.compile('<a[^>]+href=["\'](.*?)["\']',re.IGNORECASE)
return webpage_regex.findall(html)
##############不要忘记com后边的斜杠,我之前忘了斜杠,导致一直出错
link_crawler('http://example.webscraping.com/','/(index|view)')