python获取网页上特定的链接_读取指定页面中的超链接-Python 3.7

#!/usr/bin/env python

#coding: utf-8

from bs4 import BeautifulSoup

import urllib

import urllib.request

import sys

from imp import reload

reload(sys)

#sys.setdefaultencoding("utf-8")

# the url of the page

url = 'https://www.wikipedia.org/'

def findAllLink(url):

'''

Get hyperlinks from web pages

'''

# agreement, domain name

proto, rest = urllib.request.splittype(url)

domain = urllib.request.splithost(rest)[0]

# read the page

html = urllib.request.urlopen(url).read()

# Extract hyperlinks

a = BeautifulSoup(html).findAll('a')

# filter

alist = [i.attrs['href'] for i in a if i.attrs['href'][0] != 'j']

# 将形如#comment-text的锚点补全成http://www.ruanyifeng.com/blog/2015/05/co.html,将形如/feed.html补全为http://www.ruanyifeng.com/feed.html

alist = map(lambda i: proto + '://' + domain + i if i[0] == '/' else url + i if i[0] == '#' else i, alist)

return alist

if __name__ == '__main__':

for i in findAllLink(url):

print(i)