爬取网页中所有的链接

1、导入库

from bs4 import BeautifulSoup
import requests

2、requests请求

url = 'https://bj.58.com/sale.shtml'
 wb_data = requests.get(url)

3、抓取所有HTML元素
    soup = BeautifulSoup(wb_data.text,'lxml')
    #ymenu-side > ul > li:nth-child(4) > ul > li:nth-child(4) > b > a

4、选取链接元素，找到唯一定位路径
    links = soup.select('ul.ym-submnu > li > b > a')
    for link in links:
        page_url = url_host + link.get('href')

        print(page_url)
get_page(url)