python实现爬取京东商品页面的超链接-嵩天老师爬取淘宝商品举一反三

python实现爬取京东商品页面的超链接

技术路线：requests-bs4
通过bs4对标签进行查找，查找到a标签属性href，通过if语句判断该属性值是否满足要求

import requests
from bs4 import BeautifulSoup


def getHTMLText(url):
    try:
        header = {'user-agent': 'Mozilla/5.0'}
        r = requests.get(url,headers=header)
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        return r.text
    except:
        return ""


def getStockList(lst, html):
    soup = BeautifulSoup(html, 'html.parser')
    a = soup.find_all('a')
    for i in a:
        try:
            if i.attrs['href'][0:13] == '//item.jd.com':
                href = i.attrs['href']
                lst.append(href)
        except:
            continue


def printGoodsList(ilt):
    tplt = "{:4}\t{:8}\t"
    print(tplt.format("序号", "超链接"))
    count = 0
    for g in ilt:
        count = count + 1
        print(tplt.format(count, g))


def main():
    slist = []
    words = '书包'
    stock_list_url = 'https://search.jd.com/Search?keyword='
    url = stock_list_url + words
    html = getHTMLText(url)
    getStockList(slist, html)
    printGoodsList(slist)

main()