import requests import re headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36"} """ <a class="title-content" href="https://www.baidu.com/s?cl=3&tn=baid%9D%A">世界杯八强出炉</a> """ def getUrl(url): html = getHtml(url) # 清洗出新的url # urlre = "<a .*href=\"(https?://.*?)\".*>" urlre = "<a .*href=\"(https?://.*?)\".*>" urlList = re.findall(urlre, html) return urlList def getHtml(url): response = requests.get(url, headers=headers).content.decode('utf-8', 'ignore') return response def getInfo(): pass def depthCtl(url, depth): # 判断是否超出深度,超出就结束 if depthDict[url] > depth: return print('\t\t\t' * depthDict[url], '抓取了第%d层,%s' % (depthDict[url], url)) # getInfo(url) sonUrllist = getUrl(url) for newUrl in sonUrllist: if newUrl not in depthDict: depthDict[newUrl] = depthDict[url] + 1 depthCtl(newUrl, depth) if __name__ == '__main__': # 种子url startUrl = 'https://www.baidu.com/s?wd=世界杯' depthDict = {} depthDict[startUrl] = 1 depthCtl(startUrl, 4)
爬虫(19):深度抓取策略(1)
最新推荐文章于 2024-07-22 19:59:01 发布