python里的url是什么意思_python new_url = link['href'] 这句话是什么意思

#!/usr/bin/envpython2#-*-coding:UTF-8-*-importreimporturlparsefrombs4importBeautifulSoupclassHtmlParser(object):defparse(self,page_url,html_cont):ifpage_urlisNoneorhtml_c...

#!/usr/bin/env python2

# -*- coding: UTF-8 -*-

import re

import urlparse

from bs4 import BeautifulSoup

class HtmlParser(object):

def parse(self, page_url, html_cont):

if page_url is None or html_cont is None:

return

soup = BeautifulSoup(html_cont, 'html.parser', from_encoding='utf-8')

new_urls = self._get_new_urls(page_url, soup)

new_data = self._get_new_data(page_url, soup)

return new_urls, new_data

def _get_new_urls(self, page_url, soup):

new_urls = set()

links = soup.find_all('a', href=re.compile(r"/view/\d+\.htm"))

for link in links:

new_url = link['href']

new_full_url = urlparse.urljoin(page_url, new_url)

new_urls.add(new_full_url)

return new_urls

def _get_new_data(self, page_url, soup):

res_data = {}

# url

res_data['url'] = page_url

#

Python

title_node = soup.find('dd', class_= "lemmaWgt-lemmaTitle-title").find("h1")

res_data['title'] = title_node.get_text()

summary_node = soup.find('div', class_="lemma-summary")

res_data['summary'] = summary_node.get_text()

return res_data

展开