python爬虫系列(3.4-使用xpath和lxml爬取伯乐在线)

一、爬取的代码

1、网站地址

2、具体实现代码

import requests

from lxml import etree

class JobBole(object):

def __init__(self):

self.url = 'http://python.jobbole.com/all-posts/'

self.headers = {

'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.162 Safari/537.36',

}

@property

def get_article_link(self):

"""

抓取网页

:return:

"""

response = requests.get(url=self.url, headers=self.headers)

if response.status_code == 200:

html = etree.HTML(response.text)

# 现在就取第一个的访问详情页面

article = html.xpath('//div[@id="archive"]/div[@class="post floated-thumb"]')[0]

# 查找到当前标签上的a标签

a_link = article.xpath('./div[@class="post-meta"]/p[@class="align-right"]//a/@href')[0]

return a_link

def get_html(self):

"""

获取详情页面的数据

:return:

"""

if self.get_article_link:

response = requests.get(url=self.get_article_link, headers=self.headers)

if response.status_code == 200:

html = etree.HTML(response.text)

title = html.xpath('//div[@class="entry-header"]/h1/text()')[0]

article = html.xpath('//div[@id="article_content"]')[0]

print({"title": title, "article": etree.tostring(article, encoding="utf8").decode("utf8")})

if __name__ == '__main__':

jobbole = JobBole()

jobbole.get_html()

转载于:https://juejin.im/post/5be3ef0ff265da6165011137