分析:首先通过scrapy内置的xpath提取内容,发现为空,所以不行咯
采用正则re匹配出所有的<li>标签,也就是需要提取的所有内容
在把li标签通过resultTree = lxml.etree.HTML(articleBody),变成'lxml.etree._Element'
在通过resultTree.xpath()进行提取
注意 此时的xpath与scrapy的xpath是不一样的
# -*- coding: utf-8 -*-
import scrapy
from ..settings import MAX_PAGE
from ..items import TiebaBaiduItem
import re
import lxml.html
import lxml.etree
import json
class TiebaSpider(scrapy.Spider):
name = 'tieba'
allowed_domains = ['tieba.baidu.com']
start_urls = ['https://tieba.baidu.com/f?kw=%E9%83%91%E5%AE%B8&ie=utf-8&pn={}'.format(str(page * 50)) for page in range(MAX_PAGE + 1)]
def parse(self, response):
# 关键 是正则匹配出 那一段需要有用的 html代码 如下 就是把那一部分 <li>标签全取出来
articleBodyRe = re.search('<ul id="thread_list" class="threadlist_bright j_threadlist_bright">(.*?)<div class="thread_list_bottom clearfix">', response.text, re.DOTALL)
articleBody = ''
if articleBodyRe:
articleBody = articleBodyRe.group(1)
# 通过lxml.etree.HTML(articleBody) 变成html对象 再利用xpath进行提取
# 此时的xpath与scrapy使用的xpath略有不同
# 这是lxml模块中xpath的使用方式
resultTree = lxml.etree.HTML(articleBody)
articleList = resultTree.xpath('//li[contains(@class,"j_thread_list")]')
for articleElem in articleList:
articleInfo = {}
data_field = articleElem.xpath("@data-field")[0]
dataFieldJson = json.loads(data_field)
articleInfo['id'] = dataFieldJson['id']
articleInfo['author'] = dataFieldJson['author_name']
articleInfo['title'] = articleElem.xpath(".//div[@class='t_con cleafix']//a/@title")[0]
articleInfo['href'] = \
articleElem.xpath(".//div[@class='t_con cleafix']//a/@href")[0]
yield response.follow(
url = articleInfo['href'] + "?see_lz=1",
meta={'dont_redirect': True, 'articleInfo': articleInfo},
callback = self.parseArticleDetail,
errback = self.errorHandle
)
def parseArticleDetail(self, response):
print(
f"parseArticleDetail: statusCode = {response.status}, url = {response.url}")
contentLst = response.xpath(
"//div[contains(@id, 'post_content')]//text()").extract()
imgHrefLst = response.xpath(
"//div[contains(@id, 'post_content')]//img/@src").extract()
dateLst = response.xpath(
"//div[contains(@class, 'post_content_firstfloor')]//span[@class='tail-info']/text()").extract()
content = ''
for contentElem in contentLst:
content += contentElem.replace('\n', ',').replace(" ", '').strip()
content += ', '
print(f"content = {content}")
print(f"imgHrefLst = {imgHrefLst}")
articleInfo = response.meta['articleInfo']
articleItem = TiebaBaiduItem()
articleItem['item_type'] = 'articleDetail'
articleItem['_id'] = articleInfo['id']
articleItem['title'] = articleInfo['title']
articleItem['author'] = articleInfo['author']
articleItem['content'] = content
articleItem['fromUrl'] = response.url
articleItem['picHrefLst'] = imgHrefLst
articleItem['date'] = dateLst[1]
yield articleItem
# 请求错误处理:可以打印,写文件,或者写到数据库中
def errorHandle(self, failure):
print(f"request error: {failure.value.response}")