scrapy 爬取新民网
scrapy编写
1.建立scrapy项目
scrapy startproject xinmin
项目结构如图:
2.编写items,定义爬取内容
# 发布者
publishername = scrapy.Field()
# 分类
category = scrapy.Field()
# 标题
title = scrapy.Field()
# 正文
text = scrapy.Field()
# 网址链接
linkurl = scrapy.Field()
# 发布时间
publishtime = scrapy.Field()
3.编写spilder
# -*- coding: utf-8 -*-
import scrapy
from scrapy.spiders import CrawlSpider
from scrapy.selector import Selector
from xinmin.items import XinminItem
class XinminSpider(CrawlSpider):
#初始化
name = "xinmin"
allowed_domains = ["shanghai.xinmin.cn"]
start_urls = ['http://shanghai.xinmin.cn/t/gdbd/']
#srartUrl = 'http://shanghai.xinmin.cn/t/gdbd/'
#解析页面的所以新闻链接
def parse(self, response):
selector = Selector(response)
# 获取当前页面所有新闻的链接
newslink_list = selector.xpath("//div[contains(@class,'list_list')]/a/@href").extract()
# 读取每一个链接
for link_url in newslink_list:
#print("新闻链接:", link_url)
yield scrapy.Request(link_url, callback=self.parse_item)
# 下一页
next_url = self.start_urls[0] + response.xpath('//a[@class="unselect"]/@href').extract_first() # 翻页
print('next_url', next_url)
if next_url:
yield scrapy.Request(next_url, callback=self.parse)
# 抓取新闻详细内容
def parse_item(self, response):
#print('开始处理item...')
item = XinminItem()
selector = Selector(response)
# 网址链接
item['linkurl'] = response.url
# 分类 +
item['category'] = selector.xpath("//div[contains(@class,'xinminMianbaoxue')]/a[3]/text()").extract()
# 标题
item['title'] = selector.xpath("//h1[contains(@class,'article_title')]/text()").extract()
# 正文
item['text'] = selector.xpath("//div[contains(@class,'a_p')]/p/text()").extract()
# 发布时间 这两个会变化
info=selector.xpath("html/body/div[4]/div[1]/div[2]/div[2]/div/span[4]/text()").extract()
item['publishtime'] = selector.xpath("//div[contains(@class,'info')]/span[3]/text()").extract()
# 发布者
# xpath语句太长 下一版修改
item['publishername'] = selector.xpath("//div[contains(@class,'info')]/span[4]/text()").extract()
yield item
4.settings 修改设置
# Obey robots.txt rules
ROBOTSTXT_OBEY = False
DOWNLOAD_DELAY = 4
COOKIES_ENABLED = False
5.编写
# -*- coding: utf-8 -*-
class XinminPipeline(object):
#去掉空格换行符
@staticmethod
def get_list(data_list):
data_str = ''
for i in data_list:
i = str(i).strip()
data_str = data_str + i
return data_str
#写入文件
def process_item(self, item, spider):
today = time.strftime("%Y-%m-%d", time.localtime())
# 文件路径
file_path = "../data/"+today+".txt"
with open(file_path, "a", encoding="utf-8") as f:
print('开始写入文件....')
f.write("linkurl:" + self.get_list(item['linkurl']) + "\n")
f.write("title:" + self.get_list(item['title']) + "\n")
f.write("category:" + self.get_list(item['category']) + "\n")
f.write("publishername:" + self.get_list(item['publishername']) + "\n")
f.write("publishtime:" + self.get_list(item['publishtime']) + "\n")
f.write("text:" + self.get_list(item['text']) + "\n")
f.write("\n")
print('成功写入...')
return item
未完善的地方
1.有两个抓取的指标(作者,发布日期)会变,没有处理
2.没有考虑反爬的问题