创建一个scrapy项目
本次爬取网站为:https://wz.sun0769.com/app/politics/index
cmd切换目录scrapy startproject sun0769
切换创建的项目cd sun0769
创建spider目录下py文件scrapy genspider sun sun0769.com
分析网页
网站采用ajax加载,每条新闻包括在div标签中,不断往下滑动,会不断出现div,用xpath提取即可。
如何提取多页呢?
答:每一页都附带表单数据,用urlencode构造url,在scrapy框架中可轻松回调访问!
start_urls = 'https://wz.sun0769.com/app/politics/index?'
data = {
'max_id':'525117',
'page': page,
}
params = urlencode(data)
url = start_urls + params
yield Request(url, self.parse)
完成代码,保存CSV文件
sun.py
import scrapy
#from sun0769.items import Sun0769Item
import scrapy
from urllib.parse import urlencode
#from sun0769.items import Sun0769Item
from scrapy import Request
from sun0769.settings import MAX_PAGE
class SunSpider(scrapy.Spider):
name = 'sun'
allowed_domains = ['sun0769.com']
start_urls = ['https://wz.sun0769.com/app/politics/index']
def parse(self, response):
#for 循环输出10页
for page in range(1,self.settings.get('MAX_PAGE')+1):
# 定位信息页面
total=response.xpath('//div[@class="political_list"]/div')
for items in total:
item= {}
#title
item['title']=items.xpath('./a//p/text()').extract_first()
item['status']=items.xpath('./span[@class="status1"]/text()').extract_first()
item['anwser']=items.xpath('./a[2]/span/text()').extract_first()
# item['uri']=items.xpath('./a/@href').extract_first()
item['start_url']='https://wz.sun0769.com/'+items.xpath('./a/@href').extract_first()
yield scrapy.Request(
item['start_url'],
callback=self.parse_detail,
meta={"item":item},
)
#获取page,重新发送请求,直到循环介绍
start_urls = 'https://wz.sun0769.com/app/politics/index?'
data = {
'max_id':'525117',
'page': page,
}
params = urlencode(data)
url = start_urls + params
yield Request(url, self.parse)
#定位详情页
def parse_detail(self,response):
item=response.meta["item"]
item['content']=response.xpath('//div[@class="content"]/p/text()').extract_first()
yield item
pipelines.py
class Sun0769Pipeline:
def process_item(self, item, spider):
print(item)
return item
settings.py配置,照着源文件开就ok了。
# Scrapy settings for sun0769 project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
# https://docs.scrapy.org/en/latest/topics/settings.html
# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
BOT_NAME = 'sun0769'
SPIDER_MODULES = ['sun0769.spiders']
NEWSPIDER_MODULE = 'sun0769.spiders'
# Crawl responsibly by identifying yourself (and your website) on the user-agent
#设置代理
USER_AGENT = 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) ' \
'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 ' \
'Mobile Safari/537.36 Edg/94.0.992.31'
# Obey robots.txt rules
#遵守机器协议,默认True
ROBOTSTXT_OBEY = False
# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32
# Configure a delay for requests for the same website (default: 0)
# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
DOWNLOAD_DELAY =2 #与sleep原理相似,避免访问过快被拦截
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16
# Disable cookies (enabled by default)
#COOKIES_ENABLED = False
# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False
# Override the default request headers:
#DEFAULT_REQUEST_HEADERS = {
# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
# 'Accept-Language': 'en',
#}
# Enable or disable spider middlewares
# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
# 'sun0769.middlewares.Sun0769SpiderMiddleware': 543,
#}
# Enable or disable downloader middlewares
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
#DOWNLOADER_MIDDLEWARES = {
# 'sun0769.middlewares.Sun0769DownloaderMiddleware': 543,
#}
# Enable or disable extensions
# See https://docs.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
# 'scrapy.extensions.telnet.TelnetConsole': None,
#}
# Configure item pipelines
# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
#打开pipelines通道
ITEM_PIPELINES = {
'sun0769.pipelines.Sun0769Pipeline': 300,
}
# Enable and configure the AutoThrottle extension (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False
# Enable and configure HTTP caching (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
#终端直接输出结果,不输出日志
LOG_LEVEL="WARNING"
#设置编码,避免保存csv文件出现乱码
FEED_EXPORT_ENCODING = 'gb18030'
运行结果:
cmd输入scrapy crawl sun -o sun.csv
状态码200表示每一页访问成功!
会在终端输出结果,在目录下保存csv文件
自学一年半来,学习了爬取静态网页再到动态网页,selenium模拟到scrapy框架,从易到难,由深到浅!
总之,scrapy框架对于我来说,算是遇到了瓶颈了!琢磨了一个星期,也终于能正式更新这篇文章了!