编乎,分享你刚编的故事ε=ε=ε=(~ ̄▽ ̄)~
虽然只是一句调侃,但小破乎有些回答确实有些“编”的嫌疑。不过既然都是编故事,那么我们不妨去薅一下简书的羊毛。(俺觉得简书上的故事比咱这精彩(→_→))
来看看简书的网页布局
看样子有一个动态加载的按钮,如果用只用scrapy的话是无法加载动态网页的,怎么办呢
只好请selenium来协助喽( $ _ $ )
可以用selenium不停的模拟点击直到没有动态加载的按钮为止。
到底之后就可以用通用爬虫来逐个爬取啦
每个li标签都是一篇文章
文章内部我们挑有代表性的抓取
好滴上代码(由于小的第一次用selenium中间件所以上网咨询,找到了一篇文章加以优化。以下代码与CSDN作者s_kangkang_A高度重复,在此向作者致歉,擅自借用了您的代码。如果有想向这位大佬学习的小伙伴请看这里https://blog.csdn.net/s_kangkang_A)
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from jianshu.items import JianshuItem
class JsSpider(CrawlSpider):
name = 'js'
allowed_domains = ['www.jianshu.com']
start_urls = ['https://www.jianshu.com/']
# 提取规则根据首页和文章页下的推荐url获取,放入parse_detail解析,跟进爬取
rules = (
Rule(LinkExtractor(allow=r'.*/p/[0-9a-z]{12}.*'),
callback='parse_detail', follow=True),
)
def parse_detail(self, response):
item=JianshuItem()
item["title"] = response.xpath("//h1[@class='title']/text()").extract_first()
item["author"] = response.xpath("//span[@class='name']/a/text()").extract_first()
item["url"]=response.url
# item["content"] = "".join(response.xpath("//div[@class='show-content-free']/p/text()").extract()).strip()
# tags 返回的是列表,用join方法将元素隔开,转化为字符串
item["tags"] = " ".join(response.css(" div .name::text ").extract()).strip()
yield item
from jianshu.settings import *
import pymysql
import logging
class JianshuPipeline(object):
def __init__(self):
self.connect=pymysql.connect(MYSQL_HOST,MYSQL_ROOT,MYSQL_PASSWORD,MYSQL_DATABASE)
self.cursor=self.connect.cursor()
self.cursor.execute(USE) # 选定数据库
self.cursor.execute(DROP)
self.cursor.execute(CREATE)
def process_item(self, item, spider):
try:
self.cursor.execute(SAVEIN,(item["title"],item["author"],item["url"],item["tags"]))
self.connect.commit()
except Exception as error:
logging.log(error)
return item
def close_spider(self,spider):
self.connect.close()
from scrapy import signals
from selenium import webdriver
import time
from scrapy.http.response.html import HtmlResponse
import random
import logging
class SeleniumDownloadMiddleware(object):
def __init__(self):
self.driver = webdriver.Chrome()
def process_request(self, request, spider):
self.driver.get(request.url)
time.sleep(2)
# 如果有多个专题收入该文章,获取所有专题
# 点击“展开更多”,直到没有为止
try:
while True:
tags_btn = self.driver.find_element_by_class_name('load-more')
tags_btn.click()
time.sleep(2)
if not tags_btn:
break
except:
pass
source = self.driver.page_source
response = HtmlResponse(url=self.driver.current_url, body=source, request=request, encoding='utf-8')
return response
class RandomUserAgentMiddleware():
def __init__(self):
self.user_agents=[
"Mozilla/5.0(Windows;U;MSIE 9.0;Windows NT 9.0; en-US)","Mozilla/5.0(Windows NT 6.1)AppleWebKit/537.2(KHTML,like Gecko)Chrome/22.0.1216.0 Safari/537.2","Mozilla/5.0(X11;Ubuntu;Linux i686;rv:15.0)Gecko/20100101 Firefox/15.0.1" ,
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Acoo Browser; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)",
"Mozilla/4.0 (compatible; MSIE 7.0; AOL 9.5; AOLBuild 4337.35; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
"Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)",
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
"Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
"Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30)",
"Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)",
"Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6",
"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1",
]
def process_request(self,request,spider):
request.headers["User-Agent"]=random.choice(self.user_agents)
DOWNLOADER_MIDDLEWARES = {
'jianshu.middlewares.RandomUserAgentMiddleware': 544,
'jianshu.middlewares.SeleniumDownloadMiddleware':545,
}
ITEM_PIPELINES = {
'jianshu.pipelines.JianshuPipeline': 300,
}
MYSQL_HOST='localhost'
MYSQL_DATABASE='spider'
MYSQL_ROOT='root'
MYSQL_PASSWORD='123'
USE='use spider'
TABLE='jianshu'
DROP="drop table if exists %s"%TABLE
CREATE='create table %s(title varchar(255) NOT NULL,author varchar(255),url varchar(255),tags varchar(255))'%TABLE
SAVEIN='insert into '+TABLE+' (title,author,url,tags) values(%s,%s,%s,%s)'
import scrapy
class JianshuItem(scrapy.Item):
title = scrapy.Field()
author = scrapy.Field()
url=scrapy.Field()
content = scrapy.Field()
tags = scrapy.Field()
上个数据库的结果图看看
以上就是这样,小的最近也要期末了,为了不挂科只能暂时先泡图书馆,期待暑假再做出项目来分享\( ̄︶ ̄*\))