1、setting.py
import os
BOT_NAME = 'ArticleSpider'
SPIDER_MODULES = ['ArticleSpider.spiders']
NEWSPIDER_MODULE = 'ArticleSpider.spiders'
ROBOTSTXT_OBEY = False
ITEM_PIPELINES = {
'ArticleSpider.pipelines.ArticlespiderPipeline': 300,
'ArticleSpider.pipelines.ArticleImagePipeline': 1,
}
IMAGES_URLS_FIELD = "front_image_url"
project_dir = os.path.abspath(os.path.dirname(__file__))
IMAGES_STORE = os.path.join(project_dir, 'images')
2、items.py
# -*- coding: utf-8 -*-
import scrapy
class ArticlespiderItem(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
pass
class JobBoleArticleItem(scrapy.Item):
title = scrapy.Field()
create_data = scrapy.Field()
url = scrapy.Field()
url_boject_id = scrapy.Field()
front_image_url = scrapy.Field()
front_image_path = scrapy.Field()
praise_nums = scrapy.Field()
favor_nums = scrapy.Field()
comment_nums = scrapy.Field()
tags = scrapy.Field()
content = scrapy.Field()
3、pipeline.py
from scrapy.pipelines.images import ImagesPipeline
class ArticlespiderPipeline(object):
def process_item(self, item, spider):
return item
class ArticleImagePipeline(ImagesPipeline):
def item_completed(self, results, item, info):
for ok, value in results:
image_file_path = value["path"]
item["front_image_path"] = image_file_path
return item
4、jobbole.py
import scrapy
import re
from scrapy.http import Request
import urlparse
from ArticleSpider.items import JobBoleArticleItem
from ArticleSpider.utils.common import get_md5
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
class JobboleSpider(scrapy.Spider):
name = 'jobbole'
allowed_domains = ['blog.jobbole.com']
start_urls = ['http://blog.jobbole.com/all-posts/']
def parse(self, response):
"""
1.获取文章列表页中的文章url并交给scrapy下载后并进行解析
2.获取下一页的url并交给scrapy进行下载,下载完成后交给parse
"""
post_nodes = response.css("#archive .floated-thumb .post-thumb a")
for post_node in post_nodes:
image_url = post_node.css("img::attr(src)").extract_first("")
post_url = post_node.css("::attr(href)").extract_first("")
yield Request(url=urlparse.urljoin(response.url, post_url), meta={"front_image_url":image_url}, callback=self.parse_detail)
print(post_url)
next_url = response.css(".next.page-numbers::attr(href)").extract_first("")
if next_url:
yield Request(url=urlparse.urljoin(response.url, post_url), callback=self.parse)
def parse_detail(self, response):
article_item = JobBoleArticleItem()
front_image_url = response.meta.get("front_image_url", "")
title2 = response.css(".entry-header h1::text").extract()[0]
create_data2 = response.css("p.entry-meta-hide-on-mobile::text").extract()[0].strip().replace('·', '')
praise_nums2 = int(response.css(".href-style h10::text").extract()[0])
favor_nums2 = response.css(".bookmark-btn::text").extract()[0]
match_re3 = re.match(".*?(\d+).*", favor_nums2)
if match_re3:
favor_nums2 = int(match_re3.group(1))
else:
favor_nums2 = 0
comment_nums2 = response.css("span.hide-on-480::text").extract()[0]
match_re4 = re.match(".*?(\d+).*", comment_nums2)
if match_re4:
comment_nums2 = int(match_re4.group(1))
else:
comment_nums2 = 0
content2 = response.css('div.entry').extract()[0]
tag_list2 = response.css("p.entry-meta-hide-on-mobile a::text").extract()
[element for element in tag_list2 if not element.strip().endswith("评论")]
tags2 = ",".join(tag_list2)
article_item["url_object_id"] = get_md5(response.url)
article_item["title"] = title2
article_item["url"] = response.url
article_item["create_data"] = create_data2
article_item["front_image_url"] = [front_image_url]
article_item["praise_nums"] = praise_nums2
article_item["favor_nums"] = favor_nums2
article_item["comment_nums"] = comment_nums2
article_item["tags"] = tags2
article_item["content"] = content2
yield article_item
5、新建common.py
import hashlib
def get_md5(url):
if isinstance(url, str):
url = url.encode("utf-8")
m = hashlib.md5()
m.update(url)
return m.hexdigest()
if __name__ == "__main__":
print(get_md5("http://jobbole.com"))
6、新建main.py
# coding:utf-8
from scrapy.cmdline import execute
import sys
import os
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
#print(os.path.abspath(__file__)) # 自身路径
#print(os.path.dirname(os.path.abspath(__file__))) # 父目录路径
execute(["scrapy", "crawl", "jobbole"])