使用ruia来爬取一个网站博客
from ruia import AttrField, Item, Request, Spider, TextField
from ruia_ua import middleware
class ArchivesItem(Item):
"""
eg: http://www.ruanyifeng.com/blog/archives.html
"""
target_item = TextField(css_select='div#beta-inner li.module-list-item')
href = AttrField(css_select='li.module-list-item>a', attr='href')
class ArticleListItem(Item):
"""
eg: http://www.ruanyifeng.com/blog/essays/
"""
target_item = TextField(css_select='div#alpha-inner li.module-list-item')
title = TextField(css_select='li.module-list-item>a')
href = AttrField(css_select='li.module-list-item>a', attr='href')
class BlogSpider(Spider):
"""
针对博客源 http://www.ruanyifeng.com/blog/archives.html 的爬虫
这里为了模拟ua,引入了一个ruia的第三方扩展
- ruia-ua: https://github.com/howie6879/ruia-ua
- pipenv install ruia-ua
- 此扩展会自动为每一次请求随机添加 User-Agent
"""
start_urls = ['http://www.ruanyifeng.com/blog/archives.html']
request_config = {
'RETRIES': 3,
'DELAY': 0,
'TIMEOUT': 20
}
concurrency = 40
blog_nums = 0
async def parse(self, res):
items = await ArchivesItem.get_items(html=res.html)
for item in items:
yield Request(
item.href,
callback=self.parse_item
)
async def parse_item(self, res):
items = await ArticleListItem.get_items(html=res.html)
BlogSpider.blog_nums += len(items)
if __name__ == '__main__':
BlogSpider.start(middleware=middleware)
print(f"博客总数为:{BlogSpider.blog_nums} 篇")