python爬虫架构图解_python系列整理---爬虫架构简单代码实现

爬虫spider流程示意图

根据以上流程简单实现爬虫功能，只是一种简单的做事风格，实际更复杂，不做具体讨论。

1. 目录

2. engine.py

# encoding=utf-8

import os

from spider.scheduler import Scheduler

def read_urls(file_path):

with open(file_path, 'r+', encoding='utf-8') as fp:

lines = fp.readlines()

return [line.strip() for line in lines if line.strip()]

def engine():

path = os.path.dirname(__file__) + '/urls.txt'

urls = read_urls(path)

htmls = Scheduler.download(urls)

data = Scheduler.analysis(htmls)

Scheduler.storage(data)

if __name__ == '__main__':

engine()

3. scheduler.py

# encoding=utf-8

from spider.downloader import Download