文章目录
Crawlab部署教程
提示:首先确保你的电脑上面安装了docker,如果没有,建议看我之前的文章
1.下载 Crawlaw镜像
docker pull tikazyq/crawlab:latest
2. 安装 Docker-Compose
pip3 install docker-compose
3. 安装并启动 Crawlab,新建 docker-compose.yml
version: '3.3'
services:
master:
image: tikazyq/crawlab:latest
container_name: master
environment:
CRAWLAB_SERVER_MASTER: "Y"
CRAWLAB_MONGO_HOST: "mongo"
CRAWLAB_REDIS_ADDRESS: "redis"
ports:
- "8080:8080" # frontend
- "8000:8000" # backend
depends_on:
- mongo
- redis
mongo:
image: mongo:latest
restart: always
ports:
- "27017:27017"
redis:
image: redis:latest
restart: always
ports:
- "6381:6381"
3.1 官网yml文件
version: '3.3'
services:
master:
image: tikazyq/crawlab:latest
container_name: master
environment:
# CRAWLAB_API_ADDRESS: "https://<your_api_ip>:<your_api_port>" # backend API address 后端 API 地址. 适用于 https 或者源码部署
CRAWLAB_SERVER_MASTER: "Y" # whether to be master node 是否为主节点,主节点为 Y,工作节点为 N
CRAWLAB_MONGO_HOST: "mongo" # MongoDB host address MongoDB 的地址,在 docker compose 网络中,直接引用服务名称
# CRAWLAB_MONGO_PORT: "27017" # MongoDB port MongoDB 的端口
# CRAWLAB_MONGO_DB: "crawlab_test" # MongoDB database MongoDB 的数据库
# CRAWLAB_MONGO_USERNAME: "username" # MongoDB username MongoDB 的用户名
# CRAWLAB_MONGO_PASSWORD: "password" # MongoDB password MongoDB 的密码
# CRAWLAB_MONGO_AUTHSOURCE: "admin" # MongoDB auth source MongoDB 的验证源
CRAWLAB_REDIS_ADDRESS: "redis" # Redis host address Redis 的地址,在 docker compose 网络中,直接引用服务名称
# CRAWLAB_REDIS_PORT: "6379" # Redis port Redis 的端口
# CRAWLAB_REDIS_DATABASE: "1" # Redis database Redis 的数据库
# CRAWLAB_REDIS_PASSWORD: "password" # Redis password Redis 的密码
CRAWLAB_LOG_LEVEL: "info" # log level 日志级别. 默认为 info
CRAWLAB_LOG_ISDELETEPERIODICALLY: "N" # whether to periodically delete log files 是否周期性删除日志文件. 默认不删除
CRAWLAB_LOG_DELETEFREQUENCY: "@hourly" # frequency of deleting log files 删除日志文件的频率. 默认为每小时
# CRAWLAB_TASK_WORKERS: 8 # number of task executors 任务执行器个数(并行执行任务数)
# CRAWLAB_SERVER_REGISTER_TYPE: "mac" # node register type 节点注册方式. 默认为 mac 地址,也可设置为 ip(防止 mac 地址冲突)
# CRAWLAB_SERVER_REGISTER_IP: "127.0.0.1" # node register ip 节点注册IP. 节点唯一识别号,只有当 CRAWLAB_SERVER_REGISTER_TYPE 为 "ip" 时才生效
# CRAWLAB_SERVER_LANG_NODE: "Y" # whether to pre-install Node.js 预安装 Node.js 语言环境
# CRAWLAB_SERVER_LANG_JAVA: "Y" # whether to pre-install Java 预安装 Java 语言环境
# CRAWLAB_SERVER_LANG_DOTNET: "Y" # whether to pre-install .Net core 预安装 .Net Core 语言环境
# CRAWLAB_SERVER_LANG_PHP: "Y" # whether to pre-install PHP 预安装 PHP 语言环境
# CRAWLAB_SERVER_LANG_GO: "Y" # whether to pre-install Golang 预安装 Golang 语言环境
# CRAWLAB_SETTING_ALLOWREGISTER: "N" # whether to allow user registration 是否允许用户注册
# CRAWLAB_SETTING_ENABLETUTORIAL: "N" # whether to enable tutorial 是否启用教程
# CRAWLAB_SETTING_RUNONMASTER: "N" # whether to run on master node 是否在主节点上运行任务
# CRAWLAB_SETTING_DEMOSPIDERS: "Y" # whether to init demo spiders 是否使用Demo爬虫
# CRAWLAB_SETTING_CHECKSCRAPY: "Y" # whether to automatically check if the spider is scrapy 是否自动检测爬虫为scrapy
# CRAWLAB_NOTIFICATION_MAIL_SERVER: smtp.exmaple.com # STMP server address STMP 服务器地址
# CRAWLAB_NOTIFICATION_MAIL_PORT: 465 # STMP server port STMP 服务器端口
# CRAWLAB_NOTIFICATION_MAIL_SENDEREMAIL: admin@exmaple.com # sender email 发送者邮箱
# CRAWLAB_NOTIFICATION_MAIL_SENDERIDENTITY: admin@exmaple.com # sender ID 发送者 ID
# CRAWLAB_NOTIFICATION_MAIL_SMTP_USER: username # SMTP username SMTP 用户名
# CRAWLAB_NOTIFICATION_MAIL_SMTP_PASSWORD: password # SMTP password SMTP 密码
ports:
- "8080:8080" # frontend port mapping 前端端口映射
depends_on:
- mongo
- redis
# volumes:
# - "/var/crawlab/log:/var/logs/crawlab" # log persistent 日志持久化
worker:
image: tikazyq/crawlab:latest
container_name: worker
environment:
CRAWLAB_SERVER_MASTER: "N"
CRAWLAB_MONGO_HOST: "mongo"
CRAWLAB_REDIS_ADDRESS: "redis"
depends_on:
- mongo
- redis
# volumes:
# - "/var/crawlab/log:/var/logs/crawlab" # log persistent 日志持久化
mongo:
image: mongo:latest
restart: always
# environment:
# MONGO_INITDB_ROOT_USERNAME: username
# MONGO_INITDB_ROOT_PASSWORD: password
volumes:
- "/opt/crawlab/mongo/data/db:/data/db" # make data persistent 持久化
# ports:
# - "27017:27017" # expose port to host machine 暴露接口到宿主机
redis:
image: redis:latest
restart: always
# command: redis-server --requirepass "password" # set redis password 设置 Redis 密码
volumes:
- "/opt/crawlab/redis/data:/data" # make data persistent 持久化
# ports:
# - "6379:6379" # expose port to host machine 暴露接口到宿主机
# splash: # use Splash to run spiders on dynamic pages
# image: scrapinghub/splash
# container_name: splash
# ports:
# - "8050:8050"
4.开始启动,访问http://localhost:8080 就可以看到界面。
更新里面的配置设置即可:
docker-compose up -d
5. 更新 / 重启 Crawlab
# 关闭并删除 Docker 容器
docker-compose down
# 拉取最新镜像
docker pull tikazyq/crawlab:latest
# 启动 Docker 容器
docker-compose up -d
查看容器:docker ps
进入容器:docker exec -it "ID" /bin/bash
6.官方文件,docker-compose.yml
version: '3.3'
services:
master:
image: tikazyq/crawlab:latest
container_name: master
environment:
# CRAWLAB_API_ADDRESS: "https://<your_api_ip>:<your_api_port>" # backend API address 后端 API 地址. 适用于 https 或者源码部署
CRAWLAB_SERVER_MASTER: "Y" # whether to be master node 是否为主节点,主节点为 Y,工作节点为 N
CRAWLAB_MONGO_HOST: "mongo" # MongoDB host address MongoDB 的地址,在 docker compose 网络中,直接引用服务名称
CRAWLAB_MONGO_PORT: "27018" # MongoDB port MongoDB 的端口
# CRAWLAB_MONGO_DB: "crawlab_test" # MongoDB database MongoDB 的数据库
# CRAWLAB_MONGO_USERNAME: "username" # MongoDB username MongoDB 的用户名
# CRAWLAB_MONGO_PASSWORD: "password" # MongoDB password MongoDB 的密码
# CRAWLAB_MONGO_AUTHSOURCE: "admin" # MongoDB auth source MongoDB 的验证源
CRAWLAB_REDIS_ADDRESS: "redis" # Redis host address Redis 的地址,在 docker compose 网络中,直接引用服务名称
CRAWLAB_REDIS_PORT: "6380" # Redis port Redis 的端口
# CRAWLAB_REDIS_DATABASE: "1" # Redis database Redis 的数据库
# CRAWLAB_REDIS_PASSWORD: "password" # Redis password Redis 的密码
# CRAWLAB_LOG_LEVEL: "info" # log level 日志级别. 默认为 info
# CRAWLAB_LOG_ISDELETEPERIODICALLY: "N" # whether to periodically delete log files 是否周期性删除日志文件. 默认不删除
# CRAWLAB_LOG_DELETEFREQUENCY: "@hourly" # frequency of deleting log files 删除日志文件的频率. 默认为每小时
# CRAWLAB_SERVER_REGISTER_TYPE: "mac" # node register type 节点注册方式. 默认为 mac 地址,也可设置为 ip(防止 mac 地址冲突)
# CRAWLAB_SERVER_REGISTER_IP: "127.0.0.1" # node register ip 节点注册IP. 节点唯一识别号,只有当 CRAWLAB_SERVER_REGISTER_TYPE 为 "ip" 时才生效
# CRAWLAB_TASK_WORKERS: 8 # number of task executors 任务执行器个数(并行执行任务数)
# CRAWLAB_RPC_WORKERS: 16 # number of RPC workers RPC 工作协程个数
# CRAWLAB_SERVER_LANG_NODE: "Y" # whether to pre-install Node.js 预安装 Node.js 语言环境
# CRAWLAB_SERVER_LANG_JAVA: "Y" # whether to pre-install Java 预安装 Java 语言环境
# CRAWLAB_SETTING_ALLOWREGISTER: "N" # whether to allow user registration 是否允许用户注册
# CRAWLAB_SETTING_ENABLETUTORIAL: "N" # whether to enable tutorial 是否启用教程
# CRAWLAB_NOTIFICATION_MAIL_SERVER: smtp.exmaple.com # STMP server address STMP 服务器地址
# CRAWLAB_NOTIFICATION_MAIL_PORT: 465 # STMP server port STMP 服务器端口
# CRAWLAB_NOTIFICATION_MAIL_SENDEREMAIL: admin@exmaple.com # sender email 发送者邮箱
# CRAWLAB_NOTIFICATION_MAIL_SENDERIDENTITY: admin@exmaple.com # sender ID 发送者 ID
# CRAWLAB_NOTIFICATION_MAIL_SMTP_USER: username # SMTP username SMTP 用户名
# CRAWLAB_NOTIFICATION_MAIL_SMTP_PASSWORD: password # SMTP password SMTP 密码
ports:
- "8080:8080" # frontend port mapping 前端端口映射
depends_on:
- mongo
- redis
# volumes:
# - "/var/crawlab/log:/var/logs/crawlab" # log persistent 日志持久化
worker:
image: tikazyq/crawlab:latest
container_name: worker
environment:
CRAWLAB_SERVER_MASTER: "N"
CRAWLAB_MONGO_HOST: "mongo"
CRAWLAB_REDIS_ADDRESS: "redis"
depends_on:
- mongo
- redis
# environment:
# MONGO_INITDB_ROOT_USERNAME: username
# MONGO_INITDB_ROOT_PASSWORD: password
# volumes:
# - "/var/crawlab/log:/var/logs/crawlab" # log persistent 日志持久化
mongo:
image: mongo:latest
restart: always
# volumes:
# - "/opt/crawlab/mongo/data/db:/data/db" # make data persistent 持久化
# ports:
# - "27017:27017" # expose port to host machine 暴露接口到宿主机
redis:
image: redis:latest
restart: always
# command: redis-server --requirepass "password" # set redis password 设置 Redis 密码
# volumes:
# - "/opt/crawlab/redis/data:/data" # make data persistent 持久化
# ports:
# - "6379:6379" # expose port to host machine 暴露接口到宿主机
# splash: # use Splash to run spiders on dynamic pages
# image: scrapinghub/splash
# container_name: splash
# ports:
# - "8050:8050"