Dolphinscheduler伪集群安装部署（海豚调度）

你不是我的二狗子

已于 2024-07-19 12:06:36 修改

阅读量352

点赞数 3

文章标签：运维 linux

于 2024-07-10 19:27:35 首次发布

本文链接：https://blog.csdn.net/qq_43142346/article/details/140332118

版权

点击访问Dolphinscheduler官网

点击访问官方文档

为什么要用？

统一执行任务调度且拥有可视化界面，因为机器只有一台，所以使用伪集群部署。

1.创建用户并设置权限

# 创建用户需使用 root 登录
useradd dolphinscheduler

# 添加密码
echo "dolphinscheduler" | passwd --stdin dolphinscheduler

# 配置 sudo 免密
sed -i '$adolphinscheduler  ALL=(ALL)  NOPASSWD: NOPASSWD: ALL' /etc/sudoers
sed -i 's/Defaults    requirett/#Defaults    requirett/g' /etc/sudoers

# 修改目录权限，使得部署用户对二进制包解压后的 apache-dolphinscheduler-*-bin 目录有操作权限
chown -R dolphinscheduler:dolphinscheduler apache-dolphinscheduler-*-bin
chmod -R 755 apache-dolphinscheduler-*-bin

2.配置机器SSH免密登录

su dolphinscheduler

ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
chmod 600 ~/.ssh/authorized_keys

3.启动zookeeper

进入 zookeeper 的安装目录，将 zoo_sample.cfg 配置文件复制到 conf/zoo.cfg，并将 conf/zoo.cfg 中 dataDir 中的值改成 dataDir=./tmp/zookeeper

# 启动 zookeeper
./bin/zkServer.sh start

4.修改配置文件

4.1修改install_env.sh文件

# ---------------------------------------------------------
# INSTALL MACHINE

# ---------------------------------------------------------

# Due to the master, worker, and API server being deployed on a single node, the IP of the server is the machine IP or localhost

ips="localhost"

sshPort="22"

masters="localhost"

workers="localhost:default"

alertServer="localhost"

apiServers="localhost"



# DolphinScheduler installation path, it will auto-create if not exists

installPath=~/dolphinscheduler



# Deploy user, use the user you create in section **Configure machine SSH password-free login**

deployUser="dolphinscheduler"

4.2修改dolphinscheduler_env.sh文件

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# JAVA_HOME, will use it to start DolphinScheduler server
# 注意修改java路径，mysql地址和用户名密码  下面的其他组件伪集群部署不需要  也就没必要  也可以不删除
export JAVA_HOME=${JAVA_HOME:-/opt/openjdk/java}

# Database related configuration, set database type, username and password
export DATABASE=${DATABASE:-mysql}
export SPRING_PROFILES_ACTIVE=${DATABASE}
export SPRING_DATASOURCE_URL="jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8&useSSL=false"
export SPRING_DATASOURCE_USERNAME={user}
export SPRING_DATASOURCE_PASSWORD={password}

# DolphinScheduler server related configuration
export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none}
export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC}
export MASTER_FETCH_COMMAND_NUM=${MASTER_FETCH_COMMAND_NUM:-10}

# Registry center configuration, determines the type and link of the registry center
export REGISTRY_TYPE=${REGISTRY_TYPE:-zookeeper}
export REGISTRY_ZOOKEEPER_CONNECT_STRING=${REGISTRY_ZOOKEEPER_CONNECT_STRING:-localhost:2181}

# Tasks related configurations, need to change the configuration if you use the related tasks.
export HADOOP_HOME=${HADOOP_HOME:-/opt/soft/hadoop}
export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/opt/soft/hadoop/etc/hadoop}
export SPARK_HOME1=${SPARK_HOME1:-/opt/soft/spark1}
export SPARK_HOME2=${SPARK_HOME2:-/opt/soft/spark2}
export PYTHON_HOME=${PYTHON_HOME:-/opt/soft/python}
export HIVE_HOME=${HIVE_HOME:-/opt/soft/hive}
export FLINK_HOME=${FLINK_HOME:-/opt/soft/flink}
export DATAX_HOME=${DATAX_HOME:-/opt/soft/datax}
export SEATUNNEL_HOME=${SEATUNNEL_HOME:-/opt/soft/seatunnel}
export CHUNJUN_HOME=${CHUNJUN_HOME:-/opt/soft/chunjun}

export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$SEATUNNEL_HOME/bin:$CHUNJUN_HOME/bin:$PATH
~

5.初始化数据库

装个数据库5.7（给dolphinscheduler）这个自行百度吧很多

数据库安装后，为调度器创建用户和数据库

mysql -uroot -p

mysql> CREATE DATABASE dolphinscheduler DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;

# 修改 {user} 和 {password} 为你希望的用户名和密码
mysql> GRANT ALL PRIVILEGES ON dolphinscheduler.* TO '{user}'@'%' IDENTIFIED BY '{password}';
mysql> GRANT ALL PRIVILEGES ON dolphinscheduler.* TO '{user}'@'localhost' IDENTIFIED BY '{password}';

mysql> flush privileges;

注意下载数据库连接工具并放到 master-server/libs/ worker-server/libs/ alert-server/libs/ api-server/libs/ tools/libs/

wget https://downloads.mysql.com/archives/get/p/3/file/mysql-connector-java-8.0.16.tar.gz

tar -zxvf mysql-connector-java-8.0.16.tar.gz
cp mysql-connector-java-8.0.16/mysql-connector-java-8.0.16.jar /你的调度器路径/api-cp mysql-connector-java-8.0.16/mysql-connector-java-8.0.16.jar /你的调度器路径/server/libs/
cp mysql-connector-java-8.0.16/mysql-connector-java-8.0.16.jar /你的调度器路径/alert-server/libs/
cp mysql-connector-java-8.0.16/mysql-connector-java-8.0.16.jar /你的调度器路径/worker-server/libs/
cp mysql-connector-java-8.0.16/mysql-connector-java-8.0.16.jar /你的调度器路径/master-server/libs/
cp mysql-connector-java-8.0.16/mysql-connector-java-8.0.16.jar /你的调度器路径/tools/libs/


#这一步是将数据库文件导入到mysql内
bash tools/bin/upgrade-schema.sh

特别注意一定要在tools/libs中放入一份，不然会报数据库无法连接的错误（如果没有tools就不用放了)

6.部署服务

#注意了：这一步才是真正的部署步骤，会将目录复制到dolphinscheduler用户的家目录内
su - dolphinscheduler
cd /home/data/dolphinscheduler/ && bash ./bin/install.sh

7.启停服务

# 一键停止集群所有服务
bash ./bin/stop-all.sh


# 一键开启集群所有服务

bash ./bin/start-all.sh


# 启停 Master

bash ./bin/dolphinscheduler-daemon.sh stop master-server

bash ./bin/dolphinscheduler-daemon.sh start master-server


# 启停 Worker

bash ./bin/dolphinscheduler-daemon.sh start worker-server

bash ./bin/dolphinscheduler-daemon.sh stop worker-server


# 启停 Api

bash ./bin/dolphinscheduler-daemon.sh start api-server

bash ./bin/dolphinscheduler-daemon.sh stop api-server


# 启停 Alert

bash ./bin/dolphinscheduler-daemon.sh start alert-server

bash ./bin/dolphinscheduler-daemon.sh stop alert-server