Dolphinscheduler配置

集群部署 

[root@node01 ~]#  yum install -y psmisc

上传解压移动 

[root@node01 ~]# tar -zxvf apache-dolphinscheduler-3.1.4-bin.tar.gz

配置install文件 

[root@node01 ~]# vim apache-dolphinscheduler-3.1.4-bin/bin/env/install_env.sh

# 部署master、worker、API server等相关服务
25 ips="node01,node02,node03"
29 sshPort="22"
34 masters="node01,node02"
39 workers="node01:default,node02:default,node03:default"
44 alertServer="node01"
49 apiServers="node01"
# DolphinScheduler安装路径，如果不存在会创建
53 installPath="/opt/apache-dolphinscheduler-3.1.4"
# 部署用户
58 deployUser="root"
# Zookeeper的存储位置
61 zkRoot="/dolphinscheduler"

配置env文件

 [root@node01 ~]# vim apache-dolphinscheduler-3.1.4-bin/bin/env/dolphinscheduler_env.sh

 18 # JAVA_HOME, will use it to start DolphinScheduler server
 19 export JAVA_HOME=/usr/java/jdk1.8.0_351-amd64
 20 
 21 # Database related configuration, set database type, username and password
 22 export DATABASE="mysql"
 23 export SPRING_PROFILES_ACTIVE=${DATABASE}
 24 export SPRING_DATASOURCE_URL="jdbc:mysql://ip:port/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8&useSSL=false&allowPublicKeyRetrieval=true"
 25 export SPRING_DATASOURCE_USERNAME="dolphinscheduler"
 26 export SPRING_DATASOURCE_PASSWORD="Ds@1234"
 27 
 28 # DolphinScheduler server related configuration
 29 export SPRING_CACHE_TYPE="none"
 30 export SPRING_JACKSON_TIME_ZONE="Asia/Shanghai"
 31 export MASTER_FETCH_COMMAND_NUM="10"
 32 
 33 # Registry center configuration, determines the type and link of the registry center
 34 export REGISTRY_TYPE="zookeeper"
 35 export REGISTRY_ZOOKEEPER_CONNECT_STRING="node01:2181,node02:2181,node03:2181"
 36 
 37 #Tasks related configurations, need to change the configuration if you use the related tasks.
 38 export HADOOP_HOME=/opt/hadoop-3.3.4
 39 export HADOOP_CONF_DIR=/opt/hadoop-3.3.4/etc/hadoop
 40 export SPARK_HOME1=/opt/spark-3.3.2
 41 export SPARK_HOME2=${SPARK_HOME2:-/opt/soft/spark2}
 42 export PYTHON_HOME=${PYTHON_HOME:-/opt/soft/python}
 43 export HIVE_HOME=/opt/apache-hive-3.1.3-bin
 44 export FLINK_HOME=${FLINK_HOME:-/opt/soft/flink}
 45 export DATAX_HOME=/opt/datax
 46 export SEATUNNEL_HOME=${SEATUNNEL_HOME:-/opt/soft/seatunnel}
 47 export CHUNJUN_HOME=${CHUNJUN_HOME:-/opt/soft/chunjun}

配置common文件 

[root@node01 ~]# vim apache-dolphinscheduler-3.1.4-bin/api-server/conf/common.properties

25 resource.storage.type=HDFS
52 resource.hdfs.root.user=root
54 resource.hdfs.fs.defaultFS= 填写自己的配置
75 yarn.resourcemanager.ha.rm.ids=node01,node02

[root@node01 ~]# vim apache-dolphinscheduler-3.1.4-bin/worker-server/conf/common.properties

25 resource.storage.type=HDFS
52 resource.hdfs.root.user=root
54 resource.hdfs.fs.defaultFS=  填写自己的配置
75 yarn.resourcemanager.ha.rm.ids=node01,node03

将 Hadoop 集群下的 core-site.xml 和 hdfs-site.xml 复制到 worker-server/conf 以及 api-server/conf

[root@node01 ~]# cp $HADOOP_HOME/etc/hadoop/core-site.xml   ~/apache-dolphinscheduler-3.1.4-bin/worker-server/conf/
[root@node01 ~]# cp $HADOOP_HOME/etc/hadoop/hdfs-site.xml ~/apache-dolphinscheduler-3.1.4-bin/worker-server/conf/ 
[root@node01 ~]# cp $HADOOP_HOME/etc/hadoop/core-site.xml ~/apache-dolphinscheduler-3.1.4-bin/api-server/conf/ 
[root@node01 ~]# cp $HADOOP_HOME/etc/hadoop/hdfs-site.xml ~/apache-dolphinscheduler-3.1.4-bin/api-server/conf/

配置Mysql

拷贝Jar包

[root@node01 ~]# cp mysql-connector-java-8.0.20.jar apache-dolphinscheduler-3.1.4-bin/api-server/libs/ 
[root@node01 ~]# cp mysql-connector-java-8.0.20.jar apache-dolphinscheduler-3.1.4-bin/alert-server/libs/ 
[root@node01 ~]# cp mysql-connector-java-8.0.20.jar apache-dolphinscheduler-3.1.4-bin/master-server/libs/
[root@node01 ~]# cp mysql-connector-java-8.0.20.jar apache-dolphinscheduler-3.1.4-bin/worker-server/libs/ 
[root@node01 ~]# cp mysql-connector-java-8.0.20.jar apache-dolphinscheduler-3.1.4-bin/tools/libs/

[root@node01 ~]# mysql -uroot -p
mysql> CREATE DATABASE dolphinscheduler DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
# 修改 {user} 和 {password} 为你希望的用户名和密码
mysql> CREATE USER 'dolphinscheduler'@'%' IDENTIFIED BY 'Ds@1234';
mysql> GRANT ALL PRIVILEGES ON dolphinscheduler.* TO 'dolphinscheduler'@'%';
mysql> CREATE USER 'dolphinscheduler'@'localhost' IDENTIFIED BY 'Ds@1234';
mysql> GRANT ALL PRIVILEGES ON dolphinscheduler.* TO 'dolphinscheduler'@'localhost';
mysql> FLUSH PRIVILEGES;
mysql> exit;

启动所有的zookeeper

[root@node123 ~]# zkServer.sh start 

启动Hadoop

[root@node01 ~]# start-all.sh 

初始化数据库

[root@node01 ~]# bash apache-dolphinscheduler-3.1.4-bin/tools/bin/upgrade-schema.sh

安装部署Dolphin

[root@node01 ~]# bash apache-dolphinscheduler-3.1.4-bin/bin/install.sh

访问服务

浏览器访问地址 http://ip:12345/dolphinscheduler/ui 即可登录系统UI。 

默认的用户名和密码是 admin/dolphinscheduler123

# 一键停止集群所有服务

[root@node01 ~]# sh /opt/apache-dolphinscheduler-3.1.4/bin/stop-all.sh

# 一键开启集群所有服务

[root@node01 ~]# sh /opt/apache-dolphinscheduler-3.1.4/bin/start-all.sh

# 启停 Master

[root@node01 ~]# sh /opt/apache-dolphinscheduler-3.1.4/bin/dolphinscheduler-daemon.sh stop master-server

[root@node01 ~]# sh /opt/apache-dolphinscheduler-3.1.4/bin/dolphinscheduler-daemon.sh start master-server

# 启停 Worker

[root@node01 ~]# sh /opt/apache-dolphinscheduler-3.1.4/bin/dolphinscheduler-daemon.sh start worker-server

[root@node01 ~]# sh /opt/apache-dolphinscheduler-3.1.4/bin/dolphinscheduler-daemon.sh stop worker-server

# 启停 Api 

[root@node01 ~]# sh /opt/apache-dolphinscheduler-3.1.4/bin/dolphinscheduler-daemon.sh start api-server 

[root@node01 ~]# sh /opt/apache-dolphinscheduler-3.1.4/bin/dolphinscheduler-daemon.sh stop api-server 

# 启停 Logger 

[root@node01 ~]# sh /opt/apache-dolphinscheduler-3.1.4/bin/dolphinscheduler-daemon.sh start logger-server 

[root@node01 ~]# sh /opt/apache-dolphinscheduler-3.1.4/bin/dolphinscheduler-daemon.sh stop logger-server 

# 启停 Alert 

[root@node01 ~]# sh /opt/apache-dolphinscheduler-3.1.4/bin/dolphinscheduler-daemon.sh start alert-server 

[root@node01 ~]# sh /opt/apache-dolphinscheduler3.1.4/bin/dolphinscheduler-daemon.sh stop alert-server 

# 启停 Python Gateway 

[root@node01 ~]# sh /opt/apache-dolphinscheduler-3.1.4/bin/dolphinscheduler-daemon.sh start python-gateway-server 

[root@node01 ~]# sh /opt/apache-dolphinscheduler-3.1.4/bin/dolphinscheduler-daemon.sh stop python-gateway-server