dolphinscheduler-3.1.4

1、相关环境

1.1、创建用户，配置免密

useradd hadoop;
echo "Hadoop#149" | passwd --stdin hadoop

#配置sudo免密
sed -i '$ahadoop  ALL=(ALL)  NOPASSWD: NOPASSWD: ALL' /etc/sudoers
sed -i 's/Defaults    requirett/#Defaults    requirett/g' /etc/sudoers

1.2、ssh免密配置

#切换到部署用户并配置ssh本机免密登录
su hadoop;

ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
chmod 600 ~/.ssh/authorized_keys

1.3、关闭防火墙 firewalld 和 Selinux

#关闭防火墙：
systemctl stop firewalld
systemctl status firewalld
systemctl disable firewalld

#修改selinux的enforcing为disabled：
sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config

#查看selinux状态：
sestatus

1.4、创建数据库

在MySQL数据库中：
set global validate_password_policy=0;
set global validate_password_length=1;

CREATE DATABASE dolphinscheduler DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE USER ds @'%' IDENTIFIED BY 'Changxin*8';

GRANT ALL PRIVILEGES ON dolphinscheduler.* TO 'ds'@'%' IDENTIFIED BY 'Changxin*8';
GRANT ALL PRIVILEGES ON dolphinscheduler.* TO 'ds'@'localhost' IDENTIFIED BY 'Changxin*8';
flush privileges;

2、下载

wget https://mirrors.tuna.tsinghua.edu.cn/apache/dolphinscheduler/3.1.4/apache-dolphinscheduler-3.1.4-bin.tar.gz --no-check-certificate 

3、部署

3.1、创建目录并赋权

#创建目录
mkdir -p /opt/dolphin
#赋权
chown -R hadoop:hadoop /opt/dolphin
cd /opt/dolphin
tar -zxvf apache-dolphinscheduler-3.1.4-bin.tar.gz -C /opt/dolphin
#修改目录权限，使得部署用户对dolphin-backend目录有操作权限
chown -R hadoop:hadoop apache-dolphinscheduler-3.1.4-bin

3.2、修改配置dolphinscheduler_env.sh

cd /opt/software/dolphinscheduler-3.1.4/bin/env

# JAVA_HOME, will use it to start DolphinScheduler server
export JAVA_HOME=/usr/java/jdk1.8.0_201

# Database related configuration, set database type, username and password
export DATABASE=${DATABASE:-mysql}
export SPRING_PROFILES_ACTIVE=${DATABASE}
export SPRING_DATASOURCE_URL="jdbc:mysql://192.168.47.40:3306/ds?useUnicode=true&characterEncoding=UTF-8&useSSL=false"
export SPRING_DATASOURCE_USERNAME=dolphin
export SPRING_DATASOURCE_PASSWORD=Dolphin*020

# DolphinScheduler server related configuration
export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none}
export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC}
export MASTER_FETCH_COMMAND_NUM=${MASTER_FETCH_COMMAND_NUM:-10}

# Registry center configuration, determines the type and link of the registry center
export REGISTRY_TYPE=${REGISTRY_TYPE:-zookeeper}
export REGISTRY_ZOOKEEPER_CONNECT_STRING=${REGISTRY_ZOOKEEPER_CONNECT_STRING:-node-04:2181}

# Tasks related configurations, need to change the configuration if you use the related tasks.
export HADOOP_HOME=${HADOOP_HOME:-/opt/soft/hadoop}
export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/opt/soft/hadoop/etc/hadoop}
export SPARK_HOME1=${SPARK_HOME1:-/opt/soft/spark1}
export SPARK_HOME2=${SPARK_HOME2:-/opt/soft/spark2}
export PYTHON_HOME=${PYTHON_HOME:-/opt/soft/python}
export HIVE_HOME=${HIVE_HOME:-/opt/soft/hive}
export FLINK_HOME=${FLINK_HOME:-/opt/soft/flink}
export DATAX_HOME=${DATAX_HOME:-/opt/soft/datax}
export SEATUNNEL_HOME=${SEATUNNEL_HOME:-/opt/soft/seatunnel}
export CHUNJUN_HOME=${CHUNJUN_HOME:-/opt/soft/chunjun}

export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$SEATUNNEL_HOME/bin:$CHUNJUN_HOME/bin:$PATH

3.3、修改 install_env.sh

cd /opt/software/dolphinscheduler-3.1.4/bin/env

# ---------------------------------------------------------
# INSTALL MACHINE
# ---------------------------------------------------------
# A comma separated list of machine hostname or IP would be installed DolphinScheduler,
# including master, worker, api, alert. If you want to deploy in pseudo-distributed
# mode, just write a pseudo-distributed hostname
# Example for hostnames: ips="ds1,ds2,ds3,ds4,ds5", Example for IPs: ips="192.168.8.1,192.168.8.2,192.168.8.3,192.168.8.4,192.168.8.5"
ips=${ips:-"node-04"}

# Port of SSH protocol, default value is 22. For now we only support same port in all `ips` machine
# modify it if you use different ssh port
sshPort=${sshPort:-"22"}

# A comma separated list of machine hostname or IP would be installed Master server, it
# must be a subset of configuration `ips`.
# Example for hostnames: masters="ds1,ds2", Example for IPs: masters="192.168.8.1,192.168.8.2"
masters=${masters:-"node-04"}

# A comma separated list of machine <hostname>:<workerGroup> or <IP>:<workerGroup>.All hostname or IP must be a
# subset of configuration `ips`, And workerGroup have default value as `default`, but we recommend you declare behind the hosts
# Example for hostnames: workers="ds1:default,ds2:default,ds3:default", Example for IPs: workers="192.168.8.1:default,192.168.8.2:default,192.168.8.3:default"
workers=${workers:-"node-04:default"}

# A comma separated list of machine hostname or IP would be installed Alert server, it
# must be a subset of configuration `ips`.
# Example for hostname: alertServer="ds3", Example for IP: alertServer="192.168.8.3"
alertServer=${alertServer:-"node-04"}

# A comma separated list of machine hostname or IP would be installed API server, it
# must be a subset of configuration `ips`.
# Example for hostname: apiServers="ds1", Example for IP: apiServers="192.168.8.1"
apiServers=${apiServers:-"node-04"}

# The directory to install DolphinScheduler for all machine we config above. It will automatically be created by `install.sh` script if not exists.
# Do not set this configuration same as the current path (pwd). Do not add quotes to it if you using related path.
installPath=${installPath:-"/opt/module/ds-3.1.4"}

# The user to deploy DolphinScheduler for all machine we config above. For now user must create by yourself before running `install.sh`
# script. The user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled than the root directory needs
# to be created by this user
deployUser=${deployUser:-"hadoop"}

# The root of zookeeper, for now DolphinScheduler default registry server is zookeeper.
zkRoot=${zkRoot:-"/dolphinscheduler"}

3.4、添加MySQL驱动

cp ../mysql-connector-j-8.0.32.jar master-server/libs/
cp ../mysql-connector-j-8.0.32.jar worker-server/libs/
cp ../mysql-connector-j-8.0.32.jar alert-server/libs/
cp ../mysql-connector-j-8.0.32.jar api-server/libs/
cp ../mysql-connector-j-8.0.32.jar tools/libs

3.5、初始化数据库

bash tools/bin/upgrade-schema.sh

3.6、修改JVM参数

vi master-server/bin/start.sh

JAVA_OPTS=${JAVA_OPTS:-"-server -Duser.timezone=${SPRING_JACKSON_TIME_ZONE} -Xms1g -Xmx1g -Xmn512m -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof"}

3.7、一键安装&启动ds

bash ./bin/install.sh

4、登录操作

4.1、登录

 http://localhost:12345/dolphinscheduler/ui
 默认的用户名和密码： admin/dolphinscheduler123