kyuubi部署

安乐lly

于 2024-09-04 15:22:27 发布

阅读量278

点赞数 2

文章标签： hadoop spark

本文链接：https://blog.csdn.net/rachelfffy/article/details/141894886

版权

1.解压

tar -xvf apache-kyuubi-1.8.0-bin.tgz -C /opt

2.进入/opt/kyuubi/conf目录修改配置文件

cd /opt/kyuubi/conf
cp kyuubi-env.sh.template kyuubi-env.sh
cp kyuubi-defaults.conf.template kyuubi-defaults.conf

kyuubi-env.sh添加如下配置

export JAVA_HOME=/usr/local/jdk1.8.0_192
export SPARK_HOME=/opt/spark-3.2.2-bin-hadoop3.2
#export FLINK_HOME=/opt/flink
export HIVE_HOME=/opt/hive
export HADOOP_CONF_DIR=/opt/hadoop/etc/hadoop
#export FLINK_HADOOP_CLASSPATH=/opt/hadoop-3.3.2/share/hadoop/client/hadoop-client-runtime-3.3.2.jar:/opt/hadoop-3.3.2/share/hadoop/client/hadoop-client-api-3.3.2.jar
#如果开启kerberos，需要添加以下参数,指定krb5cache文件名称，防止名称相同互相冲突
#export KRB5CCNAME=/tmp/krb5cc_kyuubi
export KYUUBI_JAVA_OPTS="-Xmx10g -XX:+UnlockDiagnosticVMOptions -XX:ParGCCardsPerStrideChunk=4096 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -XX:+CMSClassUnloadingEnabled -XX:+CMSParallelRemarkEnabled -XX:+UseCondCardMark -XX:MaxDirectMemorySize=1024m  -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=./logs -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintTenuringDistribution -Xloggc:./logs/kyuubi-server-gc-%t.log -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=5M -XX:NewRatio=3 -XX:MetaspaceSize=512m"

kyuubi-defaults.conf配置如下

kyuubi.frontend.bind.host                hadoop001
kyuubi.frontend.protocols                THRIFT_BINARY,REST
kyuubi.frontend.thrift.binary.bind.port  10009
kyuubi.frontend.rest.bind.port           10099
#
kyuubi.engine.type                       SPARK_SQL
kyuubi.engine.share.level                USER
kyuubi.session.engine.initialize.timeout PT3M
#
kyuubi.ha.addresses                      hadoop001:2181,hadoop002:2181,hadoop003:2181
kyuubi.ha.namespace                      kyuubi

3.启动kyuubi

cd ../bin
hadoop@hadoop001:/opt/kyuubi/bin$ ./kyuubi start
Using kyuubi environment file /opt/kyuubi/conf/kyuubi-env.sh to initialize...
JAVA_HOME: /usr/local/jdk1.8.0_192
KYUUBI_HOME: /opt/kyuubi
KYUUBI_CONF_DIR: /opt/kyuubi/conf
KYUUBI_LOG_DIR: /opt/kyuubi/logs
KYUUBI_PID_DIR: /opt/kyuubi/pid
KYUUBI_WORK_DIR_ROOT: /opt/kyuubi/work
FLINK_HOME: 
FLINK_ENGINE_HOME: /opt/kyuubi/externals/engines/flink
SPARK_HOME: /opt/spark-3.2.2-bin-hadoop3.2
SPARK_CONF_DIR: /opt/spark-3.2.2-bin-hadoop3.2/conf
SPARK_ENGINE_HOME: /opt/kyuubi/externals/engines/spark
TRINO_ENGINE_HOME: /opt/kyuubi/externals/engines/trino
HIVE_ENGINE_HOME: /opt/kyuubi/externals/engines/hive
HADOOP_CONF_DIR: /opt/hadoop/etc/hadoop
YARN_CONF_DIR: 
Starting org.apache.kyuubi.server.KyuubiServer, logging to /opt/kyuubi/logs/kyuubi-hadoop-org.apache.kyuubi.server.KyuubiServer-hadoop001.out
Welcome to


            __  __                            __                          
           /` \/` \                          /` \       __                
           \ \ \/ /    __  __  __  __   __ __\ \ \____ /\_\               
            \ \ , <   /` \/` \/` \/` \/` \/` \\ \ \__` \/` \              
             \ \ \ \`\\ \ \_\ \\ \_\  \\ \_\  \\ \ \L\  \ \ \             
              \ \_\ \_ \/`____ \ \____/ \ \____`\ \_,__/ \ \_\            
               \/_/\/_/ `/___/> \/___/   \/___/  \/___/   \/_/            
                           /\___/                                         
                           \/__\/