spark on hive=＞spark3.1.2 on hive3.1.2 重新编译

hive on spark参考这篇文章

wget https://archive.apache.org/dist/spark/spark-3.1.2/spark-3.1.2.tgz

编译Saprk源码前置条件

Maven 3.3.9 or newer

Java 8+

Scala

修改文件 make-distribution.sh

MVN="/data/java/apache-maven-3.8.1/bin/mvn"

先通过mvn下载相应的jar包

mvn -Pyarn -Phive -Phive-thriftserver -Psparkr -DskipTests clean package

编译spark

hadoop版本根据自身安装hadoop版本，

./dev/make-distribution.sh --name custom-spark --tgz -Psparkr -Phadoop-3.2 -Dhadoop.version=3.2.0 -Phive -Phive-thriftserver -Pyarn

cp /data/java/compire/spark-3.1.2/spark-3.1.2-bin-custom-spark.tgz /data/java

vim spark-env.xml

scala版本看pom.xml的version

<scala.version>2.12.10</scala.version>

export JAVA_HOME=/usr/lib/jvm/jdk1.8.0_301
export SCALA_HOME=/data/scala/scala-2.12.10
export HADOOP_HOME=/data/java/hadoop-3.2.2
export HADOOP_HDFS_HOME=${HADOOP_HOME}
export HADOOP_CONF_DIR=${HADOOP_HOME}/etc/hadoop
export SPARK_HOME=/data/java/spark-3.1.2-bin-custom-spark
export SPARK_DIST_CLASSPATH=$(hadoop classpath)
export HIVE_HOME=/data/java/apache-hive-3.1.2-bin
export MASTER_WEBUI_PORT=8079
export SPARK_LOG_DIR=/data/java/spark-3.1.2-bin-custom-spark/logs
export SPARK_LIBRARY_PATH=${SPARK_HOME}/jars
export PATH=$SCALA_HOME/bin:$SPARK_HOME/bin:$PATH

vim hive-defaults.conf

# Example:
# spark.master                     spark://master:7077
# spark.eventLog.enabled           true
# spark.eventLog.dir               hdfs://namenode:8021/directory
# spark.serializer                 org.apache.spark.serializer.KryoSerializer
# spark.driver.memory              5g
spark.executor.extraJavaOptions  -XX:+PrintGCDetails -Dkey=value -Dnumbers="one two three"
spark.executor.extraClassPath     /data/java/apache-hive-3.1.2-bin/lib/mysql-connector-java-8.0.26.jar 
spark.driver.extraClassPath       /data/java/apache-hive-3.1.2-bin/lib/mysql-connector-java-8.0.26.jar 

vim log4j.properties

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# Set everything to be logged to the console
log4j.rootCategory=WARN, console
log4j.appender.console=org.apache.log4j.ConsoleAppender
log4j.appender.console.target=System.err
log4j.appender.console.layout=org.apache.log4j.PatternLayout
log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n

# Set the default spark-shell log level to WARN. When running the spark-shell, the
# log level for this class is used to overwrite the root logger's log level, so that
# the user can have different defaults for the shell and regular Spark apps.
log4j.logger.org.apache.spark.repl.Main=WARN

# Settings to quiet third party logs that are too verbose
log4j.logger.org.sparkproject.jetty=WARN
log4j.logger.org.sparkproject.jetty.util.component.AbstractLifeCycle=ERROR
log4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=INFO
log4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=INFO
log4j.logger.org.apache.parquet=ERROR
log4j.logger.parquet=ERROR

# SPARK-9183: Settings to avoid annoying messages when looking up nonexistent UDFs in SparkSQL with Hive support
log4j.logger.org.apache.hadoop.hive.metastore.RetryingHMSHandler=FATAL
log4j.logger.org.apache.hadoop.hive.ql.exec.FunctionRegistry=ERROR

# For deploying Spark ThriftServer
# SPARK-34128：Suppress undesirable TTransportException warnings involved in THRIFT-4805
log4j.appender.console.filter.1=org.apache.log4j.varia.StringMatchFilter
log4j.appender.console.filter.1.StringToMatch=Thrift error occurred during processing of message
log4j.appender.console.filter.1.AcceptOnMatch=false

cd /data/java/apache-hive-3.1.2-bin/conf/

cp hive-site.xml /data/java/spark-3.1.2-bin-custom-spark/conf/

export SPARK_HOME=/data/java/spark-3.1.2-bin-custom-spark

测试是否生效

spark-sql

show tables;