一、环境
组件 | 版本 |
---|
Apache Hive | 3.1.3 |
Postgres JDBC | 42.3.6 |
Apache Hadoop | 3.3.2 |
Apache Iceberg | 0.14.0 |
Java | 1.8.0_111 |
Apache maven | 3.8.6 |
二、Docker 镜像构建
BASE_DIR=/opt/soft/lakehouse/hive
HARBOR_PATH=core.harbor.apps.g66666.hk.my-demo.tech/lakehouse
HIVE_VERSION=3.1.3
HADOOP_VERSION=3.3.2
POSTGRES_JDBC_VERSION=42.3.6
ICEBERG_VERSION=0.14.0
cd ${BASE_DIR}
wget https://dlcdn.apache.org/hive/hive-${HIVE_VERSION}/apache-hive-${HIVE_VERSION}-src.tar.gz
tar -zxf apache-hive-${HIVE_VERSION}-src.tar.gz
cd apache-hive-${HIVE_VERSION}-src/standalone-metastore/
mvn clean package
${BASE_DIR}/apache-hive-${HIVE_VERSION}-src/standalone-metastore/target/apache-hive-metastore-3.1.3-bin.tar.gz
cd ${BASE_DIR}
wget https://dlcdn.apache.org/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz
tar -zxf hadoop-${HADOOP_VERSION}.tar.gz
cd hadoop-${HADOOP_VERSION}/share
rm -rf doc
tar -zxf ${BASE_DIR}/apache-hive-${HIVE_VERSION}-src/standalone-metastore/target/apache-hive-metastore-${HIVE_VERSION}-bin.tar.gz \
-C ${BASE_DIR}/
cd apache-hive-metastore-${HIVE_VERSION}/bin/lib/
rm -rf guava-*.jar
cp ${BASE_DIR}/hadoop-${HADOOP_VERSION}/share/hadoop/common/lib/guava-*-jre.jar .
cp ${BASE_DIR}/hadoop-${HADOOP_VERSION}/share/hadoop/tools/lib/aws-java-sdk-bundle-*.jar .
cp ${BASE_DIR}/hadoop-${HADOOP_VERSION}/share/hadoop/tools/lib/hadoop-aws-${HADOOP_VERSION}.jar .
wget https://repo1.maven.org/maven2/org/postgresql/postgresql/${POSTGRES_JDBC_VERSION}/postgresql-${POSTGRES_JDBC_VERSION}.jar
wget https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-hive-runtime/${ICEBERG_VERSION}/iceberg-hive-runtime-${ICEBERG_VERSION}.jar
cd ${BASE_DIR}
cat << EOF | tee Dockerfile
FROM openjdk:8-slim
ARG hive_uid=10000
USER root
ENV HIVE_HOME /opt/hive-metastore
ENV HADOOP_HOME /opt/hadoop
RUN set -ex && \
sed -i 's/http:\/\/deb.\(.*\)/https:\/\/deb.\1/g' /etc/apt/sources.list && \
apt-get update && \
apt install -y bash tini libc6 libpam-modules krb5-user libnss3 procps && \
useradd -u \${hive_uid} -g root hive && \
useradd -u 10001 -g root luke && \
mkdir -p \${HIVE_HOME} \${HADOOP_HOME} && \
chmod ug+rw -R \${HIVE_HOME} && \
chmod ug+rw -R \${HADOOP_HOME} && \
rm -rf /var/cache/apt/*
COPY apache-hive-metastore-${HIVE_VERSION}-bin \${HIVE_HOME}
COPY hadoop-${HADOOP_VERSION} \${HADOOP_HOME}
WORKDIR \${HIVE_HOME}
CMD ["./bin/start-metastore", "-p", "9083"]
USER \${hive_uid}
EOF
cd ${BASE_DIR}
sudo docker build -t hive-metastore-iceberg:${HIVE_VERSION} .
sudo docker tag hive-metastore-iceberg:${HIVE_VERSION} ${HARBOR_PATH}/hive-metastore-iceberg:${HIVE_VERSION}
sudo docker push ${HARBOR_PATH}/hive-metastore-iceberg:${HIVE_VERSION}
![](https://i-blog.csdnimg.cn/blog_migrate/edf1147ea157b598b61e47522b490a4e.jpeg)