基于docker-compose的hive搭建

docker-compose.yml

这个文件是参考big-data-europe配置在之前博客中hadoop配置文件的基础上增加几个服务,另外把配置文件名字都改成env.env
注意新的hadoop版本中端口和之前的不一样:

Namenode ports: 50470 --> 9871, 50070 --> 9870, 8020 --> 9820
Secondary NN ports: 50091 --> 9869, 50090 --> 9868
Datanode ports: 50020 --> 9867, 50010 --> 9866, 50475 --> 9865, 50075 --> 9864

  hive-server:
    image: bde2020/hive:2.3.2-postgresql-metastore
    env_file:
      - ./env.env
    environment:
      HIVE_CORE_CONF_javax_jdo_option_ConnectionURL: "jdbc:postgresql://hive-metastore/metastore"
      SERVICE_PRECONDITION: "hive-metastore:9083"
    ports:
      - "10000:10000"

  hive-metastore:
    image: bde2020/hive:2.3.2-postgresql-metastore
    env_file:
      - ./env.env
    command: /opt/hive/bin/hive --service metastore
    environment:
      SERVICE_PRECONDITION: "namenode:9870 datanode:9864 hive-metastore-postgresql:5432"
    ports:
      - "9083:9083"

  hive-metastore-postgresql:
    image: bde2020/hive-metastore-postgresql:2.3.0
    ports:
      - "5432:5432"

  presto-coordinator:
    image: shawnzhu/prestodb:0.181
    ports:
      - "8080:8080"

整合后的docker-compose.yml如下

version: "3"

networks:
  default:
    driver: bridge
    
services:
  namenode:
    image: bde2020/hadoop-namenode
    container_name: namenode
    ports:
      - 9870:9870
      - 9000:9000
    environment:
      - CLUSTER_NAME=Qbit
    env_file:
      - ./env.env

  datanode:
    image: bde2020/hadoop-datanode
    container_name: datanode
    depends_on:
      - namenode
    ports:
      - 9864:9864
    environment:
      SERVICE_PRECONDITION: "namenode:9870"
    links:
      - "namenode:namenode"
    env_file:
      - ./env.env
  
  resourcemanager:
    image: bde2020/hadoop-resourcemanager
    container_name: resourcemanager
    ports:
      - 8088:8088
    links:
      - "namenode:namenode"
      - "datanode:datanode"
    depends_on:
      - namenode
      - datanode
    environment:
      SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864"
    env_file:
      - ./env.env

  nodemanager:
    image: bde2020/hadoop-nodemanager
    container_name: nodemanager
    links:
      - "namenode:namenode"
      - "datanode:datanode"
      - "resourcemanager:resourcemanager"
    depends_on:
      - namenode
      - datanode
      - resourcemanager
    environment:
      SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864 resourcemanager:8088"
    env_file:
      - ./env.env
  
  historyserver:
    image: bde2020/hadoop-historyserver
    container_name: historyserver
    links:
      - "namenode:namenode"
      - "datanode:datanode"
      - "resourcemanager:resourcemanager"
    depends_on:
      - namenode
      - datanode
      - resourcemanager
    environment:
      SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864 resourcemanager:8088"
    env_file:
      - ./env.env
      
  hive-metastore-postgresql:
    image: bde2020/hive-metastore-postgresql:2.3.0
    container_name: hive-metastore-postgresql
    ports:
      - "5432:5432"
      
  hive-metastore:
    image: bde2020/hive:2.3.2-postgresql-metastore
    container_name: hive-metastore
    env_file:
      - ./env.env
    command: /opt/hive/bin/hive --service metastore
    links:
      - "namenode:namenode"
      - "datanode:datanode"
      - "hive-metastore-postgresql:hive-metastore-postgresql"
    depends_on:
      - namenode
      - datanode
      - hive-metastore-postgresql
    environment:
      SERVICE_PRECONDITION: "namenode:9870 datanode:9864 hive-metastore-postgresql:5432"
    ports:
      - "9083:9083"
  
  hive-server:
    image: bde2020/hive:2.3.2-postgresql-metastore
    container_name: hive-server
    env_file:
      - ./env.env
    links:
      - "hive-metastore:hive-metastore"
      - "hive-metastore-postgresql:hive-metastore-postgresql"
    depends_on:
      - hive-metastore
      - hive-metastore-postgresql
    environment:
      SERVICE_PRECONDITION: "hive-metastore:9083 hive-metastore-postgresql:5432"
    ports:
      - "10000:10000"
      
  presto:
    image: shawnzhu/prestodb:0.181
    container_name: presto
    ports:
      - "8080:8080"

env.env

这个文件是参考了big-data-europe配置在之前博客的handoop.env这个配置文件的基础上增加了HIVE_SITE_CONF开头的几项,另外把这个配置文件的名字改成了env.env

HIVE_SITE_CONF_javax_jdo_option_ConnectionURL=jdbc:postgresql://hive-metastore-postgresql:5432/metastore
HIVE_SITE_CONF_javax_jdo_option_ConnectionDriverName=org.postgresql.Driver
HIVE_SITE_CONF_javax_jdo_option_ConnectionUserName=hive
HIVE_SITE_CONF_javax_jdo_option_ConnectionPassword=hive
HIVE_SITE_CONF_datanucleus_autoCreateSchema=false
HIVE_SITE_CONF_hive_metastore_uris=thrift://hive-metastore:9083