CDH镜像构建

CDH从节点镜像构建

前期准备

已构建centos67-ssh53-java:1.8

limits.conf内容

#用于修改文件句柄数(nofile)、进程数(nproc)、最大锁定内存(memlock)

#cloudera-scm为用户名

#软件限制

cloudera-scm soft nofile 32768
cloudera-scm soft nproc 65536

#硬件限制

cloudera-scm hard nofile 1048576
cloudera-scm hard nproc unlimited
cloudera-scm hard memlock unlimited

rclocal.conf内容

#用于关闭透明大页面

echo never > /sys/kernel/mm/transparent_hugepage/defrag
echo never > /sys/kernel/mm/transparent_hugepage/enabled

supervisord.conf内容

#用于容器启动时启动相关服务或运行脚本

# priority进程启动优先级，默认999，值小的优先启动

#startretries启动失败自动重试次数，默认是3

#autorestart程序退出后自动重启

[program:sshd]
command=service sshd start
[program:crond]
command=service crond start
[program:rsyslog]
command=service rsyslog start
[program:scm-config]
priority=50
startretries=0
autorestart=false
command=sh /root/scm-config.sh
[program:cloudera-scm-agent]
priority=80
command=service cloudera-scm-agent start

scm-config.sh内容

#容器启动时执行的命令

#授予用户cloudera-scm访问对应目录权限

chown cloudera-scm:cloudera-scm $CMF_PARCELS_HOME/csd
chown cloudera-scm:cloudera-scm $CMF_PARCELS_HOME/data
chown cloudera-scm:cloudera-scm $CMF_PARCELS_HOME/parcels
chown cloudera-scm:cloudera-scm $CMF_PARCELS_HOME/parcel-repo
chown cloudera-scm:cloudera-scm $CMF_MANAGER_HOME/run/cloudera-scm-agent
chown cloudera-scm:cloudera-scm $CMF_MANAGER_HOME/run/cloudera-scm-server

#配置ntp时钟同步

sed -i ‘/^server.*/d’ /etc/ntp.conf
echo -e 'server '$CLOUDERA_MASTER ‘prefer\nfudge ‘$CLOUDERA_MASTER’
stratum 10’>> /etc/ntp.conf
echo '*/1 * * * * /usr/sbin/ntpdate -q '$CLOUDERA_MASTER >>
/var/spool/cron/root

#修改CDH参数

sed -i ‘s/^server_host=.*/server_host=’$CLOUDERA_MASTER’/g’
$CMF_MANAGER_HOME/etc/cloudera-scm-agent/config.ini

Dockerfile内容

FROM centos67-ssh53-java:1.8
ADD limits.conf /root/limits.conf
ADD rclocal.conf /root/rclocal.conf
ADD scm-config.sh /root/scm-config.sh
ADD supervisord.conf /root/supervisord.conf
ADD mysql-libs-5.1.73-8.el6_8.x86_64.rpm /root/mysql-libs-5.1.73-8.el6_8.x86_64.rpm
ADD mysql-community-release-el6-5.noarch.rpm /root/mysql-community-release-el6-5.noarch.rpm
COPY cloudera-manager-el6-cm5.15.0_x86_64.tar.gz /root/cloudera-manager-el6-cm5.15.0_x86_64.tar.gz
RUN echo ‘proxy=http://xzproxy.cnsuning.com:8080’ >> /etc/yum.conf \
&& echo -e ‘SELINUX=disabled’ >> /etc/sysconfig/selinux \
&& echo -e ‘UTC=false\nARC=false’ >> /etc/sysconfig/clock \
&& sed -i ‘s#^ZONE=.*#ZONE=“Asia/Shanghai”#g’ /etc/sysconfig/clock \
&& echo -e ‘cron.* /var/log/cron.log’ >> /etc/rsyslog.d/50-default.conf
\
&& sed -i ‘/^server.*/d’ /etc/ntp.conf \
&& echo -e ‘vm.swappiness = 10’ >> /etc/sysctl.conf \
&& sed -i ‘s#^OPTIONS=.*#OPTIONS="-g -l /var/log/ntpstats/ntpd.log"#g’
/etc/sysconfig/ntpd \
&& sed -i ‘s/^NETWORKING=.*/NETWORKING=yes/g’ /etc/sysconfig/network \
&& rpm -ivh /root/mysql-libs-5.1.73-8.el6_8.x86_64.rpm \
&& rpm -ivh /root/mysql-community-release-el6-5.noarch.rpm \
&& yum install -y chkconfig python bind-utils psmisc libxslt zlib sqlite
fuse fuse-libs portmap \
redhat-lsb cyrus-sasl-plain cyrus-sasl-gssapi python-psycopg2 bzip2-devel
supervisor \
&& yum clean all \
&& sed -i ‘/^proxy=.*/d’ /etc/yum.conf \
&& sed -i ‘s/^nodaemon=.*/nodaemon=true/g’ /etc/supervisord.conf \
&& cat /root/supervisord.conf >> /etc/supervisord.conf \
&& rm -rf /root/mysql-libs-5.1.73-8.el6_8.x86_64.rpm \
&& rm -rf /root/mysql-community-release-el6-5.noarch.rpm \
&& rm -rf /root/supervisord.conf \
&& rm -rf /var/cache/yum/* \
&& ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
&& ln -sf /usr/lib64/mysql/libmysqlclient.so.16.0.0
/usr/lib64/libmysqlclient.so.16 \
&& ln -sf /usr/lib64/mysql/libmysqlclient_r.so.16.0.0
/usr/lib64/libmysqlclient_r.so.16 \
&& tar -zxf /root/cloudera-manager-el6-cm5.15.0_x86_64.tar.gz -C /opt/ \
&& mv /opt/cm-5.15.0 /opt/cloudera-manager
ENV CMF_MANAGER_HOME=/opt/cloudera-manager \
CMF_PARCELS_HOME=/opt/cloudera \
CLOUDERA_MASTER=localhost
WORKDIR $CMF_MANAGER_HOME
RUN rm -rf /root/cloudera-manager-el6-cm5.15.0_x86_64.tar.gz \
&& sed -i ‘1i\export JAVA_HOME=’$JAVA_HOME
$CMF_MANAGER_HOME/etc/default/cloudera-scm-server \
&& sed -i
‘s#CMF_DEFAULTS:-/etc/default#CMF_DEFAULTS:-/opt/cloudera-manager/etc/default#g’
$CMF_MANAGER_HOME/etc/init.d/cloudera-scm-server \
&& sed -i ‘1i\export JAVA_HOME=’$JAVA_HOME
$CMF_MANAGER_HOME/etc/default/cloudera-scm-agent \
&& sed -i
‘s#CMF_DEFAULTS:-/etc/default#CMF_DEFAULTS:-/opt/cloudera-manager/etc/default#g’
$CMF_MANAGER_HOME/etc/init.d/cloudera-scm-agent \
&& sed -i ‘s/h:,P:,u:,p::,f,v,/h:,P:,u:,p:,f,v,/g’
$CMF_MANAGER_HOME/share/cmf/schema/scm_prepare_database.sh \
&& sed -i ‘s/user:,password::,scm-host:/user:,password:,scm-host:/g’
$CMF_MANAGER_HOME/share/cmf/schema/scm_prepare_database.sh \
&& cat /root/rclocal.conf >> /etc/rc.d/rc.local \
&& cat /root/limits.conf >> /etc/security/limits.conf \
&& rm -rf /root/rclocal.conf \
&& rm -rf /root/limits.conf \
&& ssh-keygen -t rsa -P ‘’ -f ~/.ssh/id_rsa \
&& cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys \
&& chmod u+rw ~/.ssh/authorized_keys \
&& chmod u+x /etc/rc.d/rc.local \
&& chmod u+x /root/scm-config.sh \
&& mkdir -p /usr/share/java \
&& mkdir /var/lib/cloudera-scm-server \
&& mkdir /var/lib/cloudera-scm-agent \
&& mkdir $CMF_MANAGER_HOME/run/cloudera-scm-agent \
&& mkdir $CMF_PARCELS_HOME/parcels \
&& mkdir $CMF_PARCELS_HOME/data \
&& useradd -r -d $CMF_MANAGER_HOME/run/cloudera-scm-server -M -c “Cloudera
SCM User” cloudera-scm \
&& cp $CMF_MANAGER_HOME/etc/init.d/cloudera-scm-server
/etc/init.d/cloudera-scm-server \
&& cp $CMF_MANAGER_HOME/etc/init.d/cloudera-scm-agent
/etc/init.d/cloudera-scm-agent \
&& cp $CMF_MANAGER_HOME/etc/init.d/cloudera-scm-server-db
/etc/init.d/cloudera-scm-server-db \
&& chkconfig cloudera-scm-agent on \
&& chkconfig supervisord on \
&& chkconfig rsyslog on \
&& chkconfig ntpd on
VOLUME ["/opt/cloudera-manager/log/cloudera-scm-agent",
“/opt/cloudera-manager/log/cmf”, “/opt/cloudera/data”,
“/opt/cloudera/parcel-repo”, “/opt/cloudera/csd”,
“/opt/cloudera/parcels”,"/var/lib/cloudera-scm-agent", “/usr/share/java”]
EXPOSE 22
CMD ["/usr/bin/supervisord"]

其他文件

mysql-community-release-el6-5.noarch.rpm用于添加安装mysql源

如果mysql没有安装于CDH主节点，需要运行mysql-libs-5.1.73-8.el6_8.x86_64.rpm

安装libmysqlclient.so.16.0.0、libmysqlclient_r.so.16.0.0，添加HUE组件时会用到

构建命令

进入到Dockerfile文件所在目录，执行以下命令：

sudo docker build -t cloudera-slaves:5.15.0 .

注意:构建镜像时必须联网

CDH主节点镜像构建

前期准备

已构建cloudera-slaves:5.15.0

scm-config.sh内容

#容器启动时执行的命令

#授予用户cloudera-scm访问对应目录权限

chown cloudera-scm:cloudera-scm $CMF_PARCELS_HOME/csd
chown cloudera-scm:cloudera-scm $CMF_PARCELS_HOME/data
chown cloudera-scm:cloudera-scm $CMF_PARCELS_HOME/parcels
chown cloudera-scm:cloudera-scm $CMF_PARCELS_HOME/parcel-repo
chown cloudera-scm:cloudera-scm $CMF_MANAGER_HOME/run/cloudera-scm-agent
chown cloudera-scm:cloudera-scm $CMF_MANAGER_HOME/run/cloudera-scm-server

#修改主机名

sed -i ‘s/^HOSTNAME=.*/HOSTNAME=’$HOSTNAME’/g’ /etc/sysconfig/network

#修改CDH连接数据库参数

sed -i ‘s/^server_host=.*/server_host=’$CLOUDERA_MASTER’/g’
$CMF_MANAGER_HOME/etc/cloudera-scm-agent/config.ini
sed -i
‘s/.*com.cloudera.cmf.db.type=.*/com.cloudera.cmf.db.type=’$DB_TYPE’/g’
$CMF_MANAGER_HOME/etc/cloudera-scm-server/db.properties
sed -i
‘s/.*com.cloudera.cmf.db.host=.*/com.cloudera.cmf.db.host=’$DB_HOST’/g’
$CMF_MANAGER_HOME/etc/cloudera-scm-server/db.properties
sed -i
‘s/.*com.cloudera.cmf.db.name=.*/com.cloudera.cmf.db.name=’$SCM_DB’/g’
$CMF_MANAGER_HOME/etc/cloudera-scm-server/db.properties
sed -i
‘s/.*com.cloudera.cmf.db.user=.*/com.cloudera.cmf.db.user=’$SCM_USER’/g’
$CMF_MANAGER_HOME/etc/cloudera-scm-server/db.properties
sed -i
‘s/.*com.cloudera.cmf.db.password=.*/com.cloudera.cmf.db.password=’$SCM_PWDS’/g’
$CMF_MANAGER_HOME/etc/cloudera-scm-server/db.properties
sed -i
‘s/.*com.cloudera.cmf.db.setupType=.*/com.cloudera.cmf.db.setupType=’$DB_SETUPTYPE’/g’
$CMF_MANAGER_HOME/etc/cloudera-scm-server/db.properties

scm-prepare-db.sh内容

#用于创建CDH元数据数据库

nohup $CMF_MANAGER_HOME/share/cmf/schema/scm_prepare_database.sh $DB_TYPE
-h $DB_HOST -u $DB_USER -p $DB_PWDS --scm-host $SCM_HOST $SCM_DB
$SCM_USER $SCM_PWDS
>$CMF_MANAGER_HOME/log/cloudera-scm-server/prepare_database.out 2>&1 &

Dockerfile内容

FROM cloudera-slaves:5.15.0
ENV CMF_MANAGER_HOME=/opt/cloudera-manager \
CMF_PARCELS_HOME=/opt/cloudera \
CLOUDERA_MASTER=localhost \
DB_SETUPTYPE=EXTERNAL \
DB_HOST=localhost \
DB_TYPE=mysql \
DB_USER=root \
DB_PWDS=root \
SCM_HOST=localhost \
SCM_DB=scm_base \
SCM_USER=scm \
SCM_PWDS=scm
WORKDIR $CMF_MANAGER_HOME
RUN rm -rf /root/scm-config.sh
ADD scm-config.sh /root/scm-config.sh
ADD scm-prepare-db.sh /root/scm-prepare-db.sh
#ADD mysql-connector-java-5.1.44.jar
/usr/share/java/mysql-connector-java.jar
#ADD manifest.json $CMF_PARCELS_HOME/parcel-repo/manifest.json
#ADD CDH-5.15.0-1.cdh5.15.0.p0.21-el6.parcel.sha
$CMF_PARCELS_HOME/parcel-repo/CDH-5.15.0-1.cdh5.15.0.p0.21-el6.parcel.sha
#ADD KAFKA-3.1.0-1.3.1.0.p0.35-el6.parcel.sha
$CMF_PARCELS_HOME/parcel-repo/KAFKA-3.1.0-1.3.1.0.p0.35-el6.parcel.sha
#COPY CDH-5.15.0-1.cdh5.15.0.p0.21-el6.parcel
$CMF_PARCELS_HOME/parcel-repo/CDH-5.15.0-1.cdh5.15.0.p0.21-el6.parcel
#COPY KAFKA-3.1.0-1.3.1.0.p0.35-el6.parcel
$CMF_PARCELS_HOME/parcel-repo/KAFKA-3.1.0-1.3.1.0.p0.35-el6.parc
#COPY KAFKA-1.2.0.jar $CMF_PARCELS_HOME/csd/KAFKA-1.2.0.jar
RUN sed -i ‘/^server.*/d’ /etc/ntp.conf \
&& sed -i ‘/.*priority=80.*/d’ /etc/supervisord.conf \
&& sed -i ‘/.*cloudera-scm-agent.*/d’ /etc/supervisord.conf \
&& echo -e ‘[program:ntp]\ncommand=service ntpd
start\n\n[program:scm-prepare-db]\npriority=60\nstartsecs=1\nstartretries=0\nautorestart=false\ncommand=sh
/root/scm-prepare-db.sh\n\n[program:cloudera-scm-server]\npriority=70\ncommand=service
cloudera-scm-server
start\n\n[program:cloudera-scm-agent]\npriority=80\ncommand=service
cloudera-scm-agent start’ >> /etc/supervisord.conf \
&& echo -e ‘\nserver 127.127.1.0\n\nfudge 127.127.1.0 stratum 10’ >>
/etc/ntp.conf \
&& chmod u+x /root/scm-config.sh \
&& chkconfig cloudera-scm-server on \
&& chkconfig ntpd on
VOLUME ["/opt/cloudera-manager/log/cloudera-scm-server",
“/opt/cloudera-manager/log/cloudera-scm-agent”,
“/opt/cloudera-manager/log/cmf”,
“/opt/cloudera-manager/run/cloudera-scm-server”,
“/opt/cloudera-manager/run/cloudera-scm-agent”, “/opt/cloudera/data”,
“/opt/cloudera/parcel-repo”, “/opt/cloudera/csd”, “/opt/cloudera/cache”,
“/opt/cloudera/parcels”,"/var/lib/cloudera-scm-server",
“/usr/share/java”,"/var/log/supervisord"]
EXPOSE 22 7180
CMD ["/usr/bin/supervisord"]

构建命令

进入到Dockerfile文件所在目录，执行以下命令：

sudo docker build -t cloudera-master:5.15.0 .

启动CDH节点

前期准备

配置主节点宿主机

创建目录

sudo mkdir -p /home/hadoop/cloudera/master/csd
sudo mkdir -p /home/hadoop/cloudera/master/data
sudo mkdir -p /home/hadoop/cloudera/master/java
sudo mkdir -p /home/hadoop/cloudera/master/parcel-repo
sudo mkdir -p /home/hadoop/cloudera/master/parcels
sudo mkdir -p /home/hadoop/cloudera/master/log/cmf
sudo mkdir -p /home/hadoop/cloudera/master/log/cloudera-scm-server
sudo mkdir -p /home/hadoop/cloudera/master/log/cloudera-scm-agent

其中hadoop是当前宿主机用户名，master是CDH主节点容器主机名

以上目录用于容器目录映射

配置manifest.json

其中CDH-5.15.0-1.cdh5.15.0.p0.21-el6.parcel和KAFKA-3.1.0-1.3.1.0.p0.35-el6.parcel是对应不同的的manifest.json文件，需要把kafka文件中的

{
“parcelName”: “KAFKA-3.1.0-1.3.1.0.p0.35-el6.parcel”,
“components”: [{
“pkg_version”: “1.0.1+kafka3.1.0”,
“pkg_release”: “1.3.1.0.p0.40”,
“name”: “kafka”,
“version”: “1.0.1-kafka3.1.0”
}],
“depends”: “CDH (>= 5.2), CDH (<< 6.0)”,
“replaces”: “CLABS_KAFKA”,
“hash”: “efa101bae7d94a5dc7b57ff5a09dfb041dc1546a”
}

添加到CDH-5.15.0-1.cdh5.15.0.p0.21-el6.parcel的manifest.json中

添加parcel-repo包

复制以下文件

CDH-5.15.0-1.cdh5.15.0.p0.21-el6.parcel
CDH-5.15.0-1.cdh5.15.0.p0.21-el6.parcel.sha
KAFKA-3.1.0-1.3.1.0.p0.35-el6.parcel
KAFKA-3.1.0-1.3.1.0.p0.35-el6.parcel.sha
manifest.json

到目录/home/hadoop/cloudera/master/parcel-repo目录

添加csd包

复制KAFKA-1.2.0.jar到/home/hadoop/cloudera/master/csd目录

添加数据库驱动

复制mysql-connector-java-5.1.44.jar到/home/hadoop/cloudera/master/java目录

修改mysql-connector-java-5.1.44.jar文件名为mysql-connector-java.jar

启动CDH容器

启动cloudera-master

sudo docker run -d -p 1023:22 -p 7180:7180 --name cloudera-master -v
/home/hadoop/cloudera/master/parcel-repo:/opt/cloudera/parcel-repo -v
/home/hadoop/cloudera/master/parcels:/opt/cloudera/parcels -v
/home/hadoop/cloudera/master/csd:/opt/cloudera/csd -v
/home/hadoop/cloudera/master/log/cloudera-scm-server:/opt/cloudera-manager/log/cloudera-scm-server
–v
/home/hadoop/cloudera/master/log/cloudera-scm-agent:/opt/cloudera-manager/log/cloudera-scm-agent
-v /home/hadoop/cloudera/master/log/cmf:/opt/cloudera-manager/log/cmf -v
/home/hadoop/cloudera/master/data:/opt/cloudera/data -v
/home/hadoop/cloudera/master/java:/usr/share/java --hostname cloudera-master
–restart=always --net shadownet --ip 172.18.0.3 --add-host
cloudera-slave1:172.18.0.4 --add-host cloudera-slave2:172.18.0.5 --add-host
cloudera-slave3:172.18.0.6 -e CLOUDERA_MASTER=cloudera-master -e
DB_HOST=10.49.2.114 -e DB_USER=root -e DB_PWDS=root#1234 -e SCM_HOST=% -e
SCM_DB=scm_base -e SCM_USER=scm -e SCM_PWDS=suning cloudera-master:5.15.0

注意：参数说明可参看mysql，如果mysql是单独镜像，需先启动mysql镜像

CLOUDERA_MASTER是cdh主节点容器主机名

DB_HOST是数据库IP

DB_USER是数据库用户

DB_PWDS是数据库密码

SCM_HOST是创建新用户允许访问的主机，%表示所有

SCM_USER是创建新用户名

SCM_PWDS是创建新用户密码

启动cloudera-slave1

sudo docker run -d -p 1024:22 --name cloudera-slave1 -v
/home/hadoop/cloudera/slave1/parcel-repo:/opt/cloudera/parcel-repo -v
/home/hadoop/cloudera/slave1/parcels:/opt/cloudera/parcels -v
/home/hadoop/cloudera/slave1/csd:/opt/cloudera/csd -v
/home/hadoop/cloudera/slave1/log/cloudera-scm-agent:/opt/cloudera-manager/log/cloudera-scm-agent
-v /home/hadoop/cloudera/slave1/log/cmf:/opt/cloudera-manager/log/cmf -v
/home/hadoop/cloudera/slave1/data:/opt/cloudera/data -v
/home/hadoop/cloudera/slave1/java:/usr/share/java --hostname cloudera-slave1
–restart=always --net shadownet --ip 172.18.0.4 --add-host
cloudera-master:172.18.0.3 --add-host cloudera-slave2:172.18.0.5 --add-host
cloudera-slave3:172.18.0.6 -e CLOUDERA_MASTER=cloudera-master
cloudera-slaves:5.15.0

启动cloudera-slave2

sudo docker run -d -p 1025:22 --name cloudera-slave2 -v
/home/hadoop/cloudera/slave2/parcel-repo:/opt/cloudera/parcel-repo -v
/home/hadoop/cloudera/slave2/parcels:/opt/cloudera/parcels -v
/home/hadoop/cloudera/slave2/csd:/opt/cloudera/csd -v
/home/hadoop/cloudera/slave2/log/cloudera-scm-agent:/opt/cloudera-manager/log/cloudera-scm-agent
-v /home/hadoop/cloudera/slave2/log/cmf:/opt/cloudera-manager/log/cmf -v
/home/hadoop/cloudera/slave2/data:/opt/cloudera/data -v
/home/hadoop/cloudera/slave2/java:/usr/share/java --hostname cloudera-slave2
–restart=always --net shadownet --ip 172.18.0.5 --add-host
cloudera-master:172.18.0.3 --add-host cloudera-slave1:172.18.0.4 --add-host
cloudera-slave3:172.18.0.6 -e CLOUDERA_MASTER=cloudera-master
cloudera-slaves:5.15.0

启动cloudera-slave3

sudo docker run -d -p 1026:22 --name cloudera-slave3 -v
/home/hadoop/cloudera/slave3/parcel-repo:/opt/cloudera/parcel-repo -v
/home/hadoop/cloudera/slave3/parcels:/opt/cloudera/parcels -v
/home/hadoop/cloudera/slave3/csd:/opt/cloudera/csd -v
/home/hadoop/cloudera/slave3/log/cloudera-scm-agent:/opt/cloudera-manager/log/cloudera-scm-agent
-v /home/hadoop/cloudera/slave3/log/cmf:/opt/cloudera-manager/log/cmf -v
/home/hadoop/cloudera/slave3/data:/opt/cloudera/data -v
/home/hadoop/cloudera/slave3/java:/usr/share/java --hostname cloudera-slave3
–restart=always --net shadownet --ip 172.18.0.6 --add-host
cloudera-master:172.18.0.3 --add-host cloudera-slave1:172.18.0.4 --add-host
cloudera-slave2:172.18.0.5 -e CLOUDERA_MASTER=cloudera-master
cloudera-slaves:5.15.0

CDH配置

打开浏览器访问cdh ui：

http://192.168.245.145:7180/

192.168.245.145是cdh主节点容器所在宿主机ip

后续按照页面提示一步步操作就可以

后续组件涉及到存储需将目录指定到

/home/hadoop/cloudera/master/data

/home/hadoop/cloudera/slave1/data

其中master、slave1对应主、从容器节点主机名