CDH 6.3.1安装

一、准备工作

1 安装lrzsz

[root@hadoop001 ~]# yum -y install lrzsz 
[root@hadoop002 ~]# yum -y install lrzsz
[root@hadoop003 ~]# yum -y install lrzsz

---

2 上传安装文件

[root@hadoop001 ~]# rz

[root@hadoop001 cdh63]# ll -h
total 3.9G
-rw-r--r-- 1 root root 2.0G Oct 28 11:54 CDH-6.3.1-1.cdh6.3.1.p0.1470567-el7.parcel
-rw-r--r-- 1 root root   40 Oct 28 11:47 CDH-6.3.1-1.cdh6.3.1.p0.1470567-el7.parcel.sha1
-rw-r--r-- 1 root root 1.4G Oct 28 19:22 cm6.3.1-redhat7.tar.gz
-rw-r--r-- 1 root root  78M Oct 28 19:24 jre-8u181-linux-x64.tar.gz
-rw-r--r-- 1 root root  34K Oct 28 11:48 manifest.json
-rw-r--r-- 1 root root 523M Oct 28 19:24 mysql-5.7.11-linux-glibc2.5-x86_64.tar.gz
-rw-r--r-- 1 root root 984K Oct 28 11:49 mysql-connector-java-5.1.47.jar

---

3 修改文件名

CDH-6.3.1-1.cdh6.3.1.p0.1470567-el7.parcel.sha1 这个去掉 1 的后缀。若不去掉，cdh认为文件没有下载完成，会重新从官网下载

[root@hadoop001 cdh63]# mv CDH-6.3.1-1.cdh6.3.1.p0.1470567-el7.parcel.sha1 CDH-6.3.1-1.cdh6.3.1.p0.1470567-el7.parcel.sha

mysql-connector-java-5.1.47.jar，不去掉版本号，cdh会找不到mysql - connector

[root@hadoop001 cdh63]# mv mysql-connector-java-5.1.47.jar mysql-connector-java.jar

将mysql-connector 传到 cdh server所在的机器
[root@hadoop003 cdh63]# mkdir -p /usr/share/java/
[root@hadoop003 cdh63]# cp mysql-connector-java.jar /usr/share/java/mysql-connector-java.jar

在CDH-6.3.1-1.cdh6.3.1.p0.1470567-el7.parcel中 bin 目录下，有关于mysql-connector的配置

statestored 中有这种设置/usr/share/java/mysql-connector-java.jar
export MYSQL_CONNECTOR_JAR=${MYSQL_CONNECTOR_JAR:-/usr/share/java/mysql-connector-java.jar}

catalogd 中有这种设置/usr/share/java/mysql-connector-java.jar
export MYSQL_CONNECTOR_JAR=${MYSQL_CONNECTOR_JAR:-/usr/share/java/mysql-connector-java.jar}

impalad 中有这种设置/usr/share/java/mysql-connector-java.jar
export MYSQL_CONNECTOR_JAR=${MYSQL_CONNECTOR_JAR:-/usr/share/java/mysql-connector-java.jar}

---

二.集群节点初始化

1 配置 /etc/hosts

echo "172.26.251.196 hadoop001">> /etc/hosts 
echo "172.26.251.195 hadoop002">> /etc/hosts 
echo "172.26.251.194 hadoop003">> /etc/hosts 

[root@hadoop001 ~]# echo "172.26.251.196 hadoop001">> /etc/hosts 
[root@hadoop001 ~]# echo "172.26.251.195 hadoop002">> /etc/hosts 
[root@hadoop001 ~]# echo "172.26.251.194 hadoop003">> /etc/hosts

[root@hadoop002 ~]# echo "172.26.251.196 hadoop001">> /etc/hosts 
[root@hadoop002 ~]# echo "172.26.251.195 hadoop002">> /etc/hosts 
[root@hadoop002 ~]# echo "172.26.251.194 hadoop003">> /etc/hosts

[root@hadoop003 ~]# echo "172.26.251.196 hadoop001">> /etc/hosts 
[root@hadoop003 ~]# echo "172.26.251.195 hadoop002">> /etc/hosts 
[root@hadoop003 ~]# echo "172.26.251.194 hadoop003">> /etc/hosts

---

2 关闭所有节点的防火墙及清空规则

systemctl stop firewalld 
systemctl disable firewalld 
iptables -F

[root@hadoop001 bin]# systemctl stop firewalld 
[root@hadoop001 bin]# systemctl disable firewalld 
[root@hadoop001 bin]# iptables -F

[root@hadoop002 bin]# systemctl stop firewalld 
[root@hadoop002 bin]# systemctl disable firewalld 
[root@hadoop002 bin]# iptables -F

[root@hadoop003 bin]# systemctl stop firewalld 
[root@hadoop003 bin]# systemctl disable firewalld 
[root@hadoop003 bin]# iptables -F

---

3 每个节点部署JDK

1）新建 /usr/java

[root@hadoop001 ~]# mkdir /usr/java
[root@hadoop002 ~]# mkdir /usr/java
[root@hadoop003 ~]# mkdir /usr/java

2） 解压

[root@hadoop001 ~]# tar -xzvf jre-8u181-linux-x64.tar.gz -C /usr/java/
[root@hadoop002 ~]# tar -xzvf jre-8u181-linux-x64.tar.gz -C /usr/java/
[root@hadoop003 ~]# tar -xzvf jre-8u181-linux-x64.tar.gz -C /usr/java/

3）修正所属用户及用户组

[root@hadoop003 java]# ll
total 4
drwxr-xr-x 6 10 143 4096 Jul  7  2018 jre1.8.0_181

[root@hadoop001 java]# chown -R root:root /usr/java/jre1.8.0_181
[root@hadoop002 java]# chown -R root:root /usr/java/jre1.8.0_181
[root@hadoop003 java]# chown -R root:root /usr/java/jre1.8.0_181

[root@hadoop003 java]# ll
total 4
drwxr-xr-x 6 root root 4096 Jul  7  2018 jre1.8.0_181

4）配置jdk环境

[root@hadoop001 jre1.8.0_181]# echo "export JAVA_HOME=/usr/java/jre1.8.0_181" >> /etc/profile 
[root@hadoop001 jre1.8.0_181]# echo "export PATH=/usr/java/jre1.8.0_181/bin:${PATH}" >> /etc/profile 
[root@hadoop001 jre1.8.0_181]# source /etc/profile 
[root@hadoop001 jre1.8.0_181]# which java
/usr/java/jre1.8.0_181/bin/java

[root@hadoop002 jre1.8.0_181]# echo "export JAVA_HOME=/usr/java/jre1.8.0_181" >> /etc/profile 
[root@hadoop002 jre1.8.0_181]# echo "export PATH=/usr/java/jre1.8.0_181/bin:${PATH}" >> /etc/profile 
[root@hadoop002 jre1.8.0_181]# source /etc/profile 
[root@hadoop002 jre1.8.0_181]# which java
/usr/java/jre1.8.0_181/bin/java

[root@hadoop003 jre1.8.0_181]# echo "export JAVA_HOME=/usr/java/jre1.8.0_181" >> /etc/profile 
[root@hadoop003 jre1.8.0_181]# echo "export PATH=/usr/java/jre1.8.0_181/bin:${PATH}" >> /etc/profile 
[root@hadoop003 jre1.8.0_181]# source /etc/profile 
[root@hadoop003 jre1.8.0_181]# which java
/usr/java/jre1.8.0_181/bin/java

---

4 部署 mysql

mysql选择部署在第三台机器上 hadoop003

---

5 创建CDH的元数据库和用户、amon服务的数据库及用户

切换用户
[root@hadoop003 cdh63]# sudo su - mysqladmin
启动mysql服务
hadoop003:mysqladmin:/usr/local/mysql:>service mysql restart
登录msyql
hadoop003:mysqladmin:/usr/local/mysql:>mysql -uroot -pxxxx

create database cmf DEFAULT CHARACTER SET utf8; 
create database amon DEFAULT CHARACTER SET utf8; 
grant all on cmf.* TO 'cmf'@'%' IDENTIFIED BY '123456'; 
grant all on amon.* TO 'amon'@'%' IDENTIFIED BY '123456'; 
flush privileges;

mysql> create database cmf DEFAULT CHARACTER SET utf8; 
Query OK, 1 row affected (0.00 sec)

mysql> create database amon DEFAULT CHARACTER SET utf8; 
Query OK, 1 row affected (0.00 sec)

mysql> grant all on cmf.* TO 'cmf'@'%' IDENTIFIED BY '123456'; 
Query OK, 0 rows affected, 1 warning (0.00 sec)

mysql> grant all on amon.* TO 'amon'@'%' IDENTIFIED BY '123456'; 
Query OK, 0 rows affected, 1 warning (0.00 sec)

mysql> flush privileges;
Query OK, 0 rows affected (0.00 sec)

mysql> show databases;

---

三.CDH部署

1.离线部署cm server及agent

1.1.所有节点创建⽬目录及解压

mkdir /opt/cloudera-manager 
tar -xzvf cm6.3.1-redhat7.tar.gz -C /opt/cloudera-manager/

1.2.选择hadoop001为cm server，不不下载依赖包直接部署

cd /opt/cloudera-manager/cm6.3.1/RPMS/x86_64 
rpm -ivh cloudera-manager-daemons-6.3.1-1466458.el7.x86_64.rpm --nodeps --force 
rpm -ivh  cloudera-manager-server-6.3.1-1466458.el7.x86_64.rpm --nodeps --force

1.3.所有节点(包含hadoop001)为cm agent，不下载依赖包直接部署

cd /opt/cloudera-manager/cm6.3.1/RPMS/x86_64 
rpm -ivh cloudera-manager-daemons-6.3.1-1466458.el7.x86_64.rpm --nodeps --force 
rpm -ivh cloudera-manager-agent-6.3.1-1466458.el7.x86_64.rpm  --nodeps --force

1.4.所有节点修改agent的配置，指向server的节点

sed -i "s/server_host=localhost/server_host=hadoop001/g" /etc/cloudera-scm-agent/config.ini

1.5.主节点修改server的配置:

vi /etc/cloudera-scm-server/db.properties 
com.cloudera.cmf.db.type=mysql 
com.cloudera.cmf.db.host=hadoop001 
com.cloudera.cmf.db.name=cmf 
com.cloudera.cmf.db.user=cmf 
com.cloudera.cmf.db.password=123456 
com.cloudera.cmf.db.setupType=EXTERNAL

---

2.hadoop001节点部署离线parcel源

2.1.安装httpd服务

yum install -y httpd

2.2.部署离线parcel源

$ mkdir -p /var/www/html/cdh6_parcel 
[root@ruozedata001 CDH6.3.1]# cp  CDH-6.3.1-1.cdh6.3.1.p0.1470567-el7.parcel /var/ www/html/cdh6_parcel/
 [root@ruozedata001 CDH6.3.1]# mv CDH-6.3.1-1.cdh6.3.1.p0.1470567-el7.parcel.sha1 / var/www/html/cdh6_parcel/CDH-6.3.1-1.cdh6.3.1.p0.1470567-el7.parcel.sha [root@ruozedata001 CDH6.3.1]# mv manifest.json  /var/www/html/cdh6_parcel/

$ ll 
total 3081664 
-rw-r--r-- 1 root root 2083878000 Oct 23 13:44 CDH-6.3.1-1.cdh6.3.1.p0.1470567-el7 .parcel 
-rw-r--r-- 1 root root         40 Oct 23 13:44 CDH-6.3.1-1.cdh6.3.1.p0.1470567-el7 .parcel.sha1 
-rw-r--r-- 1 root root 1411444147 Oct 23 13:43 cm6.3.1-redhat7.tar.gz 
-rw-r--r-- 1 root root  173271626 Oct 23 13:43 jdk-8u45-linux-x64.gz 
-rw-r--r-- 1 root root      33887 Oct 23 13:43 manifest.json 
-rw-r--r-- 1 root root  548193637 Oct 23 13:42 mysql-5.7.11-linux-glibc2.5-x86_64. tar.gz 
-rw-r--r-- 1 root root    1007502 Sep  1 12:31 mysql-connector-java-5.1.47.jar

$ mv CDH-6.3.1-1.cdh6.3.1.p0.1470567-el7.parcel /var/www/html/cdh6_parcel/ 
$ mv CDH-6.3.1-1.cdh6.3.1.p0.1470567-el7.parcel.sha1 /var/www/html/cdh6_parcel/CDH -6.3.1-1.cdh6.3.1.p0.1470567-el7.parcel.sha 
$ mv manifest.json  /var/www/html/cdh6_parcel/ 
sha1名称⽂文件时，重命名去掉1，不不然在部署过程CM认为如上⽂文件下载未完整，会持续下载

2.3.启动httpd

systemctl start httpd
window/mac:
http://hadoop001/cdh6_parcel/ 查看是否可以访问

---

3.hadoop001节点启动Server

3.1.启动server

systemctl start cloudera-scm-server
查看⽇日志:

cd /var/log/cloudera-scm-server/ 

$ ll 
total 20 
-rw-r----- 1 cloudera-scm cloudera-scm 18265 Oct 23 15:37 cloudera-scm-server.log 
-rw-r----- 1 cloudera-scm cloudera-scm     0 Oct 23 15:37 cmf-server-nio.log 
-rw-r----- 1 cloudera-scm cloudera-scm     0 Oct 23 15:37 cmf-server-perf.log 

$ tail -F cloudera-scm-server.log
有错误就根据错误解决，没有错误，等待1min，出现7180端⼝，表明是成功的。

3.2.阿⾥里里云web界⾯面，设置该hadoop001节点防⽕火墙放开7180端⼝

3.3.等待1min，打开 http://hadoop001:7180 账号密码:admin/admin

3.4.假如打不不开，去看server的log，根据错误仔细排查错误

---

4.所有节点启动Agent

systemctl start cloudera-scm-agent

---

5.接下来，全部Web界⾯面操作

http://hadoop001:7180/
账号密码:admin/admin

---

6.选择存储库

选择http://hadoop001/cdh6_parcel/,并把其他存储库全去掉。

---

四 操作 WEBUI界面安装CDH时报错

1.找不到集群中的机器

agent配置指向参数错误

---