环境准备
1.1 服务器准备
3台虚拟机、centos7.9、配置好对应主机的网络IP、主机名称、关闭防火墙、JDK服务、http服务。
1.2 SSH免密登录
1）生成公钥和私钥：
ssh-keygen -t rsa
2）将公钥拷贝到要免密登录的目标机器上
ssh-copy-id 主机名
注：每台服务器都需要操作1、2步骤。
1.3 chrony服务
1）查看chrony是否安装
rpm -qa | grep chrony
# 如果已安装，可忽略
2）修改配置文件vi /etc/chrony.conf
server 0.centos.pool.ntp.org iburst
server 1.centos.pool.ntp.org iburst
server 2.centos.pool.ntp.org iburst
server 3.centos.pool.ntp.org iburst
pool ntp1.aliyun.com iburst
pool ntp2.aliyun.com iburst
pool ntp3.aliyun.com iburst
pool ntp4.aliyun.com iburst
pool ntp5.aliyun.com iburst
pool ntp6.aliyun.com iburst
3）启动&关闭
systemctl start chronyd
systemctl stop chronyd
4）状态
systemctl status chronyd
5）查看是否同步成功
chronyc sources
^*开头表示成功
1.4 安装mysql，先卸载MySQL再安装
1）查看MySQL是否安装
[root@cdh1 bin]# rpm -qa | grep -i mysql
mysql-libs-5.1.73-7.el6.x86_64
2）如果安装了MySQL，就先卸载
[root@cdh1 bin]# rpm -e --nodeps mysql-libs-5.1.73-7.el6.x86_64
3）删除阿里云原有MySql依赖
[root@cdh1 bin]# yum remove mysql-libs
4）下载MySql依赖并安装
[root@cdh1 bin]# yum install libaio
[root@cdh1 bin]# yum -y install autoconf
[root@cdh1 mysql-5.6.24]# pwd
/home/gbs/software/mysql-5.6.24
[root@cdh1 mysql-5.6.24]# wget https://downloads.mysql.com/archives/get/p/23/file/MySQL-shared-compat-5.6.24-1.el6.x86_64.rpm
[root@cdh1 mysql-5.6.24]# wget https://downloads.mysql.com/archives/get/p/23/file/MySQL-shared-5.6.24-1.el6.x86_64.rpm
[root@cdh1 mysql-5.6.24]# rpm -ivh MySQL-shared-5.6.24-1.el6.x86_64.rpm
[root@cdh1 mysql-5.6.24]# rpm -ivh MySQL-shared-compat-5.6.24-1.el6.x86_64.rpm
5）上传mysql-libs.zip(百度网盘中)到cdh1的/home/gbs/software/mysql-5.6.24目录，并解压文件到当前目录
[root@cdh1 mysql-5.6.24]# yum install unzip
[root@cdh1 mysql-5.6.24]# unzip mysql-libs.zip
[root@cdh1 mysql-5.6.24]# ll
mysql-libs
mysql-libs.zip
MySQL-shared-5.6.24-1.el6.x86_64.rpm
MySQL-shared-compat-5.6.24-1.el6.x86_64.rpm
6）进入到mysql-libs文件夹下
[root@cdh1 mysql-5.6.24]# cd mysql-libs
[root@cdh1 mysql-libs]# ll
MySQL-client-5.6.24-1.el6.x86_64.rpm
mysql-connector-java-5.1.27.tar.gz
MySQL-server-5.6.24-1.el6.x86_64.rpm
1.5 安装MySQL服务器
1）安装MySQL服务端
[root@cdh1 mysql-libs]# rpm -ivh MySQL-server-5.6.24-1.el6.x86_64.rpm
2）查看产生的随机密码
[root@cdh1 mysql-libs]# cat /root/.mysql_secret
3）查看MySQL状态
[root@cdh1 mysql-libs]# service mysql status
ERROR! MySQL is not running
4）启动MySQL
[root@cdh1 mysql-libs]# service mysql start
Starting MySQL. SUCCESS!
[root@cdh1 mysql-libs]# service mysql status
SUCCESS! MySQL running (25869)
1.6 安装MySQL客户端
1）安装MySQL客户端
[root@cdh1 mysql-libs]# rpm -ivh MySQL-client-5.6.24-1.el6.x86_64.rpm
2）链接MySQL（密码替换成产生的随机密码）
[root@cdh1 mysql-libs]# mysql -u root -pt9On7OJEB7gpY2Ny
3）修改密码
mysql> SET PASSWORD=PASSWORD('123456.');
4）退出MySQL
mysql> exit;
1.7 MySQL中user表中主机配置
配置只要是root用户+密码，在任何主机上都能登录MySQL数据库。
（1）进入MySQL
[root@cdh1 mysql-libs]# mysql -u root -p123456.
（2）显示数据库
mysql>show databases;
（3）使用MySQL数据库
mysql>use mysql;
（4）展示MySQL数据库中的所有表
mysql>show tables;
（5）展示user表的结构
mysql>desc user;
（6）查询user表
mysql>select User, Host, Password from user;
（7）修改user表，把Host表内容修改为%
mysql>update user set host='%' where host='localhost';
（8）删除root用户的其他host
mysql> delete from user where host!='%';
（9）刷新
mysql>flush privileges;
（10）退出
mysql>quit;
CM安装
1.1 MySQL中建库
1）创建各组件需要的数据库
mysql> GRANT ALL ON scm.* TO 'scm'@'%' IDENTIFIED BY 'scm';
mysql> CREATE DATABASE scm DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
mysql> CREATE DATABASE hive DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
mysql> CREATE DATABASE oozie DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
mysql> CREATE DATABASE hue DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
1.2 CM安装
1）将mysql-connector-java-5.1.27-bin.jar拷贝到/usr/share/java路径下，并重命名
tar -zxvf mysql-connector-java-5.1.27.tar.gz
cd mysql-connector-java-5.1.27
mv mysql-connector-java-5.1.27-bin.jar mysql-connector-java.jar
mkdir /usr/share/java
cp mysql-connector-java.jar /usr/share/java/
scp -r /usr/share/java/ hadoop102:/usr/share/
scp -r /usr/share/java/ hadoop103:/usr/share/
2）创建cloudera-manager目录,存放cdh安装文件
[root@hadoop101 mysql-connector-java-5.1.27]# mkdir /opt/cloudera-manager
[root@hadoop101 mysql-connector-java-5.1.27]# cd /opt/software/
[root@hadoop101 software]# tar -zxvf cm6.3.1-redhat7.tar.gz
[root@hadoop101 software]# cd cm6.3.1/RPMS/x86_64/
[root@hadoop101 x86_64]# mv cloudera-manager-agent-6.3.1-1466458.el7.x86_64.rpm /opt/cloudera-manager/
[root@hadoop101 x86_64]# mv cloudera-manager-server-6.3.1-1466458.el7.x86_64.rpm /opt/cloudera-manager/
[root@hadoop101 x86_64]# mv cloudera-manager-daemons-6.3.1-1466458.el7.x86_64.rpm /opt/cloudera-manager/
[root@hadoop101 x86_64]# cd /opt/cloudera-manager/
[root@hadoop101 cloudera-manager]# ll
total 1185876
-rw-r--r-- 1 2001 2001 10483568 Sep 25 2019 cloudera-manager-agent-6.3.1-1466458.el7.x86_64.rpm
-rw-r--r-- 1 2001 2001 1203832464 Sep 25 2019 cloudera-manager-daemons-6.3.1-1466458.el7.x86_64.rpm
-rw-r--r-- 1 2001 2001 11488 Sep 25 2019 cloudera-manager-server-6.3.1-1466458.el7.x86_64.rpm
3）安装cloudera-manager-daemons,安装完毕后多出/opt/cloudera目录
[root@hadoop101 cloudera-manager]# rpm -ivh cloudera-manager-daemons-6.3.1-1466458.el7.x86_64.rpm
[root@hadoop101 cloudera-manager]# cd /opt/cloudera
cloudera/ cloudera-manager/
[root@hadoop101 cloudera-manager]# cd ..
[root@hadoop101 opt]# scp -r /opt/cloudera-manager/ hadoop102:/opt/
[root@hadoop101 opt]# scp -r /opt/cloudera-manager/ hadoop103:/opt/
[root@hadoop102 ~]# cd /opt/cloudera-manager/
[root@hadoop102 cloudera-manager]# rpm -ivh cloudera-manager-daemons-6.3.1-1466458.el7.x86_64.rpm
[root@hadoop103 ~]# cd /opt/cloudera-manager/
[root@hadoop103 cloudera-manager]# rpm -ivh cloudera-manager-daemons-6.3.1-1466458.el7.x86_64.rpm
4）安装cloudera-manager-agent
安装第三方依赖针对centos版本
[root@hadoop101 cloudera-manager]# yum install bind-utils psmisc cyrus-sasl-plain cyrus-sasl-gssapi fuse portmap fuse-libs /lib/lsb/init-functions httpd mod_ssl openssl-devel python-psycopg2 MySQL-python libxslt
[root@hadoop101 cloudera-manager]# rpm -ivh cloudera-manager-agent-6.3.1-1466458.el7.x86_64.rpm
[root@hadoop102 cloudera-manager]# yum install bind-utils psmisc cyrus-sasl-plain cyrus-sasl-gssapi fuse portmap fuse-libs /lib/lsb/init-functions httpd mod_ssl openssl-devel python-psycopg2 MySQL-python libxslt
[root@hadoop102 cloudera-manager]# rpm -ivh cloudera-manager-agent-6.3.1-1466458.el7.x86_64.rpm
[root@hadoop103 cloudera-manager]# yum install bind-utils psmisc cyrus-sasl-plain cyrus-sasl-gssapi fuse portmap fuse-libs /lib/lsb/init-functions httpd mod_ssl openssl-devel python-psycopg2 MySQL-python libxslt
[root@hadoop103 cloudera-manager]# rpm -ivh cloudera-manager-agent-6.3.1-1466458.el7.x86_64.rpm
5）安装agent的server节点
配置从节点Cloudera的-食槽剂指向主节点服务器
[root@hadoop101 cloudera-manager]# vim /etc/cloudera-scm-agent/config.ini
server_host=hadoop101
[root@hadoop102 cloudera-manager]# vim /etc/cloudera-scm-agent/config.ini
server_host=hadoop101
[root@hadoop103 cloudera-manager]# vim /etc/cloudera-scm-agent/config.ini
server_host=hadoop101
6）安装cloudera-manager-server
[root@hadoop101 cloudera-manager]# rpm -ivh cloudera-manager-server-6.3.1-1466458.el7.x86_64.rpm
7）上传CDH包导parcel-repo
[root@hadoop101 parcel-repo]# pwd
/opt/cloudera/parcel-repo
[root@hadoop101 parcel-repo]# ll
CDH-6.3.1-1.cdh6.3.1.p0.1605554-el7.parcel
CDH-6.3.1-1.cdh6.3.1.p0.1605554-el7.parcel.sha1
manifest.json
注意：其中CDH-5.15.1-cdh5.15.1.p0.18-el6.parcel.sha1 后缀要把1去掉
[root@hadoop101 parcel-repo]# mv CDH-6.3.1-1.cdh6.3.1.p0.1605554-el7.parcel.sha1 CDH-6.3.1-1.cdh6.3.1.p0.1605554-el7.parcel.sha
8）修改server的db.properties
[root@hadoop101 parcel-repo]# vim /etc/cloudera-scm-server/db.properties
com.cloudera.cmf.db.type=mysql
com.cloudera.cmf.db.host=hadoop101:3306
com.cloudera.cmf.db.name=scm
com.cloudera.cmf.db.user=scm
com.cloudera.cmf.db.password=scm
com.cloudera.cmf.db.setupType=EXTERNAL
初始脚本配置数据库scm_prepare_database.sh（在主节点上）
[root@hadoop101 log]# /opt/cloudera/cm/schema/scm_prepare_database.sh mysql scm scm
9）启动server服务
[root@hadoop101 software]# systemctl start cloudera-scm-server
tail -f /var/log/cloudera-scm-server/cloudera-scm-server.log
10）启动agent节点
[root@hadoop101 software]# systemctl start cloudera-scm-agent
[root@hadoop102 software]# systemctl start cloudera-scm-agent
[root@hadoop103 software]# systemctl start cloudera-scm-agent
CM的集群部署
1.1 接受条款和协议
1.2 集群安装
1.3 指定主机
1.4 选择CDH版本6.3.1
1.5 等待下载安装
1.6检查网络性能，检查主机
注：关闭Linux THG服务
检查集群中的各个主机的THG（对虚拟化等的内存资源分配是有好处的，但是对hadoop离线计算IO密集型操作是没有优势的，关闭THG可加快处理速度）
[root@hadoop101 software]# echo never > /sys/kernel/mm/transparent_hugepage/defrag
[root@hadoop101 software]# echo never > /sys/kernel/mm/transparent_hugepage/enabled
[root@hadoop102 software]# echo never > /sys/kernel/mm/transparent_hugepage/defrag
[root@hadoop102 software]# echo never > /sys/kernel/mm/transparent_hugepage/enabled
[root@hadoop103 software]# echo never > /sys/kernel/mm/transparent_hugepage/defrag
[root@hadoop103 software]# echo never > /sys/kernel/mm/transparent_hugepage/enabled
设置linux内核参数：vm.swappiness
vm.swappiness值的范围为0~100，作用是控制应用数据在物理内存和虚拟内存之间的交换，值越低，交换的越少。默认值为60。
查看集群各个主机的此参数值：
cat /proc/sys/vm/swappiness
建议调整值为1：
sysctl -w vm.swappiness=1
1.7群集设置
1.8 选择自定义安装
基础组件HDFS、YARN、Zookeeper安装
1.9 选择安装服务
1.10 分配节点
1.11 集群设置全部选默认即可
1.12 自动启动进程
扫一扫
2247
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
