文章目录
一、集群规划
软件 | 版本 |
---|---|
centos | 7.6-1810 |
cloudera manager | 6.3.1 |
cdh | 6.3.2 |
主机名 | IP | 功能 |
---|---|---|
cdh01 | 192.168.3.26 | server管理 |
cdh02 | 192.168.3.27 | agent节点 |
cdh03 | 192.168.3.28 | agent节点 |
二、下载安装包
manager下载:
https://archive.cloudera.com/cm6/6.3.1/redhat7/yum/RPMS/x86_64/
allkeys.asc下载
https://archive.cloudera.com/cm6/6.3.1/
cdh下载:
三、基础环境配置
3.1 主机名、网络配置
(主机名和IP配置略过,在exsi创建虚拟机时已完成)
3.2 配置hosts文件、域名解析
编辑/etc/hosts
文件
cat >> /etc/hosts <<EOF
192.168.3.26 cdh01
192.168.3.27 cdh02
192.168.3.28 cdh03
EOF
域名解析
cat >> /etc/resolv.conf <<EOF
nameserver 8.8.8.8
nameserver 8.8.4.4
EOF
3.3 关闭防火墙
[root@cdh01 ~]# systemctl stop firewalld && systemctl disable firewalld
Removed symlink /etc/systemd/system/multi-user.target.wants/firewalld.service.
Removed symlink /etc/systemd/system/dbus-org.fedoraproject.FirewallD1.service.
3.4 关闭SELinux
[root@cdh01 ~]# setenforce 0
[root@cdh01 ~]# sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config
3.5 安装ntp、配置时间同步
安装完操作系统后默认的yum仓库
[root@master ~]# ll /etc/yum.repos.d/
total 28
-rw-r--r--. 1 root root 1664 Aug 30 2017 CentOS-Base.repo
-rw-r--r--. 1 root root 1309 Aug 30 2017 CentOS-CR.repo
-rw-r--r--. 1 root root 649 Aug 30 2017 CentOS-Debuginfo.repo
-rw-r--r--. 1 root root 314 Aug 30 2017 CentOS-fasttrack.repo
-rw-r--r--. 1 root root 630 Aug 30 2017 CentOS-Media.repo
-rw-r--r--. 1 root root 1331 Aug 30 2017 CentOS-Sources.repo
-rw-r--r--. 1 root root 3830 Aug 30 2017 CentOS-Vault.repo
[root@manager ~]# cd /etc/yum.repos.d/
[root@manager yum.repos.d]# rename .repo .repo.bak *.repo
[root@manager yum.repos.d]# ll
total 28
-rw-r--r--. 1 root root 1664 Aug 30 2017 CentOS-Base.repo.bak
-rw-r--r--. 1 root root 1309 Aug 30 2017 CentOS-CR.repo.bak
-rw-r--r--. 1 root root 649 Aug 30 2017 CentOS-Debuginfo.repo.bak
-rw-r--r--. 1 root root 314 Aug 30 2017 CentOS-fasttrack.repo.bak
-rw-r--r--. 1 root root 630 Aug 30 2017 CentOS-Media.repo.bak
-rw-r--r--. 1 root root 1331 Aug 30 2017 CentOS-Sources.repo.bak
-rw-r--r--. 1 root root 3830 Aug 30 2017 CentOS-Vault.repo.bak
创建本地仓库
一般光盘安装的操作系统,都可以采用这种方式
先挂载光盘映像
[root@manager yum.repos.d]# mkdir -p /media/centos7
[root@manager yum.repos.d]# mount -o loop /dev/cdrom /media/centos7/
[root@manager ~]# ll /media/centos7/
total 1586
-rw-rw-r--. 3 root root 14 Sep 5 2017 CentOS_BuildTag
drwxr-xr-x. 3 root root 2048 Sep 5 2017 EFI
-rw-rw-r--. 3 root root 227 Aug 30 2017 EULA
-rw-rw-r--. 3 root root 18009 Dec 10 2015 GPL
drwxr-xr-x. 3 root root 2048 Sep 6 2017 images
drwxr-xr-x. 2 root root 2048 Sep 5 2017 isolinux
drwxr-xr-x. 2 root root 2048 Sep 5 2017 LiveOS
drwxrwxr-x. 2 root root 1585152 Sep 6 2017 Packages
drwxrwxr-x. 2 root root 4096 Sep 6 2017 repodata
-rw-rw-r--. 3 root root 1690 Dec 10 2015 RPM-GPG-KEY-CentOS-7
-rw-rw-r--. 3 root root 1690 Dec 10 2015 RPM-GPG-KEY-CentOS-Testing-7
-r--r--r--. 1 root root 2883 Sep 6 2017 TRANS.TBL
创建centos7.repo文件
cat >> /etc/yum.repos.d/centos7.repo <<EOF
[centos7]
name=centos7
baseurl=file:///media/centos7/
gpgcheck=0
enabled=1
EOF
配置完本地仓库后,安装ntp
[root@cdh01 yum.repos.d]# yum -y install ntp
手动对时
[root@cdh02 yum.repos.d]# ntpdate ntp.aliyun.com
4 Feb 10:22:27 ntpdate[25949]: step time server 203.107.6.88 offset 210.602459 sec
自动对时,编辑/etc/crontab
文件,让虚拟机每天0点30自动对时
SHELL=/bin/bash
PATH=/sbin:/bin:/usr/sbin:/usr/bin
MAILTO=root
# For details see man 4 crontabs
# Example of job definition:
# .---------------- minute (0 - 59)
# | .------------- hour (0 - 23)
# | | .---------- day of month (1 - 31)
# | | | .------- month (1 - 12) OR jan,feb,mar,apr ...
# | | | | .---- day of week (0 - 6) (Sunday=0 or 7) OR sun,mon,tue,wed,thu,fri,sat
# | | | | |
# * * * * * user-name command to be executed
30 0 * * * root ntpdate ntp.aliyun.com
3.6 配置免密登录
配置从manager
节点到所有节点的免密登录**(只需在manager节点操作)**
######在manager节点执行,一路回车
[root@cdh01 yum.repos.d]# ssh-keygen -t rsa
Generating public/private rsa key pair.
Enter file in which to save the key (/root/.ssh/id_rsa):
Created directory '/root/.ssh'.
Enter passphrase (empty for no passphrase):
Enter same passphrase again:
Your identification has been saved in /root/.ssh/id_rsa.
Your public key has been saved in /root/.ssh/id_rsa.pub.
The key fingerprint is:
SHA256:/TKBFms/jSpijFEHal1qN7CkWKWCoaGFj8OA4FuD2do root@cdh01
The key's randomart image is:
+---[RSA 2048]----+
|*.... |
|O+=.+ . |
|BO.O * . |
|+oX B + = |
| = E o .S o |
| . o . = |
| + * o |
| . + . . + |
| . . .. |
+----[SHA256]-----+
####在manager节点执行
####先打yes,再手动输入一次密码
[root@cdh01 yum.repos.d]# ssh-copy-id cdh01
[root@cdh01 yum.repos.d]# ssh-copy-id cdh02
[root@cdh01 yum.repos.d]# ssh-copy-id cdh03
验证
[root@cdh01 yum.repos.d]# ssh cdh02
Last login: Thu Feb 4 10:01:22 2021 from 192.168.3.125
[root@cdh02 ~]# exit
logout
Connection to cdh02 closed.
[root@cdh01 yum.repos.d]#
3.7 JDK安装
使用下载的oracle-j2sdk1.8-1.8.0+update181-1.x86_64.rpm
rpm -qa | grep java # 查询已安装的java
yum remove java* # 卸载
rpm -ivh oracle-j2sdk1.8-1.8.0+update181-1.x86_64.rpm
安装后进入/usr/java
目录查看
[root@cdh01 cdh_rpms]# cd /usr/java/
[root@cdh01 java]# ll
total 0
drwxr-xr-x. 7 root root 245 Feb 4 15:37 jdk1.8.0_181-cloudera
[root@cdh01 java]# cd jdk1.8.0_181-cloudera/
[root@cdh01 jdk1.8.0_181-cloudera]# ll
total 25976
drwxr-xr-x. 2 root root 4096 Feb 4 15:37 bin
-r--r--r--. 1 root root 3244 Jan 5 2019 COPYRIGHT
drwxr-xr-x. 3 root root 132 Feb 4 15:37 include
-rw-r--r--. 1 root root 5207314 Jan 5 2019 javafx-src.zip
drwxr-xr-x. 5 root root 185 Feb 4 15:37 jre
drwxr-xr-x. 5 root root 245 Feb 4 15:37 lib
-r--r--r--. 1 root root 40 Jan 5 2019 LICENSE
drwxr-xr-x. 4 root root 47 Feb 4 15:37 man
-r--r--r--. 1 root root 159 Jan 5 2019 README.html
-rw-r--r--. 1 root root 424 Jan 5 2019 release
-rw-r--r--. 1 root root 21099920 Jan 5 2019 src.zip
-rw-r--r--. 1 root root 106782 Jan 5 2019 THIRDPARTYLICENSEREADME-JAVAFX.txt
-r--r--r--. 1 root root 154222 Jan 5 2019 THIRDPARTYLICENSEREADME.txt
配置环境变量,编辑/etc/profile
export JAVA_HOME=/usr/java/jdk1.8.0_181-cloudera
export CLASS_PATH=$JAVA_HOME/lib:$CLASS_PATH
export PATH=$JAVA_HOME/bin:$PATH
[root@cdh02 jdk_180rpms]# source /etc/profile
[root@cdh02 jdk_180rpms]# java -version
java version "1.8.0_181"
Java(TM) SE Runtime Environment (build 1.8.0_181-b13)
Java HotSpot(TM) 64-Bit Server VM (build 25.181-b13, mixed mode)
3.8 swappiness警告
不设置此项,在页面安装时,会有以下告警信息
临时修改:
sysctl vm.swappiness=10
cat /proc/sys/vm/swappiness
如果我们重启了系统,又会变成30
永久修改:
在/etc/sysctl.conf
文件里添加如下参数:
vm.swappiness=10
或者:
echo 'vm.swappiness=10'>> /etc/sysctl.conf
让刚写入的立即也生效可以执行以下
sysctl -p
3.9 关闭透明大页
不设置此项,在页面安装时,会有以下告警信息
根据提示,执行以下操作
echo never > /sys/kernel/mm/transparent_hugepage/defrag
echo never > /sys/kernel/mm/transparent_hugepage/enabled
编辑/etc/rc.local
文件
#!/bin/bash
# THIS FILE IS ADDED FOR COMPATIBILITY PURPOSES
#
# It is highly advisable to create own systemd services or udev rules
# to run scripts during boot instead of using this file.
#
# In contrast to previous versions due to parallel execution during boot
# this script will NOT be run after all other services.
#
# Please note that you must run 'chmod +x /etc/rc.d/rc.local' to ensure
# that this script will be executed during boot.
touch /var/lock/subsys/local
# 增加以下两句
echo never > /sys/kernel/mm/transparent_hugepage/defrag
echo never > /sys/kernel/mm/transparent_hugepage/enabled
四、安装数据库
4.1 下载mysql数据库
https://dev.mysql.com/downloads/mysql/5.7.html#downloads
下载后,上传manager
节点,解压,使用yum
方式安装
[root@cdh01 mysql_rpms]# ll
total 530372
-rw-r--r--. 1 root root 543098880 Feb 4 11:39 mysql-5.7.33-1.el7.x86_64.rpm-bundle.tar
[root@cdh01 mysql_rpms]# tar -xvf mysql-5.7.33-1.el7.x86_64.rpm-bundle.tar
mysql-community-client-5.7.33-1.el7.x86_64.rpm
mysql-community-common-5.7.33-1.el7.x86_64.rpm
mysql-community-devel-5.7.33-1.el7.x86_64.rpm
mysql-community-embedded-5.7.33-1.el7.x86_64.rpm
mysql-community-embedded-compat-5.7.33-1.el7.x86_64.rpm
mysql-community-embedded-devel-5.7.33-1.el7.x86_64.rpm
mysql-community-libs-5.7.33-1.el7.x86_64.rpm
mysql-community-libs-compat-5.7.33-1.el7.x86_64.rpm
mysql-community-server-5.7.33-1.el7.x86_64.rpm
mysql-community-test-5.7.33-1.el7.x86_64.rpm
[root@cdh01 mysql_rpms]# ll
total 1060748
-rw-r--r--. 1 root root 543098880 Feb 4 11:39 mysql-5.7.33-1.el7.x86_64.rpm-bundle.tar
-rw-r--r--. 1 7155 31415 26468960 Dec 11 13:20 mysql-community-client-5.7.33-1.el7.x86_64.rpm
-rw-r--r--. 1 7155 31415 315280 Dec 11 13:20 mysql-community-common-5.7.33-1.el7.x86_64.rpm
-rw-r--r--. 1 7155 31415 3918736 Dec 11 13:20 mysql-community-devel-5.7.33-1.el7.x86_64.rpm
-rw-r--r--. 1 7155 31415 47513604 Dec 11 13:20 mysql-community-embedded-5.7.33-1.el7.x86_64.rpm
-rw-r--r--. 1 7155 31415 23261952 Dec 11 13:20 mysql-community-embedded-compat-5.7.33-1.el7.x86_64.rpm
-rw-r--r--. 1 7155 31415 131045776 Dec 11 13:21 mysql-community-embedded-devel-5.7.33-1.el7.x86_64.rpm
-rw-r--r--. 1 7155 31415 2458780 Dec 11 13:21 mysql-community-libs-5.7.33-1.el7.x86_64.rpm
-rw-r--r--. 1 7155 31415 1260364 Dec 11 13:21 mysql-community-libs-compat-5.7.33-1.el7.x86_64.rpm
-rw-r--r--. 1 7155 31415 181817592 Dec 11 13:21 mysql-community-server-5.7.33-1.el7.x86_64.rpm
-rw-r--r--. 1 7155 31415 125021984 Dec 11 13:21 mysql-community-test-5.7.33-1.el7.x86_64.rpm
[root@cdh01 mysql_rpms]# yum -y install createrepo
[root@cdh01 mysql_rpms]# createrepo ./
Spawning worker 0 with 3 pkgs
Spawning worker 1 with 3 pkgs
Spawning worker 2 with 2 pkgs
Spawning worker 3 with 2 pkgs
Workers Finished
Saving Primary metadata
Saving file lists metadata
Saving other metadata
Generating sqlite DBs
Sqlite DBs complete
[root@cdh01 mysql_rpms]# ll
total 1060752
-rw-r--r--. 1 root root 543098880 Feb 4 11:39 mysql-5.7.33-1.el7.x86_64.rpm-bundle.tar
-rw-r--r--. 1 7155 31415 26468960 Dec 11 13:20 mysql-community-client-5.7.33-1.el7.x86_64.rpm
-rw-r--r--. 1 7155 31415 315280 Dec 11 13:20 mysql-community-common-5.7.33-1.el7.x86_64.rpm
-rw-r--r--. 1 7155 31415 3918736 Dec 11 13:20 mysql-community-devel-5.7.33-1.el7.x86_64.rpm
-rw-r--r--. 1 7155 31415 47513604 Dec 11 13:20 mysql-community-embedded-5.7.33-1.el7.x86_64.rpm
-rw-r--r--. 1 7155 31415 23261952 Dec 11 13:20 mysql-community-embedded-compat-5.7.33-1.el7.x86_64.rpm
-rw-r--r--. 1 7155 31415 131045776 Dec 11 13:21 mysql-community-embedded-devel-5.7.33-1.el7.x86_64.rpm
-rw-r--r--. 1 7155 31415 2458780 Dec 11 13:21 mysql-community-libs-5.7.33-1.el7.x86_64.rpm
-rw-r--r--. 1 7155 31415 1260364 Dec 11 13:21 mysql-community-libs-compat-5.7.33-1.el7.x86_64.rpm
-rw-r--r--. 1 7155 31415 181817592 Dec 11 13:21 mysql-community-server-5.7.33-1.el7.x86_64.rpm
-rw-r--r--. 1 7155 31415 125021984 Dec 11 13:21 mysql-community-test-5.7.33-1.el7.x86_64.rpm
drwxr-xr-x. 2 root root 4096 Feb 4 12:36 repodata
创建mysql.repo
文件(里面的baseurl
根据自己的上传路径修改)
cat >> /etc/yum.repos.d/mysql.repo <<EOF
[mysql]
name=mysql
baseurl=file:///opt/mysql_rpms/
gpgcheck=0
enabled=1
EOF
4.2 安装
根据需要依次安装如下服务,注意mysql
是小写。
yum install mysql-server
yum install mysql-devel
yum install mysql-embedded
4.3 初始化及启动
参考
https://blog.csdn.net/zz_aiytag/article/details/89917832
4.4 建库并授权
mysql> CREATE DATABASE scm DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
Query OK, 1 row affected (0.00 sec)
mysql> CREATE DATABASE hive DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
Query OK, 1 row affected (0.00 sec)
mysql> CREATE DATABASE rmon DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
Query OK, 1 row affected (0.00 sec)
mysql> CREATE DATABASE oozie DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
Query OK, 1 row affected (0.00 sec)
mysql> CREATE DATABASE hue DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
Query OK, 1 row affected (0.00 sec)
mysql> CREATE USER 'cdh'@'%' IDENTIFIED BY '123456';
Query OK, 0 rows affected (0.00 sec)
mysql> GRANT ALL ON *.* TO 'cdh'@'%';
Query OK, 0 rows affected (0.00 sec)
4.5 配置jdbc
将mysql
驱动包mysql-connector-java-5.1.25.jar
拷贝到manager
节点下的目录: /usr/share/java/
重命名
mv mysql-connector-java-5.1.25.jar mysql-connector-java.jar
[root@cdh01 java]# ll
total 832
-rw-r--r--. 1 root root 848399 Apr 28 2019 mysql-connector-java.jar
[root@cdh01 java]# pwd
/usr/share/java
五、CM安装
5.1 配置httpd
安装httpd
[root@cdh01 java]# yum -y install httpd
[root@cdh01 java]# systemctl start httpd
[root@cdh01 java]# systemctl enable httpd
Created symlink from /etc/systemd/system/multi-user.target.wants/httpd.service to /usr/lib/systemd/system/httpd.service.
创建cm仓库目录
mkdir -p /var/www/html/cloudera-repos/
将下载的安装包移动到此处,并执行createrepo
[root@cdh01 cloudera-repos]# ll
total 1199784
-rw-r--r--. 1 root root 14041 Feb 4 16:59 allkeys.asc
-rw-r--r--. 1 root root 10483568 Feb 4 14:34 cloudera-manager-agent-6.3.1-1466458.el7.x86_64.rpm
-rw-r--r--. 1 root root 1203832464 Feb 4 14:34 cloudera-manager-daemons-6.3.1-1466458.el7.x86_64.rpm
-rw-r--r--. 1 root root 11488 Feb 4 14:34 cloudera-manager-server-6.3.1-1466458.el7.x86_64.rpm
-rw-r--r--. 1 root root 10996 Feb 4 14:34 cloudera-manager-server-db-2-6.3.1-1466458.el7.x86_64.rpm
-rw-r--r--. 1 root root 14209868 Feb 4 14:34 enterprise-debuginfo-6.3.1-1466458.el7.x86_64.rpm
drwxr-xr-x. 2 root root 4096 Feb 4 16:59 repodata
在浏览器上直接输入http://192.168.3.26/cloudera-repos/
查看配置结果
创建repo
文件
cat >> /etc/yum.repos.d/cloudera-manager.repo <<EOF
[cloudera-manager]
name=cloudera-manager
baseurl=http://192.168.3.26/cloudera-repos
gpgcheck=0
enabled=1
EOF
将上述创建的文件,或者上述指令在另外两上agent
节点上也执行一下
5.2 安装CM Server
[root@cdh01 yum.repos.d]# yum install cloudera-manager-daemons
[root@cdh01 yum.repos.d]# yum -y install cloudera-manager-server
[root@cdh01 yum.repos.d]# yum -y install cloudera-manager-agent
安装完CM
后/opt/
目录下会出现cloudera
文件夹
进入/opt/cloudera/parcel-repo
目录下,将下载的parcel文件上传至此处
[root@cdh01 parcel-repo]# ll
total 2033392
-rw-r--r--. 1 root root 2082186246 Feb 4 14:58 CDH-6.3.2-1.cdh6.3.2.p0.1605554-el7.parcel
-rw-r--r--. 1 root root 40 Jan 6 15:47 CDH-6.3.2-1.cdh6.3.2.p0.1605554-el7.parcel.sha1
将sha1
文件重命名,把1去掉,变为.sha
文件
[root@cdh01 parcel-repo]# ll
total 2033392
-rw-r--r--. 1 root root 2082186246 Feb 4 14:58 CDH-6.3.2-1.cdh6.3.2.p0.1605554-el7.parcel
-rw-r--r--. 1 root root 40 Jan 6 15:47 CDH-6.3.2-1.cdh6.3.2.p0.1605554-el7.parcel.sha
进入/opt/cloudera/cm/schema
目录
[root@cdh01 cm]# cd schema/
[root@cdh01 schema]# ll
total 60
drwxr-xr-x. 4 root root 8192 Feb 4 14:48 mysql
drwxr-xr-x. 4 root root 8192 Feb 4 14:48 oracle
drwxr-xr-x. 4 root root 12288 Feb 4 14:48 postgresql
-rw-r--r--. 1 root root 1437 Sep 25 2019 scm_database_functions.sh
-rwxr-xr-x. 1 root root 12450 Sep 25 2019 scm_prepare_database.sh
执行数据库连接命令
/opt/cloudera/cm/schema/scm_prepare_database.sh mysql scm cdh
更完整的指令是
/opt/cloudera/cm/schema/scm_prepare_database.sh -h cdh01 -P 3306 mysql scm cdh 123456
######### -h Host -P Port dbType dbName dbUser dbPasswd
[root@cdh01 schema]# /opt/cloudera/cm/schema/scm_prepare_database.sh mysql scm cdh
Enter SCM password: ###输入密码
JAVA_HOME=/opt/jdk_180rpms/jdk1.8.0_121
Verifying that we can write to /etc/cloudera-scm-server
Creating SCM configuration file in /etc/cloudera-scm-server
Executing: /opt/jdk_180rpms/jdk1.8.0_121/bin/java -cp /usr/share/java/mysql-connector-java.jar:/usr/share/java/oracle-connector-java.jar:/usr/share/java/postgresql-connector-java.jar:/opt/cloudera/cm/schema/../lib/* com.cloudera.enterprise.dbutil.DbCommandExecutor /etc/cloudera-scm-server/db.properties com.cloudera.cmf.db.
[ main] DbCommandExecutor INFO Successfully connected to database.
All done, your SCM database is configured correctly!
5.3 安装CM Agent
所有agent节点执行
yum -y install cloudera-manager-daemons
yum -y install cloudera-manager-agent
所有agent
节点修改agent
配置文件中server_host
参数
vim /etc/cloudera-scm-agent/config.ini
# If you have whitespace in front of a parameter name, it will be
# read as a continuation of the previous parameter value. Please
# be careful not to leave spaces in front of parameter names.
#
# To check if this file has spaces in front of parameters names
# you can do a grep like this:
# grep '^[[:blank:]]' /etc/cloudera-scm-agent/config.ini
[General]
# Hostname of the CM server.
server_host=cdh01 #修改为manager的hostname
# Port that the CM server is listening on.
server_port=7182
5.4 启动CM服务
manager节点:
systemctl start cloudera-scm-server
systemctl enable cloudera-scm-server
agent节点:
systemctl start cloudera-scm-agent
systemctl enable cloudera-scm-agent
启动过程中的日志在/var/log/cloudera-scm-*
中
打开页面,http://managerIP:7180
默认admin/admin
进行登录
六、 页面安装
参数文章
https://blog.csdn.net/weixin_40004348/article/details/106886347
https://www.cnblogs.com/swordfall/p/10816797.html
https://www.jianshu.com/p/39699ad131e3
https://blog.csdn.net/qq_39680564/article/details/99191318