最近在学习CDH，这里记录一下CDH的搭建步骤
一、版本选择
1、操作系统 Centos7
2、Cloudera Manager5.14.0
采用rpm包，后续测试升级到6.2，我在官网发现6以后好像没有提供二进制包，如果有同学知道，希望可以留言告知。
下载地址 http://archive.cloudera.com/cm5/redhat/7/x86_64/cm/5.14.0/RPMS/x86_64/
3、CDH5.14.0
下载地址 http://archive.cloudera.com/cdh5/parcels/5.14.0/
4、jdk1.8
二、规划
| 122-120 | hadoop001 | hadoop002 | hadoop003 | hadoop004 | hadoop005 | hadoop006 |
| 192.168.122.120 | 192.168.122.121 | 192.168.122.122 | 192.168.122.123 | 192.168.122.124 | 192.168.122.125 | 192.168.122.126 |
| Mysql5.7.25 | CM Server | |||||
| httpd | CM Agent | CM Agent | CM Agent | CM Agent | CM Agent | CM Agent |
| ZK | ZK | ZK | ||||
| oozie | NN | NN(HA) | DN | DN | DN | |
| hue | RN | RN(HA) | NM | NM | NM | |
| 其他服务后续添加 | HMaster | RegionServer | RegionServer | RegionServer | ||
| HIVE | ||||||
| flume | flume | flume | ||||
| kafka | kafka | kafka |
三、操作系统优化
yum -y install chkconfig python bind-utils psmisc libxslt zlib sqlite cyrus-sasl-plain cyrus-sasl-gssapi fuse fuse-libs redhat-lsb httpd mod_ssl
echo vm.swappiness=1 >> /etc/sysctl.conf
echo vm.min_free_kbytes=4194304 >> /etc/sysctl.conf
echo vm.zone_reclaim_mode=0 >> /etc/sysctl.conf
echo vm.max_map_count=655360 >> /etc/sysctl.conf # ES相关参数
echo fs.file-max=655360 >> /etc/sysctl.conf # ES相关参数
echo 'if test -f /sys/kernel/mm/transparent_hugepage/enabled; then
echo never > /sys/kernel/mm/transparent_hugepage/enabled
fi
if test -f /sys/kernel/mm/transparent_hugepage/defrag; then
echo never > /sys/kernel/mm/transparent_hugepage/defrag
fi' >> /etc/rc.d/rc.local
sed -i 's/enforcing/disabled/g' /etc/selinux/config
systemctl disable firewalld
systemctl stop tuned
systemctl disable tuned
# ES相关参数
echo '* hard nofile 65536
* soft nofile 131072
* hard nproc 4096
* soft nproc 2048 ' >> /etc/security/limits.conf
tee /etc/hosts <<-'EOF'
192.168.122.121 hadoop001
192.168.122.122 hadoop002
192.168.122.123 hadoop003
192.168.122.124 hadoop004
192.168.122.125 hadoop005
192.168.122.126 hadoop006
EOF
免密登录配置
hadoop001-003机器执行
ssh-keygen -t rsa #一路回车
ssh-copy-id -i ~/.ssh/id_rsa.pub hadoop001
ssh-copy-id -i ~/.ssh/id_rsa.pub hadoop002
ssh-copy-id -i ~/.ssh/id_rsa.pub hadoop003
ssh-copy-id -i ~/.ssh/id_rsa.pub hadoop004
ssh-copy-id -i ~/.ssh/id_rsa.pub hadoop005
ssh-copy-id -i ~/.ssh/id_rsa.pub hadoop006四、ntp安装（每个节点）
为了方便测试，使用的是阿里云ntp服务器，生产环境需要自建ntp
[root@122-120 ~]# yum -y install ntp
[root@122-120 ~]# vi /etc/ntp.conf
#server 0.rhel.pool.ntp.org iburst
#server 1.rhel.pool.ntp.org iburst
#server 2.rhel.pool.ntp.org iburst
#server 3.rhel.pool.ntp.org iburst
server ntp.aliyun.com
[root@122-120 ~]# systemctl start ntpd
[root@122-120 ~]# systemctl status ntpd
[root@122-120 ~]# systemctl enable ntpd
[root@122-120 ~]# ntpq -p
# 如果配置了ntp自启无效果，关闭chronyd
[root@122-120 ~]# systemctl disable chronyd五、MySQL安装
[root@122-120 mysql5.7.25]# ll
总用量 198448
-rw-r--r--. 1 root root 25402568 5月 14 17:10 mysql-community-client-5.7.25-1.el7.x86_64.rpm
-rw-r--r--. 1 root root 280904 5月 14 17:09 mysql-community-common-5.7.25-1.el7.x86_64.rpm
-rw-r--r--. 1 root root 2271668 5月 14 17:09 mysql-community-libs-5.7.25-1.el7.x86_64.rpm
-rw-r--r--. 1 root root 2116628 5月 14 17:09 mysql-community-libs-compat-5.7.25-1.el7.x86_64.rpm
-rw-r--r--. 1 root root 173130520 5月 14 17:09 mysql-community-server-5.7.25-1.el7.x86_64.rpm
[root@122-120 mysql5.7.25]# yum localinstall *.rpm -y
[root@122-120 mysql5.7.25]# service mysqld start
Redirecting to /bin/systemctl start mysqld.service
[root@122-120 mysql5.7.25]# chkconfig mysqld on
#查看密码
[root@122-120 mysql5.7.25]# cat /var/log/mysqld.log | grep pass
2020-05-14T09:22:49.740487Z 1 [Note] A temporary password is generated for root@localhost: ,8L-+WVaIk>j
#修改密码
[root@122-120 ~]# mysql_secure_installation
Securing the MySQL server deployment.
Enter password for user root:
The existing password for the user account root has expired. Please set a new password.
New password:
Re-enter new password:
Estimated strength of the password: 100
Do you wish to continue with the password provided?(Press y|Y for Yes, any other key for No) : Y
By default, a MySQL installation has an anonymous user,
allowing anyone to log into MySQL without having to have
a user account created for them. This is intended only for
testing, and to make the installation go a bit smoother.
You should remove them before moving into a production
environment.
Remove anonymous users? (Press y|Y for Yes, any other key for No) : y
Success.
Normally, root should only be allowed to connect from
'localhost'. This ensures that someone cannot guess at
the root password from the network.
Disallow root login remotely? (Press y|Y for Yes, any other key for No) : n
... skipping.
By default, MySQL comes with a database named 'test' that
anyone can access. This is also intended only for testing,
and should be removed before moving into a production
environment.
Remove test database and access to it? (Press y|Y for Yes, any other key for No) : y
- Dropping test database...
Success.
- Removing privileges on test database...
Success.
Reloading the privilege tables will ensure that all changes
made so far will take effect immediately.
Reload privilege tables now? (Press y|Y for Yes, any other key for No) : y
Success.
All done!
#MySQL建库，为了方便测试，授权用户密码统一
create database cm DEFAULT CHARACTER SET utf8;
create database amon DEFAULT CHARACTER SET utf8;
create database hive DEFAULT CHARACTER SET utf8;
create database oozie DEFAULT CHARACTER SET utf8;
create database hue DEFAULT CHARACTER SET utf8;
grant all privileges on *.* to 'cdh'@'%' identified by '******';
grant all privileges on *.* to 'cdh'@'localhost' identified by '******';
flush privileges;六、本地CDH仓库
[root@122-120 ~]# yum -y install httpd createrepo
[root@122-120 ~]# systemctl start httpd.service
[root@122-120 ~]# systemctl enable httpd.service
[root@122-120 ~]# cd /var/www/html/
[root@122-120 html]# mkdir -p cloudera-parcels/cdh-5.15.1
[root@122-120 html]# cd cloudera-parcels/cdh-5.15.1
[root@122-120 cdh-5.14.0]# createrepo . #生成RPM元数据
[root@122-120 cdh-5.14.0]# cp /src/5.14.0/CDH-5.14.0-1.cdh5.14.0.p0.4-el7.parcel .
[root@122-120 cdh-5.14.0]# cp /src/5.14.0/CDH-5.14.0-1.cdh5.14.0.p0.4-el7.parcel.sha1 CDH-5.14.0-1.cdh5.14.0.p0.4-el7.parcel.sha
[root@122-120 cdh-5.14.0]# cp /src/5.14.0/manifest.json .
[root@122-120 cdh-5.14.0]# tee /etc/yum.repos.d/cdh-5.14.0.repo <<-'EOF'
[cdh-5.14.0]
name=cdh-5.14.0
baseurl=http://192.168.122.120/cloudera-parcels/cdh-5.14.0
gpgcheck=0
enabled=1
EOF
[root@122-120 ]# yum clean all
[root@122-120 ]# yum makecache
扫一扫
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
