cdh安装

cdh安装

cdh安装

软件包下载

http://archive.cloudera.com/cm5/redhat/7/x86_64/cm/

parcel包下载

http://archive.cloudera.com/cdh5/parcels/

#在我的百度云盘里面也可以去下载

安装前准备：规划好ip，datanode存储，Cloudera的mysql，namenode的/opt存储

注意：硬件要求

/tmp必须大于10G

/var/log必须大于10G

最小生产配置：hdfs：

搭建好yum源

ip规划

10.0.0.175    master1

10.0.0.174    master2

10.0.0.173    node1

10.0.0.172    node2

10.0.0.171    node3

10.0.0.170    node4

#配置主机名，检测主机的防火墙，selinux，时间同步，yum源（如果需要使用ssh免密码登陆需要配置ssh）

#配置主机名

hostnamectl set-hostname hadoop-master1

#配置hosts列表

10.0.0.175    hadoop-master1

10.0.0.174    hadoop-master2

10.0.0.173    hadoop-slave1

10.0.0.172    hadoop-slave2

10.0.0.171    hadoop-slave3

#关闭防火墙

systemctl stop firewalld

systemctl disable firewalld

#关闭selinux

setenforce 0

sed -i 's/enforcing/disabled/g' /etc/sysconfig/selinux

#安装ntp时间服务

yum -y install ntp

systemctl restart ntpd

systemctl enable ntpd

修改页面检查的错误

echo never > /sys/kernel/mm/transparent_hugepage/defrag

echo never > /sys/kernel/mm/transparent_hugepage/enabled

sysctl vm.swappiness=10

cat /etc/sysctl.conf

kernel.shmall = 4294967296

vm.swappiness=10

cat /proc/sys/vm/swappiness

#添加用户

useradd tarining

echo "trainging" | passwd tarining --stdin

#修改yum源

#cdh的源：http:10.0.0.176/cdh5.10.0

#parcel源： http://10.0.0.176/parcel5.10.0

#查看cdh和parcel的源是否可用

curl -v http://10.0.0.176/cdh5.10.0/

curl -v http://10.0.0.176/parcel5.10.0/

#配置yum源

vim /etc/yum.repos.d/clousera.repo

[cloudera-manager]

# Packages for Cloudera Manager, Version 5, on RedHat or CentOS 7 x86_64

name=Cloudera Manager

baseurl= http://10.0.0.176/cdh5.10.0

#gpgkey = https://archive.cloudera.com/cm5/redhat/7/x86_64/cm/RPM-GPG-KEY-cloudera

gpgcheck = 0

#查看yum源

yum list |grep cloudera

**********************************************************************

cloudera-manager-agent.x86_64               5.10.0-1.cm5100.p0.85.el7  cloudera-manager

cloudera-manager-daemons.x86_64             5.10.0-1.cm5100.p0.85.el7  cloudera-manager

cloudera-manager-server.x86_64              5.10.0-1.cm5100.p0.85.el7  cloudera-manager

cloudera-manager-server-db-2.x86_64         5.10.0-1.cm5100.p0.85.el7  cloudera-manager

enterprise-debuginfo.x86_64                 5.10.0-1.cm5100.p0.85.el7  cloudera-manager

jdk.x86_64                                  2000:1.6.0_31-fcs          cloudera-manager

oracle-j2sdk1.7.x86_64                      1.7.0+update67-1           cloudera-manager

**********************************************************************

#配置数据库

yum -y install mariadb mariadb-server

systemctl start mariadb

systemctl enable mariadb

#设置数据库密码

mysqladmin -uroot -p password 'root'

#配置mysql的驱动

yum -y install wget

wget https://cdn.mysql.com//Downloads/Connector-J/mysql-connector-java-5.1.46.tar.gz

tar -zxvf mysql-connector-java-5.1.46.tar.gz

mkdir -p  /usr/share/java/

mv mysql-connector-java-5.1.46/mysql-connector-java-5.1.46-bin.jar /usr/share/java/

ln -s /usr/share/java/mysql-connector-java-5.1.46-bin.jar /usr/share/java/mysql-connector-java.jar

#创建数据库

mysql -uroot -proot -e "create database cm default character set 'utf8';"

mysql -uroot -proot -e "grant all privileges on cm.* to 'cm'@'%' identified by 'password';"

mysql -uroot -proot -e "grant all privileges on cm.* to 'cm'@'hadoop-master1' identified by 'password';"

mysql -uroot -proot -e "create database rm default character set 'utf8';"

mysql -uroot -proot -e "grant all privileges on rm.* to 'rm'@'%' identified by 'password';"

mysql -uroot -proot -e "grant all privileges on rm.* to 'rm'@'hadoop-master1' identified by 'password';"

mysql -uroot -proot -e "create database hive default character set 'utf8';"

mysql -uroot -proot -e "grant all privileges on hive.* to 'hive'@'%' identified by 'password';"

mysql -uroot -proot -e "grant all privileges on hive.* to 'hive'@'hadoop-master2' identified by 'password';"

mysql -uroot -proot -e "create database oozie default character set 'utf8';"

mysql -uroot -proot -e "grant all privileges on oozie.* to 'oozie'@'%' identified by 'password';"

mysql -uroot -proot -e "grant all privileges on oozie.* to 'oozie'@'hadoop-master1' identified by 'password';"

mysql -uroot -proot -e "create database hue default character set 'utf8';"

mysql -uroot -proot -e "grant all privileges on hue.* to 'hue'@'%' identified by 'password';"

mysql -uroot -proot -e "grant all privileges on hue.* to 'hue'@'hadoop-master1' identified by 'password';"

#安装cloudera软件包

yum -y install cloudera-manager-server cloudera-manager-daemons jdk oracle-j2sdk1.7 httpd

#注册cloudera的服务到数据库

/usr/share/cmf/schema/scm_prepare_database.sh mysql -h hadoop-master2 cm cm password

#安装http服务

yum -y install httpd

systemctl start httpd

systemctl enable httpd

#启动cloudera服务

systemctl start cloudera-scm-server

#tail -f  /var/log/cloudera-scm-server/cloudera-scm-server.log

#登陆浏览器

10.0.0.175：7180

用户名：admin

密码：admin

#增加节点

#配置主机名，检测主机的防火墙，selinux，时间同步，swap交换分区，yum源（如果需要使用ssh免密码登陆需要配置ssh）

#配置主机名

hostnamectl set-hostname hadoop-slave4

#配置hosts列表

10.0.0.175    hadoop-master1

10.0.0.174    hadoop-master2

10.0.0.173    hadoop-slave1

10.0.0.172    hadoop-slave2

10.0.0.171    hadoop-slave3

10.0.0.170    hadoop-slave4

#关闭防火墙

systemctl stop firewalld

systemctl disable firewalld

#关闭selinux

setenforce 0

sed -i 's/enforcing/disabled/g' /etc/sysconfig/selinux

#安装ntp时间服务

yum -y install ntp

systemctl restart ntpd

systemctl enable ntpd

#关闭swap交换分区

swapoff -a

#修改fstab配置文件

#---------------------------------------------------------------------------------------------------------

#自建yum源

#直接把cdh的tar包下载到本地，然后解压到http的服务目录就行

cdh建议把/proc/sys/vm/swappiness改为1

sudo sysctl -w vm.swappiness=1

cdh卸载