Centos7.0
jdk 安装
网络 (要求能上网):
vi /etc/sysconfig/network 根据自己的配置修改hostname,这个需要重启虚拟机才会生效
# Created by anaconda
#
HOSTNAME=hadoop3
vi /etc/resolv.conf 添加:nameserver 8.8.8.8 或者nameserver 网关地址
# Generated by NetworkManager
nameserver 8.8.8.8
nameserver 8.8.4.4
vi /etc/hosts 根据自己的ip 进行相应配置
127.0.0.1 localhost
192.168.45.136 hadoop3
192.168.45.134 hadoop1
192.168.45.135 hadoop2
[root@hadoop1 /]# uuidgen ens33
579889a2-536b-4232-aa0a-6c6ae3f3f501 (每台不用)
而且 BOOTPROTO=static 不然会安装出错
1 实现SSH
2 实现ntp
3 python2.7安装 (重点 CHD是PHP开发) pip 安装 ,supervisor安装
4 cloudera
4.1创建目录
mkdir /opt/cloudera-manager
4.2解压CM到/opt/cloudera-manager目录
http://archive.cloudera.com/cm5/cm/5/cloudera-manager-centos7-cm5.10.0_x86_64.tar.gz
解压
tar -zxvf cloudera-manager-centos7-cm5.10.0_x86_64.tar.gz -C /opt/cloudera-manager
离线安装 还要下载以下文件 放在(cd opt/cloudera/parcel-repo/5.0)
下载(放的路径 opt/cloudera/parcel-repo/5.0)
http://archive.cloudera.com/cdh5/parcels/5.10.0/CDH-5.10.0-1.cdh5.10.0.p0.41-el5.parcel
http://archive.cloudera.com/cdh5/parcels/5.10.0/CDH-5.10.0-1.cdh5.10.0.p0.41-el5.parcel.sha1
http://archive.cloudera.com/cdh5/parcels/5.10.0/manifest.json
sha1sum CDH-5.10.0-1.cdh5.10.0.p0.41-el5.parcel | cut -d ' ' -f 1 > CDH-5.10.0-1.cdh5.10.0.p0.41-el5.parcel.sha
下载(所有文件)
http://archive.cloudera.com/cm5/redhat/7/x86_64/cm/5.6/RPMS/x86_64/
https://archive.cloudera.com/cm5/redhat/7/x86_64/cm/5.6.0/repodata/
(1 )vim /etc/yum.repos.d/cloudera-manager.repo
[cloudera-manager]
name = Cloudera Manager, Version 5.10.0
baseurl = http://192.168.45.134:8081/parcels/5.0/ (httpd 要安装,安装完成后 配至访问路径 是: opt/cloudera/parcel-repo,下载安装方地址 )
gpgcheck =0
~
~
~
下载 (放的路径 opt/cloudera/parcel-repo/5.0/repodata/)
http://archive.cloudera.com/cdh5/redhat/7/x86_64/cdh/5.10.0/repodata/
(2)yum clean all && yum makecache
新建 cloudera-repo.repo一个本地Cloudera yum ( vi cloudera-repo.repo)
1
2 [cloudera-cdh-repo]
3 name=Cloudera's Distribution for Hadoop, Version 5
4 baseurl=http://192.168.45.134:8081/5.6/cdh/5/ (本地 存储地址)
5 gpgkey =http://192.168.45.134:8081/5.6/cdh/5/RPM-GPG-KEY-cloudera
6 enabled=1
7 gpgcheck=0
8
9 [cloudera-cm-repo]
10 name=Cloudera Manager
11 baseurl=http://192.168.45.134:8081/5.6/cm/5/
12 gpgkey =http://192.168.45.134:8081/5.6/cm/5/RPM-GPG-KEY-cloudera
13 enabled=1
14 gpgcheck=0
15
~
~
~
~
~
~
~
~
(3)yum list cloudera* 验证一下yum 是否成功
rpm -ivh
4.3创建cloudera-scm用户
useradd --system --home=/opt/cloudera-manager/cm-5.7.1/run/cloudera-scm-server/ --no-create-home --shell=/bin/false --comment "Cloudera SCM User" cloudera-scm
创建本地元数据存放目录
mkdir /var/cloudera-scm-server
4.4修改文件夹所属用户
chown cloudera-scm:cloudera-scm /var/cloudera-scm-server/
chown cloudera-scm:cloudera-scm /opt/cloudera-manager/
4.5创建系统用户
useradd --system --home=/opt/cloudera-manager/cm-5.10.0/run/cloudera-scm-server --no-create-home --shell=/bin/false --comment "Cloudera SCM User" cloudera-scm
4.6创建 与mysql()连接 (重点)#创建相关的数据库(主机配置)
Mysql
grant all on *.* to temp@'%' Identified by 'temp' with grant option;
Mysql 与 cloudera 连接
./scm_prepare_database.sh mysql temp -h localhost -uroot -p1987323smg --scm-host localhost scm scm scm
5 如果是mysql8.0以上可以用
5.1 CREATE USER 'scm‘@'%' IDENTIFIED with mysql_native_password BY 'shumeiG@123'; // 创建admini 用户和密码
5.2 GRANT ALL PRIVILEGES ON *.* TO 'scm'@'%' WITH GRANT OPTION;
5.3 flush privileges;
5.4新建 scm 数据库 同时给 scm 数据库 加上 scm 用户访问的权限
Create database scm DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
5.5./scm_prepare_database.sh mysql scm scm shumeiG@123
/opt/module/cm/cm-5.12.1/share/cmf/schema/scm_prepare_database.sh
验证用户是否创建成功:
cat /etc/passwd | grep cloudera-scm
6 另两台同步
scp -r ./opt/cloudera-manager/cm-5.10.0 root@192.168.45.131: /opt/cloudera-manager
scp -r cm-5.10.0 root@192.168.45.136:/opt/cloudera-manager
创建server存储目录
mkdir /var/cloudera-scm-server
chown
cloudera-scm:cloudera-scm /var/cloudera-scm-server
创建hadoop离线安装包存储目录
mkdir -p /opt/cloudera/parcels
chown cloudera-scm:cloudera-scm /opt/cloudera/parcels
7配置
[root@hdfs1 ~]# vi /opt/cloudera-manager/cm-5.10.0/etc/cloudera-scm-agent/config.ini
将server_host修改为cloudera manager server的主机名,对于本示例而言,也就是server主机
8启动Cloudera Manager Agent
[root@hdfs1 ~]# cd /opt/cloudera-manager/cm-5.10.0/etc/init.d/
./cloudera-scm-server start
./cloudera-scm-server start 重起(主机)
sudo service cloudera-scm-server stop
./cloudera-scm-agent start 重起(其他两台)
Starting cloudera-scm-agent:[FAILED]
十一、测试agent和server是否通信成功
service cloudera-scm-server status
service cloudera-scm-agent status
netstat –anp|grep 7182
netstat -apn|grep 7182
# server 端开启的是7182端口,用于和agent进行通讯
启动失败时可以查看日志
server 日志 /var/log/cloudera-scm-server
agent 日志 /var/log/cloudera-scm-agen
4 常见问题
cloudera-scm-agent start >> This script is being executed in another terminal. Exit.
cloudera-scm-agent python2.7.11: No such file or directory
cloudera-scm-agent依赖的python版本
cloudera python
1 安装CDH5,启动命令之后 7182端口访问不了
1.1 netstat -nlp|grep 7182 主要查 mysql 有配置好没
2 起动失败:/usr/bin/env: python2.7: No such file or directory
python2.7安装
3 OpenJDK 64-Bit Server VM warning: ignoring option MaxPermSize=256m; support was removed in 8.0
http://apache.fayea.com/tomcat/tomcat-9/v9.0.30/bin/apache-tomcat-9.0.30.tar.gz
JAVA_OPTS="-Xms512m -Xmx1024m -Xss1024K -XX:PermSize=512m -XX:MaxPermSize=1024m"
export TOMCAT_HOME=/usr/local/tomcat9/apache-tomcat-9.0.30
export CATALINA_HOME=/usr/local/tomcat9/apache-tomcat-9.0.30
export JRE_HOME=/usr/lib/jvm/jre-1.8.0-openjdk.x86_64/bin/java
export JAVA_HOME=/usr/lib/jvm/jre-1.8.0-openjdk.x86_64/bin/java
4 BEGIN python -c 'import socket; import sys; s = socket.socket(socket.AF_INET); s.settimeout(5.0); s.connect((sys.argv[1], int(sys.argv[2]))); s.close();' 192.168.45.134 7182
Traceback (most recent call last):
File "<string>", line 1, in <module>
解决办法:
yum downgrade openssl
yum install openssl-devel
5 ERROR Caught unexpected exception in main loop.
找到下面这个文件 client_configs.py (路径参考 /opt/cloudera-manager/cm-5.10.0/lib64/cmf/agent/build/env/lib/python2.7/site-pac kages/cmf-5.10.0-py2.7.egg/lib/cmf/client_configs.py)
新添加
if len(line.rstrip().split(" "))<=4:
5 运行结果 http://192.168.45.134:7180/cmf/login 服务
[root@hadoop01 /]# yum clean all && yum makecache (运行试一下)
报错:http://hadoop01/html/cm/repodata/repomd.xml:[Errno 14] HTTP Error 404 - Not Found
1、Apache httpd服务器上,安装mod_ssl,执行命令
yum -y install mod_ssl
然后重启 httpd服务
2、在本地的yum配置中关闭ssl校验
编辑:/etc/yum.conf
添加: sslverify=false
https://docs.cloudera.com/documentation/enterprise/5-10-x/topics/how_to_configure_cm_tls.html