1)生成秘钥文件（在test1上执行） for h in ${all[@]} do ssh ${h} -C "ssh-keygen -t rsa -P '' -f /root/.ssh/id_rsa" ssh ${h} -C "cp ~/.ssh/id_rsa.pub ~/.ssh/authorized_keys" done 2)分发秘钥(在test1上执行) for h in ${sync[@]} do ssh-copy-id ${h} done
3、集群时间同步
一般使用集群中的某一个节点作为时间同步服务器，本例中选择Ambari所在的机器作为ntp服务端。
安装： for h in ${all[@]} do ssh ${h} -C " yum install ntp -y " done 修改服务端/etc/ntp.conf: $ vi /etc/ntp.conf # 取消此处restrict前面的注释，并将对应的ip段设置为自己主机所在的ip段（如果是公司openstack，则要配置内部地址） # Hosts on local network are less restricted. restrict 172.16.0.0 mask 255.255.255.0 nomodify notrap 例如IP是172.17.109.11 则修改为restrict 172.17.0.0 mask 255.255.255.0 nomodify notrap # 将下面的外网server均注释掉，加下面的两行，使用本地clock作为，时间服务器 # Use public servers from the pool.ntp.org project. # Please consider joining the pool (http://www.pool.ntp.org/join.html). #server 0.rhel.pool.ntp.org iburst #server 1.rhel.pool.ntp.org iburst #server 2.rhel.pool.ntp.org iburst #server 3.rhel.pool.ntp.org iburst server 127.127.1.0 fudge 127.127.1.0 stratum 10 启动服务端ntp centos 6： service ntpd start centos 7： systemctl start ntpd 修改客户端(集群中其他节点)/etc/ntp.conf，为了方便，先拷贝一份到本地/tmp目录下，修改好再同步: $ scp root@bigdata-nn-01:/etc/ntp.conf /tmp $ vi /tmp/ntp.conf # 将文件中的public servers全部注释掉，填写服务端的地址(ip 或者 hostname 均可) # Use public servers from the pool.ntp.org project. # Please consider joining the pool (http://www.pool.ntp.org/join.html). #server 0.rhel.pool.ntp.org iburst #server 1.rhel.pool.ntp.org iburst #server 2.rhel.pool.ntp.org iburst #server 3.rhel.pool.ntp.org iburst server 172.16.0.118 iburst 同步配置文件到其他服务器： for h in ${sync[@]} do scp /tmp/ntp.conf ${h}:/etc done 同步集群的时间： for h in ${sync[@]} do ssh $h -C "ntpdate <AMBARI HOSTNAME/IP>" done 配置时间自动同步： centos6： for h in ${all[@]} do ssh ${h} -C "chkconfig ntpd on ; service ntpd start ;" ssh ${h} -C "chkconfig –list | grep ntpd" done centos7: for h in ${all[@]} do ssh ${h} -C " systemctl enable ntpd ; systemctl start ntpd;" ssh ${h} -C " systemctl is-enabled ntpd " done
1)准备安装包jdk-8u65-linux-x64.tar.gz并解压 tar -xvf jdk-8u65-linux-x64.tar.gz -C /opt/ 2)下载jce_policy-8.zip安装jce unzip -o -j -q jce_policy-8.zip -d /opt/jdk1.8.0_65/jre/lib/security/ chown 10:143 /opt/jdk1.8.0_65/jre/lib/security/local_policy.jar chown 10:143 /opt/jdk1.8.0_65/jre/lib/security/US_export_policy.jar 3)配置环境变量 # vi /etc/profile export JAVA_HOME=/opt/jdk1.8.0_65 export PATH=$JAVA_HOME/bin:$PATH export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar 4)分发 for h in ${sync[@]} do scp -r /opt/ jdk1.8.0_65 ${h}:/opt scp -r /etc/profile ${h}:/etc ssh ${h} -C “source /etc/profile” done 5) 检查 for h in ${syncall[@]} do ssh ${h} -C “java -version” done
6、关闭防火墙
Centos6：
chkconfig iptables off --永久关闭 /etc/init.d/iptables stop --临时关闭 分发： for h in ${all[@]} do ssh ${h} -C "chkconfig iptables off; /etc/init.d/iptables stop" ssh ${h} -C " /etc/init.d/iptables status " done
Centos7：
systemctl disable firewalld systemctl stop firewalld 分发： for h in ${all[@]} do ssh ${h} -C " systemctl disable firewalld;sytemctl stop firewalld" ssh ${h} -C "systemctl status firewalld " done
7、关闭Selinux（centos6和centos7 通用）
在test1上做如下配置： # setenforce 0 ----临时禁止 # vi /etc/selinux/config ---永久禁止 SELINUX=disabled 分发配置 for h in ${sync[@]} do scp -r /etc/selinux/config ${h}:/etc/selinux done
8、关闭PackageKit
vim /etc/yum/pluginconf.d/refresh-packagekit.conf [main] enabled=0 分发到其他机器上： for h in ${sync[@]} do scp -r /etc/yum/pluginconf.d/refresh-packagekit.conf ${h}:/etc/yum/pluginconf.d/ done
9、设置umask
1) 临时改变： umask 0022 2) 永久改变所有用户的umask echo umask 0022 >>/etc/profile 3) 分发： for h in ${all[@]} do ssh root@${h} "echo umask 0022 >>/etc/profile" done
10、设置最大文件数
Centos6：
临时修改： ulimit -n 65535 永久修改： vim /etc/security/limits.conf * soft nproc 65535 * hard nproc 65535 * soft nofile 65535 * hard nofile 65535 分发： for h in ${sync[@]} do scp -r /etc/security/limits.conf ${h}:/etc/security done
Centos7:
临时修改： ulimit -n 65535 永久修改： 1) vim /etc/security/limits.conf * soft nproc 65535 * hard nproc 65535 * soft nofile 65535 * hard nofile 65535 2) vim /etc/systemd/system.conf 和 vim /etc/systemd/user.conf DefaultLimitCORE=infinity DefaultLimitNOFILE=65535 DefaultLimitNPROC=65535 分发： for h in ${sync[@]} do scp /etc/security/limits.conf ${h}:/etc/security scp /etc/systemd/system.conf ${h}:/etc/systemd scp /etc/systemd/user.conf ${h}:/etc/system ssh $h -C "systemctl daemon-reload" done
临时关闭： # echo never > /sys/kernel/mm/redhat_transparent_hugepage/enabled # cat /sys/kernel/mm/redhat_transparent_hugepage/enabled always madvise [never] 永久关闭： [root@getlnx06 ~]# vi /etc/rc.local #!/bin/sh # This script will be executed *after* all the other init scripts. # You can put your own initialization stuff in here if you don't # want to do the full Sys V style init stuff. touch /var/lock/subsys/local if test -f /sys/kernel/mm/redhat_transparent_hugepage/enabled; then echo never > /sys/kernel/mm/redhat_transparent_hugepage/enabled fi 分发： for h in ${sync[@]} do scp /etc/rc.local ${h}:/etc/ done
Centos7：
临时禁止： echo never > /sys/kernel/mm/transparent_hugepage/enabled echo never > /sys/kernel/mm/transparent_hugepage/defrag 永久禁止： vi /etc/systemd/system/disable-thp.service ，如果此文件不存在，创建即可 [Unit] Description=Disable Transparent Huge Pages (THP) [Service] Type=simple ExecStart=/bin/sh -c "echo 'never' > /sys/kernel/mm/transparent_hugepage/enabled && echo 'never' > /sys/kernel/mm/transparent_hugepage/defrag" [Install] WantedBy=multi-user.target 分发： for h in ${sync[@]} do scp /etc/systemd/system/disable-thp.service ${h}:/etc/systemd/system/ ssh ${h} -C "systemctl daemon-reload;systemctl start disable-thp;systemctl enable disable-thp" done
1) 集群之间可以root免密（ambari-server上的机器上有amabri repo库就行）： cp ambari.repo /etc/yum.repos.d/ 即可 2) 集群之间不可以root免密（集群中每台机器都需要配置ambari repo库）： for h in ${sync[@]} do scp -r /etc/yum.repos.d/ambari.repo ${h}:/etc/yum.repos.d done
启动：ambari-server start 访问页面：http://<ambari server host>:8080/
2、安装ambari-agent (optional)
如果集群之间不能root免密，还需要在每台机器上安装ambari-agent。 1)在每台机器上安装ambari agent for h in ${sync[@]} do ssh $h -C “yum install ambari-agent -y” done 2) 修改ambari-agent.ini文件的配置 #vi /etc/ambari-agent/conf/ambari-agent.ini [server] Hostname=<ambary server host> url_port=8440 secured_url_port=8441 3) 分发ambari-agent.ini文件 for h in ${sync[@]} do scp /etc/ambari-agent/conf/ambari-agent.ini $h:/etc/ambari-agent/conf done 4)启动ambari agent # ambari-agent start
上图底下有个“Click hereto see the warnings.” 按钮，打开后会显示主机上的各种警告信息，虽然已经安装了 ambari－agent,可是还有其他可能导致安装集群失败的潜在不足，比如 ntp没做，或防火墙规则存在，虽然放行了 ssh，但是等安装hadoop集群，需要打开很多的tcp 端口，可能会导致错误发生。自己视情况检查吧（我这个环境因为安装过HDP,许多提示都是说已经安装了某包，有某些配置文件已存在等等）。一定要确保所有的检测都通过，否则安装过程中会出现错误。
su - hadoop hive hive> create table people(id string, name string); OK hive> insert into people values("1","louis"); Query ID = hadoop_20170620112529_356f40fa-60d4-4c25-9728-ee3ff6adb49c Total jobs = 1 Launching Job 1 out of 1 Status: Running (Executing on YARN cluster with App id application_1497866533048_0004) -------------------------------------------------------------------------------- VERTICES STATUS TOTAL COMPLETED RUNNING PENDING FAILED KILLED -------------------------------------------------------------------------------- Map 1 .......... SUCCEEDED 1 1 0 0 0 0 -------------------------------------------------------------------------------- VERTICES: 01/01 [==========================>>] 100% ELAPSED TIME: 7.23 s -------------------------------------------------------------------------------- Loading data to table default.people Table default.people stats: [numFiles=1, numRows=1, totalSize=8, rawDataSize=7] OK Time taken: 11.96 seconds hive> select * from people; OK 1 louis Time taken: 0.393 seconds, Fetched: 1 row(s)
4.Hbase检查
su - hadoop hbase shell //创建表 hbase(main):001:0> create 'test','cf' 0 row(s) in 4.8910 seconds => Hbase::Table - test //查看所有表 hbase(main):002:0> list TABLE test 1 row(s) in 0.0560 seconds => ["test"] //往表里面插入数据 hbase(main):003:0> put 'test','row1','cf:a','value1' 0 row(s) in 0.3720 seconds hbase(main):004:0> put 'test','row2','cf:b','value2' 0 row(s) in 0.0270 seconds hbase(main):005:0> put 'test','row3','cf:3','value3' 0 row(s) in 0.0220 seconds //扫描表中的所有数据 hbase(main):006:0> scan 'test' ROW COLUMN+CELL row1 column=cf:a, timestamp=1497929482464, value=value1 row2 column=cf:b, timestamp=1497929489695, value=value2 row3 column=cf:3, timestamp=1497929495188, value=value3 3 row(s) in 0.0680 seconds //获取表中的一条数据 hbase(main):007:0> get 'test','row1' COLUMN CELL cf:a timestamp=1497929482464, value=value1 1 row(s) in 0.0550 seconds
5.spark检查
su - hadoop // 使用spark-submit提交任务 spark-submit --class org.apache.spark.examples.SparkPi \ --master yarn --deploy-mode client \ --driver-memory 1G --executor-memory 1G \ --executor-cores 1 /usr/hdp/2.6.0.3-8/spark/lib/spark-examples-1.6.3.2.6.0.3-8-hadoop2.7.3.2.6.0.3-8.jar 40 输出的结果中会有： 17/06/20 11:48:02 INFO DAGScheduler: Job 0 finished: reduce at SparkPi.scala:36, took 4.955870 s Pi is roughly 3.1410777852694465 // 使用spark-shell连接hive spark-shell scala> val result = sqlContext.sql("select * from default.people") scala> result.show +---+-----+ | id| name| +---+-----+ | 1|louis| +---+-----+
638
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
