CentOs7 安装Hadoop&Hbase
1.SSH无密码登录
ssh-keygen -t rsa //生成公钥,一路回车
cd ~/.ssh
cat id_rsa.pub >> authorized_keys //两个 >> 表示追加
chmod 600 authorized_keys //更改权限
ssh-copy-id root@c0 //将生成的公钥分发至所有节点,期间输入yes再输入密码
ssh-copy-id root@c1
ssh-copy-id root@c2
#在每一台主机上重复上述操作,即完成所有机器之间的免密登录
ssh c1 //登录测试(主机名)
hostname //查看当前连接的主机名,如果无误,用exit退出
2.安装Hadoop(在一台主机安装即可)
2.1准备工作
cd /opt/modules/softwares
tar -zxvf hadoop-2.8.5.tar.gz //解压
cd /opt/modules/softwares/hadoop-2.8.5/etc/hadoop //进入目录
mkdir -p /opt/modules/softwares/hadoop-2.8.5/tmp/dfs/name //创建三个目录
vi /etc/profile //将Hadoop添加至环境变量
export HADOOP_HOME=/opt/modules/softwares/hadoop-2.8.5
export PATH=$HADOOP_HOME/bin:$PATH
source /etc/profile
hadoop version //检测是否成功
2.2修改配置文件(注意修改主机名和路径)
vi core-site.xml //修改配置文件 core-site.xml 如下
<configuration>
<property>
<name>fs.default.name</name>
<value>hdfs://c0:9000</value>
</property>
<property>
<name>hadoop.tmp.dir</name>
<value>/opt/modules/softwares/hadoop-2.8.5/tmp</value>
</property>
</configuration>
vi hdfs-site.xml //修改配置文件 hdfs-site.xml 如下
<configuration>
<property>
<name>dfs.replication</name>
<value>1</value>
</property>
<property>
<name>dfs.namenode.name.dir</name>
<value>/opt/modules/softwares/hadoop-2.8.5/tmp/dfs/name</value>
</property>
<property>
<name>dfs.datanode.data.dir</name>
<value>/opt/modules/softwares/hadoop-2.8.5/tmp/dfs/data</value>
</property>
<property>
<name>dfs.namenode.secondary.http-address</name>
<value>c0:9001</value>
</property>
</configuration>
cp -i mapred-site.xml.template mapred-site.xml //复制文件mapred-site.xml.template新文件命名为mapred-site.xml
vi mapred-site.xml //修改配置文件 mapred-site.xml
<configuration>
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
</configuration>
vi yarn-site.xml //修改配置文件 yarn-site.xml
<configuration>
<property>
<name>yarn.resourcemanager.hostname</name>
<value>c0</value>
</property>
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
<property>
<name>yarn.log-aggregation-enable</name>
<value>true</value>
</property>
<property>
<name>yarn.log-aggregation.retain-seconds</name>
<value>604800</value>
</property>
</configuration>
vi slaves //修改配置文件 slaves
#内容如下(三台主机名)
c0
c1
c2
vi /etc/profile //确认自己的JAVA_HOME
(export JAVA_HOME=/usr/java/jdk1.8.0_221)
vi hadoop-env.sh //修改其中的JAVA_HOME
(# The java implementation to use.
export JAVA_HOME=/usr/java/jdk1.8.0_221)
vi mapred-env.sh //修改其中的JAVA_HOME
(export JAVA_HOME=/usr/java/jdk1.8.0_221
export HADOOP_JOB_HISTORYSERVER_HEAPSIZE=1000
export HADOOP_MAPRED_ROOT_LOGGER=INFO,RFA)
vi yarn-env.sh //修改其中的JAVA_HOME
(# some Java parameters
export JAVA_HOME=/usr/java/jdk1.8.0_221
if [ "$JAVA_HOME" != "" ]; then
#echo "run java in $JAVA_HOME"
JAVA_HOME=$JAVA_HOME
fi)
2.3发送安装包,将 Hadoop 复制到其他机器上
scp -r /opt/modules/softwares/hadoop-2.8.5 c1:/opt/modules/softwares
scp -r /opt/modules/softwares/hadoop-2.8.5 c2:/opt/modules/softwares
#分发完毕!
2.4启动Hadoop集群
cd /opt/modules/softwares/hadoop-2.8.5 //进入Hadoop目录
#启动Hadoop之前需要先启动HDFS和YARN两个模块
#首次启动hdfs时需要在c0上先将其格式化
bin/hdfs namenode -format //格式化hdfs
sbin/start-dfs.sh //格式化完成后启动hdfs
sbin/start-yarn.sh //启动yarn
jps //查看进程是否启动成功
#c0应有SecondaryNameNode,ResourceManager,DataNode,NodeManager
#c1,c2应有DataNode,NodeManager
2.5访问hdfs,yarn。在c0(主节点)上打开浏览器输入
c0:50070 //访问hdfs
#可查看相应信息,其中可点击 Live Nodes查看两个从节点
c0:8088 //访问yarn
至此,Hadoop集群搭建完毕!
2.6PS:关闭Hadoop
如果已经安装了HBase,请务必先关闭HBase再关闭Hadoop!
#stop-hbase.sh //停止HBase命令(bin/stop-hbase.sh)
sbin/stop-yarn.sh
sbin/stop-dfs.sh
3.安装HBase
3.1准备环境
cd /opt/modules/softwares
tar -zxvf hbase-1.2.6-bin.tar.gz //解压
vi /etc/profile //添加环境变量
export HBASE_HOME=/opt/modules/softwares/hbase-1.2.6
export PATH=$PATH:$HBASE_HOME/bin
source /etc/profile
hbase version //检验是否成功
3.2修改HBase配置信息(注意主机名)
cd /opt/modules/softwares/hbase-1.2.6/conf //进入目录
vi hbase-env.sh //设置JAVA_HOME为实际jdk路径
(# The java implementation to use. Java 1.7+ required.
export JAVA_HOME=/usr/java/jdk1.8.0_221)
vi regionservers //修改其内容为三台主机名
(c0
c1
c2)
vi hbase-site.xml //修改配置文件 hbase-site.xml
<configuration>
<property>
<name>hbase.rootdir</name>
<value>hdfs://c0:9000/hbase</value>
</property>
<property>
<name>hbase.cluster.distributed</name>
<value>true</value>
</property>
<property>
<name>hbase.master</name>
<value>hdfs://c0:60000</value>
</property>
<property>
<name>hbase.zookeeper.quorum</name>
<value>c0,c1,c2</value>
</property>
<property>
<name>hbase.tmp.dir</name>
<value>file:/opt/modules/softwares/hbase-1.2.6/tmp</value>
</property>
</configuration>
3.3将配置好的HBase分发到c1,c2
scp -r /opt/modules/softwares/hbase-1.2.6 c1:/opt/modules/softwares
scp -r /opt/modules/softwares/hbase-1.2.6 c2:/opt/modules/softwares
#分发完毕!
3.4启动HBase(Hadoop必须已经启动,最好在Hadoop启动两分钟后再启动HBase)
start-hbase.sh //几秒
jps //查看进程是否启动成功
(c0应有HMaster、HQuorumPeer、HRegionServer
c1和c2应有HQuorumPeer、HRegionServer)
hbase shell //进入hbase命令模式(可用exit退出)
status //查看状态
(1 active master, 0 backup masters, 3 servers, 0 dead, 0.6667 average load)
c0:16010 //浏览器输入,即可查看HBase配置
#stop-hbase.sh //停止HBase命令(bin/stop-hbase.sh)
4.1打jar包时
在src/main/resouces目录下建立log4j.properties 其内容如下
hadoop.root.logger=DEBUG, console
log4j.rootLogger = DEBUG, console
log4j.appender.console=org.apache.log4j.ConsoleAppender
log4j.appender.console.target=System.out
log4j.appender.console.layout=org.apache.log4j.PatternLayout
log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
4.2运行jar包
cd /opt/modules/softwares
java -jar 1kafkaToRedis.jar //启动消费者监听程序
java -jar 2jsonToKafka.jar //(新开一个控制端)58w条数据,等待结束
java -jar 3HBaseCreateOP.jar //等待
java -jar 4redisToHbase.jar
PS:stopping hbasecat: /tmp/hbase-root-master.pid: 没有那个文件或目录
c1: no zookeeper to stop because no pid file /tmp/hbase-root-zookeeper.pid
mkdir -p /var/hadoop/pids //各个结点都要运行下述代码
chown -R c:c /var/hadoop //c是用户名
cd /opt/modules/softwares/hadoop-2.8.5/etc/hadoop
vi hadoop-env.sh //增加一行如下
export HADOOP_PID_DIR=/var/hadoop/pids
vi yarn-env.sh //增加一行如下
export YARN_PID_DIR=/var/hadoop/pids
cd /opt/modules/softwares/hbase-1.2.6/conf
vi hbase-env.sh //增加一行如下
export HBASE_PID_DIR=/var/hadoop/pids
cd /opt/modules/softwares/hadoop-2.8.5
sbin/start-all.sh //启动Hadoop
cd /opt/modules/softwares/hbase-1.2.6
bin/start-hbase.sh //启动HBase