虚拟机

最新推荐文章于 2023-06-04 21:12:42 发布

点点滴滴的淡淡的

最新推荐文章于 2023-06-04 21:12:42 发布

阅读量104

点赞数

分类专栏： java后台文章标签：虚拟机

本文链接：https://blog.csdn.net/qq503758762/article/details/111830171

版权

java后台专栏收录该内容

19 篇文章 0 订阅

订阅专栏

1、设置虚拟机网络
vi /etc/sysconfig/network-scripts/ifcfg-eth0
ONBOOT从NO改成yes

2、安装java
tar -zxvf jdk-8u271-linux-x64.tar.gz
vi /etc/profile

set java environment
JAVA_HOME=/home/jdk1.8.0_271
JRE_HOME=/home/jdk1.8.0_271/jre
CLASS_PATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar:$JRE_HOME/lib
PATH=$PATH:$JAVA_HOME/bin:$JRE_HOME/bin
export JAVA_HOME JRE_HOME CLASS_PATH PATH

source /etc/profile

3、安装Hadoop
tar -zxvf hadoop-3.3.0.tar.gz
vi /etc/profile
export HADOOP_HOME=/home/hadoop-3.3.0
export PATH=$PATH:$HADOOP_HOME/bin:/$HADOOP_HOME/sbin

最终环境变量
set java environment
JAVA_HOME=/home/jdk1.8.0_271
JRE_HOME=/home/jdk1.8.0_271/jre
CLASS_PATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar:$JRE_HOME/lib
export HADOOP_HOME=/home/hadoop-3.3.0
export HBASE_HOME=/home/hbase-2.4.0
export JAVA_HOME JRE_HOME CLASS_PATH PATH
PATH=$PATH:$JAVA_HOME/bin:$JRE_HOME/bin:$HADOOP_HOME/bin:/$HADOOP_HOME/sbin:$HBASE_HOME/bin:/$HBASE_HOME/sbin

vi /home/hadoop-3.3.0/etc/hadoop

vi core-site.xml
<property>
   <name>hadoop.tmp.dir</name>
   <value>/home/hadoop-3.3.0/tmp</value>
   <description>Abase for other temporary directories.</description>
</property>
<property>
   <name>fs.default.name</name>
   <value>hdfs://192.168.110.129:9000</value>
</property>

vi hadoop-env.sh
将export JAVA_HOME=${JAVA_HOME}
修改为：export JAVA_HOME=/home/jdk1.8.0_271

vi hdfs-site.xml
<property>
<name>dfs.name.dir</name>
<value>/home/hadoop-3.3.0/dfs/name</value>
<description>Path on the local filesystem where theNameNode stores the namespace and transactions logs persistently.</description>
</property>
<property>
<name>dfs.data.dir</name>
<value>/home/hadoop-3.3.0/dfs/data</value>
<description>Comma separated list of paths on the localfilesystem of a DataNode where it should store its blocks.</description>
</property>
<property>
<name>dfs.replication</name>
<value>2</value>
</property>
<property>
<name>dfs.permissions</name>
<value>false</value>
<description>need not permissions</description>
</property>

cp mapred-site.xml.template mapred-site.xml
vi mapred-site.xml

<property>
<name>mapred.job.tracker</name>
<value>192.168.110.129:49001</value>
</property>
<property>
<name>mapred.local.dir</name>
<value>/home/hadoop-3.3.0/var</value>
</property>
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
<property>
   <name>mapreduce.jobhistory.address</name>
   <value>192.168.110.129:10020</value>
   <description>MR JobHistory Server管理的日志的存放位置</description>
</property>
<property>
   <name>mapreduce.jobhistory.webapp.address</name>
   <value>192.168.110.129:19888</value>
   <description>查看历史服务器已经运行完的Mapreduce作业记录的web地址，需要启动该服务
   才行</description>
</property>
<property>
   <name>mapreduce.jobhistory.done-dir</name>
   <value>/history/done</value>
   <description>MR JobHistory Server管理的日志的存放位置,默认:/mr-history/done</description>
</property>
<property>
   <name>mapreduce.jobhistory.intermediate-done-dir</name>
   <value>/history/done_intermediate</value>
   <description>MapReduce作业产生的日志存放位置，默认值:/mr-history/tmp</description>
</property>

vi slaves
192.168.110.130
192.168.110.139

vi yarn-site.xml

<name>yarn.resourcemanager.hostname</name>
<value>192.168.110.129</value>
</property>
<property>
<description>The address of the applications manager interface in the RM.</description>
<name>yarn.resourcemanager.address</name>
<value>${yarn.resourcemanager.hostname}:8032</value>
</property>
<property>
<description>The address of the scheduler interface.</description>
<name>yarn.resourcemanager.scheduler.address</name>
<value>${yarn.resourcemanager.hostname}:8030</value>
</property>
<property>
<description>The http address of the RM web application.</description>
<name>yarn.resourcemanager.webapp.address</name>
<value>${yarn.resourcemanager.hostname}:8088</value>
</property>
<property>
<description>The https adddress of the RM web application.</description>
<name>yarn.resourcemanager.webapp.https.address</name>
<value>${yarn.resourcemanager.hostname}:8090</value>
</property>
<property>
<name>yarn.resourcemanager.resource-tracker.address</name>
<value>${yarn.resourcemanager.hostname}:8031</value>
</property>
<property>
<description>The address of the RM admin interface.</description>
<name>yarn.resourcemanager.admin.address</name>
<value>${yarn.resourcemanager.hostname}:8033</value>
</property>
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
<property>
<name>yarn.scheduler.maximum-allocation-mb</name>
<value>2048</value>
<discription>每个节点可用内存,单位MB,默认8182MB</discription>
</property>
<property>
<name>yarn.nodemanager.vmem-pmem-ratio</name>
<value>2.1</value>
</property>
<property>
<name>yarn.nodemanager.resource.memory-mb</name>
<value>2048</value>
</property>
<property>
<name>yarn.nodemanager.vmem-check-enabled</name>
<value>false</value>
</property>

scp -r /home/hadoop-3.3.0 root@192.168.110.130:/home/
scp -r /home/hadoop-3.3.0 root@192.168.110.139:/home/

scp -r /home/jdk1.8.0_271 root@192.168.110.130:/home/
scp -r /home/jdk1.8.0_271 root@192.168.110.139:/home/

hadoop namenode -format
start-dfs.sh start
start-yarn.sh start
./mr-jobhistory-daemon.sh start historyserver 启动历史记录服务

start-all.sh //启动
stop-all.sh //关闭

4、安装部署zookeeper
tar -zxvf apache-zookeeper-3.6.2-bin.tar.gz
cd /home/apache-zookeeper-3.6.2-bin/conf
cp zoo_sample.cfg zoo.cfg
vi zoo.cfg
server.1=192.168.110.129:2888:3888
server.2=192.168.110.130:2888:3888
server.3=192.168.110.139:2888:3888

mkdir /data
mkdir /data/zookeeper
vi myid

scp -r /home/apache-zookeeper-3.6.2-bin root@192.168.110.130:/home/
scp -r /home/apache-zookeeper-3.6.2-bin root@192.168.110.139:/home/
cd /home/apache-zookeeper-3.6.2-bin/bin
./zkServer.sh start

5、安装hbase
tar -zxvf hbase-2.4.0-bin.tar.gz

vi /etc/profile
export HBASE_HOME=/home/hbase-2.4.0
export PATH=$PATH:$HBASE_HOME/bin:/$HBASE_HOME/sbin

cd /home/hbase-2.4.0/conf/

vi hbase-env.sh
export JAVA_HOME=/home/jdk1.8.0_271/
export HBASE_CLASSPATH=/home/hadoop-3.3.0/etc/hadoop
export HBASE_MANAGES_ZK=false

vi hbase-site.xml

<property> 
<name>hbase.rootdir</name>
<value>hdfs://192.168.110.129:9000/hbase</value>
</property>
<property> 
<name>hbase.cluster.distributed</name>
<value>true</value>
</property>
<property> 
<name>hbase.master.info.port</name>
<value>16010</value>
</property>
<property> 
<name>hbase.regionserver.info.port</name>
<value>16030</value>
</property>
<property>
<name>hbase.zookeeper.quorum</name>
<value>192.168.110.129:2181,192.168.110.130:2181,192.168.110.139:2181</value>
</property>
<property>
　　<name>hbase.zookeeper.property.dataDir</name>
　　<value>/home/hbase-2.4.0/tmp/zk/data</value>
　</property>
<property>
<name>hbase.coprocessor.abortonerror</name>
<value>false</value>
</property>

vi regionservers

scp -r /home/hbase-2.4.0 root@192.168.110.130:/home/
scp -r /home/hbase-2.4.0 root@192.168.110.139:/home/

start-hbase.sh
stop-hbase.sh

安装组件netstat
yum -y install net-tools

rpm -qa telnet-server
rpm -qa xinetd
yum -y install telnet-server.x86_64
yum -y install telnet.x86_64
yum -y install xinetd.x86_64
chkconfig telnet on

bin/yarn jar share/hadoop/mapreduce/hadoop-mapreduce-examples-3.3.0.jar wordcount /wordcountdemo/input /wordcountdemo/output

<property>
<name>dfs.namenode.name.dir</name>
<value>file:/home/hadoop-3.3.0/data/namenode</value>
</property>
<property>
<name>dfs.datanode.data.dir</name>
<value>file:/home/hadoop-3.3.0/data/datanode</value>
</property>

hadoop namenode -format

start-all.sh //启动
stop-all.sh

cd /home/hadoop-3.3.0
rm -rf tmp/
mkdir tmp
chmod 755 tmp

ssh-keygen -t rsa -P ""
cat /root/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys

vi core-site.xml
vi hdfs-site.xml
vi mapred-site.xml
vi yarn-site.xml

cd sbin
vi start-dfs.sh
vi stop-dfs.sh

HDFS_DATANODE_USER=root
HADOOP_SECURE_DN_USER=hdfs
HDFS_NAMENODE_USER=root
HDFS_SECONDARYNAMENODE_USER=root

vi start-yarn.sh
vi stop-yarn.sh
YARN_RESOURCEMANAGER_USER=root
HADOOP_SECURE_DN_USER=yarn
YARN_NODEMANAGER_USER=root

ssh-keygen
ssh-keygen -t dsa
ssh-copy-id A1
ssh-copy-id A2
ssh-copy-id A3
ssh-copy-id -i .ssh/id_rsa.pub root@192.168.110.130
ssh-copy-id -i .ssh/id_rsa.pub root@192.168.110.139

Hadoop JAVA_HOME is not set and could not be found
vi hadoop-env.sh

export JAVA_HOME=/home/jdk1.8.0_271
export HADOOP_HOME=/home/hadoop-3.3.0

hadoop fs -ls -r /
hadoop fs -mkdir -p /wordcount/input
hadoop fs -put a.txt /wordcount/input/

找不到加载主类
vi $HADOOP_HOME/etc/hadoop/yarn-site.xml
<property>
<name>yarn.application.classpath</name>
<value>hadoop classpath返回信息</value>
</property>

/home/hadoop-3.3.0/etc/hadoop:/home/hadoop-3.3.0/share/hadoop/common/lib/*:/home/hadoop-3.3.0/share/hadoop/common/*:/home/hadoop-3.3.0/share/hadoop/hdfs:/home/hadoop-3.3.0/share/hadoop/hdfs/lib/*:/home/hadoop-3.3.0/share/hadoop/hdfs/*:/home/hadoop-3.3.0/share/hadoop/mapreduce/*:/home/hadoop-3.3.0/share/hadoop/yarn:/home/hadoop-3.3.0/share/hadoop/yarn/lib/*:/home/hadoop-3.3.0/share/hadoop/yarn/*

点点滴滴的淡淡的

关注

0
点赞
踩
0

收藏

觉得还不错? 一键收藏
0
评论
虚拟机

1、设置虚拟机网络vi /etc/sysconfig/network-scripts/ifcfg-eth0ONBOOT从NO改成yes2、安装javatar -zxvf jdk-8u271-linux-x64.tar.gzvi /etc/profileset java environmentJAVA_HOME=/home/jdk1.8.0_271JRE_HOME=/home/jdk1.8.0_271/jreCLASS_PATH=.:$JAVA_HOME/lib/dt.jar:$JAVA
复制链接

扫一扫

专栏目录