exec 6&-关闭
exec 6<>/dev/tcp/ip.xx.port读写
exec 6<&-
exec 9<>/dev/tcp/ip.xx.port
echo -ne "GET /index.html HTTP/1.1/r/n" >&9
echo -ne "HOST:www.baidu.com/r/n" >&9
echo 9<&-
echo -e "DEVICE="eth0"\nONBOOT=yes\nNETBOOT=yes\nIPV6INIT=no\nBOOTPROTO=none\nTYPE=Ethernet\nIPADDR=192.168.1.11\nPREFIX=24\nGATEWAY=192.168.1.254">/etc/sysconfig/network-scripts/ifcfg-eth0
安装
yum install java-1.8.0-openjdk
yum -y install java-1.8.0-openjdk-devel
jps
主机名字ping /etc/hosts
拷贝scp hadoop-2.7.6.tar.gz 192.168.1.1:/usr/local/hadoop
修改/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.131-11.b12.el7.x86_64/jre
修改/usr/local/hadoop/etc/hadoop/hadoop-env.sh
验证./bin/hadoop version 在/usr/local/hadoop
/usr/local/hadoop/etc/hadoop路径
jar xx.jar
bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-2.7.6.jar wordcount oo xx
xml格式vim core-site.xml
<property>
<name></name>
<value></value>
</property>
<description></description>
vim /root/.ssh/known_hosts
core-site.xml
http://hadoop.apache.org/Documentation/Release 2.7.6/Configuration
网站有默认配置文件http://hadoop.apache.org/docs/r2.7.6/
http://hadoop.apache.org/docs/r2.7.6/hadoop-project-dist/hadoop-common/core-default.xml
192.168.1.1
u1 NameNode
SecondaryNameNode
ResourceManager HDFS
YARN
192.168.1.2
u3 DataNode
NodeManager HDFS
YARN
192.168.1.3
u3 DataNode
NodeManager HDFS
YARN
192.168.1.4
u4 DataNode
NodeManager HDFS
YARN
vim /etc/ssh/ssh_config
GSSAPIAuthentication no
StrictHostKeyChecking no
/usr/local/hadoop/etc/hadoop 添加vim slaves主机名
hdfs-site.xml
<configuration>
<property>
<name>dfs.namenode.http-address</name>
<value>u1:50070</value>
</property>
<property>
<name>dfs.namenode.secondary.http-address</name>
<value>u1:50090</value>
</property>
<property>
<name>dfs.replication</name>
<value>2</value>
</property>
</configuration>
所有机器scp -r /usr/local/hadoop/ 192.168.1.4:/usr/local/
./bin/hdfs u1 -format
./sbin/start-dfs.sh
jps
./sbin/stop-dfs.sh
rm -rf /usr/local/hadoop/logs
/usr/local/dfsadmin -report
firewalld
systemctl stop firewalld
yum remove
rpm -qa | grep firewalld
iptables
vim mapred-site.xml
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
<configuration>
<property>
<name>yarn.resourcemanager.hostname</name>
<value>h1</value>
</property>
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
</configuration>
rsync -aSH --delete /usr/
./sbin/start-yarn.sh
./bin/yarn node -list
验证
1.1:50070
1.1:50090
1.1:50075
1.2:8088
1.2:8042
for i in {2..4}; do scp yarn-site.xml 192.168.1.$i:/usr/local/hadoop/etc/hadoop/; done
./sbin/start-yarn.sh
./bin/yarn node -list
./bin/hdfs dfsadmin -report
./bin/hadoop fs -put/get
准备1.5 网关 1.6 客户端
关闭selinux卸载firewalld
nfsgw /etc/hosts
name
uid gid ===
adduser -g 100 nsd1802
adduser -u 1000 -g 100 nsd1802
停止集群服务 ./sbin/stop-all.sh
/usr/local/hadoop/etc/hadoop
vim core-site.xml
hadoop.proxyuser.nsd1802.groups *
hadoop.proxyuser.nsd1802.hosts *
启动./sbin/start-dfs.sh
配置网关
echo -e "DEVICE="eth0"\nONBOOT=yes\nNETBOOT=yes\nIPV6INIT=no\nBOOTPROTO=none\nTYPE=Ethernet\nIPADDR=192.168.1.5\nPREFIX=24\nGATEWAY=192.168.1.254">/etc/sysconfig/network-scripts/ifcfg-eth0
cat /etc/hosts
adduser -u 1000 -g 100 nsd1802
yum -y install java-1.8.0-openjdk-devel
setfacl -m user:nsd1802:rwx logs
scp /etc/yum.repos.d/dvd.repo 192.168.1.6:/etc/yum.repos.d/
scp -r /usr/local/hadoop/ 192.168.1.5:/usr/local/
yum remove -y rpcbind nfs-utils
参数
vim hdfs-site.xml
mkdir /var/nfstemp
chown nsd1802:users /var/nfstemp
<property>
<name>nfs.exports.allowed.hosts</name>
<value>* rw</value>
</property>
<property>
<name>nfs.rtmax</name>
<value>4194304</value>
</property>
<property>
<name>nfs.wtmax</name>
<value>1048576</value>
</property>
<property>
<name>nfs.port.monitoring.disabled</name>
<value>false</value>
</property>
<property>
<name>nfs.dump.dir</name>
<value>/var/nfstemp</value>
</property>
<property>
<name>dfs.namenode.accesstime.precision</name>
<value>3600000</value>
</property>
./sbin/stop-all.sh
./sbin/start-dfs.sh
./sbin/hadoop-daemon.sh --script ./bin/hdfs start portmap
jps
su -l nsd1802
cd /usr/local/hadoop
./sbin/hadoop-daemon.sh --script ./bin/hdfs start nfs3
jps
mount -t nfs -o vers=3,proto=tcp,nolock,noacl,noatime,sync 192.168.1.5:/ /mount/
git clone https://github.com/MrZhangzhg/nsd2018.git
安装openjdk
配置/etc/hosts
拷贝/usr/local/hadoop/复制
slavers添加节点
ssh免密码登陆
cd /root/.ssh/
ssh-copy-id -i id_rsa.pub newnode
启动./sbin/hadoop-daemon.sh start datanode
设置带宽./bin/hdfs dfsadmin -setBalancerBandwidth 67108864
同步数据./sbin/start-balancer.sh -threshold 5
验证 ./bin/hdfs dfsadmin -report
配置hdfs-site.xml增加
<property>
<name>dfs.hosts.exclude</name>
<value>/usr/local/hadoop/etc/hadoop/exclude</value>
</property>
touch /usr/local/hadoop/exclude
server.1=u2:2888:3888
server.2=u3:2888:3888
server.3=u4:2888:3888
server.4=u1:2888:3888:observer
mkdir /tmp/zookeeper
cd /tmp/zookeeper
echo 4 > myid
/usr/local/zookeeper-3.4.10/bin/zkServer.sh startus
cd /usr/local/kafka/config
vim server.properties
/usr/local/kafka/bin/kafka-server-start.sh -daemon /usr/local/kafka/config/server.properties
./bin/kafka-topics.sh --create --partitions 2 --replication-
factor 2 --zookeeper localhost:2181 --topic nsd1802
./bin/kafka-topics.sh --list --zookeeper localhost:2181
/bin/kafka-topics.sh --describe --zookeeper localhost:2181 --
topic nsd1802
目录
hadoop-env.sh
slaves u2 u3 u4
配置文件core-site.xml
hdfd-site.xml
yarn-site.xml
mapred-site.xml
usr/local/
core-site.xml
<property>
<name>fs.defaultFS</name>
<value>hdfs://mycluster</value>
</property>
<property>
<name>hadoop.tmp.dir</name>
<value>/var/hadoop</value>
</property>
<property>
<name>ha.zookeeper.quorum</name>
<value>node2:2181,node3:2181,node4:2181</value>
</property>
#######################################################
ALL 同步配置 所有机器配置
NN1 初始化ZK集群./bin/hdfs zkfc -formatZK
NODE启动服务 ./sbin/hadoop-daemon.sh start journalnode
successfully format
NN1 格式化 ./bin/hdfs namenode –format
NN2 同步数据本地 ./var/hadoop/dfs
NN1 初始化JNS ./bin/hdfs namenode –initializeSharedEdits
NODE 停止服务./sbin/hadoop-daemon.sh stop journalnode
############################################################
NN1: ./sbin/start-all.sh
NN2: ./sbin/yarn-daemon.sh start resourcemanager
bin/hdfs haadmin -getServiceState nn1
bin/hdfs haadmin -getServiceState nn2
bin/yarn rmadmin -getServiceState nn1
bin/yarn rmadmin -getServiceState nn2
./bin/hadoop dfsadmin –report
./bin/yarn node -list
./bin/hadoop fs -ls hdfs:/
exec 6<>/dev/tcp/ip.xx.port读写
exec 6<&-
exec 9<>/dev/tcp/ip.xx.port
echo -ne "GET /index.html HTTP/1.1/r/n" >&9
echo -ne "HOST:www.baidu.com/r/n" >&9
echo 9<&-
echo -e "DEVICE="eth0"\nONBOOT=yes\nNETBOOT=yes\nIPV6INIT=no\nBOOTPROTO=none\nTYPE=Ethernet\nIPADDR=192.168.1.11\nPREFIX=24\nGATEWAY=192.168.1.254">/etc/sysconfig/network-scripts/ifcfg-eth0
安装
yum install java-1.8.0-openjdk
yum -y install java-1.8.0-openjdk-devel
jps
主机名字ping /etc/hosts
拷贝scp hadoop-2.7.6.tar.gz 192.168.1.1:/usr/local/hadoop
修改/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.131-11.b12.el7.x86_64/jre
修改/usr/local/hadoop/etc/hadoop/hadoop-env.sh
验证./bin/hadoop version 在/usr/local/hadoop
/usr/local/hadoop/etc/hadoop路径
jar xx.jar
bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-2.7.6.jar wordcount oo xx
xml格式vim core-site.xml
<property>
<name></name>
<value></value>
</property>
<description></description>
vim /root/.ssh/known_hosts
core-site.xml
http://hadoop.apache.org/Documentation/Release 2.7.6/Configuration
网站有默认配置文件http://hadoop.apache.org/docs/r2.7.6/
http://hadoop.apache.org/docs/r2.7.6/hadoop-project-dist/hadoop-common/core-default.xml
192.168.1.1
u1 NameNode
SecondaryNameNode
ResourceManager HDFS
YARN
192.168.1.2
u3 DataNode
NodeManager HDFS
YARN
192.168.1.3
u3 DataNode
NodeManager HDFS
YARN
192.168.1.4
u4 DataNode
NodeManager HDFS
YARN
vim /etc/ssh/ssh_config
GSSAPIAuthentication no
StrictHostKeyChecking no
/usr/local/hadoop/etc/hadoop 添加vim slaves主机名
hdfs-site.xml
<configuration>
<property>
<name>dfs.namenode.http-address</name>
<value>u1:50070</value>
</property>
<property>
<name>dfs.namenode.secondary.http-address</name>
<value>u1:50090</value>
</property>
<property>
<name>dfs.replication</name>
<value>2</value>
</property>
</configuration>
所有机器scp -r /usr/local/hadoop/ 192.168.1.4:/usr/local/
./bin/hdfs u1 -format
./sbin/start-dfs.sh
jps
./sbin/stop-dfs.sh
rm -rf /usr/local/hadoop/logs
/usr/local/dfsadmin -report
firewalld
systemctl stop firewalld
yum remove
rpm -qa | grep firewalld
iptables
vim mapred-site.xml
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
<configuration>
<property>
<name>yarn.resourcemanager.hostname</name>
<value>h1</value>
</property>
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
</configuration>
rsync -aSH --delete /usr/
./sbin/start-yarn.sh
./bin/yarn node -list
验证
1.1:50070
1.1:50090
1.1:50075
1.2:8088
1.2:8042
for i in {2..4}; do scp yarn-site.xml 192.168.1.$i:/usr/local/hadoop/etc/hadoop/; done
./sbin/start-yarn.sh
./bin/yarn node -list
./bin/hdfs dfsadmin -report
./bin/hadoop fs -put/get
准备1.5 网关 1.6 客户端
关闭selinux卸载firewalld
nfsgw /etc/hosts
name
uid gid ===
adduser -g 100 nsd1802
adduser -u 1000 -g 100 nsd1802
停止集群服务 ./sbin/stop-all.sh
/usr/local/hadoop/etc/hadoop
vim core-site.xml
hadoop.proxyuser.nsd1802.groups *
hadoop.proxyuser.nsd1802.hosts *
启动./sbin/start-dfs.sh
配置网关
echo -e "DEVICE="eth0"\nONBOOT=yes\nNETBOOT=yes\nIPV6INIT=no\nBOOTPROTO=none\nTYPE=Ethernet\nIPADDR=192.168.1.5\nPREFIX=24\nGATEWAY=192.168.1.254">/etc/sysconfig/network-scripts/ifcfg-eth0
cat /etc/hosts
adduser -u 1000 -g 100 nsd1802
yum -y install java-1.8.0-openjdk-devel
setfacl -m user:nsd1802:rwx logs
scp /etc/yum.repos.d/dvd.repo 192.168.1.6:/etc/yum.repos.d/
scp -r /usr/local/hadoop/ 192.168.1.5:/usr/local/
yum remove -y rpcbind nfs-utils
参数
vim hdfs-site.xml
mkdir /var/nfstemp
chown nsd1802:users /var/nfstemp
<property>
<name>nfs.exports.allowed.hosts</name>
<value>* rw</value>
</property>
<property>
<name>nfs.rtmax</name>
<value>4194304</value>
</property>
<property>
<name>nfs.wtmax</name>
<value>1048576</value>
</property>
<property>
<name>nfs.port.monitoring.disabled</name>
<value>false</value>
</property>
<property>
<name>nfs.dump.dir</name>
<value>/var/nfstemp</value>
</property>
<property>
<name>dfs.namenode.accesstime.precision</name>
<value>3600000</value>
</property>
./sbin/stop-all.sh
./sbin/start-dfs.sh
./sbin/hadoop-daemon.sh --script ./bin/hdfs start portmap
jps
su -l nsd1802
cd /usr/local/hadoop
./sbin/hadoop-daemon.sh --script ./bin/hdfs start nfs3
jps
mount -t nfs -o vers=3,proto=tcp,nolock,noacl,noatime,sync 192.168.1.5:/ /mount/
git clone https://github.com/MrZhangzhg/nsd2018.git
安装openjdk
配置/etc/hosts
拷贝/usr/local/hadoop/复制
slavers添加节点
ssh免密码登陆
cd /root/.ssh/
ssh-copy-id -i id_rsa.pub newnode
启动./sbin/hadoop-daemon.sh start datanode
设置带宽./bin/hdfs dfsadmin -setBalancerBandwidth 67108864
同步数据./sbin/start-balancer.sh -threshold 5
验证 ./bin/hdfs dfsadmin -report
配置hdfs-site.xml增加
<property>
<name>dfs.hosts.exclude</name>
<value>/usr/local/hadoop/etc/hadoop/exclude</value>
</property>
touch /usr/local/hadoop/exclude
server.1=u2:2888:3888
server.2=u3:2888:3888
server.3=u4:2888:3888
server.4=u1:2888:3888:observer
mkdir /tmp/zookeeper
cd /tmp/zookeeper
echo 4 > myid
/usr/local/zookeeper-3.4.10/bin/zkServer.sh startus
cd /usr/local/kafka/config
vim server.properties
/usr/local/kafka/bin/kafka-server-start.sh -daemon /usr/local/kafka/config/server.properties
./bin/kafka-topics.sh --create --partitions 2 --replication-
factor 2 --zookeeper localhost:2181 --topic nsd1802
./bin/kafka-topics.sh --list --zookeeper localhost:2181
/bin/kafka-topics.sh --describe --zookeeper localhost:2181 --
topic nsd1802
目录
hadoop-env.sh
slaves u2 u3 u4
配置文件core-site.xml
hdfd-site.xml
yarn-site.xml
mapred-site.xml
usr/local/
core-site.xml
<property>
<name>fs.defaultFS</name>
<value>hdfs://mycluster</value>
</property>
<property>
<name>hadoop.tmp.dir</name>
<value>/var/hadoop</value>
</property>
<property>
<name>ha.zookeeper.quorum</name>
<value>node2:2181,node3:2181,node4:2181</value>
</property>
#######################################################
ALL 同步配置 所有机器配置
NN1 初始化ZK集群./bin/hdfs zkfc -formatZK
NODE启动服务 ./sbin/hadoop-daemon.sh start journalnode
successfully format
NN1 格式化 ./bin/hdfs namenode –format
NN2 同步数据本地 ./var/hadoop/dfs
NN1 初始化JNS ./bin/hdfs namenode –initializeSharedEdits
NODE 停止服务./sbin/hadoop-daemon.sh stop journalnode
############################################################
NN1: ./sbin/start-all.sh
NN2: ./sbin/yarn-daemon.sh start resourcemanager
bin/hdfs haadmin -getServiceState nn1
bin/hdfs haadmin -getServiceState nn2
bin/yarn rmadmin -getServiceState nn1
bin/yarn rmadmin -getServiceState nn2
./bin/hadoop dfsadmin –report
./bin/yarn node -list
./bin/hadoop fs -ls hdfs:/