部署好CentOS 6.5 X86_64位 Hadoop后，测试怎么也不通过，防火墙是关闭的，Selinux也是关闭的，检查core-site.xml、hdfs-site.xml配置都没有问题，查看磁盘空间还有400多M，尝试多种处理办法后无果，增加Hadoop dfs.namenode.name.dir和dfs.datanode.data.dir所在磁盘空间后问题解决，希望对遇到同样问题的朋友有所帮助，具体处理过程如下：
[root@MasterSlave ~]# hdfs dfs -mkdir /tmp
[root@MasterSlave ~]# hdfs dfs -ls /
Found 1 items
drwxr-xr-x - root supergroup 0 2014-03-14 12:32 /tmp
[root@MasterSlave ~]# hdfs dfs -copyFromLocal /usr/hadoop/hduser/pg20417.txt /tmp
14/03/14 12:18:33 WARN hdfs.DFSClient: DataStreamer Exception
org.apache.hadoop.ipc.RemoteException(java.io.IOException): File /tmp/pg20417.txt._COPYING_ could only be replicated to 0 nodes instead of minReplication (=1). There are 1 datanode(s) running and no node(s) are excluded in this operation.
at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.chooseTarget(BlockManager.java:1384)
at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:2477)
at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:555)
at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.addBlock(ClientNamenodeProtocolServerSideTranslatorPB.java:387)
at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java:59582)
at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:585)
at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:928)
at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2048)
at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2044)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:415)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1491)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2042)
at org.apache.hadoop.ipc.Client.call(Client.java:1347)
at org.apache.hadoop.ipc.Client.call(Client.java:1300)
at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:206)
at com.sun.proxy.$Proxy9.addBlock(Unknown Source)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:186)
at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:102)
at com.sun.proxy.$Proxy9.addBlock(Unknown Source)
at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.addBlock(ClientNamenodeProtocolTranslatorPB.java:330)
at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.locateFollowingBlock(DFSOutputStream.java:1226)
at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.nextBlockOutputStream(DFSOutputStream.java:1078)
at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:514)
copyFromLocal: File /tmp/pg20417.txt._COPYING_ could only be replicated to 0 nodes instead of minReplication (=1). There are 1 datanode(s) running and no node(s) are excluded in this operation.
14/03/14 12:18:33 ERROR hdfs.DFSClient: Failed to close file /tmp/pg20417.txt._COPYING_
org.apache.hadoop.ipc.RemoteException(java.io.IOException): File /tmp/pg20417.txt._COPYING_ could only be replicated to 0 nodes instead of minReplication (=1). There are 1 datanode(s) running and no node(s) are excluded in this operation.
at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.chooseTarget(BlockManager.java:1384)
at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:2477)
at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:555)
at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.addBlock(ClientNamenodeProtocolServerSideTranslatorPB.java:387)
at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java:59582)
at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:585)
at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:928)
at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2048)
at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2044)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:415)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1491)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2042)
at org.apache.hadoop.ipc.Client.call(Client.java:1347)
at org.apache.hadoop.ipc.Client.call(Client.java:1300)
at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:206)
at com.sun.proxy.$Proxy9.addBlock(Unknown Source)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:186)
at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:102)
at com.sun.proxy.$Proxy9.addBlock(Unknown Source)
at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.addBlock(ClientNamenodeProtocolTranslatorPB.java:330)
at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.locateFollowingBlock(DFSOutputStream.java:1226)
at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.nextBlockOutputStream(DFSOutputStream.java:1078)
at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:514)
[root@MasterSlave ~]#
[root@Hadoop hadoop]# df -h
Filesystem Size Used Avail Use% Mounted on
/dev/mapper/VolGroup-LogVol04 5.0G 625M 4.1G 14% /
tmpfs 495M 224K 495M 1% /dev/shm
/dev/sda1 485M 57M 403M 13% /boot
/dev/mapper/VolGroup-LogVol05 26G 7.0G 18G 29% /home
/dev/mapper/VolGroup-LogVol01 504M 17M 462M 4% /tmp
/dev/mapper/VolGroup-LogVol02 3.0G 2.4G 414M 86% /usr
/dev/mapper/VolGroup-LogVol03 1008M 284M 673M 30% /var
[root@MasterSlave ~]# fdisk -l
Disk /dev/sda: 42.9 GB, 42949672960 bytes
255 heads, 63 sectors/track, 5221 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk identifier: 0x00029e4a
Device Boot Start End Blocks Id System
/dev/sda1 * 1 64 512000 83 Linux
Partition 1 does not end on cylinder boundary.
/dev/sda2 64 5222 41430016 8e Linux LVM
Disk /dev/sdb: 21.5 GB, 21474836480 bytes
255 heads, 63 sectors/track, 2610 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk identifier: 0x00000000
Disk /dev/mapper/VolGroup-LogVol04: 5368 MB, 5368709120 bytes
255 heads, 63 sectors/track, 652 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk identifier: 0x00000000
Disk /dev/mapper/VolGroup-lv_swap: 4294 MB, 4294967296 bytes
255 heads, 63 sectors/track, 522 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk identifier: 0x00000000
Disk /dev/mapper/VolGroup-LogVol05: 27.9 GB, 27925676032 bytes
255 heads, 63 sectors/track, 3395 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk identifier: 0x00000000
Disk /dev/mapper/VolGroup-LogVol01: 536 MB, 536870912 bytes
255 heads, 63 sectors/track, 65 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk identifier: 0x00000000
Disk /dev/mapper/VolGroup-LogVol03: 1073 MB, 1073741824 bytes
255 heads, 63 sectors/track, 130 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk identifier: 0x00000000
Disk /dev/mapper/VolGroup-LogVol02: 3221 MB, 3221225472 bytes
255 heads, 63 sectors/track, 391 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk identifier: 0x00000000
[root@MasterSlave ~]# fdisk /dev/sdb
Device contains neither a valid DOS partition table, nor Sun, SGI or OSF disklabel
Building a new DOS disklabel with disk identifier 0x646eeb23.
Changes will remain in memory only, until you decide to write them.
After that, of course, the previous content won't be recoverable.
Warning: invalid flag 0x0000 of partition table 4 will be corrected by w(rite)
WARNING: DOS-compatible mode is deprecated. It's strongly recommended to
switch off the mode (command 'c') and change display units to
sectors (command 'u').
Command (m for help): p
Disk /dev/sdb: 21.5 GB, 21474836480 bytes
255 heads, 63 sectors/track, 2610 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk identifier: 0x646eeb23
Device Boot Start End Blocks Id System
Command (m for help): n
Command action
e extended
p primary partition (1-4)
p
Partition number (1-4): 1
First cylinder (1-2610, default 1):
Using default value 1
Last cylinder, +cylinders or +size{K,M,G} (1-2610, default 2610):
Using default value 2610
Command (m for help): p
Disk /dev/sdb: 21.5 GB, 21474836480 bytes
255 heads, 63 sectors/track, 2610 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk identifier: 0x646eeb23
Device Boot Start End Blocks Id System
/dev/sdb1 1 2610 20964793+ 83 Linux
Command (m for help): t
Selected partition 1
Hex code (type L to list codes): 8e
Changed system type of partition 1 to 8e (Linux LVM)
Command (m for help): p
Disk /dev/sdb: 21.5 GB, 21474836480 bytes
255 heads, 63 sectors/track, 2610 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk identifier: 0x646eeb23
Device Boot Start End Blocks Id System
/dev/sdb1 1 2610 20964793+ 8e Linux LVM
Command (m for help): w
The partition table has been altered!
Calling ioctl() to re-read partition table.
Syncing disks.
[root@MasterSlave ~]# partprobe /dev/sdb1
[root@MasterSlave ~]# pvdisplay
--- Physical volume ---
PV Name /dev/sda2
VG Name VolGroup
PV Size 39.51 GiB / not usable 3.00 MiB
Allocatable yes (but full)
PE Size 4.00 MiB
Total PE 10114
Free PE 0
Allocated PE 10114
PV UUID yOoqMa-lfnq-9odT-xD1m-Cd1h-ARQw-XbTM2Z
[root@MasterSlave ~]# vgextend VolGroup /dev/sdb1
Volume group "VolGroup" successfully extended
[root@MasterSlave ~]# lvdisplay
--- Logical volume ---
LV Path /dev/VolGroup/LogVol05
LV Name LogVol05
VG Name VolGroup
LV UUID 3JhPxA-0E4j-QJSE-Bdgp-XGka-PjMB-Va7XMb
LV Write Access read/write
LV Creation host, time localhost.localdomain, 2014-03-08 16:45:24 +0800
LV Status available
# open 1
LV Size 26.01 GiB
Current LE 6658
Segments 1
Allocation inherit
Read ahead sectors auto
- currently set to 256
Block device 253:2
--- Logical volume ---
LV Path /dev/VolGroup/LogVol04
LV Name LogVol04
VG Name VolGroup
LV UUID eHUcVX-yUbK-2xy9-hxRL-F3U2-3NLQ-wqYUu1
LV Write Access read/write
LV Creation host, time localhost.localdomain, 2014-03-08 16:45:26 +0800
LV Status available
# open 1
LV Size 5.00 GiB
Current LE 1280
Segments 1
Allocation inherit
Read ahead sectors auto
- currently set to 256
Block device 253:0
--- Logical volume ---
LV Path /dev/VolGroup/LogVol01
LV Name LogVol01
VG Name VolGroup
LV UUID w2E8nW-tIdM-HygX-ZiJJ-DtZu-9ttW-uPWyE1
LV Write Access read/write
LV Creation host, time localhost.localdomain, 2014-03-08 16:45:26 +0800
LV Status available
# open 1
LV Size 512.00 MiB
Current LE 128
Segments 1
Allocation inherit
Read ahead sectors auto
- currently set to 256
Block device 253:3
--- Logical volume ---
LV Path /dev/VolGroup/LogVol03
LV Name LogVol03
VG Name VolGroup
LV UUID taFRAd-IN3r-fTp3-MdqF-Oz3o-S1Yb-xNAqkg
LV Write Access read/write
LV Creation host, time localhost.localdomain, 2014-03-08 16:45:27 +0800
LV Status available
# open 1
LV Size 1.00 GiB
Current LE 256
Segments 1
Allocation inherit
Read ahead sectors auto
- currently set to 256
Block device 253:4
--- Logical volume ---
LV Path /dev/VolGroup/LogVol02
LV Name LogVol02
VG Name VolGroup
LV UUID 0AJV5d-xNDN-Dvp2-Jdtb-o1Qp-tea4-FXWPSb
LV Write Access read/write
LV Creation host, time localhost.localdomain, 2014-03-08 16:45:27 +0800
LV Status available
# open 1
LV Size 3.00 GiB
Current LE 768
Segments 1
Allocation inherit
Read ahead sectors auto
- currently set to 256
Block device 253:5
--- Logical volume ---
LV Path /dev/VolGroup/lv_swap
LV Name lv_swap
VG Name VolGroup
LV UUID B4DO0f-yZm0-fp3L-waeZ-3z3d-r0rD-6GHLLL
LV Write Access read/write
LV Creation host, time localhost.localdomain, 2014-03-08 16:45:28 +0800
LV Status available
# open 1
LV Size 4.00 GiB
Current LE 1024
Segments 1
Allocation inherit
Read ahead sectors auto
- currently set to 256
Block device 253:1
[root@MasterSlave ~]# mount
/dev/mapper/VolGroup-LogVol04 on / type ext4 (rw)
proc on /proc type proc (rw)
sysfs on /sys type sysfs (rw)
devpts on /dev/pts type devpts (rw,gid=5,mode=620)
tmpfs on /dev/shm type tmpfs (rw)
/dev/sda1 on /boot type ext4 (rw)
/dev/mapper/VolGroup-LogVol05 on /home type ext4 (rw)
/dev/mapper/VolGroup-LogVol01 on /tmp type ext4 (rw)
/dev/mapper/VolGroup-LogVol02 on /usr type ext4 (rw)
/dev/mapper/VolGroup-LogVol03 on /var type ext4 (rw)
none on /proc/sys/fs/binfmt_misc type binfmt_misc (rw)
[root@MasterSlave ~]# lvextend -L 10G /dev/VolGroup/LogVol02
Extending logical volume LogVol02 to 10.00 GiB
Logical volume LogVol02 successfully resized
[root@MasterSlave ~]# resize2fs /dev/VolGroup/LogVol02
resize2fs 1.41.12 (17-May-2010)
Filesystem at /dev/VolGroup/LogVol02 is mounted on /usr; on-line resizing required
old desc_blocks = 1, new_desc_blocks = 1
Performing an on-line resize of /dev/VolGroup/LogVol02 to 2621440 (4k) blocks.
The filesystem on /dev/VolGroup/LogVol02 is now 2621440 blocks long.
[root@MasterSlave ~]# df -h
Filesystem Size Used Avail Use% Mounted on
/dev/mapper/VolGroup-LogVol04 5.0G 625M 4.1G 14% /
tmpfs 495M 72K 495M 1% /dev/shm
/dev/sda1 485M 57M 403M 13% /boot
/dev/mapper/VolGroup-LogVol05 26G 7.5G 17G 31% /home
/dev/mapper/VolGroup-LogVol01 504M 17M 462M 4% /tmp
/dev/mapper/VolGroup-LogVol02 9.9G 2.5G 7.0G 26% /usr
/dev/mapper/VolGroup-LogVol03 1008M 145M 812M 16% /var
[root@MasterSlave ~]# hdfs dfs -copyFromLocal /usr/hadoop/hduser/pg20417.txt /tmp
[root@MasterSlave ~]# hdfs dfs -ls /tmp
Found 1 items
-rw-r--r-- 1 root supergroup 674570 2014-03-14 12:32 /tmp/pg20417.txt
2503
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
