在本机上装的CentOS 5.5 虚拟机,
软件准备:jdk 1.6 U26
hadoop:hadoop-0.20.203.tar.gz
ssh检查配置
- [root@localhost~]#ssh-keygen-trsa
- Generatingpublic/privatersakeypair.
- Enterfileinwhichtosavethekey(/root/.ssh/id_rsa):
- Createddirectory'/root/.ssh'.
- Enterpassphrase(emptyfornopassphrase):
- Entersamepassphraseagain:
- Youridentificationhasbeensavedin/root/.ssh/id_rsa.
- Yourpublickeyhasbeensavedin/root/.ssh/id_rsa.pub.
- Thekeyfingerprintis:
- a8:7a:3e:f6:92:85:b8:c7:be:d9:0e:45:9c:d1:36:3broot@localhost.localdomain
- [root@localhost~]#
- [root@localhost~]#cd..
- [root@localhost/]#cdroot
- [root@localhost~]#ls
- anaconda-ks.cfgDesktopinstall.loginstall.log.syslog
- [root@localhost~]#cd.ssh
- [root@localhost.ssh]#catid_rsa.pub>authorized_keys
- [root@localhost.ssh]#
- [root@localhost.ssh]#sshlocalhost
- Theauthenticityofhost'localhost(127.0.0.1)'can'tbeestablished.
- RSAkeyfingerprintis41:c8:d4:e4:60:71:6f:6a:33:6a:25:27:62:9b:e3:90.
- Areyousureyouwanttocontinueconnecting(yes/no)?yes
- Warning:Permanentlyadded'localhost'(RSA)tothelistofknownhosts.
- Lastlogin:TueJun2122:40:312011
- [root@localhost~]#
安装jdk
- [root@localhostjava]#chmod+xjdk-6u26-linux-i586.bin
- [root@localhostjava]#./jdk-6u26-linux-i586.bin
- ......
- ......
- ......
- FormoreinformationonwhatdataRegistrationcollectsand
- howitismanagedandused,see:
- http://java.sun.com/javase/registration/JDKRegistrationPrivacy.html
- PressEntertocontinue.....
- Done.
安装完成后生成文件夹:jdk1.6.0_26
配置环境变量
- [root@localhostjava]#vi/etc/profile
- #添加如下信息
- #setjavaenvironment
- exportJAVA_HOME=/usr/java/jdk1.6.0_26
- exportCLASSPATH=$CLASSPATH:$JAVA_HOME/lib:$JAVA_HOME/jre/lib
- exportPATH=$JAVA_HOME/lib:$JAVA_HOME/jre/bin:$PATH:$HOME/bin
- exportHADOOP_HOME=/usr/local/hadoop/hadoop-0.20.203
- exportPATH=$PATH:$HADOOP_HOME/bin
- [root@localhostjava]#chmod+x/etc/profile
- [root@localhostjava]#source/etc/profile
- [root@localhostjava]#
- [root@localhostjava]#java-version
- javaversion"1.6.0_26"
- Java(TM)SERuntimeEnvironment(build1.6.0_26-b03)
- JavaHotSpot(TM)ClientVM(build20.1-b02,mixedmode,sharing)
- [root@localhostjava]#
修改hosts
- [root@localhostconf]#vi/etc/hosts
- #Donotremovethefollowingline,orvariousprograms
- #thatrequirenetworkfunctionalitywillfail.
- 127.0.0.1localhost.localdomainlocalhost
- ::1localhost6.localdomain6localhost6
- 127.0.0.1namenodedatanode01
解压安装hadoop
- [root@localhosthadoop]#tarzxvfhadoop-0.20.203.tar.gz
- ......
- ......
- ......
- hadoop-0.20.203.0/src/contrib/ec2/bin/image/create-hadoop-image-remote
- hadoop-0.20.203.0/src/contrib/ec2/bin/image/ec2-run-user-data
- hadoop-0.20.203.0/src/contrib/ec2/bin/launch-hadoop-cluster
- hadoop-0.20.203.0/src/contrib/ec2/bin/launch-hadoop-master
- hadoop-0.20.203.0/src/contrib/ec2/bin/launch-hadoop-slaves
- hadoop-0.20.203.0/src/contrib/ec2/bin/list-hadoop-clusters
- hadoop-0.20.203.0/src/contrib/ec2/bin/terminate-hadoop-cluster
- [root@localhosthadoop]#
进入hadoop配置conf
- ####################################
- [root@localhostconf]#vihadoop-env.sh
- #添加代码
- #setjavaenvironment
- exportJAVA_HOME=/usr/java/jdk1.6.0_26
- #####################################
- [root@localhostconf]#vicore-site.xml
- <?xmlversion="1.0"?>
- <?xml-stylesheettype="text/xsl"href="configuration.xsl"?>
- <!--Putsite-specificpropertyoverridesinthisfile.-->
- <configuration>
- <property>
- <name>fs.default.name</name>
- <value>hdfs://namenode:9000/</value>
- </property>
- <property>
- <name>hadoop.tmp.dir</name>
- <value>/usr/local/hadoop/hadooptmp</value>
- </property>
- </configuration>
- #######################################
- [root@localhostconf]#vihdfs-site.xml
- <?xmlversion="1.0"?>
- <?xml-stylesheettype="text/xsl"href="configuration.xsl"?>
- <!--Putsite-specificpropertyoverridesinthisfile.-->
- <configuration>
- <property>
- <name>dfs.name.dir</name>
- <value>/usr/local/hadoop/hdfs/name</value>
- </property>
- <property>
- <name>dfs.data.dir</name>
- <value>/usr/local/hadoop/hdfs/data</value>
- </property>
- <property>
- <name>dfs.replication</name>
- <value>1</value>
- </property>
- </configuration>
- #########################################
- [root@localhostconf]#vimapred-site.xml
- <?xmlversion="1.0"?>
- <?xml-stylesheettype="text/xsl"href="configuration.xsl"?>
- <!--Putsite-specificpropertyoverridesinthisfile.-->
- <configuration>
- <property>
- <name>mapred.job.tracker</name>
- <value>namenode:9001</value>
- </property>
- <property>
- <name>mapred.local.dir</name>
- <value>/usr/local/hadoop/mapred/local</value>
- </property>
- <property>
- <name>mapred.system.dir</name>
- <value>/tmp/hadoop/mapred/system</value>
- </property>
- </configuration>
- #########################################
- [root@localhostconf]#vimasters
- #localhost
- namenode
- #########################################
- [root@localhostconf]#vislaves
- #localhost
- datanode01
启动 hadoop
- #####################<spanstyle="font-size:small;">格式化namenode##############</span>
- [root@localhostbin]#hadoopnamenode-format
- 11/06/2300:43:54INFOnamenode.NameNode:STARTUP_MSG:
- /************************************************************
- STARTUP_MSG:StartingNameNode
- STARTUP_MSG:host=localhost.localdomain/127.0.0.1
- STARTUP_MSG:args=[-format]
- STARTUP_MSG:version=0.20.203.0
- STARTUP_MSG:build=http://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.20-security-203-r1099333;compiledby'oom'onWedMay407:57:50PDT2011
- ************************************************************/
- 11/06/2300:43:55INFOutil.GSet:VMtype=32-bit
- 11/06/2300:43:55INFOutil.GSet:2%maxmemory=19.33375MB
- 11/06/2300:43:55INFOutil.GSet:capacity=2^22=4194304entries
- 11/06/2300:43:55INFOutil.GSet:recommended=4194304,actual=4194304
- 11/06/2300:43:56INFOnamenode.FSNamesystem:fsOwner=root
- 11/06/2300:43:56INFOnamenode.FSNamesystem:supergroup=supergroup
- 11/06/2300:43:56INFOnamenode.FSNamesystem:isPermissionEnabled=true
- 11/06/2300:43:56INFOnamenode.FSNamesystem:dfs.block.invalidate.limit=100
- 11/06/2300:43:56INFOnamenode.FSNamesystem:isAccessTokenEnabled=falseaccessKeyUpdateInterval=0min(s),accessTokenLifetime=0min(s)
- 11/06/2300:43:56INFOnamenode.NameNode:Cachingfilenamesoccuringmorethan10times
- 11/06/2300:43:57INFOcommon.Storage:Imagefileofsize110savedin0seconds.
- 11/06/2300:43:57INFOcommon.Storage:Storagedirectory/usr/local/hadoop/hdfs/namehasbeensuccessfullyformatted.
- 11/06/2300:43:57INFOnamenode.NameNode:SHUTDOWN_MSG:
- /************************************************************
- SHUTDOWN_MSG:ShuttingdownNameNodeatlocalhost.localdomain/127.0.0.1
- ************************************************************/
- [root@localhostbin]#
- ###########################################
- [root@localhostbin]#./start-all.sh
- startingnamenode,loggingto/usr/local/hadoop/hadoop-0.20.203/bin/../logs/hadoop-root-namenode-localhost.localdomain.out
- datanode01:startingdatanode,loggingto/usr/local/hadoop/hadoop-0.20.203/bin/../logs/hadoop-root-datanode-localhost.localdomain.out
- namenode:startingsecondarynamenode,loggingto/usr/local/hadoop/hadoop-0.20.203/bin/../logs/hadoop-root-secondarynamenode-localhost.localdomain.out
- startingjobtracker,loggingto/usr/local/hadoop/hadoop-0.20.203/bin/../logs/hadoop-root-jobtracker-localhost.localdomain.out
- datanode01:startingtasktracker,loggingto/usr/local/hadoop/hadoop-0.20.203/bin/../logs/hadoop-root-tasktracker-localhost.localdomain.out
- [root@localhostbin]#jps
- 11971TaskTracker
- 11807SecondaryNameNode
- 11599NameNode
- 12022Jps
- 11710DataNode
- 11877JobTracker
查看集群状态
- [root@localhostbin]#hadoopdfsadmin-report
- ConfiguredCapacity:4055396352(3.78GB)
- PresentCapacity:464142351(442.64MB)
- DFSRemaining:464089088(442.59MB)
- DFSUsed:53263(52.01KB)
- DFSUsed%:0.01%
- Underreplicatedblocks:0
- Blockswithcorruptreplicas:0
- Missingblocks:0
- -------------------------------------------------
- Datanodesavailable:1(1total,0dead)
- Name:127.0.0.1:50010
- DecommissionStatus:Normal
- ConfiguredCapacity:4055396352(3.78GB)
- DFSUsed:53263(52.01KB)
- NonDFSUsed:3591254001(3.34GB)
- DFSRemaining:464089088(442.59MB)
- DFSUsed%:0%
- DFSRemaining%:11.44%
- Lastcontact:ThuJun2301:11:15PDT2011
- [root@localhostbin]#
其他问题:1
- ####################启动报错##########
- [root@localhostbin]#./start-all.sh
- startingnamenode,loggingto/usr/local/hadoop/hadoop-0.20.203/bin/../logs/hadoop-root-namenode-localhost.localdomain.out
- Theauthenticityofhost'datanode01(127.0.0.1)'can'tbeestablished.
- RSAkeyfingerprintis41:c8:d4:e4:60:71:6f:6a:33:6a:25:27:62:9b:e3:90.
- Areyousureyouwanttocontinueconnecting(yes/no)?y
- Pleasetype'yes'or'no':yes
- datanode01:Warning:Permanentlyadded'datanode01'(RSA)tothelistofknownhosts.
- datanode01:startingdatanode,loggingto/usr/local/hadoop/hadoop-0.20.203/bin/../logs/hadoop-root-datanode-localhost.localdomain.out
- <strong><spanstyle="color:#ff0000;">datanode01:Unrecognizedoption:-jvm
- datanode01:CouldnotcreatetheJavavirtualmachine.</span>
- </strong>
- namenode:startingsecondarynamenode,loggingto/usr/local/hadoop/hadoop-0.20.203/bin/../logs/hadoop-root-secondarynamenode-localhost.localdomain.out
- startingjobtracker,loggingto/usr/local/hadoop/hadoop-0.20.203/bin/../logs/hadoop-root-jobtracker-localhost.localdomain.out
- datanode01:startingtasktracker,loggingto/usr/local/hadoop/hadoop-0.20.203/bin/../logs/hadoop-root-tasktracker-localhost.localdomain.out
- [root@localhostbin]#jps
- 10442JobTracker
- 10533TaskTracker
- 10386SecondaryNameNode
- 10201NameNode
- 10658Jps
- ################################################
- [root@localhostbin]#vihadoop
- elif["$COMMAND"="datanode"];then
- CLASS='org.apache.hadoop.hdfs.server.datanode.DataNode'
- if[[$EUID-eq0]];then
- HADOOP_OPTS="$HADOOP_OPTS-jvmserver$HADOOP_DATANODE_OPTS"
- else
- HADOOP_OPTS="$HADOOP_OPTS-server$HADOOP_DATANODE_OPTS"
- fi
- #http://javoft.net/2011/06/hadoop-unrecognized-option-jvm-could-not-create-the-java-virtual-machine/
- #改为
- elif["$COMMAND"="datanode"];then
- CLASS='org.apache.hadoop.hdfs.server.datanode.DataNode'
- #if[[$EUID-eq0]];then
- #HADOOP_OPTS="$HADOOP_OPTS-jvmserver$HADOOP_DATANODE_OPTS"
- #else
- HADOOP_OPTS="$HADOOP_OPTS-server$HADOOP_DATANODE_OPTS"
- #fi
- #或者换非root用户启动
- #启动成功
2,启动时要关闭防火墙
查看运行情况:
http://localhost:50070
- NameNode'localhost.localdomain:9000'
- Started:ThuJun2301:07:18PDT2011
- Version:0.20.203.0,r1099333
- Compiled:WedMay407:57:50PDT2011byoom
- Upgrades:Therearenoupgradesinprogress.
- Browsethefilesystem
- NamenodeLogs
- ClusterSummary
- 6filesanddirectories,1blocks=7total.HeapSizeis31.38MB/966.69MB(3%)
- ConfiguredCapacity:3.78GB
- DFSUsed:52.01KB
- NonDFSUsed:3.34GB
- DFSRemaining:442.38MB
- DFSUsed%:0%
- DFSRemaining%:11.44%
- LiveNodes:1
- DeadNodes:0
- DecommissioningNodes:0
- NumberofUnder-ReplicatedBlocks:0
- NameNodeStorage:
- StorageDirectoryTypeState
- /usr/local/hadoop/hdfs/nameIMAGE_AND_EDITSActive
http://localhost:50030
- namenodeHadoopMap/ReduceAdministration
- QuickLinks
- *SchedulingInfo
- *RunningJobs
- *RetiredJobs
- *LocalLogs
- State:RUNNING
- Started:ThuJun2301:07:30PDT2011
- Version:0.20.203.0,r1099333
- Compiled:WedMay407:57:50PDT2011byoom
- Identifier:201106230107
- ClusterSummary(HeapSizeis15.31MB/966.69MB)
- RunningMapTasksRunningReduceTasksTotalSubmissionsNodesOccupiedMapSlotsOccupiedReduceSlotsReservedMapSlotsReservedReduceSlotsMapTaskCapacityReduceTaskCapacityAvg.Tasks/NodeBlacklistedNodesGraylistedNodesExcludedNodes
- 00010000224.00000
- SchedulingInformation
- QueueNameStateSchedulingInformation
- defaultrunningN/A
- Filter(Jobid,Priority,User,Name)
- Example:'user:smith3200'willfilterby'smith'onlyintheuserfieldand'3200'inallfields
- RunningJobs
- none
- RetiredJobs
- none
- LocalLogs
- Logdirectory,JobTrackerHistoryThisisApacheHadooprelease0.20.203.0
测试:
- ##########建立目录名称##########
- [root@localhostbin]#hadoopfs-mkdirtestFolder
- ###############拷贝文件到文件夹中
- [root@localhostlocal]#ls
- binetcgameshadoopincludeliblibexecsbinsharesrcSSH_key_file
- [root@localhostlocal]#hadoopfs-copyFromLocalSSH_key_filetestFolder
- 进入web页面即可查看
参考:http://bxyzzy.blog.51cto.com/854497/352692
附: 准备FTP :yum install vsftpd (方便文件传输 和hadoop无关)
关闭防火墙:service iptables start
启动FTP:service vsftpd start