hadoop复习 1搭建

参考http://hadoop.apache.org/docs/r1.0.4/cn/quickstart.html

环境变量

hadoop-env.sh
export JAVA_HOME=/usr/local/java/jdk1.6.0_45/

修改两个配置文件
# cat hdfs-site.xml

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
    <property>
        <name>fs.default.name</name>
        <value>localhost:9000</value>
    </property>
    <property>
        <name>dfs.replication</name>
        <value>1</value>                                                                                                                                         
    </property>
</configuration>

# cat mapred-site.xml

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
    <property>$
        <name>mapred.job.tracker</name>$
        <value>localhost:9001</value>$
    </property>$
</configuration>

ssh本机互信，略
为了ssh

vim /etc/hosts
::1^I^Ilocalhost6.localdomain localhost

export HADOOP_HOME=/data/hadoop/hadoop-1.2.1
export HADOOP_CONF_DIR=$HADOOP_HOME/conf

初始化hdfs
bin/hadoop namenode -format
bin/start-all.sh

启动后几个进程

root@centos54 bin]# jps
23275 NameNode
23396 DataNode
23543 SecondaryNameNode

23641 JobTracker
23768 TaskTracker

运行wordcount的例子
mkdir haoning
cd haoning

../bin/hadoop fs -put ../conf input  
../bin/hadoop jar ../hadoop-examples-1.2.1.jar grep input output 'dfs[a-z.]+'

[code="java"]# tree
.
|-- input
| |-- capacity-scheduler.xml
| |-- configuration.xsl
| |-- core-site.xml
| |-- fair-scheduler.xml
| |-- hadoop-env.sh
| |-- hadoop-metrics2.properties
| |-- hadoop-policy.xml
| |-- hdfs-site.xml
| |-- log4j.properties
| |-- mapred-queue-acls.xml
| |-- mapred-site.xml
| |-- masters
| |-- slaves
| |-- ssl-client.xml.example
| |-- ssl-server.xml.example
| |-- task-log4j.properties
| `-- taskcontroller.cfg
`-- output
|-- _SUCCESS
`-- part-00000[/code]

[code="java"]
得到的结果
# cat output/*
1 dfs.replication
1 dfs.server.namenode.
1 dfsadmin
[/code]

验证结果

[code="java"]
cd input
# grep dfs *
hadoop-env.sh:# The following applies to multiple commands (fs, dfs, fsck, distcp etc)
hadoop-policy.xml: dfsadmin and mradmin commands to refresh the security policy in-effect.
hdfs-site.xml: <name>dfs.replication</name>
log4j.properties:log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=WARN
#
[/code]

NameNode - http://localhost:50070/
JobTracker - http://localhost:50030/
~