单机,简单测试
#############################################1#######################################
[root@localhost ~]# uname -aLinux localhost.localdomain 2.6.39.3 #1 SMP Tue Aug 2 13:02:40 CST 2011 x86_64 x86_64 x86_64 GNU/Linux
[root@localhost ~]# java -v
bash: java: command not found
[root@localhost ~]# wget http://download.oracle.com/otn-pub/java/jdk/7u1-b08/jdk-7u1-linux-x64.rpm
[root@localhost ~]# rpm -ivh jdk-7u1-linux-x64.rpm
[root@localhost ~]# wget http://labs.renren.com/apache-mirror//hadoop/common/hadoop-0.20.2/hadoop-0.20.2.tar.gz
[root@localhost ~]#tar zxvf hadoop-0.20.2.tar.gz
[root@localhost ~]#cd hadoop-0.20.2
[root@localhost ~]# export JAVA_HOME=/usr/java/jdk1.7.0_01
[root@localhost hadoop-0.20.2]# bin/hadoop
Usage: hadoop [--config confdir] COMMAND
where COMMAND is one of:
namenode -format format the DFS filesystem
secondarynamenode run the DFS secondary namenode
namenode run the DFS namenode
datanode run a DFS datanode
dfsadmin run a DFS admin client
mradmin run a Map-Reduce admin client
fsck run a DFS filesystem checking utility
fs run a generic filesystem user client
balancer run a cluster balancing utility
jobtracker run the MapReduce job Tracker node
pipes run a Pipes job
tasktracker run a MapReduce task Tracker node
job manipulate MapReduce jobs
queue get information regarding JobQueues
version print the version
jar <jar> run a jar file
distcp <srcurl> <desturl> copy file or directories recursively
archive -archiveName NAME <src>* <dest> create a hadoop archive
daemonlog get/set the log level for each daemon
or
CLASSNAME run the class named CLASSNAME
Most commands print help when invoked w/o parameters.
#############################################2#######################################
[root@localhost ~]# cat 1.txt y_sn
y_name
y_mail
y_wtel
y_mtel
y_check1
y_dadang1
y_sex
y_check2
y_dadang2
[root@localhost ~]# hadoop-0.20.2/bin/hadoop jar hadoop-0.20.2/contrib/streaming/hadoop-0.20.2-streaming.jar -input 1.txt -output 2 -mapper "awk -F '\t' '{print \$1}'" -reducer "wc -l"
11/12/05 18:23:42 INFO jvm.JvmMetrics: Initializing JVM Metrics with processName=JobTracker, sessionId=
11/12/05 18:23:42 WARN mapred.JobClient: No job jar file set. User classes may not be found. See JobConf(Class) or JobConf#setJar(String).
11/12/05 18:23:42 INFO mapred.FileInputFormat: Total input paths to process : 1
11/12/05 18:23:42 INFO streaming.StreamJob: getLocalDirs(): [/tmp/hadoop-root/mapred/local]
11/12/05 18:23:42 INFO streaming.StreamJob: Running job: job_local_0001
11/12/05 18:23:42 INFO streaming.StreamJob: Job running in-process (local Hadoop)
11/12/05 18:23:42 INFO mapred.FileInputFormat: Total input paths to process : 1
11/12/05 18:23:42 INFO mapred.MapTask: numReduceTasks: 1
11/12/05 18:23:42 INFO mapred.MapTask: io.sort.mb = 100
11/12/05 18:23:43 INFO mapred.MapTask: data buffer = 79691776/99614720
11/12/05 18:23:43 INFO mapred.MapTask: record buffer = 262144/327680
11/12/05 18:23:43 INFO streaming.PipeMapRed: PipeMapRed exec [/usr/bin/awk, -F, \t, {print $1}]
11/12/05 18:23:43 INFO streaming.PipeMapRed: R/W/S=1/0/0 in:NA [rec/s] out:NA [rec/s]
11/12/05 18:23:43 INFO streaming.PipeMapRed: R/W/S=10/0/0 in:NA [rec/s] out:NA [rec/s]
11/12/05 18:23:43 INFO streaming.PipeMapRed: MRErrorThread done
11/12/05 18:23:43 INFO streaming.PipeMapRed: Records R/W=10/1
11/12/05 18:23:43 INFO streaming.PipeMapRed: MROutputThread done
11/12/05 18:23:43 INFO streaming.PipeMapRed: mapRedFinished
11/12/05 18:23:43 INFO mapred.MapTask: Starting flush of map output
11/12/05 18:23:43 INFO mapred.MapTask: Finished spill 0
11/12/05 18:23:43 INFO mapred.TaskRunner: Task:attempt_local_0001_m_000000_0 is done. And is in the process of commiting
11/12/05 18:23:43 INFO mapred.LocalJobRunner: Records R/W=10/1
11/12/05 18:23:43 INFO mapred.TaskRunner: Task 'attempt_local_0001_m_000000_0' done.
11/12/05 18:23:43 INFO mapred.LocalJobRunner:
11/12/05 18:23:43 INFO mapred.Merger: Merging 1 sorted segments
11/12/05 18:23:43 INFO mapred.Merger: Down to the last merge-pass, with 1 segments left of total size: 109 bytes
11/12/05 18:23:43 INFO mapred.LocalJobRunner:
11/12/05 18:23:43 INFO streaming.PipeMapRed: PipeMapRed exec [/usr/bin/wc, -l]
11/12/05 18:23:43 INFO streaming.PipeMapRed: R/W/S=1/0/0 in:NA [rec/s] out:NA [rec/s]
11/12/05 18:23:43 INFO streaming.PipeMapRed: R/W/S=10/0/0 in:NA [rec/s] out:NA [rec/s]
11/12/05 18:23:43 INFO streaming.PipeMapRed: MRErrorThread done
11/12/05 18:23:43 INFO streaming.PipeMapRed: Records R/W=10/1
11/12/05 18:23:43 INFO streaming.PipeMapRed: MROutputThread done
11/12/05 18:23:43 INFO streaming.PipeMapRed: mapRedFinished
11/12/05 18:23:43 INFO mapred.TaskRunner: Task:attempt_local_0001_r_000000_0 is done. And is in the process of commiting
11/12/05 18:23:43 INFO mapred.LocalJobRunner:
11/12/05 18:23:43 INFO mapred.TaskRunner: Task attempt_local_0001_r_000000_0 is allowed to commit now
11/12/05 18:23:43 INFO mapred.FileOutputCommitter: Saved output of task 'attempt_local_0001_r_000000_0' to file:/usr/home/wenjie3/2
11/12/05 18:23:43 INFO mapred.LocalJobRunner: Records R/W=10/1 > reduce
11/12/05 18:23:43 INFO mapred.TaskRunner: Task 'attempt_local_0001_r_000000_0' done.
11/12/05 18:23:43 INFO streaming.StreamJob: map 100% reduce 100%
11/12/05 18:23:43 INFO streaming.StreamJob: Job complete: job_local_0001
11/12/05 18:23:43 INFO streaming.StreamJob: Output: 2
[root@localhost ~]# ls 2
part-00000
[root@localhost ~]# cat 2/part-00000
10
[root@localhost ~]#
##############################################
把 export JAVA_HOME=/usr/java/jdk1.7.0_01
加入到 /etc/rc.local
【理解错误,最后分拆后的消耗时间,比原始执行大,以后再试验下】