Streming 模式解析Mapreduce使用简例

stream_jar="/home/hadoop/hadoop-0.20.2/contrib/streaming/hadoop-0.20.2-streaming.jar";
hadoop jar ${stream_jar} \
    -D hadoop.job.ugi="admin" \
    -D mapred.job.name="job_test_desc_ \
    -D mapred.reduce.tasks="0" \
    -D mapred.output.compress="true" \
    -D mapred.output.compression.codec="org.apache.hadoop.io.compress.GzipCodec" \
    -input ${input} \
    -output ${output} \
    -mapper ./mapper.sh \
    -file ./mapper.sh \

-file mapper程序的本地目录

maper.sh

sed 's/\t/#/g'