1、命令样式:
hadoop jar $STREAM \
-files ./mapper.py,./reducer.py \
-mapper ./mapper.py \
-reducer ./reducer.py \
-input /user/$(whoami)/input/*.txt \
-output /user/$(whoami)/output
hadoop jar share/hadoop/tools/lib/hadoop-streaming-2.7.2.jar -files source/map_temperature.py,source/reduce_temperature.py -mapper map_temperature.py -reducer reduce_temperature.py -input /input/temperature1/* -output /output/MaxTemp4
2、本地测试方法:
cat 1901.txt | python map_temperature.py | sort | python reduce_temperature.py
3、源码
mapper.py
#!/usr/bin/env python
#coding=utf-8
import re
import sys
for line in sys.stdin:
val = line.strip()
try:
(year, temp, q) = (val[15:19],val[87:92],val[92:93])
except:
(year, temp, q) = (2014,25,1)
# print ("{},{},{}".format(year,temp,q))
if temp!= "+9999" and re.match("[01459]",q):
print "%s,%s"%(year,temp)
recudce.py
#!/usr/bin/env python
#coding=utf-8
import sys
(last_key, max_val) = (None, -sys.maxint)
for line in sys.stdin:
# print line
# print line.strip().split(",")
try:
(key, val) = line.strip().split(",")
if last_key and last_key!= key:
print "%s,%s"%(last_key,max_val)
(last_key, max_val) = (key, int(val))
else:
(last_key, max_val) = (key, max(max_val, int(val)))
except:
print "==================error=========================="
continue
if last_key:
print "%s,%s"%(last_key, max_val)