# mapper.py
import sys
import os
import io
sys.stdin = io.TextIOWrapper(sys.stdin.buffer,encoding='utf-8', errors='ignore')
sys.stdout = io.TextIOWrapper(sys.stdout.buffer,encoding='utf-8', errors='ignore')
for line in sys.stdin:
input_file = os.getenv('map_input_file')
line = line.strip()
words = line.split()
for word in words:
print("%s\t%s" % (word, input_file))
# reduce.py
import sys
A = []
for line in sys.stdin:
words = line.strip()
word, file = words.split("\t")
try:
if file == "afs://baihua.afs.baidu.com:9902/user/hlbrd/wangyan81/wuzhichao01/cat_test/A.txt":
current_word = word
else:
if current_word!=word and current_word!=None:
print(current_word)
current_word = None
except ValueError:
continue
sh文件
#/home/work/.hmpclient/bin/hadoop dfs -D hadoop.job.ugi=hlbrd,n3ZAM6Gl -D fs.default.name=[] -rmr []
/home/work/.hmpclient/bin/hadoop dfs -D hadoop.job.ugi=[] -D fs.default.name=[] -put A.txt []
HD_INPUT1=[]
HD_INPUT2=[]
HD_OUTPUT=[]
#执行rmr相关操作一定要小心
#/home/work/.hmpclient/bin/hadoop dfs -D hadoop.job.ugi=hlbrd,n3ZAM6Gl -D fs.default.name=[] -rmr ${HD_OUTPUT}
/home/work/.hmpclient/bin/hadoop streaming \
-D hadoop.job.ugi=hlbrd,n3ZAM6Gl \
-D fs.default.name=[] \
-D mapred.job.name="search_production_other_page_wenku_green_street" \
-D mapred.job.queue.name=defaultqueue \
-D mapred.job.tracker=[] \
-D stream.num.map.output.key.fields=1 \
-D num.key.fields.for.partition=1 \
-D mapred.job.priority=HIGH \
-D mapred.job.map.capacity=100 \
-D mapred.job.reduce.capacity=100 \
-D stream.memory.limit=10000 \
-D mapred.map.tasks=500 \
-D mapred.reduce.tasks=500 \
-partitioner org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner \
-inputformat org.apache.hadoop.mapred.TextInputFormat \
-outputformat org.apache.hadoop.mapred.TextOutputFormat \
-input ${HD_INPUT1} \
-input ${HD_INPUT2} \
-output ${HD_OUTPUT} \
-file mapper.py \
-mapper "python3 mapper.py" \
-file reduce.py \
-reducer "python3 reduce.py" \