run.sh
#!/bin/bash
HADOOP_HOME=/usr/local/webserver/hadoop
INPUT_PATH=/data/archive/app_oeudjgn5872a7c3aaa54_datamine/george/inputs
INPUT_PATH2=/user/resys/projects/image/koudai_img/d_img/reduced/2015-taobao/part-00$1*
#total 10, from may00 to may 10
OUTPUT_PATH=/data/archive/app_oeudjgn5872a7c3aaa54_datamine/george/output/may0$1
#OUTPUT_PATH=output
#echo "Clearing output path: $OUTPUT_PATH"
$HADOOP_HOME/bin/hadoop fs -rmr $OUTPUT_PATH
${HADOOP_HOME}/bin/hadoop jar\
${HADOOP_HOME}/share/hadoop/tools/lib/hadoop-streaming-2.6.0.jar\
-file mapper1.py\
-file reducer1.py\
-mapper mapper1.py\
-reducer reducer1.py\
-input $INPUT_PATH\
-input $INPUT_PATH2\
-output $OUTPUT_PATH\
-jobconf mapreduce.jobtracker.split.metainfo.maxsize=-1\
-jobconf mapred.job.queue.name="offline"\
-jobconf mapred.job.priority="NORMAL"\
-jobconf mapred.reduce.tasks="50"
mapper.py
#!/usr/bin/env python
import sys
import base64
for line in sys.stdin:
line = line.split()
if len(line)>=2:
print line[0], "\t", line[2]
reducer1.py
#!/usr/bin/env python
import sys
cn = 1
nb_line = 1
for line in sys.stdin:
line = line.split()
if nb_line == 1:
last_key, last_value = line[0], line[1]
before_cur_key, before_cur_value = line[0], line[1]
else:
cur_key, cur_value = line[0], line[1]
if last_key != cur_key:
if cn <= 1:
before_cur_key = last_key = cur_key
before_cur_value = last_value = cur_value
cn = 1
continue
else:
#to verify which is the image
if len(before_cur_value)>=1000:
print before_cur_key, '\t', before_cur_value
elif len(last_value)>=1000:
print last_key, '\t', last_value
before_cur_key = last_key = cur_key
before_cur_value = last_value = cur_value
cn = 1
else:
before_cur_key = cur_key
before_cur_value = cur_value
cn += 1
nb_line += 1
if cn != 1:
if len(before_cur_value)>=1000:
print before_cur_key, '\t', before_cur_value
elif len(last_value) >= 1000:
print last_key, '\t', last_value