文章标题

run.sh

#!/bin/bash
HADOOP_HOME=/usr/local/webserver/hadoop
INPUT_PATH=/data/archive/app_oeudjgn5872a7c3aaa54_datamine/george/inputs
INPUT_PATH2=/user/resys/projects/image/koudai_img/d_img/reduced/2015-taobao/part-00$1*

#total 10, from may00 to may 10
OUTPUT_PATH=/data/archive/app_oeudjgn5872a7c3aaa54_datamine/george/output/may0$1
#OUTPUT_PATH=output
#echo "Clearing output path: $OUTPUT_PATH"
$HADOOP_HOME/bin/hadoop fs -rmr $OUTPUT_PATH

${HADOOP_HOME}/bin/hadoop jar\
   ${HADOOP_HOME}/share/hadoop/tools/lib/hadoop-streaming-2.6.0.jar\
  -file mapper1.py\
  -file reducer1.py\
  -mapper mapper1.py\
  -reducer reducer1.py\
  -input $INPUT_PATH\
  -input $INPUT_PATH2\
  -output $OUTPUT_PATH\
  -jobconf mapreduce.jobtracker.split.metainfo.maxsize=-1\
  -jobconf mapred.job.queue.name="offline"\
  -jobconf mapred.job.priority="NORMAL"\
  -jobconf mapred.reduce.tasks="50"

mapper.py

#!/usr/bin/env python
import sys 
import base64

for line in sys.stdin:
    line = line.split()
    if len(line)>=2:
        print line[0], "\t", line[2]

reducer1.py

#!/usr/bin/env python
import sys 

cn = 1 
nb_line = 1 
for line in sys.stdin:
    line = line.split()
    if nb_line == 1:
        last_key, last_value = line[0], line[1]
        before_cur_key, before_cur_value = line[0], line[1]
    else:
        cur_key, cur_value = line[0], line[1]
        if last_key != cur_key:
            if cn <= 1:
                before_cur_key = last_key = cur_key
                before_cur_value = last_value = cur_value
                cn = 1 
                continue
            else:
                #to verify which is the image
                if len(before_cur_value)>=1000: 
                    print before_cur_key, '\t', before_cur_value 
                elif len(last_value)>=1000:
                    print last_key, '\t', last_value
                before_cur_key = last_key = cur_key
                before_cur_value = last_value = cur_value
                cn = 1 
        else:
            before_cur_key = cur_key
            before_cur_value = cur_value
            cn += 1
    nb_line += 1
if cn != 1:
    if len(before_cur_value)>=1000: 
        print before_cur_key, '\t', before_cur_value 
    elif len(last_value) >= 1000:
        print last_key, '\t', last_value