mapreduce python实例_Python MapReduce大小写,PythonMapReduce,案例

map_t.py

import sys

import re

p = re.compile(r'\w+')

for line in sys.stdin:

ss = line.strip().split(' ')

for s in ss:

if len(p.findall(s)) < 1:

continue

s_low = p.findall(s)[0].lower()

print(s_low + ',' + '1')

reduce_t.py

import sys

cur_word = None

s = 0

for line in sys.stdin:

word, val = line.strip().split(',')

if cur_word is None:

cur_word = word

if cur_word != word:

print('%s\t%s' % (cur_word, s))

cur_word = word

s = 0

s += int(val)

print('%s\t%s' % (cur_word, sum))

HADOOP_CMD="/usr/local/src/hadoop-2.6.1/bin/hadoop"

STREAM_JAR_PATH="/usr/local/src/hadoop-2.6.1/share/hadoop/tools/lib/hadoop-streaming-2.6.1.jar"

INPUT_FILE_PATH="/data/words.txt"

OUTPUT_PATH="/output/wc"

$HADOOP_CMD fs -rmr -skipTrash $OUTPUT_PATH

# Step 1.

$HADOOP_CMD jar $STREAM_JAR_PATH \

-input $INPUT_FILE_PATH \

-output $OUTPUT_PATH \

-mapper "python map_t.py" \

-reducer "python red_t.py" \

-file ./map_t.py \

-file ./red_t.py