shell编写streaming

2019独角兽企业重金招聘Python工程师标准>>>

##mapper.sh
#! /bin/bash
while read LINE; do
  for word in $LINE
  do
    echo "$word 1"
  done
done



##reducer.sh
#! /bin/bash
count=0
started=0
word=""
while read LINE;do
  newword=`echo $LINE | cut -d ' '  -f 1`
  if [ "$word" != "$newword" ];then
    [ $started -ne 0 ] && echo -e "$word \t $count"
    word=$newword
    count=1
    started=1
  else
    count=$(( $count + 1 ))
  fi
done
echo -e "$word \t $count"

转载于:https://my.oschina.net/kyo4321/blog/1358423