Spring For Hadoop学习笔记（3）

绑spring跑Hadoop的案例已经跑成功，下面贴个代码。

CountMapper.java:

CountReducer.java:

<pre name="code" class="java">import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import java.io.IOException;
import java.util.StringTokenizer;

/**
 * Created by hadoop on 15-5-21.
 */
public class CountMapper extends Mapper<Object, Text, Text, IntWritable> {

    private static final IntWritable one = new IntWritable(1);
    private Text word = new Text();

    public CountMapper() {
    }

    public void map(Object key, Text value, Context context)
            throws IOException, InterruptedException {
        StringTokenizer itr = new StringTokenizer(value.toString());

        while (itr.hasMoreTokens()) {
            this.word.set(itr.nextToken());
            context.write(this.word, one);
        }

    }

}

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;
import java.util.Iterator;

/**
 * Created by hadoop on 15-5-21.
 */
public class CountReducer extends Reducer<Text, IntWritable, Text, IntWritable> {

    private IntWritable result = new IntWritable();

    public CountReducer() {
    }

    public void reduce(Text key, Iterable<IntWritable> values,
                       Context context) throws IOException, InterruptedException {
        int sum = 0;

        IntWritable val;
        for (Iterator i$ = values.iterator(); i$.hasNext(); sum += val
                .get()) {
            val = (IntWritable) i$.next();
        }

        this.result.set(sum);
        context.write(key, this.result);
    }
}

spring-config.xml:
<pre name="code" class="html"><?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
       xmlns:hdp="http://www.springframework.org/schema/hadoop"
       xmlns:context="http://www.springframework.org/schema/context"
       xsi:schemaLocation="
        http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
        http://www.springframework.org/schema/hadoop http://www.springframework.org/schema/hadoop/spring-hadoop.xsd
        http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-4.1.xsd">


    <hdp:configuration>
        fs.defaultFS=${hd.fs}
        hadoop.tmp.dir=${java.io.tmpdir}
    </hdp:configuration>

    <hdp:job id="word-count"
             input-path="hdfs://localhost:9000/input/"
             output-path="hdfs://localhost:9000/output/"
             mapper="com.jsnu.chw.hadoop.wordcount.CountMapper"
             reducer="com.jsnu.chw.hadoop.wordcount.CountReducer"
             combiner="com.jsnu.chw.hadoop.wordcount.CountReducer"
             key="org.apache.hadoop.io.Text"
             value="org.apache.hadoop.io.IntWritable"
             />

    <hdp:job-runner id="job-runner" job-ref="word-count" run-at-startup="true"/>
    <context:property-placeholder location="./hadoop.properties"/>
    <context:property-placeholder location="./log4j.properties"/>

</beans>

hadoop.properties:

hd.fs=hdfs://localhost:9000
java.io.tmpdir=/home/hadoop/tmp

main.java:

import org.apache.hadoop.mapreduce.Job;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;

/**
 * Created by hadoop on 15-5-21.
 */
public class main {

    public static void main(String[] args) {

        ApplicationContext ctx = new ClassPathXmlApplicationContext("classpath:spring-config.xml");
        Job job = (Job) ctx.getBean("word-count");


    }
}

wordcount就是hadoop自带的例子。