hadoopMapReduce小例子

今天做了一个简单的mapreduce小程序，目的是熟悉一下怎么在真实的集群上跑自己的mr程序，之前写的都是在单机模拟运行的。功能很简单就是求一个key对应值里面最大的。

[b]开发环境：[/b]

hadoop2.5

jdk1.6

eclipse3.7

1、创建一个java工程，加入如下jar包：

hadoop-common-2.5.2.jar

hadoop-mapreduce-client-common-2.5.2.jar

hadoop-mapreduce-client-core-2.5.2.jar

hadoop-mapreduce-client-jobclient-2.5.2.jar

2、主要类


MaxNumberDriver.java用于启动mapreduce程序的入口

[code="java"]package org.qhy.mapreduce.maxnumber;


import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;


public class MaxNumberDriver extends Configured implements Tool{

@Override
public int run(String[] arg0) throws Exception {
if(arg0.length {


@Override

protected void map(LongWritable key, Text value, Mapper.Context context)

throws IOException, InterruptedException {

String lineVal = value.toString();

String[] arrayVal = lineVal.split(",");

String year= arrayVal[0],number=arrayVal[1];

context.write(new Text(year), new IntWritable(Integer.parseInt(number)));

}

}


MaxNumberReducer.java reducer实现类


package org.qhy.mapreduce.maxnumber;


import java.util.ArrayList;

import java.util.Iterator;

import java.util.List;


import org.apache.commons.lang3.math.NumberUtils;

import org.apache.hadoop.io.IntWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Reducer;


public class MaxNumberReducer extends Reducer {


// @Override

// protected void reduce(Text year, java.lang.Iterable numbers,

// org.apache.hadoop.mapreduce.Reducer.Context context)

// throws java.io.IOException, InterruptedException {

// int maxValue = Integer.MIN_VALUE;

// for(IntWritable value: numbers){

// maxValue = Math.max(maxValue,value.get());

// }

// context.write(year, new IntWritable(maxValue));

// }





/**

* {@inheritDoc}使用第三方Apache Lang包

* @see org.apache.hadoop.mapreduce.Reducer#reduce(KEYIN, java.lang.Iterable, org.apache.hadoop.mapreduce.Reducer.Context)

*/

@Override

protected void reduce(Text year, java.lang.Iterable numbers,

org.apache.hadoop.mapreduce.Reducer.Context context)

throws java.io.IOException, InterruptedException {

Iterator itr = numbers.iterator();

int length=0;

List list = new ArrayList();

while(itr.hasNext()){

IntWritable iwVal = itr.next();

list.add(iwVal.get());

length++;

}

int[] array = new int[length];

for(int i=0;iexport->jar->选择导出路径（一路next）-> 记得选一下main函数类

4、运行

将jar放到系统上linux上

./hadoop jar ~/testData/MRMaxNumber.jar hdfs://hdp01nn:9000/maxnumber hdfs://hdp01nn:9000/testOut/maxnumber4