win7环境测试mapreduce的wordcount

最新推荐文章于 2024-09-27 00:00:00 发布

xy@

最新推荐文章于 2024-09-27 00:00:00 发布

阅读量109

点赞数

文章标签： hadoop

本文链接：https://blog.csdn.net/weixin_44260021/article/details/104893080

版权

mapper类

import java.io.IOException;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;

public class Mapper extends org.apache.hadoop.mapreduce.Mapper<LongWritable, Text, Text, IntWritable>{
	
	Text k = new Text();
	IntWritable v = new IntWritable();
	@Override
	protected void map(LongWritable key, Text value,
			Context context)
			throws IOException, InterruptedException {
		//1获取一行
		String line = value.toString();
		
		//2切割
		String[] split = line.split(" ");
		
		//3输出
		for (String string : split) {
			k.set(string);
			context.write(k, v);
		}
	}
}

reducer类

import java.io.IOException;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;

public class Reducer extends org.apache.hadoop.mapreduce.Reducer<Text, IntWritable,Text, IntWritable>{
	
	@Override
	protected void reduce(Text key, Iterable<IntWritable> values,
			Context context)
			throws IOException, InterruptedException {
		//1累加求和
		int sum = 0;
		for (IntWritable count : values) {
			sum += count.get();
		}
		
		//2写出
		context.write(key, new IntWritable(sum));
	}
}

Diver类

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;



public class Driver {

	public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
		//1获取配置信息
		Configuration conf = new Configuration();
		Job job =Job.getInstance(conf);
		
		//2设置jar包路径
		job.setJarByClass(Driver.class);
		
		//3关联mapper类和reducer类
		job.setMapperClass(Mapper.class);
		job.setReducerClass(Reducer.class);
		
		//4设置map输出
		job.setMapOutputKeyClass(Text.class);
		job.setMapOutputValueClass(IntWritable.class);
		
		//5设置reduce输入
		job.setOutputKeyClass(Text.class);
		job.setOutputValueClass(IntWritable.class);
		
		//6设置输入输出路径
		FileInputFormat.setInputPaths(job, new Path(args [0]));
		FileOutputFormat.setOutputPath(job, new Path(args [1]));
		
		//7提交
		boolean result = job.waitForCompletion(true);
		System.exit(result?0:1);
	}

}