Java如何使用mapreduce_MapReduce简单使用

1、启动hadoop工程

2、MapReduce统计文本单词数量

public class WordCount {

private static class WordMapper extends

Mapper {

@Override

protected void map(LongWritable key, Text value,

Mapper.Context context)

throws IOException, InterruptedException {

String string = value.toString();

String[] strs = string.split(" ");

for (String str : strs) {

context.write(new Text(str), new IntWritable(1));

}

}

}

private static class WordReduce extends

Reducer {

// key 单词 //value：{1,1}

@Override

protected void reduce(Text key, Iterable values,

Reducer.Context context)

throws IOException, InterruptedException {

int count = 0;

for (IntWritable value : values) {

count += value.get();

}

context.write(key, new IntWritable(count));

}

}

public static void main(String[] args) throws IOException,

ClassNotFoundException, InterruptedException {

Configuration configuration = HadoopConfig.getConfiguration();

Job job = Job.getInstance(configuration, "统计单词数目");

job.setJarByClass(WordCount.class);

job.setMapperClass(WordMapper.class);

job.setMapOutputKeyClass(Text.class);

job.setMapOutputValueClass(IntWritable.class);

job.setReducerClass(WordReduce.class);

job.setOutputKeyClass(Text.class);

job.setOutputValueClass(IntWritable.class);

FileInputFormat.addInputPath(job, new Path("/data"));

FileOutputFormat.setOutputPath(job, new Path("/ouput"));

job.waitForCompletion(true);

System.exit(job.waitForCompletion(true) ? 0 : 1);

}

2、MapReduce排除文本重复数据

public class Dup {

private static class DupMapper extends

Mapper {

@Override

protected void map(LongWritable key, Text value,

Mapper.Context context)

throws IOException, InterruptedException {

context.write(new Text(value), NullWritable.get());

}

}

private static class DupReduce extends

Reducer {

@Override

protected void reduce(Text key, Iterable values,

Reducer.Context context)

throws IOException, InterruptedException {

context.write(new Text(key), NullWritable.get());

}

}

public static void main(String[] args) throws IOException,

ClassNotFoundException, InterruptedException {

Configuration configuration = HadoopConfig.getConfiguration();

Job job = Job.getInstance(configuration, "去重");

job.setJarByClass(Dup.class);

job.setMapperClass(DupMapper.class);

job.setMapOutputKeyClass(Text.class);

job.setMapOutputValueClass(NullWritable.class);

job.setOutputKeyClass(Text.class);

job.setOutputValueClass(NullWritable.class);

job.setReducerClass(DupReduce.class);

FileInputFormat.addInputPath(job, new Path("/data"));

FileOutputFormat.setOutputPath(job, new Path("/dup"));

job.waitForCompletion(true);

System.exit(job.waitForCompletion(true) ? 0 : 1);

}

3、MapReduce实线文本数据的简单排序

public class Sort {

private static class SortMapper extends

Mapper {

//输出，输入

@Override

protected void map(

LongWritable key,

Text value_text,

Mapper.Context context)

throws IOException, InterruptedException {

int value = Integer.parseInt(value_text.toString());

context.write(new IntWritable(value), new IntWritable(1));

}

}

private static class SortReduce extends

Reducer {

@Override

protected void reduce(

IntWritable key,

Iterable values,

Reducer.Context context)

throws IOException, InterruptedException {

for (IntWritable value : values) {

context.write(key, NullWritable.get());

}

}

}

public static void main(String[] args) throws IOException,

ClassNotFoundException, InterruptedException {

Configuration configuration = HadoopConfig.getConfiguration();

Job job = Job.getInstance(configuration, "排序");

job.setJarByClass(Sort.class);

job.setMapperClass(SortMapper.class);

job.setMapOutputKeyClass(IntWritable.class);

job.setMapOutputValueClass(IntWritable.class);

job.setOutputKeyClass(IntWritable.class);

job.setOutputValueClass(NullWritable.class);

job.setReducerClass(SortReduce.class);

FileInputFormat.addInputPath(job, new Path("/data"));

FileOutputFormat.setOutputPath(job, new Path("/sort"));

job.waitForCompletion(true);

}

4、MapReduce实线单表连接

文本数据如下：

child  parent

tom   lucy

tom   jack

lucy   mary

lucy   ben

public class Single {

private static class SingleMapper extends

Mapper {

@Override

protected void map(LongWritable key, Text value,

Mapper.Context context)

throws IOException, InterruptedException {

String string = value.toString();

if (!string.contains("child")) {

String[] strings = string.split(" ");

context.write(new Text(strings[0]), new Text(strings[1] + ":1"));

context.write(new Text(strings[1]), new Text(strings[0] + ":2"));

}

}

}

// reduce是执行key的次数

private static class SingleReduce extends Reducer {

@Override

protected void reduce(Text key, Iterable values,

Reducer.Context context)

throws IOException, InterruptedException {

List left = Lists.newArrayList();

List right = Lists.newArrayList();

for (Text value : values) {

String[] strings = value.toString().split(":");

if (strings[1].equals("1")) {

right.add(strings[0]);

} else {

left.add(strings[0]);

}

}

for (String lef : left) {

for (String rig : right) {

context.write(new Text(lef), new Text(rig));

}

}

}

}

public static void main(String[] args) throws IOException,

ClassNotFoundException, InterruptedException {

Configuration configuration = HadoopConfig.getConfiguration();

Job job = Job.getInstance(configuration, "单表连接");

job.setJarByClass(Sort.class);

job.setMapperClass(SingleMapper.class);

job.setMapOutputKeyClass(Text.class);

job.setMapOutputValueClass(Text.class);

job.setOutputKeyClass(Text.class);

job.setOutputValueClass(Text.class);

job.setReducerClass(SingleReduce.class);

FileInputFormat.addInputPath(job, new Path("/data"));

FileOutputFormat.setOutputPath(job, new Path("/single"));

job.waitForCompletion(true);

}

输出结果如下：

grandchild  grandparent  //额外加入的，表达思路

tom   mary

tom   ben