在eclipse上编写方法,实现一个 WordCount 程序
package cn.cqsw;
import java.io.IOException;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
public class WCMap extends Mapper<LongWritable, Text, Text, LongWritable> {
@Override
protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text,LongWritable>.Context context)
throws IOException, InterruptedException {
String[] splited=value.toString().split(" ");
for(String str :splited)
{
context.write(new Text(str),new LongWritable(1));
}
}
}
package cn.cqsw;
import java.io.IOException;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
public class WCReduce extends Reducer<Text, LongWritable,Text, LongWritable> {
@Override
protected void reduce(Text text, Iterable<LongWritable> iterable,Reducer<Text, LongWritable,Text,LongWritable>.Context context) throws IOException,InterruptedException {
long time=0;
for(LongWritable lw :iterable)
{
time+=lw.get();
}
context.write(text, new LongWritable(time));
}
}
package cn.cqsw;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
public class MapReduceDemo{
public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
Configuration conf = new Configuration();
Job job = Job.getInstance(conf);
job.setJarByClass(MapReduceDemo.class);
job.setMapperClass(WCMap.class);
job.setReducerClass(WCReduce.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(LongWritable.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(LongWritable.class);
Path inputPath = new Path("/hello");
Path outputPath = new Path("/output");
FileInputFormat.setInputPaths(job, inputPath);
FileOutputFormat.setOutputPath(job, outputPath);
boolean waitForCompletion = job.waitForCompletion(true);
System.exit(waitForCompletion?0:1);
}
}