import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import java.io.IOException;
import java.util.StringTokenizer;
public class wc {
public static class WCMapper extends Mapper<LongWritable, Text,Text,IntWritable>{
private final static IntWritable one=new IntWritable(1);
private Text words=new Text();
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
StringTokenizer stringTokenizer = new StringTokenizer(value.toString());
while (stringTokenizer.hasMoreElements()){
String word= stringTokenizer.nextToken();
words.set(word);
context.write(words,one);
}
}
}
public static class WCReduce extends Reducer<Text,IntWritable,Text,IntWritable>{
private IntWritable result=new IntWritable();
@Override
protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
int sum=0;
for (IntWritable val:values){
sum+=val.get();
}
result.set(sum);
context.write(key,result);
}
}
public static void main(String[] args) throws Exception{
//配置文件
Configuration conf = new Configuration();
Job job = Job.getInstance(conf,"wordcount");
job.setJarByClass(wc.class);
//map端和reduce端
job.setMapperClass(WCMapper.class);
job.setReducerClass(WCReduce.class);
//map端输出的字节码
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(IntWritable.class);
//reduce端输出的字节码
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
//输入文件地址
FileInputFormat.setInputPaths(job, new Path("E:\\学习\\大三第二学期课程\\大数据架构技术\\wordCount\\word.txt"));
//输出文件地址
Path path=new Path("E:\\学习\\大三第二学期课程\\大数据架构技术\\wordCount\\word_out");
FileSystem fs=FileSystem.get(conf);
if (fs.exists(path)){
fs.delete(path,true);
}
FileOutputFormat.setOutputPath(job,path);
job.waitForCompletion(true);
}
}
大数据MapReduce词频统计
最新推荐文章于 2024-05-24 09:45:00 发布