public class WordCount {
static {
System.setProperty("hadoop.home.dir", "D:\\专高三\\第四天idea实现hdfs的一些命令\\winutils-master\\hadoop-2.8.3");
}
public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
//创建job
Configuration conf = new Configuration();
Job job = Job.getInstance(conf, "word-count");
job.setJarByClass(WordCount.class);
//输入文件
FileInputFormat.addInputPath(job, new Path(args[0]));
//编写mapper处理
job.setMapperClass(MyMapper.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(LongWritable.class);
//shuffle
//编写reducer处理
job.setReducerClass(MyReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(LongWritable.class);
//输出文件
FileOutputFormat.setOutputPath(job,new Path(args[1]));
//提交
boolean result = job.waitForCompletion(true);
System.out.println(result?1:0);
}
public static class MyMapper extends Mapper<LongWritable, Text, Text, LongWritable>{
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
String words = value.toString();
String[] wordArr = words.split(" ");
for(String word:wordArr){
context.write(new Text(word),new LongWritable(1));
}
}
}
public static class MyReducer extends Reducer<Text, LongWritable, Text, LongWritable>{
@Override
protected void reduce(Text key, Iterable<LongWritable> values, Context context) throws IOException, InterruptedException {
Long sum = 0L;
for(LongWritable value : values){
sum += value.get();
}
context.write(key,new LongWritable(sum));
}
}
}
worldcount 传参
file:/D://words.txt
file:/D://out
集群运行:
job.setJarByClass(WordCount.class);
命令:
bin/yarn jar hadoop_demo-1.0-SNAPSHOT.jar mapreduce.WordCount file:/home/test/hadoop-2.9.2/NOTICE.txt file:/home/test/out