mapreduce处理多文件代码（气温处理）

最新推荐文章于 2023-09-12 11:20:50 发布

墨羽寻觅

最新推荐文章于 2023-09-12 11:20:50 发布

阅读量99

点赞数

分类专栏：代码文章标签： hadoop big data

本文链接：https://blog.csdn.net/m0_53100381/article/details/120249616

版权

代码专栏收录该内容

3 篇文章 0 订阅

订阅专栏

import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class study {
    public static class myMap extends Mapper< LongWritable, Text, Text, LongWritable> {

        public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {

            // 每行气象数据
            String line = value.toString();
            String [] info=line.split(",");

            FileSplit fileSplit = (FileSplit) context.getInputSplit();

            // 通过文件名称提取省份名称
            String weatherStationId = fileSplit.getPath().getName();
            String [] mm=weatherStationId.split("_");
            String provinceName=mm[0];

            context.write(new Text(provinceName), new LongWritable(Long.valueOf(info[2])));
        }
    }

    public static class myReduce extends Reducer< Text, LongWritable, Text, LongWritable> {

        public void reduce(Text key, Iterable< LongWritable> values,Context context) throws IOException, InterruptedException {

            Long sum = 0L;
            int count = 0;

            for (LongWritable val : values) {
                sum += val.get();
                count++;
            }
            long avg=sum/count;
            context.write(key,new LongWritable(avg));
        }
    }

    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
        Configuration conf=new Configuration();
        Job job=Job.getInstance(conf, study.class.getSimpleName());
        job.setJarByClass(study.class);
        FileInputFormat.addInputPath(job, new Path(args[0]));
        job.setMapperClass(myMap.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(LongWritable.class);
        job.setReducerClass(myReduce.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(LongWritable.class);
        FileOutputFormat.setOutputPath(job, new Path(args[1]));
        job.waitForCompletion(true);
    }
}