利用Hadoop中的Mapreducer处理数据

TSITL

已于 2023-11-08 23:20:17 修改

阅读量217

点赞数 4

文章标签： hadoop 大数据分布式

于 2023-11-08 19:48:26 首次发布

本文链接：https://blog.csdn.net/m0_73096141/article/details/134296809

版权

数据集合

1949-10-01 14:21:02   34℃
1949-10-02 14:01:02   36℃
1950-01-01 11:21:02   32℃
1950-10-01 12:21:02   37℃
1949-11-02 14:01:02   37℃
1951-12-01 12:21:02   23℃
1950-10-02 12:21:02   41℃
1950-10-03 12:21:02   27℃
1951-07-01 12:21:02   45℃
1951-07-02 12:21:02   46℃
1950-11-13 12:21:02   37℃
1951-08-08 12:21:02   48℃
1949-07-07 14:01:02   39℃
1949-05-05 14:01:02   32℃
1951-03-03 12:21:02   22℃

Mapper类

package com.neuedu.mapper;


import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import java.io.IOException;


public class temperatureMapper extends Mapper<LongWritable, Text, Text, IntWritable> {
    private Text outk = new Text();
    private IntWritable outv = new IntWritable(1);
    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        String line = value.toString();
        String[] tokens = line.split("\t");
        String date = tokens[0].trim();
        String tem = tokens[1];
        int temp;
        temp =Character.getNumericValue(tem.charAt(0))*10+Character.getNumericValue(tem.charAt(1));

        String year = date.substring(0, 4);
        outk.set(year);
        outv.set(temp);
        context.write(outk, outv);
    }
}

Reducer类

package com.neuedu.reduce;


import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableComparator;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableComparator;

public class temperatureReducer extends Reducer<Text, IntWritable,Text, IntWritable> {
    private Text outk = new Text();
    private IntWritable outv = new IntWritable();

    @Override
    protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
       int max = Integer.MIN_VALUE;

        for (IntWritable value : values) {
            if(value.get()>=max) {
                max = value.get();
            }
        }
        context.write(key, new IntWritable(max));
    }
}

Driver类

package com.neuedu.driver;

import com.neuedu.mapper.temperatureMapper;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.Job;

import com.neuedu.reduce.temperatureReducer;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;

public class temperatureDriver {
    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
        Configuration conf = new Configuration();
        Job job = Job.getInstance(conf);
        job.setJarByClass(temperatureDriver.class);
        job.setMapperClass(temperatureMapper.class);
        job.setReducerClass(temperatureReducer.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(IntWritable.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);
        Path path = new Path("C:\\Users\\72389\\Desktop\\temperature.txt");
        FileInputFormat.setInputPaths(job,path);
        FileOutputFormat.setOutputPath(job,new Path("C:\\Users\\72389\\Desktop\\output"));
        boolean b = job.waitForCompletion(true);
        System.exit(b?0:1);
    }
}