Mapreduce 入门级实验记录

开头批注：这个教程很多地方已经有一模一样的代码了，只是放在这里当个记录留个纪念吧，都是验证过的代码

1. 默认配置：

• Ubuntu 18.04
• java-1.8.0_162
• hadoop 3.2.1，下载在/usr/local/hadoop中

2.实验：MAPREDUCE进行分析数据

目标：输入美国气象局以前的天气记录（ftp://ftp.ncdc.noaa.gov/pub/data/noaa/），获得这一年/每年最低气温

1. 进入hadoop文件夹，顺手开启一下hadoop，命令 start-dfs.sh
2. 下载数据，原本数据是美国气象局的数据，网站。。。简约而不简单。
   多种下载方式：
   1. 壮士型，自己写个脚本全部弄下来，有点大，几十个G还是M来着：https://blog.csdn.net/weixin_30872867/article/details/96228453
   2. 普通型，一行代码弄个几年的做实验，下载，解压+上传hdfs：

cd /usr/local/hadoop/
mkdir temperature
cd temperature
wget http://labfile.oss.aliyuncs.com/courses/237/temperature.zip
unzip temperature.zip
cd 1971 
zcat *.gz > /usr/local/hadoop/temperature.txt
cd ../../
hadoop fs -mkdir -p /class5/in
hadoop fs -copyFromLocal temperature.txt /class5/in
hadoop fs -ls /class5/in

3. cd进myclass文件夹， vim 编辑三个文件：MinTemperature.java、MinTemperatureMapper.java、MinTemperatureReducer.java，三个文件分别如下，在hadoop文件夹下的myclass文件夹中

MinTemperature.java

import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.Job;
public class MinTemperature{
        public static void main(String[] args) throws Exception {
                if (args.length !=2) {
                        System.err.println("Usage: MinTemperature <input path> <output path>");
                        System.exit(-1);
                }
                Job job = new Job();
                job.setJarByClass(MinTemperature.class);
                job.setJobName("Min temperature");
                FileInputFormat.addInputPath(job, new Path(args[0]));
                FileOutputFormat.setOutputPath(job, new Path(args[1]));
                job.setMapperClass(MinTemperatureMapper.class);
                job.setReducerClass(MinTemperatureReducer.class);
                job.setOutputKeyClass(Text.class);
                job.setOutputValueClass(IntWritable.class);
                System.exit(job.waitForCompletion(true)?0:1);
        }
}

MinTemperatureMapper.java

import java.io.IOException;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

public class MinTemperatureMapper extends Mapper<LongWritable,Text,Text,IntWritable>{
        private static final int MISSING = 9999;
    
        @Override
        public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException{

                String line = value.toString();
                String year = line.substring(15, 19);

                int airTemperature;
                if (line.charAt(87) == '+') {
                        airTemperature = Integer.parseInt(line.substring(88 , 92));
                } else {
                        airTemperature = Integer.parseInt(line.substring(87 , 92));
                }
                //airTemperature = Integer.parseInt(line.substring(14, 19).trim());
    
                String quality = line.substring(92, 93);
                if (airTemperature != MISSING && quality.matches("[01459]")) {
                        context.write(new Text(year), new IntWritable(airTemperature));
                }
                //if(airTemperature!= MISSING){
                //      context.write(new Text(year), new IntWritable(airTemperature));
                //}
        }
}

MinTemperatureReducer.java

import java.io.IOException;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

public class MinTemperatureReducer
  extends Reducer<Text, IntWritable, Text, IntWritable>
{

  @Override
  public void reduce(Text key, Iterable<IntWritable> values,Context context)
          throws IOException, InterruptedException
        {

                int minValue= Integer.MAX_VALUE;
                for (IntWritable value : values)
                {
                        minValue= Math.min(minValue, value.get());
                }
                context.write(key, new IntWritable(minValue));
        }
}

4. 编译

javac MinTemperatur*.java

5. 打包成jar并迁移

jar cvf ./MinTemperature.jar ./Min*.class
mv MinTemperature.jar ..
hadoop fs -rm -r -skipTrash /class5/out
(注，上一句是先把上次的输出文件夹删了，否则bug，没有文件夹就不写)
cd ../
hadoop jar MinTemperature.jar MinTemperature /class5/in/temperature.txt /class5/out

6. 运行

cd /usr/local/hadoop
hadoop jar MinTemperature.jar MinTemperature /class5/in/temperature.txt /class5/out

7. 查看结果

hadoop fs -ls /class5/out
hadoop fs -cat /class5/out/part-r-00000