MapReduce在Yarn上执行流程、Yarn核心组件功能、模拟ApplicationMaster发送Task、Map端的join...

MapReduce在Yarn上执行流程

Yarn核心组件功能

模拟ApplicationMaster发送Task

在MR中，MapTask、ReduceTask
都是线程对象，因为需要在网络中传输，所以都实现了序列化接口

package com.shujia.MapReduce;

import java.io.IOException;
import java.io.ObjectOutputStream;
import java.io.OutputStream;
import java.io.Serializable;
import java.net.Socket;

public class Demo7MRAppMaster {
    // 作为MR任务运行的AM，负责任务Task的分配与调度
    public static void main(String[] args) throws IOException {
        // 创建Socket客户端
        Socket sk = new Socket("localhost", 8888);

        // 创建Task任务，即将通过Socket发送给NM
        Task task = new Task();

        System.out.println("Task已构建，准备发送");
        // 建立输出流
        OutputStream outputStream = sk.getOutputStream();

        // 将输出流转换为Object的输出流
        ObjectOutputStream objectOutputStream = new ObjectOutputStream(outputStream);

        // 直接将任务以Object的形式发送出去
        objectOutputStream.writeObject(task);

        objectOutputStream.flush();
        System.out.println("Task已发送成功");

        //关闭连接
        objectOutputStream.close();
        outputStream.close();
        sk.close();

    }
}

/**
 * 在MR中，MapTask、ReduceTask
 * 都是线程对象，因为需要在网络中传输，所以都实现了序列化接口
 * 分区、分组、排序等其他功能由MR框架提供
 */
class Task extends Thread implements Serializable {
    @Override
    public void run() {
        for (int i = 0; i < 100; i++) {
            System.out.println(i);
            try {
                Thread.sleep(1000);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }

        }
        System.out.println("Task执行完毕");
    }
}

package com.shujia.MapReduce;

import java.io.IOException;
import java.io.InputStream;
import java.io.ObjectInputStream;
import java.net.ServerSocket;
import java.net.Socket;

public class Demo8NodeManager {
    // 接收AM发送过来的Task并执行
    public static void main(String[] args) throws IOException, ClassNotFoundException {
        // 创建Socket服务端
        ServerSocket serverSocket = new ServerSocket(8888);
        System.out.println("NodeManager已经启动，等待接收任务");

        // 建立Socket连接
        Socket socket = serverSocket.accept();

        // 创建输入流
        InputStream inputStream = socket.getInputStream();

        // 将输入流转换为Object输入流
        ObjectInputStream objectInputStream = new ObjectInputStream(inputStream);

        // 直接从Object输入流获取Object对象
        Object taskObj = objectInputStream.readObject();

        System.out.println("接收到了AM发送的Task");

        // 将Object对象转换为Task对象
        Task task = (Task) taskObj;

        System.out.println("正在执行Task");
        // 执行Task
        task.start();


        // 关闭流，断开连接
        objectInputStream.close();
        inputStream.close();
        socket.close();
        serverSocket.close();

    }
}

Map端的join

#适用于 大表 关联 小表

#原理：将小表广播

package com.shujia.MapReduce;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Hashtable;

// 实现MapJoin
public class Demo9MapJoin {
    // Map端
    public static class MyMapper extends Mapper<LongWritable, Text, Text, NullWritable> {
        Hashtable<String, String> sumScoreHTable = new Hashtable<>();

        // 每个Map任务会执行一次
        @Override
        protected void setup(Mapper<LongWritable, Text, Text, NullWritable>.Context context) throws IOException {
            // 获取”小表“的数据，构建Hashtable
            URI[] cacheFiles = context.getCacheFiles();
            // 获取”小表“的路径
            URI uri = cacheFiles[0];
            // 根据uri读取小表的数据
            FileSystem fs = FileSystem.get(context.getConfiguration());
            FSDataInputStream open = fs.open(new Path(uri.toString()));
            BufferedReader br = new BufferedReader(new InputStreamReader(open));
            String line;
            while ((line = br.readLine()) != null) {
                if (line.contains("\t")) {
                    String[] sumScoreSplit = line.split("\t");
                    String id = sumScoreSplit[0];
                    String sumScore = sumScoreSplit[1];
                    // 以id作为key，总分sumScore作为value 构建HashTable
                    sumScoreHTable.put(id, sumScore);
                }
            }
        }

        @Override
        // 处理”大表“的数据
        protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, NullWritable>.Context context) throws IOException, InterruptedException {
            String[] stuSplits = value.toString().split(",");
            String id = stuSplits[0];
            String name = stuSplits[1];
            String clazz = stuSplits[4];
            // 根据学生id从HashTable中获取学生总分
            String sumScore = sumScoreHTable.getOrDefault(id, "0");

            // 将id、name、clazz、sumScore拼接并直接由Map端输出到HDFS
            String outKey = id + "," + name + "," + clazz + "," + sumScore;
            context.write(new Text(outKey), NullWritable.get());

        }
    }

    // Driver端
    public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException, URISyntaxException {
        Configuration conf = new Configuration();
        Job job = Job.getInstance(conf);

        job.setJobName("Demo9MapJoin");
        job.setJarByClass(Demo9MapJoin.class);

        // 配置Map端
        job.setMapperClass(MyMapper.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(NullWritable.class);

        // 配置输入输出路径
        // 将 ”大表“的数据作为输入路径，小表不需要作为输入路径，待会可以广播
        FileInputFormat.addInputPath(job, new Path("/student/input"));

        // 输出路径不需要提前创建，如果该目录已存在则会报错
        // 通过HDFS的JavaAPI判断输出路径是否存在
        Path outPath = new Path("/student/mapjoin/output");
        FileSystem fs = FileSystem.get(conf);
        if (fs.exists(outPath)) {
            fs.delete(outPath, true);
        }
        FileOutputFormat.setOutputPath(job, outPath);

        // 将”小表“的数据 添加到MR的CacheFile中
        job.addCacheFile(new URI("hdfs://cluster/student/score/output/part-r-00000"));

        // 将任务提交并等待完成
        job.waitForCompletion(true);

        /**
         * hadoop jar Hadoop-1.0.jar com.shujia.MapReduce.Demo9MapJoin
         */

    }
}

查看MapReduce任务输出日志