大数据--mapreduce作业提交源码

最新推荐文章于 2021-06-25 11:52:27 发布
最新推荐文章于 2021-06-25 11:52:27 发布
阅读量241 收藏
点赞数
文章标签： 大数据
原文链接：http://www.cnblogs.com/jeff190812/p/11435824.html
版权

MapReduce作业提交查看源码

1、判断job的状态

//将private Job.JobState state的值修改为DEFINE，然后执行submit（）方法；

public boolean waitForCompletion(boolean verbose) throws IOException, InterruptedException, ClassNotFoundException {
    if (this.state == Job.JobState.DEFINE) {
        this.submit();
    }

    if (verbose) {
        this.monitorAndPrintJob();
    } else {
        int completionPollIntervalMillis = getCompletionPollInterval(this.cluster.getConf());

        while(!this.isComplete()) {
            try {
                Thread.sleep((long)completionPollIntervalMillis);
            } catch (InterruptedException var4) {
            }
        }
    }

    return this.isSuccessful();
}

 

2、确认job的状态是否是DEFINE

//执行this.ensureState(Job.JobState.DEFINE);代码判断job的状态；

public void submit() throws IOException, InterruptedException, ClassNotFoundException {
    this.ensureState(Job.JobState.DEFINE);
    this.setUseNewAPI();
    this.connect();
    final JobSubmitter submitter = this.getJobSubmitter(this.cluster.getFileSystem(), this.cluster.getClient());
    this.status = (JobStatus)this.ugi.doAs(new PrivilegedExceptionAction<JobStatus>() {
        public JobStatus run() throws IOException, InterruptedException, ClassNotFoundException {
            return submitter.submitJobInternal(Job.this, Job.this.cluster);
        }
    });
    this.state = Job.JobState.RUNNING;
    LOG.info("The url to track the job: " + this.getTrackingURL());
}

 

 

3、执行ensureState方法确认job的状态是DEFINE

private void ensureState(Job.JobState state) throws IllegalStateException {
    if (state != this.state) {
        throw new IllegalStateException("Job in state " + this.state + " instead of " + state);
    } else if (state == Job.JobState.RUNNING && this.cluster == null) {
        throw new IllegalStateException("Job in state " + this.state + ", but it isn't attached to any job tracker!");
    }
}

 

4、执行setUseNewAPI方法切换新的API

private void setUseNewAPI() throws IOException {
    int numReduces = this.conf.getNumReduceTasks();
    String oldMapperClass = "mapred.mapper.class";
    String oldReduceClass = "mapred.reducer.class";
    this.conf.setBooleanIfUnset("mapred.mapper.new-api", this.conf.get(oldMapperClass) == null);
    String mode;
    if (this.conf.getUseNewMapper()) {
        mode = "new map API";
        this.ensureNotSet("mapred.input.format.class", mode);
        this.ensureNotSet(oldMapperClass, mode);
        if (numReduces != 0) {
            this.ensureNotSet("mapred.partitioner.class", mode);
        } else {
            this.ensureNotSet("mapred.output.format.class", mode);
        }
    } else {
        mode = "map compatibility";
        this.ensureNotSet("mapreduce.job.inputformat.class", mode);
        this.ensureNotSet("mapreduce.job.map.class", mode);
        if (numReduces != 0) {
            this.ensureNotSet("mapreduce.job.partitioner.class", mode);
        } else {
            this.ensureNotSet("mapreduce.job.outputformat.class", mode);
        }
    }

 

    if (numReduces != 0) {
        this.conf.setBooleanIfUnset("mapred.reducer.new-api", this.conf.get(oldReduceClass) == null);
        if (this.conf.getUseNewReducer()) {
            mode = "new reduce API";
            this.ensureNotSet("mapred.output.format.class", mode);
            this.ensureNotSet(oldReduceClass, mode);
        } else {
            mode = "reduce compatibility";
            this.ensureNotSet("mapreduce.job.outputformat.class", mode);
            this.ensureNotSet("mapreduce.job.reduce.class", mode);
        }
    }

}

 

//执行getNumReduceTasks方法获取ReduceTask的个数

public int getNumReduceTasks() {

    //ReduceTask的数目的默认值是1
    return this.getInt("mapreduce.job.reduces", 1);
}

 

5、执行connect方法

private synchronized void connect() throws IOException, InterruptedException, ClassNotFoundException {
    if (this.cluster == null) {
        this.cluster = (Cluster)this.ugi.doAs(new PrivilegedExceptionAction<Cluster>() {
            public Cluster run() throws IOException, InterruptedException, ClassNotFoundException {
                return new Cluster(Job.this.getConfiguration());
            }
        });
    }

}

 

public Cluster(InetSocketAddress jobTrackAddr, Configuration conf) throws IOException {
    this.fs = null;
    this.sysDir = null;
    this.stagingAreaDir = null;
    this.jobHistoryDir = null;
    this.providerList = null;
    this.conf = conf;

    //通过ugi获取到提交job的用户名
    this.ugi = UserGroupInformation.getCurrentUser();
    this.initialize(jobTrackAddr, conf);
}

Cluster构造器里面conf的值：

 

 

 

 

 

private void initialize(InetSocketAddress jobTrackAddr, Configuration conf) throws IOException {
    this.initProviderList();
    Iterator i$ = this.providerList.iterator();

    while(i$.hasNext()) {
        ClientProtocolProvider provider = (ClientProtocolProvider)i$.next();
        LOG.debug("Trying ClientProtocolProvider : " + provider.getClass().getName());
        ClientProtocol clientProtocol = null;

        try {
            if (jobTrackAddr == null) {
                clientProtocol = provider.create(conf);
            } else {
                clientProtocol = provider.create(jobTrackAddr, conf);
            }

            if (clientProtocol != null) {
                this.clientProtocolProvider = provider;
                this.client = clientProtocol;
                LOG.debug("Picked " + provider.getClass().getName() + " as the ClientProtocolProvider");
                break;
            }

            LOG.debug("Cannot pick " + provider.getClass().getName() + " as the ClientProtocolProvider - returned null protocol");
        } catch (Exception var7) {
            LOG.info("Failed to use " + provider.getClass().getName() + " due to error: ", var7);
        }
    }

    if (null == this.clientProtocolProvider || null == this.client) {
        throw new IOException("Cannot initialize Cluster. Please check your configuration for mapreduce.framework.name and the correspond server addresses.");
    }
}

通过ClientProtocolProvider能看到有两个子类一个是本地的一个是集群的：

 

 

 

 

 

6、创建job的提交路径

public synchronized FileSystem getFileSystem() throws IOException, InterruptedException {
    if (this.fs == null) {
        try {
            this.fs = (FileSystem)this.ugi.doAs(new PrivilegedExceptionAction<FileSystem>() {
                public FileSystem run() throws IOException, InterruptedException {
                    Path sysDir = new Path(Cluster.this.client.getSystemDir());
                    return sysDir.getFileSystem(Cluster.this.getConf());
                }
            });
        } catch (InterruptedException var2) {
            throw new RuntimeException(var2);
        }
    }

    return this.fs;
}

 

public String getSystemDir() {
    Path sysDir = new Path(this.conf.get("mapreduce.jobtracker.system.dir", "/tmp/hadoop/mapred/system"));
    return this.fs.makeQualified(sysDir).toString();
}

通过getSystemDir方法拿到系统的提交job的路径

 

 

 

//将配置信息写到分布式缓存里面去，将生成的jobID返回出去；

JobStatus submitJobInternal(Job job, Cluster cluster) throws ClassNotFoundException, InterruptedException, IOException {
    this.checkSpecs(job);
    Configuration conf = job.getConfiguration();

    //将配置信息添加到分布式缓存里
    addMRFrameworkToDistributedCache(conf);
    Path jobStagingArea = JobSubmissionFiles.getStagingDir(cluster, conf);

    //获取本地IP
    InetAddress ip = InetAddress.getLocalHost();
    if (ip != null) {
        this.submitHostAddress = ip.getHostAddress();
        this.submitHostName = ip.getHostName();
        conf.set("mapreduce.job.submithostname", this.submitHostName);
        conf.set("mapreduce.job.submithostaddress", this.submitHostAddress);
    }
    //生成新的jobID
    JobID jobId = this.submitClient.getNewJobID();

    //生成的信息的jobID添加到job里
    job.setJobID(jobId);

    //生成存放job信息的路径
    Path submitJobDir = new Path(jobStagingArea, jobId.toString());
    JobStatus status = null;

    JobStatus var24;
    try {

        //将相关的配置信息添加到缓存
        conf.set("mapreduce.job.user.name", UserGroupInformation.getCurrentUser().getShortUserName());
        conf.set("hadoop.http.filter.initializers", "org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer");
        conf.set("mapreduce.job.dir", submitJobDir.toString());
        LOG.debug("Configuring job " + jobId + " with " + submitJobDir + " as the submit dir");
        TokenCache.obtainTokensForNamenodes(job.getCredentials(), new Path[]{submitJobDir}, conf);
        this.populateTokenCache(conf, job.getCredentials());
        if (TokenCache.getShuffleSecretKey(job.getCredentials()) == null) {
            KeyGenerator keyGen;
            try {
                keyGen = KeyGenerator.getInstance("HmacSHA1");
                keyGen.init(64);
            } catch (NoSuchAlgorithmException var19) {
                throw new IOException("Error generating shuffle secret key", var19);
            }

            SecretKey shuffleKey = keyGen.generateKey();
            TokenCache.setShuffleSecretKey(shuffleKey.getEncoded(), job.getCredentials());
        }

        if (CryptoUtils.isEncryptedSpillEnabled(conf)) {
            conf.setInt("mapreduce.am.max-attempts", 1);
            LOG.warn("Max job attempts set to 1 since encrypted intermediatedata spill is enabled");
        }

        this.copyAndConfigureFiles(job, submitJobDir);
        Path submitJobFile = JobSubmissionFiles.getJobConfPath(submitJobDir);
        LOG.debug("Creating splits at " + this.jtFs.makeQualified(submitJobDir));

        //根据生成切片文件个数确定maptask的数量
        int maps = this.writeSplits(job, submitJobDir);
        conf.setInt("mapreduce.job.maps", maps);

        //这个maps的值是1
        LOG.info("number of splits:" + maps);
        String queue = conf.get("mapreduce.job.queuename", "default");
        AccessControlList acl = this.submitClient.getQueueAdmins(queue);
        conf.set(QueueManager.toFullPropertyName(queue, QueueACL.ADMINISTER_JOBS.getAclName()), acl.getAclString());
        TokenCache.cleanUpTokenReferral(conf);
        if (conf.getBoolean("mapreduce.job.token.tracking.ids.enabled", false)) {
            ArrayList<String> trackingIds = new ArrayList();
            Iterator i$ = job.getCredentials().getAllTokens().iterator();

            while(i$.hasNext()) {
                Token<? extends TokenIdentifier> t = (Token)i$.next();
                trackingIds.add(t.decodeIdentifier().getTrackingId());
            }

            conf.setStrings("mapreduce.job.token.tracking.ids", (String[])trackingIds.toArray(new String[trackingIds.size()]));
        }

        ReservationId reservationId = job.getReservationId();
        if (reservationId != null) {
            conf.set("mapreduce.job.reservation.id", reservationId.toString());
        }
        //将相关的配置信息写到磁盘job.xml里面
        this.writeConf(conf, submitJobFile);
        this.printTokens(jobId, job.getCredentials());
        status = this.submitClient.submitJob(jobId, submitJobDir.toString(), job.getCredentials());
        if (status == null) {
            throw new IOException("Could not launch job");
        }

        var24 = status;
    } finally {
        if (status == null) {
            LOG.info("Cleaning up the staging area " + submitJobDir);
            if (this.jtFs != null && submitJobDir != null) {
                this.jtFs.delete(submitJobDir, true);
            }
        }

    }

    return var24;
}

 

 

private <T extends InputSplit> int writeNewSplits(JobContext job, Path jobSubmitDir) throws IOException, InterruptedException, ClassNotFoundException {
    Configuration conf = job.getConfiguration();
    InputFormat<?, ?> input = (InputFormat)ReflectionUtils.newInstance(job.getInputFormatClass(), conf);
    List<InputSplit> splits = input.getSplits(job);
    T[] array = (InputSplit[])((InputSplit[])splits.toArray(new InputSplit[splits.size()]));
    Arrays.sort(array, new JobSubmitter.SplitComparator());
    JobSplitWriter.createSplitFiles(jobSubmitDir, conf, jobSubmitDir.getFileSystem(conf), array);
    return array.length;
}

//待处理的文件是一个小于128M的文件所以切片数是1：

 

 

 

生成的切片文件信息提交到相关路径

 

 

生成的ob.xml的信息：

 

 

 

 

 

 

7、更新job的状态为RUNNING

public void submit() throws IOException, InterruptedException, ClassNotFoundException {
    this.ensureState(Job.JobState.DEFINE);
    this.setUseNewAPI();
    this.connect();
    final JobSubmitter submitter = this.getJobSubmitter(this.cluster.getFileSystem(), this.cluster.getClient());
    this.status = (JobStatus)this.ugi.doAs(new PrivilegedExceptionAction<JobStatus>() {
        public JobStatus run() throws IOException, InterruptedException, ClassNotFoundException {
            return submitter.submitJobInternal(Job.this, Job.this.cluster);
        }
    });
    this.state = Job.JobState.RUNNING;
    LOG.info("The url to track the job: " + this.getTrackingURL());
}

 

将state从DEFINE更新为RUNNNG；

 

 

 

 

public synchronized String getTrackingUrl() {
    return this.trackingUrl;
}

获取到URL：

 

 

 

 

 

8、执行mapreduce任务

public boolean waitForCompletion(boolean verbose) throws IOException, InterruptedException, ClassNotFoundException {

//state的值是RUNNING跳过该代码块
    if (this.state == Job.JobState.DEFINE) {
        this.submit();
    }
    //verbose的值是true所以开始执行mapreduce任务

//
    if (verbose) {
        this.monitorAndPrintJob();
    } else {
        int completionPollIntervalMillis = getCompletionPollInterval(this.cluster.getConf());

        while(!this.isComplete()) {
            try {
                Thread.sleep((long)completionPollIntervalMillis);
            } catch (InterruptedException var4) {
            }
        }
    }

    return this.isSuccessful();
}

 

 

 

转载于:https://www.cnblogs.com/jeff190812/p/11435824.html

awys19931001
关注
  • 0
    点赞
  • 踩
  • 0
    收藏
    觉得还不错? 一键收藏
  • 0
    评论
【大数据之Hadoop】十、MapReduce中job提交流程源码分析
阿宁的博客
04-07 313
源码中：job提交三个信息（jar包，xml，切片信息），在本地不需要提交jar包，只有在集群上才需要提交。
大数据 hadoop 部署 分析错误
qq_41809929的博客
01-10 1765
hadoop Check MapReduce2报出错误 stderr: /var/lib/ambari-agent/data/errors-50.txt Traceback (most recent call last): File “/var/lib/ambari-agent/cache/common-services/YARN/2.1.0.2.0/package/scripts/mapred_...
Hadoop MapReduce Job 提交流程
热门推荐
小杨的技术博客
04-06 2万+
文章目录ensureState 确保状态setUseNewAPIconnectinitialize提交 jobgetStagingDirgetStagingAreaDirgetLoginUsermkdirsgetLocalHost & getNewJobIDcopyAndConfigureFileswriteSplitswriteConf(conf, submitJobFile)submi...
mapreduce执行时报java.net.ConnectException
weixin_30709929的博客
08-28 607
mapreduce执行时报java.net.ConnectException 解决方案 在执行mapreduce程序时出现java.net.ConnectException: 拒绝连接异常,在错误信息中看到" localhost:35334"字眼,怀疑是没有配置主机名的原因, (1)尝试对每台虚拟机设置主机名,同时更新ssh密钥信息。(2)更改hdf...
MapReduce——客户端提交任务源码分析
qq634506153的专栏
06-09 151
MR程序并不会在客户端执行任何的计算操作，它是为计算工作做好准备，例如计算出切片信息，直接影响到Map任务的并行度。 在Driver中提交任务时，会写到这样的语句： boolean result = job.waitForCompletion(true); 进入到waitForCompletion中： public boolean waitForCompletion(boolean verbose) throws IOException, InterruptedException, Cla
精选_大数据Hadoop平台2-2、MapReduce_源码打包
03-10
9. **YARN调度**：在Hadoop 2.x中，MapReduce作业的调度由YARN（Yet Another Resource Negotiator）负责，理解YARN的工作方式对于优化作业性能也很关键。 10. **容错性**：MapReduce具有天然的容错性，如果某个节点...
MapReduce保姆级教程源码
04-26
- 作业提交：用户提交一个Job对象到JobTracker，包含作业配置和输入数据信息。 - 任务切片：JobTracker将输入数据分割成任务，分配给TaskTracker执行。 - 任务执行：TaskTracker下载输入数据，运行map和reduce...
使用hadoop-streaming运行Python编写的MapReduce程序.rar
05-09
4. **运行作业**：执行提交脚本，Hadoop会处理剩下的工作，包括分发Python脚本到各个节点，启动Map和Reduce任务。 5. **查看结果**：作业完成后，可以在指定的输出目录下找到MapReduce处理后的结果。 6. **优化**...
MapReduce - WordCount案例 - 含各种部署方式源码
最新发布
06-25
MapReduce是Google提出的一种分布式计算模型，被广泛...通过Java实现，我们可以灵活地设计和部署MapReduce作业，无论是本地测试还是远程集群执行。理解并熟练掌握这些知识，对于从事大数据处理的IT专业人员至关重要。
大数据小型项目源码之mapreduce英语单词频次统计
01-17
同时，项目中可能包含一个主程序，用于提交MapReduce作业到Hadoop集群。 6. **输入与输出格式**： MapReduce作业通常接收HDFS上的文本文件作为输入，而输出也是键值对形式的文本文件。在这个项目中，输入可能是...
[Hadoop源码解读]（一）MapReduce篇之InputFormat
云计算?
08-23 309
平时我们写MapReduce程序的时候，在设置输入格式的时候，总会调用形如job.setInputFormatClass(KeyValueTextInputFormat.class);来保证输入文件按照我们想要的格式被读取。所有的输入格式都继承于InputFormat，这是一个抽象类，其子类有专门用于读取普通文件的FileInputFormat，用来读取数据库的DBInputFormat...
hive报错--code1/2/3
qq_46893497的博客
12-21 8342
return code 1 原因： 'Error while processing statement: FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.MoveTask' 报错是抄的，多次实践证明，执行hive sql ERROR 返回 return code 1 基本上是执行的HQL无法加入到对应的用户队列中。 解决方法 1： 执行HQL前，加入 set MapReduce.q
MapReduce任务运行到running job卡住
似水流年
06-29 1万+
一：如下异常： Starting Job 16/06/30 01:15:34 INFO client.RMProxy: Connecting to ResourceManager at master/192.168.10.50:8032 16/06/30 01:15:35 INFO input.FileInputFormat: Total input paths to process :
解决 hadoop 2.x 配置 yarn 运行任务 Running job 卡住
xzlAwin的博客
06-25 1017
解决 hadoop 2.x 配置 yarn 运行任务 Running job 卡住 错误信息： 21/06/24 18:56:40 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 21/06/24 18:56:41 INFO client.RMProxy: Connecting to Resour
MapReduce_Job提交流程源码详解
底层码农
03-07 1万+
版本：Apache Hadoop 2.7.2 运行环境：Windows10 Eclipse Debug:https://blog.csdn.net/qq_40794973/article/details/87876772 主要介绍的是写完mr(job) 如何提交到集群上，这一段过程它执行了哪些操作。写一个简答的 WordCount ，通过调试来看程序是如何执行的。 1、创建一个 Mav...
Hadoop-2.7.3源码分析：MapReduce作业提交源码跟踪
程裕强的专栏
11-26 4394
1、提交JobMapReduce程序框架请参考 http://blog.csdn.net/chengyuqiang/article/details/72804007@Override public int run(String[] args) throws Exception { //读取配置文件 Configuration conf = getConf()
Hive调用MapReduce任务失败
sky_flying1的博客
09-24 4891
1. Hive简介Hive（蜂巢）Apache Hadoop生态圈的构成之一，其主要功能是基于Hadoop提供MapReduce的类SQL查询。Hive的语法规则和Mysql中SQL的语法规则极为相似，有Mysql使用经验的同学都能轻松上手。 Hive的执行分为本地模式和集群模式。本地模式执行简单的Hive SQL，不需要启动MapReduce程序，如-- 本地模式 select col_name
Hadoop MapReduce 任务执行流程源代码详细解析
RiverM的专栏
09-27 1万+
目录 1 引言  1.1 目的  1.2 读者范围 2 综述 3 代码详细分析  3.1 启动Hadoop集群  3.2 JobTracker启动以及Job的初始化  3.3 TaskTracker启动以及发送Heartbeat  3.4 JobTracker接
MapReduce任务提交源码分析
qq_40337206的博客
08-31 179
MapReduce任务提交 API 调用顺序： waitForCompletion -》 submit -》submitJobInternal 主要的部分在 submitJobInternal 1. 任务提交 waitForCompletion job.waitForCompletion(true) 2. 任务提交 submit 3.任务提交 submitJobInternal 3.1 ...
第4章大数据技术教程-MapReduce.pdf
12-18
第四章《大数据技术教程-MapReduce.pdf》详细介绍了分布式计算框架MapReduce的基本概念和应用。在Hadoop流行之前，分布式框架虽然存在，但实现较为复杂，大公司专利化，小公司无法承担开发分布式系统的成本和人力。...

“相关推荐”对你有帮助么？

  • 非常没帮助
  • 没帮助
  • 一般
  • 有帮助
  • 非常有帮助
提交
评论
添加红包

请填写红包祝福语或标题

个

红包个数最小为10个

元

红包金额最低5元

当前余额3.43元 前往充值 >
需支付：10.00元
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付元
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明：

1.余额是钱包充值的虚拟货币，按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载，可以购买VIP、付费专栏及课程。

余额充值