yarn知识点

一、YARN主要负责集群的管理和调度，支持主从架构，主节点最多可以有2个，从节点可以有多个。
其中:ResourceManager是主节点，主要负责集群资源的分配和调度。
NodeManager是从节点，主要负责当前机器资源管理。
二、yarn主要管理内存和CPU这两种资源类型

三、YARN中支持三种调度器
1：FIFO Scheduler :先进先出（first in,first out）调度策略
2：Capacity Scheduler：FIFO Scheduler的多队列版本
3：FairScheduler：多队列，多用户共享资源

• CapacityScheduler：它是FifoScheduler的多队列版本，就是我们先把集群中的整块资源划分成多份，我们可以人为的给这些资源定义使用场景，例如图里面的queue A里面运行普通的任务，queueB中运行优先级比较高的任务。这两个队列的资源是相互对立的 ，但是注意一点，队列内部还是按照先进先出的规则。
• FairScheduler：支持多个队列，每个队列可以配置一定的资源，每个队列中的任务共享其所在队列的所有资源，不需要排队等待资源具体是这样的，假设我们向一个队列中提交了一个任务，这个任务刚开始会占用整个队列的资源，当你再提交第二个任务的时候，第一个任务会把他的资源释放出来一部分给第二个任务使用。
  四、案例YARN多资源队列配置和使用
• 希望增加2个队列，一个是online队列，一个是offline队列
  然后向offline队列中提交一个mapreduce任务，需要在修改集群中etc/hadoop目录下的capacity-scheduler.xml配置文件

<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->
<configuration>

  <property>
    <name>yarn.scheduler.capacity.maximum-applications</name>
    <value>10000</value>
    <description>
      Maximum number of applications that can be pending and running.
    </description>
  </property>

  <property>
    <name>yarn.scheduler.capacity.maximum-am-resource-percent</name>
    <value>0.1</value>
    <description>
      Maximum percent of resources in the cluster which can be used to run 
      application masters i.e. controls number of concurrent running
      applications.
    </description>
  </property>

  <property>
    <name>yarn.scheduler.capacity.resource-calculator</name>
    <value>org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator</value>
    <description>
      The ResourceCalculator implementation to be used to compare 
      Resources in the scheduler.
      The default i.e. DefaultResourceCalculator only uses Memory while
      DominantResourceCalculator uses dominant-resource to compare 
      multi-dimensional resources such as Memory, CPU etc.
    </description>
  </property>
<property>
    <name>yarn.scheduler.capacity.root.default.user-limit-factor</name>
    <value>1</value>
    <description>
      Default queue user limit a percentage from 0.0 to 1.0.
    </description>
  </property>





  <property> 
    <name>yarn.scheduler.capacity.root.queues</name> 
    <value>default,online,offline</value> 
    <description>,</description> 
  </property> 
  <property> 
    <name>yarn.scheduler.capacity.root.default.capacity</name> 
    <value>70</value> 
    <description>default70%</description> 
  </property> 
  <property> 
    <name>yarn.scheduler.capacity.root.online.capacity</name> 
    <value>10</value> 
    <description>online10%</description> 
  </property> 
  <property> 
    <name>yarn.scheduler.capacity.root.offline.capacity</name> 
    <value>20</value> 
    <description>offline20%</description> 
  </property> 
  <property> 
    <name>yarn.scheduler.capacity.root.default.maximum-capacity</name> 
    <value>70</value> 
    <description>Default.</description> 
  </property> 
  <property> 
    <name>yarn.scheduler.capacity.root.online.maximum-capacity</name> 
    <value>10</value> 
    <description>online.</description> 
  </property> 
  <property> 
    <name>yarn.scheduler.capacity.root.offline.maximum-capacity</name> 
    <value>20</value> 
    <description>offline.</description> 
  </property> 

  <property>
    <name>yarn.scheduler.capacity.root.default.state</name>
    <value>RUNNING</value>
    <description>
      The state of the default queue. State can be one of RUNNING or STOPPED.
    </description>
  </property>

  <property>
    <name>yarn.scheduler.capacity.root.default.acl_submit_applications</name>
    <value>*</value>
    <description>
      The ACL of who can submit jobs to the default queue.
    </description>
  </property>

  <property>
    <name>yarn.scheduler.capacity.root.default.acl_administer_queue</name>
    <value>*</value>
    <description>
      The ACL of who can administer jobs on the default queue.
    </description>
  </property>

  <property>
    <name>yarn.scheduler.capacity.root.default.acl_application_max_priority</name>
    <value>*</value>
    <description>
      The ACL of who can submit applications with configured priority.
      For e.g, [user={name} group={name} max_priority={priority} default_priority={priority}]
    </description>
  </property>

   <property>
     <name>yarn.scheduler.capacity.root.default.maximum-application-lifetime
     </name>
     <value>-1</value>
     <description>
        Maximum lifetime of an application which is submitted to a queue
        in seconds. Any value less than or equal to zero will be considered as
        disabled.
        This will be a hard time limit for all applications in this
        queue. If positive value is configured then any application submitted
        to this queue will be killed after exceeds the configured lifetime.
        User can also specify lifetime per application basis in
        application submission context. But user lifetime will be
        overridden if it exceeds queue maximum lifetime. It is point-in-time
        configuration.
        Note : Configuring too low value will result in killing application
        sooner. This feature is applicable only for leaf queue.
     </description>
   </property>

   <property>
     <name>yarn.scheduler.capacity.root.default.default-application-lifetime
     </name>
     <value>-1</value>
     <description>
        Default lifetime of an application which is submitted to a queue
        in seconds. Any value less than or equal to zero will be considered as
        disabled.
        If the user has not submitted application with lifetime value then this
        value will be taken. It is point-in-time configuration.
        Note : Default lifetime can't exceed maximum lifetime. This feature is
        applicable only for leaf queue.
     </description>
   </property>

  <property>
    <name>yarn.scheduler.capacity.node-locality-delay</name>
    <value>40</value>
    <description>
      Number of missed scheduling opportunities after which the CapacityScheduler 
      attempts to schedule rack-local containers.
      When setting this parameter, the size of the cluster should be taken into account.
      We use 40 as the default value, which is approximately the number of nodes in one rack.
      Note, if this value is -1, the locality constraint in the container request
      will be ignored, which disables the delay scheduling.
    </description>
  </property>

  <property>
    <name>yarn.scheduler.capacity.rack-locality-additional-delay</name>
    <value>-1</value>
    <description>
      Number of additional missed scheduling opportunities over the node-locality-delay
      ones, after which the CapacityScheduler attempts to schedule off-switch containers,
      instead of rack-local ones.
      Example: with node-locality-delay=40 and rack-locality-delay=20, the scheduler will
      attempt rack-local assignments after 40 missed opportunities, and off-switch assignments
      after 40+20=60 missed opportunities.
      When setting this parameter, the size of the cluster should be taken into account.
      We use -1 as the default value, which disables this feature. In this case, the number
      of missed opportunities for assigning off-switch containers is calculated based on
      the number of containers and unique locations specified in the resource request,
      as well as the size of the cluster.
    </description>
  </property>

  <property>
    <name>yarn.scheduler.capacity.queue-mappings</name>
    <value></value>
    <description>
      A list of mappings that will be used to assign jobs to queues
      The syntax for this list is [u|g]:[name]:[queue_name][,next mapping]*
      Typically this list will be used to map users to queues,
      for example, u:%user:%user maps all users to queues with the same name
      as the user.
    </description>
  </property>

  <property>
    <name>yarn.scheduler.capacity.queue-mappings-override.enable</name>
    <value>false</value>
    <description>
      If a queue mapping is present, will it override the value specified
      by the user? This can be used by administrators to place jobs in queues
      that are different than the one specified by the user.
      The default is false.
    </description>
  </property>

  <property>
    <name>yarn.scheduler.capacity.per-node-heartbeat.maximum-offswitch-assignments</name>
    <value>1</value>
    <description>
      Controls the number of OFF_SWITCH assignments allowed
      during a node's heartbeat. Increasing this value can improve
      scheduling rate for OFF_SWITCH containers. Lower values reduce
      "clumping" of applications on particular nodes. The default is 1.
      Legal values are 1-MAX_INT. This config is refreshable.
    </description>
  </property>


  <property>
    <name>yarn.scheduler.capacity.application.fail-fast</name>
    <value>false</value>
    <description>
      Whether RM should fail during recovery if previous applications'
      queue is no longer valid.
    </description>
  </property>

</configuration>

接下来就可以在web界面看到调度队列的信息了：

• 如果需要任务进入指定的队列中运行的话就需要再进行配置，在job的时候添加需要进入的队列的参数，在运行时加入这个参数就可以进入指定的队列了String[] remainingArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
  接下来进行代码的测试：

package com.imooc.wc;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;

public class WordCountJobQueue {
    /**
     * 组装job=map+reduce
     */
    public static void main(String[] args) {
        //args = new String[] { "g:/1.txt", "g:/out" };

        try {
            Configuration conf = new Configuration();
            //解析命令行中-D后面传递过来的参数，添加到conf中
            String[] remainingArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
            //创建一个job
            Job job = Job.getInstance(conf);

            job.setJarByClass(WordCountJobQueue.class);

            //输入路径
            FileInputFormat.setInputPaths(job, new Path(remainingArgs[0]));
            //输出路径
            FileOutputFormat.setOutputPath(job, new Path(remainingArgs[1]));

            //map代码
            job.setMapperClass(MyMapper.class);
            //k2输出的类型Text
            job.setMapOutputKeyClass(Text.class);
            //v2输出类型LongWriable
            job.setMapOutputValueClass(IntWritable.class);


            //reduce 代码
            job.setReducerClass(MyReduce.class);
            //k3输出类型Text
            job.setOutputKeyClass(Text.class);
            //v3输出类型LongWriable
            job.setOutputValueClass(IntWritable.class);
            //提交job
            job.waitForCompletion(true);


        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    public static class MyMapper extends Mapper<LongWritable, Text, Text, IntWritable> {
        Logger logger = LoggerFactory.getLogger(MyMapper.class);
        /**
         * 这个map函数就是可以接收k1,v1， 产生k2,v2
         * <p>
         * 0    hello world ---> hello  1
         * world  1
         *
         * @param //k1
         * @param //v1
         * @param context
         * @throws IOException
         * @throws InterruptedException
         */

        //Text k = new Text();
        IntWritable v2 = new IntWritable(1);

        @Override
        protected void map(LongWritable k1, Text v1, Context context) throws IOException, InterruptedException {

            // 1 获取一行
            //String line = value.toString();

            // 2 切割
            //String[] words = line.split(" ");
            String[] words = v1.toString().split(" ");

            // 3 输出
            for (String word : words) {
                Text k2 = new Text(word);
                //k.set(word);
                //System.out.println("k2:"+word+"...v2:1");
                //logger.info("<k1,v1>=<"+k1.get()+","+v1.toString()+">");
                context.write(k2, v2);
            }
        }
    }

    public static class MyReduce extends Reducer<Text, IntWritable, Text, IntWritable> {
        Logger logger = LoggerFactory.getLogger(MyReduce.class);
        /**
         * 争对v2s的数据进行累加求和，并且最终把数据转化为k3,v3
         *
         * @param key
         * @param v2s
         * @param context
         * @throws IOException
         * @throws InterruptedException
         */
        int sum;
        //IntWritable v3 = new IntWritable();

        @Override
        protected void reduce(Text k2, Iterable<IntWritable> v2s, Context context) throws IOException, InterruptedException {
            // 1 累加求
            sum = 0;
            for (IntWritable v2 : v2s) {
                //System.out.println("<k2,v2>=<"+k2.toString()+","+v2.get()+">");
                //logger.info("<k2,v2>=<"+k2.toString()+","+v2.get()+">");
                sum += v2.get();
            }
            // 2 输出
            Text k3=k2;
            //v3.set(sum);
            IntWritable v3 = new IntWritable(sum);
            //System.out.println("<k3,v3>=<"+k3.toString()+","+v3.get()+">");
            //logger.info("<k3,v3>=<"+k3.toString()+","+v3.get()+">");
            context.write(k3, v3);
        }
    }

}