hadoop 配置文件处理

hadoop 配置文件处理

[toc]

Configuration 类

Configuration 作用

1.加载配置文件 2.可以加载多个配置文件 3.支持动态修改配置 4.快速保存配置文件

构造方法

    public Configuration();
    //是否加载默认的配置文件，默认为true 加载
    public Configuration(boolean loadDefaults);
    //根据其他实例 构件一个新的实例
    public Configuration(Configuration other);

主要成员变量

    //加载配置文件过程中是否输出日志，默认为true 不输出，主要用于开发调试
    private boolean quietmode = true;
    //保存通过addResource加载的对象
    private ArrayList<Resource> resources = new ArrayList<Resource>();
    //保存final属性为true的属性值
    private Set<String> finalParameters = Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
    //保存解析配置文件后的健-值对
    private Properties properties;
    //保存通过set修改的配置项
    private Properties overlay;
    private ClassLoader classLoader;
    //保存了 健-来源数组 
    private Map<String, String[]> updatingResource;
    //被弃用的配置属性列表， AtomicReference 是确保赋值的原子操作， DeprecationContext 主要是被弃用的属性map
    private static AtomicReference<DeprecationContext> deprecationContext ；

加载资源方法

    //把其他配置文件中的name , 和value 值加载到当前配置中，其他像final等值放弃
    public void addResource(Configuration conf)
    public void addResource(InputStream in)
    //name加载在资源名称，主要在source中用到
    public void addResource(InputStream in, String name)
    public void addResource(Path file)
    //加载classpath下的文件 文件名=name
    public void addResource(String name)
    public void addResource(URL url)

addSource方法的具体实现

  public synchronized void reloadConfiguration() {
    properties = null;  // trigger reload
    finalParameters.clear(); // clear site-limits
  }
  
  private synchronized void addResourceObject(Resource resource) {
    //添加资源到资源列表的list中
    resources.add(resource); // add to resources
    //准备重新加载资源，实际上在getProps()方法中才会触发加载资源的动作
    reloadConfiguration();
  }

set方法

作用： 设置修改或添加一个键-值对

public void set(String name, String value, String source) {
    //判断name ，value 是否为null 如果为null 抛出异常IllegalArgumentException ，并携带错误信息
    Preconditions.checkArgument(
        name != null,
        "Property name must not be null");
    Preconditions.checkArgument(
        value != null,
        "The value of property " + name + " must not be null");
    name = name.trim();
    //获取被弃用的属性列表对象
    DeprecationContext deprecations = deprecationContext.get();
    //如果为空 ，调用getProps()方法 ，加载配置文件，解析配置项，加载配置文件
    if (deprecations.getDeprecatedKeyMap().isEmpty()) {
      getProps();
    }
    //添加到更新value的列表中
    getOverlay().setProperty(name, value);
    //添加到现在可用配置属性中
    getProps().setProperty(name, value);
    String newSource = (source == null ? "programatically" : source);
    //判断是否是被弃用的key
    if (!isDeprecated(name)) {
      updatingResource.put(name, new String[] {newSource});
      String[] altNames = getAlternativeNames(name);
      if(altNames != null) {
        for(String n: altNames) {
          if(!n.equals(name)) {
            getOverlay().setProperty(n, value);
            getProps().setProperty(n, value);
            updatingResource.put(n, new String[] {newSource});
          }
        }
      }
    }
    else {
      String[] names = handleDeprecation(deprecationContext.get(), name);
      String altSource = "because " + name + " is deprecated";
      for(String n : names) {
        getOverlay().setProperty(n, value);
        getProps().setProperty(n, value);
        updatingResource.put(n, new String[] {altSource});
      }
    }
  }

get方法

   public String get(String name) {
   //获取相关的name列表，包括被弃用的，或者新增的
    String[] names = handleDeprecation(deprecationContext.get(), name);
    String result = null;
    for(String n : names) {
      // 获取结果
      result = substituteVars(getProps().getProperty(n));
    }
    return result;
  }

其中substituteVars()方法 会根据传入的值，自动扩展属性，也就是说，value中包含了${key}这种格式的变量时，变量会被替换成对应的值。

解析配置文件方法 getProps()

 protected synchronized Properties getProps() {
    //判断properties 是否为空，如果为空 加载配置文件
    if (properties == null) {
      properties = new Properties();
      Map<String, String[]> backup =
          new ConcurrentHashMap<String, String[]>(updatingResource);
      //加载配置文件
      loadResources(properties, resources, quietmode);
      //加载更新过的配置，更新source列表
      if (overlay != null) {
        properties.putAll(overlay);
        for (Map.Entry<Object,Object> item: overlay.entrySet()) {
          String key = (String)item.getKey();
          String[] source = backup.get(key);
          if(source != null) {
            updatingResource.put(key, source);
          }
        }
      }
    }
    return properties;
  }
  //加载配置文件
   private void loadResources(Properties properties,
                             ArrayList<Resource> resources,
                             boolean quiet) {
    //加载默认配置文件
    if(loadDefaults) {
      for (String resource : defaultResources) {
        loadResource(properties, new Resource(resource), quiet);
      }
    
      //support the hadoop-site.xml as a deprecated case
      if(getResource("hadoop-site.xml")!=null) {
        loadResource(properties, new Resource("hadoop-site.xml"), quiet);
      }
    }
    //加载addsource（） 添加的source
    for (int i = 0; i < resources.size(); i++) {
      Resource ret = loadResource(properties, resources.get(i), quiet);
      if (ret != null) {
        resources.set(i, ret);
      }
    }
  }
  //加载单个配置文件
  private Resource loadResource(Properties properties, Resource wrapper, boolean quiet) {
    String name = UNKNOWN_RESOURCE;
    try {
      Object resource = wrapper.getResource();
      name = wrapper.getName();
      
      DocumentBuilderFactory docBuilderFactory 
        = DocumentBuilderFactory.newInstance();
      //ignore all comments inside the xml file ，是否忽略注释
      docBuilderFactory.setIgnoringComments(true);

      //allow includes in the xml file， 支持命名空间
      docBuilderFactory.setNamespaceAware(true);
      try {
          //支持include机制
          docBuilderFactory.setXIncludeAware(true);
      } catch (UnsupportedOperationException e) {
        LOG.error("Failed to set setXIncludeAware(true) for parser "
                + docBuilderFactory
                + ":" + e,
                e);
      }
      DocumentBuilder builder = docBuilderFactory.newDocumentBuilder();
      Document doc = null;
      Element root = null;
      boolean returnCachedProperties = false;
      //判断souurce 类型 ，解析资源，用dom方法，解析构建document对象
      if (resource instanceof URL) {                  // an URL resource
        doc = parse(builder, (URL)resource);
      } else if (resource instanceof String) {        // a CLASSPATH resource
        URL url = getResource((String)resource);
        doc = parse(builder, url);
      } else if (resource instanceof Path) {          // a file resource
        // Can't use FileSystem API or we get an infinite loop
        // since FileSystem uses Configuration API.  Use java.io.File instead.
        File file = new File(((Path)resource).toUri().getPath())
          .getAbsoluteFile();
        if (file.exists()) {
          if (!quiet) {
            LOG.debug("parsing File " + file);
          }
          doc = parse(builder, new BufferedInputStream(
              new FileInputStream(file)), ((Path)resource).toString());
        }
      } else if (resource instanceof InputStream) {
        doc = parse(builder, (InputStream) resource, null);
        returnCachedProperties = true;
      } else if (resource instanceof Properties) {
        overlay(properties, (Properties)resource);
      } else if (resource instanceof Element) {
        root = (Element)resource;
      }

      if (root == null) {
        if (doc == null) {
          if (quiet) {
            return null;
          }
          throw new RuntimeException(resource + " not found");
        }
        root = doc.getDocumentElement();
      }
      Properties toAddTo = properties;
      if(returnCachedProperties) {
        toAddTo = new Properties();
      }
      //判断是否为configuration 节点
      if (!"configuration".equals(root.getTagName()))
        LOG.fatal("bad conf file: top-level element not <configuration>");
      NodeList props = root.getChildNodes();
      DeprecationContext deprecations = deprecationContext.get();
      //循环子节点
      for (int i = 0; i < props.getLength(); i++) {
        Node propNode = props.item(i);
        if (!(propNode instanceof Element))
          continue;
        Element prop = (Element)propNode;
        //如果包含了其他配置文件， 递归解析配置文件
        if ("configuration".equals(prop.getTagName())) {
          loadResource(toAddTo, new Resource(prop, name), quiet);
          continue;
        }
        if (!"property".equals(prop.getTagName()))
          LOG.warn("bad conf file: element not <property>");
        NodeList fields = prop.getChildNodes();
        String attr = null;
        String value = null;
        boolean finalParameter = false;
        LinkedList<String> source = new LinkedList<String>();
        //解析 name ，value ，final，source等配置项
        for (int j = 0; j < fields.getLength(); j++) {
          Node fieldNode = fields.item(j);
          if (!(fieldNode instanceof Element))
            continue;
          Element field = (Element)fieldNode;
          if ("name".equals(field.getTagName()) && field.hasChildNodes())
            attr = StringInterner.weakIntern(
                ((Text)field.getFirstChild()).getData().trim());
          if ("value".equals(field.getTagName()) && field.hasChildNodes())
            value = StringInterner.weakIntern(
                ((Text)field.getFirstChild()).getData());
          if ("final".equals(field.getTagName()) && field.hasChildNodes())
            finalParameter = "true".equals(((Text)field.getFirstChild()).getData());
          if ("source".equals(field.getTagName()) && field.hasChildNodes())
            source.add(StringInterner.weakIntern(
                ((Text)field.getFirstChild()).getData()));
        }
        source.add(name);
        
        // Ignore this parameter if it has already been marked as 'final'
        if (attr != null) {
          if (deprecations.getDeprecatedKeyMap().containsKey(attr)) {
            DeprecatedKeyInfo keyInfo =
                deprecations.getDeprecatedKeyMap().get(attr);
            keyInfo.clearAccessed();
            for (String key:keyInfo.newKeys) {
              // update new keys with deprecated key's value 
              //更新新的key 用被弃用key的value，同时忽略final属性为ture的值
              loadProperty(toAddTo, name, key, value, finalParameter, 
                  source.toArray(new String[source.size()]));
            }
          }
          else {
            loadProperty(toAddTo, name, attr, value, finalParameter, 
                source.toArray(new String[source.size()]));
          }
        }
      }
      
      if (returnCachedProperties) {
        overlay(properties, toAddTo);
        return new Resource(toAddTo, name);
      }
      return null;
    } catch (IOException e) {
      LOG.fatal("error parsing conf " + name, e);
      throw new RuntimeException(e);
    } catch (DOMException e) {
      LOG.fatal("error parsing conf " + name, e);
      throw new RuntimeException(e);
    } catch (SAXException e) {
      LOG.fatal("error parsing conf " + name, e);
      throw new RuntimeException(e);
    } catch (ParserConfigurationException e) {
      LOG.fatal("error parsing conf " + name , e);
      throw new RuntimeException(e);
    }
  }

Configurable 接口

主要有两个方法 getConf(),setConf() 主要使用方法： 和org.apahce.hadoop.util.ReflectionUtils的newInstance()配合使用。

  public static <T> T newInstance(Class<T> theClass, Configuration conf) {
    T result;
    try {
      Constructor<T> meth = (Constructor<T>) CONSTRUCTOR_CACHE.get(theClass);
      if (meth == null) {
        meth = theClass.getDeclaredConstructor(EMPTY_ARRAY);
        meth.setAccessible(true);
        CONSTRUCTOR_CACHE.put(theClass, meth);
      }
      result = meth.newInstance();
    } catch (Exception e) {
      throw new RuntimeException(e);
    }
    setConf(result, conf);
    return result;
  }
  public static void setConf(Object theObject, Configuration conf) {
    if (conf != null) {
      //检测到对象实现了Configuable 接口，会调用setConf()方法，初始化对象
      if (theObject instanceof Configurable) {
        ((Configurable) theObject).setConf(conf);
      }
      setJobConf(theObject, conf);
    }
  }

ConfServlet

主要作用： 处理来自页面的配置文件请求

转载于:https://my.oschina.net/zwf007/blog/891062