hadoop Configuration 源码解读
一般 Configuration 使用方法：
Configuration conf = new Configuration();
String s = conf.get("fs.defaultFS");
System.out.println(s);
FileSystem fs = FileSystem.get(conf);
想必大家都对属性的加载有点疑惑，今天就来探究一番。
没有配置 hdfs 相关配置时候
Configuration conf = new Configuration();
String s = conf.get("fs.defaultFS");
System.out.println(s);
输出的 s 结果是：
file:///
因为 有一份 core-default.xml 在 hadoop-common-3.3.3.jar 的 resource 里面，Configuration 初始化的时候，会加载 它，
它里面的 fs.defaultFS 默认值就是 file:///
源码开始
new Configuration() 发生了什么
- Configuration 类的 static 属性初始化
private static final Logger LOG = LoggerFactory.getLogger(Configuration.class);
private static final Logger LOG_DEPRECATION = LoggerFactory.getLogger("org.apache.hadoop.conf.Configuration.deprecation");
private static final Set<String> TAGS = ConcurrentHashMap.newKeySet();
private static final String DEFAULT_STRING_CHECK = "testingforemptydefaultvalue";
private static boolean restrictSystemPropsDefault = false;
private static final WeakHashMap<Configuration,Object> REGISTRY = new WeakHashMap<Configuration,Object>(); //放置 configuration 对象
private static final CopyOnWriteArrayList<String> defaultResources = new CopyOnWriteArrayList<String>(); //放置默认的 资源文件名 list core-default.xml、core-site.xml、hadoop-site.xml
private static DeprecationDelta[] defaultDeprecations = //已经废弃了的 conf key 旧key与新key mapping
new DeprecationDelta[] {
new DeprecationDelta("topology.script.file.name",
CommonConfigurationKeys.NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY),
new DeprecationDelta("topology.script.number.args",
CommonConfigurationKeys.NET_TOPOLOGY_SCRIPT_NUMBER_ARGS_KEY),
new DeprecationDelta("hadoop.configured.node.mapping",
CommonConfigurationKeys.NET_TOPOLOGY_CONFIGURED_NODE_MAPPING_KEY),
new DeprecationDelta("topology.node.switch.mapping.impl",
CommonConfigurationKeys.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY),
new DeprecationDelta("dfs.df.interval",
CommonConfigurationKeys.FS_DF_INTERVAL_KEY),
new DeprecationDelta("fs.default.name",
CommonConfigurationKeys.FS_DEFAULT_NAME_KEY),
new DeprecationDelta("dfs.umaskmode",
CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY),
new DeprecationDelta("dfs.nfs.exports.allowed.hosts",
CommonConfigurationKeys.NFS_EXPORTS_ALLOWED_HOSTS_KEY)
};
//全局已经被 弃用的 key
private static AtomicReference<DeprecationContext> deprecationContext =
new AtomicReference<DeprecationContext>(
new DeprecationContext(null, defaultDeprecations));
- Configuration 类的 static 代码块执行
static {
// Add default resources
addDefaultResource("core-default.xml"); //添加 core-default.xml 文件名到 defaultResources
addDefaultResource("core-site.xml"); //添加 core-site.xml 文件名到 defaultResources；注意
// hadoop-common jar 中内置了 core-site.xml 默认空文件
// print deprecation warning if hadoop-site.xml is found in classpath
ClassLoader cL = Thread.currentThread().getContextClassLoader();
if (cL == null) {
cL = Configuration.class.getClassLoader();
}
if (cL.getResource("hadoop-site.xml") != null) { //如果 classpath 里面患有 hadoop-site.xml
LOG.warn("DEPRECATED: hadoop-site.xml found in the classpath. " +
"Usage of hadoop-site.xml is deprecated. Instead use core-site.xml, "
+ "mapred-site.xml and hdfs-site.xml to override properties of " +
"core-default.xml, mapred-default.xml and hdfs-default.xml " +
"respectively");
addDefaultResource("hadoop-site.xml");
}
}
public static synchronized void addDefaultResource(String name) { //添加文件名到 defaultResources
if(!defaultResources.contains(name)) {
defaultResources.add(name);
for(Configuration conf : REGISTRY.keySet()) {//因为 REGISTRY size 是 0，所以这里不会执行
if(conf.loadDefaults) {
conf.reloadConfiguration();
}
}
}
}
- Configuration 类的 构造方法
public Configuration() {
this(true);
}
public Configuration(boolean loadDefaults) {
this.loadDefaults = loadDefaults;
synchronized(Configuration.class) { //类锁
REGISTRY.put(this, null); //REGISTRY 中加入本身
}
}
conf.get(“fs.defaultFS”); 发生了什么
private Properties properties; //属性存放的地方
public String get(String name) {
String[] names = handleDeprecation(deprecationContext.get(), name); //处理 新旧 key
String result = null;
for(String n : names) {
result = substituteVars(getProps().getProperty(n));
//从已经读取了 core-default.xml 、 core-site.xml(如果有) 属性 到 properties ，从 properties 获取 属性值
}
return result;
}
private String[] handleDeprecation(DeprecationContext deprecations,
String name) {
if (null != name) {
name = name.trim(); //去除 两端 的 空格
}
// Initialize the return value with requested name
String[] names = new String[]{name};
// Deprecated keys are logged once and an updated names are returned 拿到已经废弃 conf 的 keys，这个 name 是否是 已经废弃的 key
DeprecatedKeyInfo keyInfo = deprecations.getDeprecatedKeyMap().get(name);
if (keyInfo != null) {
if (!keyInfo.getAndSetAccessed()) {
logDeprecation(keyInfo.getWarningMessage(name));
}
// Override return value for deprecated keys
names = keyInfo.newKeys; //替换为 新 key
}
// Update properties with deprecated key if already loaded and new
// deprecation has been added
updatePropertiesWithDeprecatedKeys(deprecations, names); //设置新 key 对应 旧 key 的值；
// 里面已经读取了 core-default.xml 、 core-site.xml(如果有) 属性 到 properties 了
// If there are no overlay values we can return early
Properties overlayProperties = getOverlay(); //是否有需要 覆盖的值
if (overlayProperties.isEmpty()) { //没有的话，直接返货 names
return names;
}
// Update properties and overlays with reverse lookup values
for (String n : names) {
String deprecatedKey = deprecations.getReverseDeprecatedKeyMap().get(n);
if (deprecatedKey != null && !overlayProperties.containsKey(n)) {
String deprecatedValue = overlayProperties.getProperty(deprecatedKey);
if (deprecatedValue != null) {
getProps().setProperty(n, deprecatedValue);
overlayProperties.setProperty(n, deprecatedValue);
}
}
}
return names;
}
private void updatePropertiesWithDeprecatedKeys(
DeprecationContext deprecations, String[] newNames) {
for (String newName : newNames) {
String deprecatedKey = deprecations.getReverseDeprecatedKeyMap().get(newName); //拿到 旧 conf key
if (deprecatedKey != null && !getProps().containsKey(newName)) {
String deprecatedValue = getProps().getProperty(deprecatedKey);
if (deprecatedValue != null) {
getProps().setProperty(newName, deprecatedValue); //旧key对应的新key设置旧值
}
}
}
}
private void updatePropertiesWithDeprecatedKeys(
DeprecationContext deprecations, String[] newNames) {
for (String newName : newNames) {
String deprecatedKey = deprecations.getReverseDeprecatedKeyMap().get(newName); //拿到 旧 conf key
if (deprecatedKey != null && !getProps().containsKey(newName)) { //执行 getProps 方法
String deprecatedValue = getProps().getProperty(deprecatedKey);
if (deprecatedValue != null) {
getProps().setProperty(newName, deprecatedValue); //旧key对应的新key设置旧值
}
}
}
}
protected synchronized Properties getProps() { //获取 core-default.xml 、 core-site.xml(如果有) 属性 保存类 properties
if (properties == null) { //首次可能为 null
properties = new Properties();
loadProps(properties, 0, true);//加载 core-default.xml 、 core-site.xml(如果有) 到 props 里面
}
return properties;
}
private synchronized void loadProps(final Properties props,
final int startIdx, final boolean fullReload) {
if (props != null) {
Map<String, String[]> backup =
updatingResource != null // 第一次执行该方法时，backup为null
? new ConcurrentHashMap<>(updatingResource) : null;
//加载 core-default.xml 、 core-site.xml(如果有) 到 props 里面；resources目前size=0
loadResources(props, resources, startIdx, fullReload, quietmode); //加载资源
if (overlay != null) {
props.putAll(overlay);
if (backup != null) {
for (Map.Entry<Object, Object> item : overlay.entrySet()) {
String key = (String) item.getKey();
String[] source = backup.get(key);
if (source != null) {
updatingResource.put(key, source);
}
}
}
}
}
}
private void loadResources(Properties properties,
ArrayList<Resource> resources,
int startIdx,
boolean fullReload,
boolean quiet) {
if(loadDefaults && fullReload) {
for (String resource : defaultResources) { //从 放置默认的 资源文件名 list
// 1. 加载 core-default.xml 属性 到 properties； 在 hadoop-common-3.3.3.jar 的 resource 里面
// 2. 记载 core-site.xml 中的属性到 properties （如果classpath配置了这个文件）
loadResource(properties, new Resource(resource, false), quiet);
}
}
for (int i = startIdx; i < resources.size(); i++) {
Resource ret = loadResource(properties, resources.get(i), quiet);
if (ret != null) {
resources.set(i, ret);
}
}
this.addTags(properties);
}
private Resource loadResource(Properties properties,
Resource wrapper, boolean quiet) {
String name = UNKNOWN_RESOURCE;
try {
Object resource = wrapper.getResource();
name = wrapper.getName();
boolean returnCachedProperties = false;
// Resource类未继承任何父类，因此并不满足以下的逻辑判断
if (resource instanceof InputStream) {
returnCachedProperties = true;
} else if (resource instanceof Properties) {
overlay(properties, (Properties)resource);
}
//如果没有手动给classpath中添加core-default.xml、core-site.xml文件，会从/org/apache/hadoop/hadoop-common/3.3.3/hadoop-common-3.2.2.jar!/core-default.xml位置读取配置文件
XMLStreamReader2 reader = getStreamReader(wrapper, quiet);
if (reader == null) { //如果没有 core-site.xml 文件在 classpath，则会返回null
if (quiet) {
return null;
}
throw new RuntimeException(resource + " not found");
}
Properties toAddTo = properties;
if(returnCachedProperties) {
toAddTo = new Properties();
}
//读取到属性值后，将属性值依次赋值到toAddTo
List<ParsedItem> items = new Parser(reader, wrapper, quiet).parse();
for (ParsedItem item : items) {
loadProperty(toAddTo, item.name, item.key, item.value,
item.isFinal, item.sources);
}
reader.close();
if (returnCachedProperties) {
overlay(properties, toAddTo);
return new Resource(toAddTo, name, wrapper.isParserRestricted());
}
return null;
} catch (IOException e) {
LOG.error("error parsing conf " + name, e);
throw new RuntimeException(e);
} catch (XMLStreamException e) {
LOG.error("error parsing conf " + name, e);
throw new RuntimeException(e);
}
}
569
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
