flink源码分析-获取JVM最大堆内存

flink版本: flink-1.11.2

代码位置: org.apache.flink.runtime.util.EnvironmentInformation#getMaxJvmHeapMemory

如果设置了-Xmx参数，就返回这个参数，如果没设置就返回机器物理内存的1/4.  这里主要看各个机器内存的获取方法。

	/**
	 * The maximum JVM heap size, in bytes.
	 *
	 * <p>This method uses the <i>-Xmx</i> value of the JVM, if set. If not set, it returns (as
	 * a heuristic) 1/4th of the physical memory size.
	 *
	 * @return The maximum JVM heap size, in bytes.
	 */
	public static long getMaxJvmHeapMemory() {
		final long maxMemory = Runtime.getRuntime().maxMemory();
		if(maxMemory != Long.MAX_VALUE) {
			// we have the proper max memory
			return maxMemory;
		} else {
			// max JVM heap size is not set - use the heuristic to use 1/4th of the physical memory
			final long physicalMemory = Hardware.getSizeOfPhysicalMemory();
			if(physicalMemory != -1) {
				// got proper value for physical memory
				return physicalMemory / 4;
			} else {
				throw new RuntimeException(
					"Could not determine the amount of free memory.\n" + "Please set the maximum memory for the JVM, e.g. -Xmx512M for 512 megabytes.");
			}
		}
	}

进入getSizeOfPhysicalMemory()方法，里面有获取各种操作系统物理内存的方法:

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.flink.runtime.util;

import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.lang.management.ManagementFactory;
import java.lang.management.OperatingSystemMXBean;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.flink.util.OperatingSystem;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * Convenience class to extract hardware specifics of the computer executing the running JVM.
 */
public class Hardware {

	private static final Logger LOG = LoggerFactory.getLogger(Hardware.class);

	private static final String LINUX_MEMORY_INFO_PATH = "/proc/meminfo";

	private static final Pattern LINUX_MEMORY_REGEX = Pattern.compile("^MemTotal:\\s*(\\d+)\\s+kB$");

	// ------------------------------------------------------------------------
	
	/**
	 * Gets the number of CPU cores (hardware contexts) that the JVM has access to.
	 * 
	 * @return The number of CPU cores.
	 */
	public static int getNumberCPUCores() {

		// TODO_MA 注释： 获取 Cpu Cores
		return Runtime.getRuntime().availableProcessors();
	}

	/**
	 * Returns the size of the physical memory in bytes.
	 * 
	 * @return the size of the physical memory in bytes or {@code -1}, if
	 *         the size could not be determined.
	 */
	public static long getSizeOfPhysicalMemory() {
		// first try if the JVM can directly tell us what the system memory is
		// this works only on Oracle JVMs
		try {
			Class<?> clazz = Class.forName("com.sun.management.OperatingSystemMXBean");
			Method method = clazz.getMethod("getTotalPhysicalMemorySize");
			OperatingSystemMXBean operatingSystemMXBean = ManagementFactory.getOperatingSystemMXBean();

			// someone may install different beans, so we need to check whether the bean
			// is in fact the sun management bean
			if (clazz.isInstance(operatingSystemMXBean)) {
				return (Long) method.invoke(operatingSystemMXBean);
			}
		}
		catch (ClassNotFoundException e) {
			// this happens on non-Oracle JVMs, do nothing and use the alternative code paths
		}
		catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) {
			LOG.warn("Access to physical memory size: " +
					"com.sun.management.OperatingSystemMXBean incompatibly changed.", e);
		}

		// we now try the OS specific access paths
		switch (OperatingSystem.getCurrentOperatingSystem()) {
			case LINUX:
				return getSizeOfPhysicalMemoryForLinux();

			case WINDOWS:
				return getSizeOfPhysicalMemoryForWindows();

			case MAC_OS:
				return getSizeOfPhysicalMemoryForMac();

			case FREE_BSD:
				return getSizeOfPhysicalMemoryForFreeBSD();

			case UNKNOWN:
				LOG.error("Cannot determine size of physical memory for unknown operating system");
				return -1;

			default:
				LOG.error("Unrecognized OS: " + OperatingSystem.getCurrentOperatingSystem());
				return -1;
		}
	}

	/**
	 * Returns the size of the physical memory in bytes on a Linux-based
	 * operating system.
	 * 
	 * @return the size of the physical memory in bytes or {@code -1}, if
	 *         the size could not be determined
	 */
	private static long getSizeOfPhysicalMemoryForLinux() {
		try (BufferedReader lineReader = new BufferedReader(new FileReader(LINUX_MEMORY_INFO_PATH))) {
			String line;
			while ((line = lineReader.readLine()) != null) {
				Matcher matcher = LINUX_MEMORY_REGEX.matcher(line);
				if (matcher.matches()) {
					String totalMemory = matcher.group(1);
					return Long.parseLong(totalMemory) * 1024L; // Convert from kilobyte to byte
				}
			}
			// expected line did not come
			LOG.error("Cannot determine the size of the physical memory for Linux host (using '/proc/meminfo'). " +
					"Unexpected format.");
			return -1;
		}
		catch (NumberFormatException e) {
			LOG.error("Cannot determine the size of the physical memory for Linux host (using '/proc/meminfo'). " +
					"Unexpected format.");
			return -1;
		}
		catch (Throwable t) {
			LOG.error("Cannot determine the size of the physical memory for Linux host (using '/proc/meminfo') ", t);
			return -1;
		}
	}

	/**
	 * Returns the size of the physical memory in bytes on a Mac OS-based
	 * operating system
	 * 
	 * @return the size of the physical memory in bytes or {@code -1}, if
	 *         the size could not be determined
	 */
	private static long getSizeOfPhysicalMemoryForMac() {
		BufferedReader bi = null;
		try {
			Process proc = Runtime.getRuntime().exec("sysctl hw.memsize");

			bi = new BufferedReader(new InputStreamReader(proc.getInputStream()));

			String line;
			while ((line = bi.readLine()) != null) {
				if (line.startsWith("hw.memsize")) {
					long memsize = Long.parseLong(line.split(":")[1].trim());
					bi.close();
					proc.destroy();
					return memsize;
				}
			}

		} catch (Throwable t) {
			LOG.error("Cannot determine physical memory of machine for MacOS host", t);
			return -1;
		} finally {
			if (bi != null) {
				try {
					bi.close();
				} catch (IOException ignored) {}
			}
		}
		return -1;
	}

	/**
	 * Returns the size of the physical memory in bytes on FreeBSD.
	 * 
	 * @return the size of the physical memory in bytes or {@code -1}, if
	 *         the size could not be determined
	 */
	private static long getSizeOfPhysicalMemoryForFreeBSD() {
		BufferedReader bi = null;
		try {
			Process proc = Runtime.getRuntime().exec("sysctl hw.physmem");

			bi = new BufferedReader(new InputStreamReader(proc.getInputStream()));

			String line;
			while ((line = bi.readLine()) != null) {
				if (line.startsWith("hw.physmem")) {
					long memsize = Long.parseLong(line.split(":")[1].trim());
					bi.close();
					proc.destroy();
					return memsize;
				}
			}
			
			LOG.error("Cannot determine the size of the physical memory for FreeBSD host " +
					"(using 'sysctl hw.physmem').");
			return -1;
		}
		catch (Throwable t) {
			LOG.error("Cannot determine the size of the physical memory for FreeBSD host " +
					"(using 'sysctl hw.physmem')", t);
			return -1;
		}
		finally {
			if (bi != null) {
				try {
					bi.close();
				} catch (IOException ignored) {}
			}
		}
	}

	/**
	 * Returns the size of the physical memory in bytes on Windows.
	 * 
	 * @return the size of the physical memory in bytes or {@code -1}, if
	 *         the size could not be determined
	 */
	private static long getSizeOfPhysicalMemoryForWindows() {
		BufferedReader bi = null;
		try {
			Process proc = Runtime.getRuntime().exec("wmic memorychip get capacity");

			bi = new BufferedReader(new InputStreamReader(proc.getInputStream()));

			String line = bi.readLine();
			if (line == null) {
				return -1L;
			}

			if (!line.startsWith("Capacity")) {
				return -1L;
			}

			long sizeOfPhyiscalMemory = 0L;
			while ((line = bi.readLine()) != null) {
				if (line.isEmpty()) {
					continue;
				}

				line = line.replaceAll(" ", "");
				sizeOfPhyiscalMemory += Long.parseLong(line);
			}
			return sizeOfPhyiscalMemory;
		}
		catch (Throwable t) {
			LOG.error("Cannot determine the size of the physical memory for Windows host " +
					"(using 'wmic memorychip')", t);
			return -1L;
		}
		finally {
			if (bi != null) {
				try {
					bi.close();
				} catch (Throwable ignored) {}
			}
		}
	}

	// --------------------------------------------------------------------------------------------

	private Hardware() {}
}

另外注意try catch的这种用法:

		try (BufferedReader lineReader = new BufferedReader(new FileReader(LINUX_MEMORY_INFO_PATH))) {
			String line;
			while ((line = lineReader.readLine()) != null) {
				Matcher matcher = LINUX_MEMORY_REGEX.matcher(line);
				if (matcher.matches()) {
					String totalMemory = matcher.group(1);
					return Long.parseLong(totalMemory) * 1024L; // Convert from kilobyte to byte
				}
			}
			// expected line did not come
			LOG.error("Cannot determine the size of the physical memory for Linux host (using '/proc/meminfo'). " +
					"Unexpected format.");
			return -1;
		}
		catch (NumberFormatException e) {
			LOG.error("Cannot determine the size of the physical memory for Linux host (using '/proc/meminfo'). " +
					"Unexpected format.");
			return -1;
		}