序
本文主要研究一下flink的BlobServer
BlobServer
flink-1.7.2/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobServer.java
public class BlobServer extends Thread implements BlobService, BlobWriter, PermanentBlobService, TransientBlobService {
/** The log object used for debugging. */
private static final Logger LOG = LoggerFactory.getLogger(BlobServer.class);
/** Counter to generate unique names for temporary files. */
private final AtomicLong tempFileCounter = new AtomicLong(0);
/** The server socket listening for incoming connections. */
private final ServerSocket serverSocket;
/** Blob Server configuration. */
private final Configuration blobServiceConfiguration;
/** Indicates whether a shutdown of server component has been requested. */
private final AtomicBoolean shutdownRequested = new AtomicBoolean();
/** Root directory for local file storage. */
private final File storageDir;
/** Blob store for distributed file storage, e.g. in HA. */
private final BlobStore blobStore;
/** Set of currently running threads. */
private final Set<BlobServerConnection> activeConnections = new HashSet<>();
/** The maximum number of concurrent connections. */
private final int maxConnections;
/** Lock guarding concurrent file accesses. */
private final ReadWriteLock readWriteLock;
/**
* Shutdown hook thread to ensure deletion of the local storage directory.
*/
private final Thread shutdownHook;
// --------------------------------------------------------------------------------------------
/**
* Map to store the TTL of each element stored in the local storage, i.e. via one of the {@link
* #getFile} methods.
**/
private final ConcurrentHashMap<Tuple2<JobID, TransientBlobKey>, Long> blobExpiryTimes =
new ConcurrentHashMap<>();
/** Time interval (ms) to run the cleanup task; also used as the default TTL. */
private final long cleanupInterval;
/**
* Timer task to execute the cleanup at regular intervals.
*/
private final Timer cleanupTimer;
/**
* Instantiates a new BLOB server and binds it to a free network port.
*
* @param config Configuration to be used to instantiate the BlobServer
* @param blobStore BlobStore to store blobs persistently
*
* @throws IOException
* thrown if the BLOB server cannot bind to a free network port or if the
* (local or distributed) file storage cannot be created or is not usable
*/
public BlobServer(Configuration config, BlobStore blobStore) throws IOException {
this.blobServiceConfiguration = checkNotNull(config);
this.blobStore = checkNotNull(blobStore);
this.readWriteLock = new ReentrantReadWriteLock();
// configure and create the storage directory
this.storageDir = BlobUtils.initLocalStorageDirectory(config);
LOG.info("Created BLOB server storage directory {}", storageDir);
// configure the maximum number of concurrent connections
final int maxConnections = config.getInteger(BlobServerOptions.FETCH_CONCURRENT);
if (maxConnections >= 1) {
this.maxConnections = maxConnections;
}
else {
LOG.warn("Invalid value for maximum connections in BLOB server: {}. Using default value of {}",
maxConnections, BlobServerOptions.FETCH_CONCURRENT.defaultValue());
this.maxConnections = BlobServerOptions.FETCH_CONCURRENT.defaultValue();
}
// configure the backlog of connections
int backlog = config.getInteger(BlobServerOptions.FETCH_BACKLOG);
if (backlog < 1) {
LOG.warn("Invalid value for BLOB connection backlog: {}. Using default value of {}",
backlog, BlobServerOptions.FETCH_BACKLOG.defaultValue());
backlog = BlobServerOptions.FETCH_BACKLOG.defaultValue();
}
// Initializing the clean up task
this.cleanupTimer = new Timer(true);
this.cleanupInterval = config.getLong(BlobServerOptions.CLEANUP_INTERVAL) * 1000;
this.cleanupTimer
.schedule(new TransientBlobCleanupTask(blobExpiryTimes, readWriteLock.writeLock(),
storageDir, LOG), cleanupInterval, cleanupInterval);
this.shutdownHook = ShutdownHookUtil.addShutdownHook(this, getClass().getSimpleName(), LOG);
// ----------------------- start the server -------------------
final String serverPortRange = config.getString(BlobServerOptions.PORT);
final Iterator<In