DataNode存放hdfs中的数据,接受客户端与其他DataNode的请求。
DataNode服务由如下线程构成,按DataNode启动各线程的先后出场顺序列出:
1、DataXceiveServer:数据接收监听守护线程,当监听有请求时,开启一个DataXceiver线程;
public void run() {
try {
while (shouldListen) {
Socket s = ss.accept();
//s.setSoTimeout(READ_TIMEOUT);
new Daemon(new DataXceiver(s)).start();
}
ss.close();
} catch (IOException ie) {
LOG.info("Exiting DataXceiveServer due to " + ie.toString());
}
}
2、DataXceiver:数据接收线程,从DataXceiveServer读数据,或是写数据到DataXceiveServer;
3、DataNode:DataNode自身还有一个线程,一直运行offerService(),直到服务器被shutdown为止;
offerService一直循环做以下几件事情:
1)每隔HEARTBEAT_INTERVAL ,默认 3 * 1000=3秒钟,向NameNode发送一次心跳信息;
2)每隔blockReportIntervalBasis - new Random().nextInt((int)(blockReportIntervalBasis/10)),默认blockReportIntervalBasis=60 * 60 * 1000=1小时,向NameNode发送最新的数据块信息,并且返回被删除的数据块列表,以便被GC进行垃圾回收;
3)只要接收到数据库列表大于0,就向NameNode发送这些数据块ID;
4)在DataNode启动后,延后DATANODE_STARTUP_PERIOD,默认是2 * 60 * 1000=2分钟,从NameNode获取数据块相关指令,对每个数据块,新启动一个DataTransfer线程来处理;
while (shouldRun) {
long now = System.currentTimeMillis();
// Every so often, send heartbeat or block-report
synchronized (receivedBlockList) {
if (now - lastHeartbeat > HEARTBEAT_INTERVAL) {
namenode.sendHeartbeat(localName, data.getCapacity(), data.getRemaining());
//LOG.info("Just sent heartbeat, with name " + localName);
lastHeartbeat = now;
}
if (now - lastBlockReport > blockReportInterval) {
// Send latest blockinfo report if timer has expired.
// Get back a list of local block(s) that are obsolete
// and can be safely GC'ed.
Block toDelete[] = namenode.blockReport(localName, data.getBlockReport());
data.invalidate(toDelete);
lastBlockReport = now;
continue;
}
if (receivedBlockList.size() > 0) {
// Send newly-received blockids to namenode
Block blockArray[] = (Block[]) receivedBlockList.toArray(new Block[receivedBlockList.size()]);
receivedBlockList.removeAllElements();
namenode.blockReceived(localName, blockArray);
}
if (now - sendStart > datanodeStartupPeriod) {
// Check to see if there are any block-instructions from the
// namenode that this datanode should perform.
BlockCommand cmd = namenode.getBlockwork(localName, xmitsInProgress);
if (cmd != null && cmd.transferBlocks()) {
//
// Send a copy of a block to another datanode
//
Block blocks[] = cmd.getBlocks();
DatanodeInfo xferTargets[][] = cmd.getTargets();
for (int i = 0; i < blocks.length; i++) {
if (!data.isValidBlock(blocks[i])) {
String errStr = "Can't send invalid block " + blocks[i];
LOG.info(errStr);
namenode.errorReport(localName, errStr);
break;
} else {
if (xferTargets[i].length > 0) {
LOG.info("Starting thread to transfer block " + blocks[i] + " to " + xferTargets[i]);
new Daemon(new DataTransfer(xferTargets[i], blocks[i])).start();
}
}
}
} else if (cmd != null && cmd.invalidateBlocks()) {
data.invalidate(cmd.getBlocks());
}
}
// There is no work to do; sleep until hearbeat timer elapses,
// or work arrives, and then iterate again.
long waitTime = HEARTBEAT_INTERVAL - (now - lastHeartbeat);
if (waitTime > 0 && receivedBlockList.size() == 0) {
try {
receivedBlockList.wait(waitTime);
} catch (InterruptedException ie) {
}
}
}
}
4、DataTransfer:数据块传输线程,将指定的数据块传到指定的DataNode;
Socket s = new Socket();
s.connect(curTarget, READ_TIMEOUT);
s.setSoTimeout(READ_TIMEOUT);
DataOutputStream out = new DataOutputStream(new BufferedOutputStream(s.getOutputStream()));
try {
long filelen = data.getLength(b);
DataInputStream in = new DataInputStream(new BufferedInputStream(data.getBlockData(b)));
try {
//
// Header info
//
out.write(OP_WRITE_BLOCK);
out.writeBoolean(true);
b.write(out);
out.writeInt(targets.length);
for (int i = 0; i < targets.length; i++) {
targets[i].write(out);
}
out.write(RUNLENGTH_ENCODING);
out.writeLong(filelen);
//
// Write the data
//
while (filelen > 0) {
int bytesRead = in.read(buf, 0, (int) Math.min(filelen, buf.length));
out.write(buf, 0, bytesRead);
filelen -= bytesRead;
}
} finally {
in.close();
}
} finally {
out.close();
}