0.1版
---
在datanode的函数
public void offerService() throws Exception {中，
有这么一段函数
if (now - lastBlockReport > blockReportInterval) {
//
// Send latest blockinfo report if timer has expired.
// Get back a list of local block(s) that are obsolete
// and can be safely GC'ed.
//
Block toDelete[] = namenode.blockReport(localName, data.getBlockReport());
data.invalidate(toDelete);
lastBlockReport = now;
continue;
}
也就是把本地的block全部发送到namenode,然后namenode返回失效的block,然后datanode再在本地将失效的block直接物理删除。
---下面就来看看namenode收到blockReport后究竟做了什么？
public Block[] blockReport(String sender, Block blocks[]) {
LOG.info("Block report from "+sender+": "+blocks.length+" blocks.");
return namesystem.processReport(blocks, new UTF8(sender));
}
可见namenode直接调用了本地的 namesystem.processReport(blocks, new UTF8(sender));
所以我们就来分析namesystem.processReport(...) 函数。
---
public synchronized Block[] processReport(Block newReport[], UTF8 name) {
DatanodeInfo node = (DatanodeInfo) datanodeMap.get(name);//从映射表中根据名字找到对应的节点信息
if (node == null) {//找不到就报错
throw new IllegalArgumentException("Unexpected exception. Received block report from node " + name + ", but there is no info for " + name);
}
//
// Modify the (block-->datanode) map, according to the difference
// between the old and new block report.
//
int oldPos = 0, newPos = 0;//因为要对两个新旧数组开始遍历，可以理解为遍历我们以前在数据结构里学的有序链表:)
要强调一点的就是Block对象本身实现了compare函数。所以产生的数组才是有序的。下面的代码才可以这么写。
Block oldReport[] = node.getBlocks();
while (oldReport != null && newReport != null && oldPos < oldReport.length && newPos < newReport.length) {
int cmp = oldReport[oldPos].compareTo(newReport[newPos]);//开始比较二者的blkid
if (cmp == 0) {//如果二者的ID一致，则说明二者是一样的一个block文件块，2个指示器都需要自增1
// Do nothing, blocks are the same
oldPos++;//不解释
newPos++;//不解释
} else if (cmp < 0) {//表明，老的有有一块新的没有，则老的这一块应该被删除，已经失效了。
// The old report has a block the new one does not
removeStoredBlock(oldReport[oldPos], node);//未完待续
oldPos++;
} else {//表明有一个新的块，则需要在本地namenode更新。
// The new report has a block the old one does not
addStoredBlock(newReport[newPos], node);//未完待续
newPos++;
}
}
while (oldReport != null && oldPos < oldReport.length) {//如果老的还有更多，则都是过时的。
// The old report has a block the new one does not
removeStoredBlock(oldReport[oldPos], node);
oldPos++;
}
while (newReport != null && newPos < newReport.length) {//如果还有新的，则都需要新增到本地。
// The new report has a block the old one does not
addStoredBlock(newReport[newPos], node);
newPos++;
}
//
// Modify node so it has the new blockreport
//
node.updateBlocks(newReport); //依据最新的block来更新本地DataNode的信息。
//
// We've now completely updated the node's block report profile.
// We now go through all its blocks and find which ones are invalid,
// no longer pending, or over-replicated.
//
// (Note it's not enough to just invalidate blocks at lease expiry
// time; datanodes can go down before the client's lease on
// the failed file expires and miss the "expire" event.)
//
// This function considers every block on a datanode, and thus
// should only be invoked infrequently.
//
Vector obsolete = new Vector();
for (Iterator it = node.getBlockIterator(); it.hasNext(); ) {
Block b = (Block) it.next();
if (! dir.isValidBlock(b) && ! pendingCreateBlocks.contains(b)) {
LOG.info("Obsoleting block " + b);
obsolete.add(b);
}
}
return (Block[]) obsolete.toArray(new Block[obsolete.size()]);
}
上面有几个函数没有解释，先解释
synchronized void removeStoredBlock(Block block, DatanodeInfo node) {
源码如下：
synchronized void removeStoredBlock(Block block, DatanodeInfo node) {
TreeSet containingNodes = (TreeSet) blocksMap.get(block);//找到对应的节点集合
if (containingNodes == null || ! containingNodes.contains(node)) {
throw new IllegalArgumentException("No machine mapping found for block " + block + ", which should be at node " + node);
}
containingNodes.remove(node);//需要的话，删除这个节点
//
// It's possible that the block was removed because of a datanode
// failure. If the block is still valid, check if replication is
// necessary. In that case, put block on a possibly-will-
// be-replicated list.
//
if (dir.isValidBlock(block) && (containingNodes.size() < this.desiredReplication)) {
synchronized (neededReplications) {
neededReplications.add(block);//如果备份数低于阀值，则需要进行复制任务。
}
}
//
// We've removed a block from a node, so it's definitely no longer
// in "excess" there.
//
TreeSet excessBlocks = (TreeSet) excessReplicateMap.get(node.getName());//excessReplicateMap是记录某个节点的块冗余了。
if (excessBlocks != null) {
excessBlocks.remove(block);//删除这个节点的这个快
if (excessBlocks.size() == 0) {//如果为0了，自然可以把这个节点从excessReplicateMap移除
excessReplicateMap.remove(node.getName());
}
}
}
接下来再分析 synchronized void addStoredBlock(Block block, DatanodeInfo node) {
synchronized void addStoredBlock(Block block, DatanodeInfo node) {
TreeSet containingNodes = (TreeSet) blocksMap.get(block);
if (containingNodes == null) {
containingNodes = new TreeSet();
blocksMap.put(block, containingNodes);
}
if (! containingNodes.contains(node)) {
containingNodes.add(node);
} else {
LOG.info("Redundant addStoredBlock request received for block " + block + " on node " + node);
}//上面的代码主要是用来更新blocksMap这个数据结构
synchronized (neededReplications) {
if (dir.isValidBlock(block)) {
if (containingNodes.size() >= this.desiredReplication) {
neededReplications.remove(block);//如果已经达到了阀值了，则不需要再复制了
pendingReplications.remove(block);//正在复制的也不需要了
} else if (containingNodes.size() < this.desiredReplication) {
if (! neededReplications.contains(block)) {
neededReplications.add(block);//如果不够阀值，则需要进行备份复制。
}
}
//
// Find how many of the containing nodes are "extra", if any.
// If there are any extras, call chooseExcessReplicates() to
// mark them in the excessReplicateMap.
//
Vector nonExcess = new Vector();
for (Iterator it = containingNodes.iterator(); it.hasNext(); ) {
DatanodeInfo cur = (DatanodeInfo) it.next();
TreeSet excessBlocks = (TreeSet) excessReplicateMap.get(cur.getName());
if (excessBlocks == null || ! excessBlocks.contains(block)) {
nonExcess.add(cur);
}
}
if (nonExcess.size() > this.maxReplication) {
chooseExcessReplicates(nonExcess, block, this.maxReplication);
}
}
}
}
想了一会，其实没必要完全理解这段代码，自己写也是没问题的，每个人都有自己的逻辑思维，相信自己的实力。
好歹咱也看过不少源码了，就那么回事吧。
扫一扫
760
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
