【HDFS写流程源码分析】
1. Client --> new Configuration(); 【参考权威指南：6.1】
-->调用无参构造器Configuration()
-->调用构造器Configuration(boolean loadDefaults)：true
说明：为Configuration对象分配空间，读取配置文件信息，
封装到对象的属性中
2.Client --> FileSystem.get(conf);【获取分布式文件系统对象DistributedFileSystem】
-->【FileSystem.class】static get(Configuration conf)
-->【FileSystem.class】static get(URI uri, Configuration conf)
{
//根据uri等一些参数从缓存队中获取FileSystem对象
return CACHE.get(uri, conf);
}
-->【FileSystem$Cache】get(URI uri, Configuration conf){
Key key = new Key(uri, conf);
return getInternal(uri, conf, key);
}
-->调用【FileSystem$Cache$Key】的构造器，
创建Key对象，封装uri,conf。
说明：除了处理uri(忽略大小写操作)之外，
获取了客户端的用户组等信息
-->【FileSystem$Cache】getInternal(URI uri, Configuration conf,
Key key){
fs = createFileSystem(uri, conf);
}
-->【FileSystem.class】createFileSystem(URI uri, Configuration conf){
//获取org.apache.hadoop.hdfs.DistributedFileSystem类名
Class<?> clazz = getFileSystemClass(uri.getScheme(), conf);
//通过反射机制创建文件系统的实例对象（DistributedFileSystem）
FileSystem fs = (FileSystem)ReflectionUtils.newInstance(clazz, conf);
//对DistributedFileSystem进行初始化
//给其中一个属性dfs赋值为new DFSClient()
fs.initialize(uri, conf);
return fs; //返回分布式文件系统对象
}
3.Client -->new Path("/r1/r2/f1.txt");//使用Path类对象来描述一个文件路径
4.Client -->fs.create(path);
-->【FileSystem.class】create(Path f)
-->【FileSystem.class】create(Path f, boolean overwrite)
-->【FileSystem.class】create(Path f, boolean overwrite, int bufferSize, short replication, long blockSize)
-->【FileSystem.class】create(Path f,boolean overwrite, int bufferSize,short replication, long blockSize,
Progressable progress)
-->【DistributedFileSystem】create(Path f, FsPermission permission, boolean overwrite, int bufferSize,
short replication, long blockSize, Progressable progress)
-->【DistributedFileSystem】create(final Path f, final FsPermission permission, final EnumSet<CreateFlag> cflags,
final int bufferSize, final short replication, final long blockSize, final Progressable progress, final ChecksumOpt checksumOpt)
说明：调用了接口【FileSystemLinkResolver】，
创建匿名内部类,重写doCall方法
-->调用doCall方法
-->【DFSClient】create(String src,FsPermission permission, EnumSet<CreateFlag> flag, short replication, long blockSize, Progressable progress,int buffersize, ChecksumOpt checksumOpt)
说明：跟namenode进行通信
-->【DFSClient】create(String src, FsPermission permission,EnumSet<CreateFlag> flag, boolean createParent,short replication,long blockSize, Progressable progress ,int buffersize,ChecksumOpt checksumOpt, InetSocketAddress[] favoredNodes)
说明：添加favorednode，提示namenode应该将文件块放在哪里，
想获取最优的存储节点
{
//获取流对象
final DFSOutputStream result = DFSOutputStream.newStreamForCreate(this,src, masked, flag, createParent, replication, blockSize, progress, buffersize, dfsClientConf.createChecksum(checksumOpt),getFavoredNodesStr(favoredNodes));
beginFileLease(result.getFileId(), result);//开始使用文件租约
return result;
}
-->【DFSOutputStream】newStreamForCreate(DFSClient dfsClient, String src,FsPermission masked, EnumSet<CreateFlag> flag, boolean createParent,short replication, long blockSize, Progressable progress, int buffersize,DataChecksum checksum, String[] favoredNodes)
{
//客户端通过rpc协议与namenode通信，namenode进行一些校验(权限，覆盖，路径)。成功后，在内存和编辑日志文件中记录一条信息，并返回封装HdfsFileStatus
stat = dfsClient.namenode.create(src, masked, dfsClient.clientName,new EnumSetWritable<CreateFlag>(flag), createParent, replication,blockSize, SUPPORTED_CRYPTO_VERSIONS);
break;
/*
主要目的是创建流对象，在内部维护着：dataQueue,ackQueue,文件状态：currentPacket, dataStreamer线程，
*/
final DFSOutputStream out = new DFSOutputStream(dfsClient, src, stat,flag, progress, checksum, favoredNodes);
out.start();//启动DataStreamer线程
return out;//返回流对象
}
-->DFSOutputStream(DFSClient dfsClient, String src, HdfsFileStatus stat,EnumSet<CreateFlag> flag, Progressable progress,DataChecksum checksum, String[] favoredNodes){
/*
调用下面构造器进行一些属性的初始化，同时给FSOutputSummer里的属性赋值
protected FSOutputSummer(DataChecksum sum) {
this.sum = sum;
// 内部缓冲区buf的大小":512*9 = 4608
this.buf = new byte[sum.getBytesPerChecksum() * BUFFER_NUM_CHUNKS];
// 存储校验和的内部缓缓冲区大小：4*9 = 36
this.checksum = new byte[getChecksumSize() * BUFFER_NUM_CHUNKS];
this.count = 0;
}
FSOutputSummer：这是一个通用的输出流，用于在将数据写入底层流之前生成数据的校验和
*/
this(dfsClient, src, progress, stat, checksum);
this.shouldSyncBlock = flag.contains(CreateFlag.SYNC_BLOCK);
//计算包中的基本数据信息：如packetSize chunkSize等等
computePacketChunkSize(dfsClient.getConf().writePacketSize, bytesPerChecksum);
streamer = new DataStreamer(stat, null);
if (favoredNodes != null && favoredNodes.length != 0) {
streamer.setFavoredNodes(favoredNodes);
}
}
-->【DFSOutputStream】computePacketChunkSize(int psize, int csize){
// bodySize = 65536-33 = 65503
final int bodySize = psize - PacketHeader.PKT_MAX_HEADER_LEN;
// chunkSize = 512+4 = 516
final int chunkSize = csize + getChecksumSize();
// chunksPerpacket = 126;
chunksPerPacket = Math.max(bodySize/chunkSize, 1);
// packetSize = 516*126 = 65016
packetSize = chunkSize*chunksPerPacket;
if (DFSClient.LOG.isDebugEnabled()) {
DFSClient.LOG.debug("computePacketChunkSize: src=" + src +
", chunkSize=" + chunkSize +
", chunksPerPacket=" + chunksPerPacket +
", packetSize=" + packetSize);
}
}
5.Client -->fsout.write(bs);
-->【FilterOutputStream】write(byte b[])
-->【DataOutputStream】write(byte b[], int off, int len)
-->【FSDataOutputStream.PositionCache】write(byte b[], int off, int len)
-->【FSOutputSummer】write(byte b[], int off, int len)
-->【FSOutputSummer】write1(byte b[], int off, int len)
说明：编写数组的一部分，如果需要，最多刷新一次底层流。
{
int bytesToCopy = buf.length-count;
bytesToCopy = (len<bytesToCopy) ? len : bytesToCopy;
//将客户端的要写的数据字节 复制到本地缓存区中
System.arraycopy(b, off, buf, count, bytesToCopy);
count += bytesToCopy;
if (count == buf.length) {
// local buffer is full 当缓存区满时，进行冲刷到管道中
flushBuffer();
}
return bytesToCopy;
}
6、Client -->fsout.close();
-->【FSDataOutputStream】close()
-->【FSDataOutputStream$PositionCache】close()
-->【DFSOutputStream】close()
-->【DFSOutputStream】closeImpl()
-->【FSOutputSummer】 flushBuffer();
-->【FSOutputSummer】flushBuffer(boolean keep,boolean flushPartial)
说明：强制缓冲输出字节进行校验和，并将其写入底层输出流。
如果缓冲区中有一个尾随的部分块，
-->【FSOutputSummer】writeChecksumChunks(byte b[], int off, int len)
说明：为给定的数据块生成校验和，并将输出块&校验和生成到底层输出流。
{
//对512个字节计算校验和，4个字节大小，存如checksum
sum.calculateChunkedSums(b, off, len, checksum, 0);
for (int i = 0; i < len; i += sum.getBytesPerChecksum()) {
int chunkLen = Math.min(sum.getBytesPerChecksum(), len - i);
int ckOffset = i / sum.getBytesPerChecksum() * getChecksumSize();
writeChunk(b, off + i, chunkLen, checksum, ckOffset, getChecksumSize());
}
}
-->【DFSOutputStream】writeChunk(byte[] b, int offset, int len,byte[] checksum, int ckoff, int cklen)
-->【DFSOutputStream】writeChunkImpl(byte[] b, int offset, int len, byte[] checksum, int ckoff, int cklen)
-->【DFSOutputStream】createPacket(int packetSize, int chunksPerPkt, long offsetInBlock,long seqno, boolean lastPacketInBlock)
说明：调用DFSPacket构造创建对象。
DFSPacket(byte[] buf, int chunksPerPkt, long offsetInBlock, long seqno,
int checksumSize, boolean lastPacketInBlock) {
this.lastPacketInBlock = lastPacketInBlock;
this.numChunks = 0;
this.offsetInBlock = offsetInBlock;
this.seqno = seqno;
// packet的缓存区：65049
this.buf = buf;
// 33
checksumStart = PacketHeader.PKT_MAX_HEADER_LEN;
// 33
checksumPos = checksumStart;
// 33+126*4 =537
dataStart = checksumStart + (chunksPerPkt * checksumSize);
// 537
dataPos = dataStart;
maxChunks = chunksPerPkt;
}
[.....CCCC.........DDDDDD..........]===65049
^ ^
| |
33 537
--> flushInternal(); //flush all data to Datanodes
-->//将文件的最后一个快持久化到Datanode的磁盘上，
然后告诉namenode此文件有哪些块组成
completeFile(lastBlock);
604
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
