[IO系统]08 IO读流程分析

       本文从整体来分析缓存IO的控制流和数据流，并基于IO系统图来解析读IO：
 
注：对上述层次图的理解参见文章《[IO系统]01 IO子系统》
       一步一步往前走。(内核代码版本4.5.4)
1.1 用户态
       程序的最终目的是要把数据写到磁盘上，如前所述，用户态有两个“打开”函数——read和fread。其中fread是glibc对系统调用read的封装。
参见《[IO系统]02 用户态的文件IO操作》
       fread的流程在用户态的操作比较复杂，涉及到更多的数据复制和处理流程，本文不做介绍，后续单独分析。
       Poxis接口read直接通过系统调用read。
1.2 read系统调用/VFS层
read系统调用是在fs/read_write.c中实现的：

SYSCALL_DEFINE3(read, unsigned int, fd, char __user *, buf, size_t, count)
{
	struct fd f = fdget_pos(fd);
	ssize_t ret = -EBADF;


	if (f.file) {
		loff_t pos = file_pos_read(f.file);
		ret = vfs_read(f.file, buf, count, &pos);
		if (ret >= 0)
			file_pos_write(f.file, pos);
		fdput_pos(f);
	}
	return ret;
}

       函数解析：
1. 通过函数fdget_pos将整形的句柄ID fd转化为内核数据结构fd，可称之为文件句柄描述符。
fd结构体如下：
struct fd {
struct file *file;/* 文件对象 */
unsigned int flags;
};
       内核中文件系统各结构体之间的关系参照文章《[IO系统]因OPEN建立的结构体关系图》
2. 获取文件file的操作位置，也可以理解光标（记录在pos）；
3. 调用VFS接口vfs_read()实现读操作：检查参数的有效性，通过__vfs_read函数调用具体文件系统（如ext4，xfs，btrfs，ocfs2）的write函数。

ssize_t vfs_read(struct file *file, char __user *buf, size_t count, loff_t *pos)
{
	ssize_t ret;


	if (!(file->f_mode & FMODE_READ)) /* 判断是否具有读权限 */
		return -EBADF;
	if (!(file->f_mode & FMODE_CAN_READ))
		return -EINVAL;
	if (unlikely(!access_ok(VERIFY_WRITE, buf, count)))
		return -EFAULT;


	ret = rw_verify_area(READ, file, pos, count);
	if (ret >= 0) {
		count = ret;
		ret = __vfs_read(file, buf, count, pos);
		if (ret > 0) {
			fsnotify_access(file);
			add_rchar(current, ret);
		}
		inc_syscr(current);
	}


	return ret;
}

当然如果没有定义具体的read函数，则通过接口read_iter实现数据读操作。

ssize_t __vfs_read(struct file *file, char __user *buf, size_t count,
		   loff_t *pos)
{
	if (file->f_op->read)
		return file->f_op->read(file, buf, count, pos);
	else if (file->f_op->read_iter)
		return new_sync_read(file, buf, count, pos);
	else
		return -EINVAL;
}
EXPORT_SYMBOL(__vfs_read);

new_sync_read是对参数进行一次封装，封装为结构体iov_iter
4. 更新file的pos变量。


1.3 具体文件系统层
如果具体文件系统，比如ext4，没有read接口，则调用read_iter接口来去读取数据：
const struct file_operations ext4_file_operations = {
.llseek = ext4_llseek,
.read_iter = generic_file_read_iter,
.write_iter = ext4_file_write_iter,
…
};
而read_iter时直接调用generic_file_read_iter函数。

/**
 * generic_file_read_iter - generic filesystem read routine
 * @iocb:	kernel I/O control block
 * @iter:	destination for the data read
 *
 * This is the "read_iter()" routine for all filesystems
 * that can use the page cache directly.
 */
ssize_t
generic_file_read_iter(struct kiocb *iocb, struct iov_iter *iter)
{
     /* 解析参数*/
	struct file *file = iocb->ki_filp;
	ssize_t retval = 0;
	loff_t *ppos = &iocb->ki_pos;
	loff_t pos = *ppos;
	size_t count = iov_iter_count(iter);


	if (!count)
		goto out; /* skip atime */


	if (iocb->ki_flags & IOCB_DIRECT) { /* 直接IO读 */
		struct address_space *mapping = file->f_mapping;
		struct inode *inode = mapping->host;
		loff_t size;


		size = i_size_read(inode);
		retval = filemap_write_and_wait_range(mapping, pos,
					pos + count - 1);/* 刷mapping下的脏页,保证数据一致性 */
		if (!retval) {/* 脏页回刷失败，则通过直接IO写回存储设备 */
			struct iov_iter data = *iter;
			retval = mapping->a_ops->direct_IO(iocb, &data, pos);
		}


		if (retval > 0) {
			*ppos = pos + retval;
			iov_iter_advance(iter, retval);
		}


		/*
		 * Btrfs can have a short DIO read if we encounter
		 * compressed extents, so if there was an error, or if
		 * we've already read everything we wanted to, or if
		 * there was a short read because we hit EOF, go ahead
		 * and return.  Otherwise fallthrough to buffered io for
		 * the rest of the read.  Buffered reads will not work for
		 * DAX files, so don't bother trying.
		 */
		if (retval < 0 || !iov_iter_count(iter) || *ppos >= size ||
		    IS_DAX(inode)) {
			file_accessed(file);
			goto out;
		}
	}
     /* 缓存IO读 */
	retval = do_generic_file_read(file, ppos, iter, retval);
out:
	return retval;
}
EXPORT_SYMBOL(generic_file_read_iter);


直接IO：
       控制流，若为直接IO，在执行读取操作时，首先会将mapping下的脏数据刷回磁盘，然后调用do_generic_file_read读取数据。
       数据流，数据依旧存放在用户态缓存中，并不需要将数据复制到page cache中，减少了数据复制次数。
缓存IO：
       控制流，若进入BufferIO，则直接调用do_generic_file_read来读取数据。
       数据流，数据从用户态复制到内核态page cache中。

       函数do_generic_file_read分析
/**
 * do_generic_file_read - generic file read routine
 * @filp:	the file to read
 * @ppos:	current file position
 * @iter:	data destination
 * @written:	already copied
 *
 * This is a generic file read routine, and uses the
 * mapping->a_ops->readpage() function for the actual low-level stuff.
 *
 * This is really ugly. But the goto's actually try to clarify some
 * of the logic when it comes to error handling etc.
 */
static ssize_t do_generic_file_read(struct file *filp, loff_t *ppos,
		struct iov_iter *iter, ssize_t written)
{
	struct address_space *mapping = filp->f_mapping;
	struct inode *inode = mapping->host;
	struct file_ra_state *ra = &filp->f_ra;
	pgoff_t index;
	pgoff_t last_index;
	pgoff_t prev_index;
	unsigned long offset;      /* offset into pagecache page */
	unsigned int prev_offset;
	int error = 0;


/* 因为读取是按照页来的，所以需要计算本次读取的第一个page*/
	index = *ppos >> PAGE_SHIFT;
	prev_index = ra->prev_pos >> PAGE_SHIFT; /* 上次预读page的起始索引 */
	prev_offset = ra->prev_pos & (PAGE_SIZE-1); /* 上次预读的起始位置 */
	last_index = (*ppos + iter->count + PAGE_SIZE-1) >> PAGE_SHIFT; /*读取最后一个页 */
	offset = *ppos & ~PAGE_MASK;


	for (;;) {
		struct page *page;
		pgoff_t end_index;
		loff_t isize;
		unsigned long nr, ret;


		cond_resched();
find_page:
		page = find_get_page(mapping, index); /*在radix树中查找相应的page*/ 
		if (!page) { /*在radix树中查找相应的page*/ 
			/* 如果没有找到page，内存中没有将数据,先进行预读 */
			page_cache_sync_readahead(mapping,
					ra, filp,
					index, last_index - index);
			page = find_get_page(mapping, index); /*在radix树中再次查找相应的page*/
			if (unlikely(page == NULL))
				goto no_cached_page;
		}
		if (PageReadahead(page)) { 
/* 发现找到的page已经是预读的情况了，再继续异步预读,此处是基于经验的优化 */
			page_cache_async_readahead(mapping,
					ra, filp, page,
					index, last_index - index);
		}
if (!PageUptodate(page)) {/* 数据内容不是最新，则需要更新数据内容 */
			/*
			 * See comment in do_read_cache_page on why
			 * wait_on_page_locked is used to avoid unnecessarily
			 * serialisations and why it's safe.
			 */
			wait_on_page_locked_killable(page);
			if (PageUptodate(page))
				goto page_ok;


			if (inode->i_blkbits == PAGE_SHIFT ||
					!mapping->a_ops->is_partially_uptodate)
				goto page_not_up_to_date;
			if (!trylock_page(page))
				goto page_not_up_to_date;
			/* Did it get truncated before we got the lock? */
			if (!page->mapping)
				goto page_not_up_to_date_locked;
			if (!mapping->a_ops->is_partially_uptodate(page,
							offset, iter->count))
				goto page_not_up_to_date_locked;
			unlock_page(page);
		}
page_ok: /* 数据内容是最新的 */
		/*
		 * i_size must be checked after we know the page is Uptodate.
		 *
		 * Checking i_size after the check allows us to calculate
		 * the correct value for "nr", which means the zero-filled
		 * part of the page is not copied back to userspace (unless
		 * another truncate extends the file - this is desired though).
		 */
/*下面这段代码是在page中的内容ok的情况下将page中的内容拷贝到用户空间去，主要的逻辑分为检查参数是否合法进性拷贝操作*/ 
/*合法性检查*/
		isize = i_size_read(inode);
		end_index = (isize - 1) >> PAGE_SHIFT;
		if (unlikely(!isize || index > end_index)) {
			put_page(page);
			goto out;
		}


		/* nr is the maximum number of bytes to copy from this page */
		nr = PAGE_SIZE;
		if (index == end_index) {
			nr = ((isize - 1) & ~PAGE_MASK) + 1;
			if (nr <= offset) {
				put_page(page);
				goto out;
			}
		}
		nr = nr - offset;


		/* If users can be writing to this page using arbitrary
		 * virtual addresses, take care about potential aliasing
		 * before reading the page on the kernel side.
		 */
		if (mapping_writably_mapped(mapping))
			flush_dcache_page(page);


		/*
		 * When a sequential read accesses a page several times,
		 * only mark it as accessed the first time.
		 */
		if (prev_index != index || offset != prev_offset)
			mark_page_accessed(page);
		prev_index = index;


		/*
		 * Ok, we have the page, and it's up-to-date, so
		 * now we can copy it to user space...
		 */
         /* 将数据从内核态复制到用户态 */
		ret = copy_page_to_iter(page, offset, nr, iter);
		offset += ret;
		index += offset >> PAGE_SHIFT;
		offset &= ~PAGE_MASK;
		prev_offset = offset;


		put_page(page);
		written += ret;
		if (!iov_iter_count(iter))
			goto out;
		if (ret < nr) {
			error = -EFAULT;
			goto out;
		}
		continue;


page_not_up_to_date:
		/* Get exclusive access to the page ...互斥访问 */
		error = lock_page_killable(page);
		if (unlikely(error))
			goto readpage_error;


page_not_up_to_date_locked:
		/* Did it get truncated before we got the lock? */
/* 获取到锁之后，发现这个page没有被映射了，
* 可能是在获取锁之前就被其它模块释放掉了，重新开始获取lock*/
		if (!page->mapping) {
			unlock_page(page);
			put_page(page);
			continue;
		}
		/* Did somebody else fill it already? */
         /* 获取到锁后发现page中的数据已经ok了，不需要再读取数据 */
		if (PageUptodate(page)) {
			unlock_page(page);
			goto page_ok;
		}


readpage:
		/* 数据读取操作
		 * A previous I/O error may have been due to temporary
		 * failures, eg. multipath errors.
		 * PG_error will be set again if readpage fails.
		 */
		ClearPageError(page);
		/* Start the actual read. The read will unlock the page. */
		error = mapping->a_ops->readpage(filp, page);
         …
		if (…error) {
			….
			goto readpage_error;
		}
          …
		goto page_ok;


readpage_error:
		/* UHHUH! A synchronous read error occurred. Report it，同步读取失败 */
		put_page(page);
		goto out;


no_cached_page:
		/* 系统中没有数据，又不进行预读的情况，显示的分配page，并读取page
		 * Ok, it wasn't cached, so we need to create a new page.. */
		page = page_cache_alloc_cold(mapping);
		…
		error = add_to_page_cache_lru(page, mapping, index,
				mapping_gfp_constraint(mapping, GFP_KERNEL));
		if (error) {
			…
			goto out;
		}
		goto readpage;
	}


out:
	ra->prev_pos = prev_index;
	ra->prev_pos <<= PAGE_SHIFT;
	ra->prev_pos |= prev_offset;


	*ppos = ((loff_t)index << PAGE_SHIFT) + offset;
	file_accessed(filp);
	return written ? written : error;
}

       文件系统通过预读机制将磁盘上的数据读入到page cache中


1.4 Page Cache层
       Page Cache是文件数据在内存中的副本，因此Page Cache管理与内存管理系统和文件系统都相关：一方面Page Cache作为物理内存的一部分，需要参与物理内存的分配回收过程，另一方面Page Cache中的数据来源于存储设备上的文件，需要通过文件系统与存储设备进行读写交互。从操作系统的角度考虑，Page Cache可以看做是内存管理系统与文件系统之间的联系纽带。因此，Page Cache管理是操作系统的一个重要组成部分，它的性能直接影响着文件系统和内存管理系统的性能。
本文不做详细介绍，后续讲解。


1.5 通用块层
       同IO写流程分析。


1.6 IO调度层
       同IO写流程分析。


1.7 设备驱动层
       同IO写流程分析。


1.8 设备层
       暂不分析。


1.9 参考文献
       [博客] http://blog.chinaunix.net/uid-28236237-id-4030381.html