项目源网站 tinywebserver。这个模块是缓冲区模块,相比于TCP UDP的循环队列,这个缓冲区的操作牺牲了一部分存储空间的重复利用,更加的直白易懂。
代码
头文件
/*
* @Author : mark
* @Date : 2020-06-26
* @copyleft Apache 2.0
*/
#ifndef BUFFER_H
#define BUFFER_H
#include <cstring> //perror
#include <iostream>
#include <unistd.h> // write
#include <sys/uio.h> //readv
#include <vector> //readv
#include <atomic>
#include <assert.h>
class Buffer {
public:
Buffer(int initBuffSize = 1024);
~Buffer() = default;
size_t WritableBytes() const;
size_t ReadableBytes() const ;
size_t PrependableBytes() const;
const char* Peek() const;
void EnsureWriteable(size_t len);
void HasWritten(size_t len);
void Retrieve(size_t len);
void RetrieveUntil(const char* end);
void RetrieveAll() ;
std::string RetrieveAllToStr();
const char* BeginWriteConst() const;
char* BeginWrite();
void Append(const std::string& str);
void Append(const char* str, size_t len);
void Append(const void* data, size_t len);
void Append(const Buffer& buff);
ssize_t ReadFd(int fd, int* Errno);
ssize_t WriteFd(int fd, int* Errno);
private:
char* BeginPtr_();
const char* BeginPtr_() const;
void MakeSpace_(size_t len);
// buffer_start readPos_ writePos_ buffer_end
// |----------------------|-------------------|-------------------|
// | | 数据 | |
std::vector<char> buffer_;
std::atomic<std::size_t> readPos_; //读指针的位置 [readPos_, writePos_) 这一块是可读的正确的数据
std::atomic<std::size_t> writePos_; //写指针的位置 [writePos_, buffer_end) 这一块是空闲区域
};
#endif //BUFFER_H
源文件
/*
* @Author : mark
* @Date : 2020-06-26
* @copyleft Apache 2.0
*/
#include "buffer.h"
Buffer::Buffer(int initBuffSize) : buffer_(initBuffSize), readPos_(0), writePos_(0) {}
size_t Buffer::ReadableBytes() const
{
// 可以读的大小
return writePos_ - readPos_;
}
size_t Buffer::WritableBytes() const
{
// 可以写的大小
return buffer_.size() - writePos_;
}
size_t Buffer::PrependableBytes() const
{
return readPos_;
}
const char *Buffer::Peek() const
{
return BeginPtr_() + readPos_;
}
void Buffer::Retrieve(size_t len)
{
//取回数据readPos_到readPos_+len-1的数据
assert(len <= ReadableBytes());
readPos_ += len;
}
void Buffer::RetrieveUntil(const char *end)
{
//取回数据readPos_到end之间的数据
assert(Peek() <= end);
Retrieve(end - Peek());
}
void Buffer::RetrieveAll()
{
//清空buffer_
bzero(&buffer_[0], buffer_.size()); // 把buffer_里的数据全部设为0
readPos_ = 0;
writePos_ = 0;
}
std::string Buffer::RetrieveAllToStr()
{
//将缓冲区所有的数据都转换成str
std::string str(Peek(), ReadableBytes());
RetrieveAll();
return str;
}
const char *Buffer::BeginWriteConst() const
{
return BeginPtr_() + writePos_;
}
char *Buffer::BeginWrite()
{
return BeginPtr_() + writePos_;
}
void Buffer::HasWritten(size_t len)
{
writePos_ += len;
}
void Buffer::Append(const std::string &str)
{
Append(str.data(), str.length());
}
void Buffer::Append(const void *data, size_t len)
{
assert(data);
Append(static_cast<const char *>(data), len);
}
void Buffer::Append(const char *str, size_t len)
{
// 把数据[str, str+len)放到buffer_里
assert(str);
EnsureWriteable(len);
std::copy(str, str + len, BeginWrite());
HasWritten(len);
}
void Buffer::Append(const Buffer &buff)
{
Append(buff.Peek(), buff.ReadableBytes());
}
void Buffer::EnsureWriteable(size_t len)
{
// 确保buffer_里有足够的空间,有的话就返回,没有的话就扩大buffer_
if (WritableBytes() < len)
{
MakeSpace_(len);
}
assert(WritableBytes() >= len);
}
ssize_t Buffer::ReadFd(int fd, int *saveErrno)
{
// 把数据从fd读入buffer
char buff[65535];
struct iovec iov[2];
const size_t writable = WritableBytes();
iov[0].iov_base = BeginPtr_() + writePos_;
iov[0].iov_len = writable;
iov[1].iov_base = buff;
iov[1].iov_len = sizeof(buff);
const ssize_t len = readv(fd, iov, 2); // 分散读,先把数据放到buffer_里,溢出了之后放到buff里
if (len < 0)
{
*saveErrno = errno;
}
else if (static_cast<size_t>(len) <= writable)
{
writePos_ += len;
}
else
{
writePos_ = buffer_.size();
Append(buff, len - writable);
}
return len;
}
ssize_t Buffer::WriteFd(int fd, int *saveErrno)
{
//将buffer_里的数据写到fd
size_t readSize = ReadableBytes();
ssize_t len = write(fd, Peek(), readSize);
if (len < 0)
{
*saveErrno = errno;
return len;
}
readPos_ += len;
return len;
}
char *Buffer::BeginPtr_()
{
return &*buffer_.begin();
}
const char *Buffer::BeginPtr_() const
{
return &*buffer_.begin();
}
void Buffer::MakeSpace_(size_t len)
{
if (WritableBytes() + PrependableBytes() < len)
{
// 如果buffer不够用了,就扩大buffer
// buffer_begin readPos_ writePos_ buffer_end -->
// buffer_begin readPos_ writePos_ buffer_end+len
buffer_.resize(writePos_ + len + 1);
}
else
{
// 够用,把数据搬移到buffer_begin
// buffer_begin readPos_ writePos_ buffer_end -->
// readPos_ writePos_ buffer_end
size_t readable = ReadableBytes();
std::copy(BeginPtr_() + readPos_, BeginPtr_() + writePos_, BeginPtr_());
readPos_ = 0;
writePos_ = readPos_ + readable;
assert(readable == ReadableBytes());
}
}
缓冲区定义,读,写
缓冲区的定义
// buffer_start readPos_ writePos_ buffer_end
// |----------------------|-------------------|-------------------|
// | | 数据 | |
std::vector<char> buffer_;
std::atomic<std::size_t> readPos_; //读指针的位置 [readPos_, writePos_) 这一块是可读的正确的数据
std::atomic<std::size_t> writePos_; //写指针的位置 [writePos_, buffer_end) 这一块是空闲区域
用vector buffer_相当于就是一串存储一个字节的数据,就是缓冲区。
readPos_ 和 writePos_之间的,就是有效的数据。
从fd里读数据到缓冲区
读数据就是把writePos_向后移动,直到移动到末尾,不够就扩容
ssize_t Buffer::ReadFd(int fd, int *saveErrno)
{
// 把数据从fd读入buffer
char buff[65535];
struct iovec iov[2];
const size_t writable = WritableBytes();
iov[0].iov_base = BeginPtr_() + writePos_;
iov[0].iov_len = writable;
iov[1].iov_base = buff;
iov[1].iov_len = sizeof(buff);
const ssize_t len = readv(fd, iov, 2); // 分散读,先把数据放到buffer_里,溢出了之后放到buff里
if (len < 0)
{
*saveErrno = errno;
}
else if (static_cast<size_t>(len) <= writable)
{
writePos_ += len;
}
else
{
writePos_ = buffer_.size();
Append(buff, len - writable);
}
return len;
}
这里他比较高明的地方在于,用了readv这个函数,先把数据读到缓冲区,缓冲区溢出了之后,扩容,再把溢出来的数据添加到缓冲区里。避免了开始的时候要对fd里的数据大小进行判断。
- Append
void Buffer::Append(const std::string &str)
{
Append(str.data(), str.length());
}
void Buffer::Append(const void *data, size_t len)
{
assert(data);
Append(static_cast<const char *>(data), len);
}
void Buffer::Append(const char *str, size_t len)
{
// 把数据[str, str+len)放到buffer_里
assert(str);
EnsureWriteable(len);
std::copy(str, str + len, BeginWrite());
HasWritten(len);
}
void Buffer::Append(const Buffer &buff)
{
Append(buff.Peek(), buff.ReadableBytes());
}
这几个Append都是一个,把const char *str里读取len的数据到缓冲区
把缓冲区的数据写到fd里
- WriteFd
ssize_t Buffer::WriteFd(int fd, int *saveErrno)
{
//将buffer_里的数据写到fd
size_t readSize = ReadableBytes();
ssize_t len = write(fd, Peek(), readSize);
if (len < 0)
{
*saveErrno = errno;
return len;
}
readPos_ += len;
return len;
}
这没啥好说的,就是把数据写到fd里
- Retrive
void Buffer::Retrieve(size_t len)
{
//取回数据readPos_到readPos_+len-1的数据
assert(len <= ReadableBytes());
readPos_ += len;
}
void Buffer::RetrieveUntil(const char *end)
{
//取回数据readPos_到end之间的数据
assert(Peek() <= end);
Retrieve(end - Peek());
}
void Buffer::RetrieveAll()
{
//清空buffer_
bzero(&buffer_[0], buffer_.size()); // 把buffer_里的数据全部设为0
readPos_ = 0;
writePos_ = 0;
}
这三个Retrive函数,就是把已经写入的数据,撤回来,将其标记为无效数据,就是把readPos_向后移size个字节。
扩容函数
这个MakeSpace_函数只在Append 的时候调用
void Buffer::MakeSpace_(size_t len)
{
if (WritableBytes() + PrependableBytes() < len)
{
// 如果buffer不够用了,就扩大buffer
// buffer_begin readPos_ writePos_ buffer_end -->
// buffer_begin readPos_ writePos_ buffer_end+len
buffer_.resize(writePos_ + len + 1);
}
else
{
// 够用,把数据搬移到buffer_begin
// buffer_begin readPos_ writePos_ buffer_end -->
// readPos_ writePos_ buffer_end
size_t readable = ReadableBytes();
std::copy(BeginPtr_() + readPos_, BeginPtr_() + writePos_, BeginPtr_());
readPos_ = 0;
writePos_ = readPos_ + readable;
assert(readable == ReadableBytes());
}
}
- 缓冲区的数据够用,writePos_后面的空间够了
- 缓冲区的数据够用,writePos_后面的空间不够 --> 将readPos_ 到writePos_的数据移动到缓冲区的头部
- 缓冲区的数据不够用,建一个新的vector,将writePos_到writePos_数据复制过去,然后扩容
这就是比较高明的地方,循环队列每次添加的时候都要判断当前指针是不是到了末尾,就很麻烦,这种简单的方法减少了判断。
测试代码
gtest
#include <gtest/gtest.h>
#include <string>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include "buffer.h"
namespace WebServerTest
{
using namespace std;
// 匿名空间,避免命名冲突
class BufferTest : public testing::Test
{
protected:
void SetUp() override
{
buffer_.RetrieveAll(); // 清空缓冲区
file_data_ = "file write test";
}
Buffer buffer_;
string file_data_;
};
TEST_F(BufferTest, InintState)
{
// 初始状态下的测试
EXPECT_EQ(buffer_.ReadableBytes(), 0); // 初始状态下,缓冲区为空
EXPECT_EQ(buffer_.WritableBytes(), 1024); // 初始状态下,可写字节数为1024
EXPECT_EQ(buffer_.PrependableBytes(), 0); // 初始状态下,头部预留字节数为0
EXPECT_EQ(buffer_.BeginWrite(), buffer_.Peek()); // 初始状态下,开始写指针与Peek指针相同
};
TEST_F(BufferTest, AppendData)
{
// 添加数据
const char *str = "Hello, World!";
buffer_.Append(str, strlen(str)); // 添加字符串
EXPECT_EQ(buffer_.ReadableBytes(), strlen(str)); // 可读字节数等于添加的字符串长度
EXPECT_EQ(buffer_.WritableBytes(), 1024 - strlen(str)); // 可写字节数等于1024减去添加的字符串长度
EXPECT_EQ(buffer_.PrependableBytes(), 0); // 头部预留字节数为0
EXPECT_EQ(0, strcmp(buffer_.Peek(), str)); // Peek指针指向添加的字符串开头
buffer_.Append("1234567890", 10); // 添加10个字符
EXPECT_EQ(buffer_.ReadableBytes(), strlen(str) + 10); // 可读字节数等于添加的字符串长度加10
EXPECT_EQ(buffer_.WritableBytes(), 1024 - strlen(str) - 10); // 可写字节数等于1024减去添加的字符串长度减去10
EXPECT_EQ(buffer_.PrependableBytes(), 0); // 头部预留字节数为0
}
TEST_F(BufferTest,AppendDataOverflow){
// 添加数据溢出
Buffer small_buffer(10);
const char *str = "aaaa aaaa aaaa";
small_buffer.Append(str, strlen(str));
EXPECT_EQ(small_buffer.ReadableBytes(), strlen(str));
EXPECT_EQ(small_buffer.WritableBytes(), 1);
string res_str=small_buffer.RetrieveAllToStr();
EXPECT_STREQ(res_str.c_str(),str);
int fd = open("append_data_overflow.txt", O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); // 创建一个名为 output.txt 的文件
if (fd < 0) {
cerr << "Failed to open file for writing.";
}
string new_data="abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
int ret = write(fd,new_data.c_str(),new_data.size());
if(ret<0){
cerr<<"Failed to write data to file."<<endl;
}
close(fd); // 关闭文件描述符
int fd_read = open("append_data_overflow.txt", O_RDONLY);
int Errno = 0;
small_buffer.ReadFd(fd_read,&Errno);
if(Errno!=0){
cerr<<"Failed to read data from file."<<endl;
}
close(fd_read); // 关闭文件描述符
EXPECT_STREQ(new_data.c_str(),small_buffer.Peek());
}
TEST_F(BufferTest, Retrive)
{
// 回退数据
string str = "Hello, World!1234567890";
buffer_.Append(str);
EXPECT_EQ(buffer_.RetrieveAllToStr(), str);
// 数据等于 初始状态下的测试
EXPECT_EQ(buffer_.ReadableBytes(), 0);
EXPECT_EQ(buffer_.WritableBytes(), 1024);
EXPECT_EQ(buffer_.PrependableBytes(), 0);
EXPECT_EQ(buffer_.BeginWrite(), buffer_.Peek());
buffer_.Append(str);
buffer_.Retrieve(5); // 回退5个字节
EXPECT_EQ(buffer_.ReadableBytes(), str.size() - 5); // 可读字节数等于添加的字符串长度减去5
EXPECT_EQ(buffer_.WritableBytes(), 1024 - str.size()); // 可写字节数等于1024减去添加的字符串长度加5
EXPECT_EQ(buffer_.PrependableBytes(), 5); // 头部预留字节数为5
EXPECT_EQ(0, strcmp(buffer_.Peek(), str.c_str() + 5)); // Peek指针指向回退后的字符串开头
}
TEST_F(BufferTest, Append)
{
// 添加数据
const char *str = "Hello, World!";
buffer_.Append(str, strlen(str)); // 添加字符串
EXPECT_STREQ(str, buffer_.Peek());
buffer_.RetrieveAll();
string str2 = "1234567890";
buffer_.Append(str2);
EXPECT_STREQ(str2.c_str(), buffer_.Peek());
buffer_.RetrieveAll();
void *ptr=(void *)str;
buffer_.Append(ptr, strlen(str));
EXPECT_STREQ(str, buffer_.Peek());
buffer_.RetrieveAll();
}
// 测试 WriteFd 函数
TEST_F(BufferTest, WriteFdTest) {
int fd = open("write_fd_test.txt", O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); // 创建一个名为 output.txt 的文件
if (fd < 0) {
cerr<< "Failed to open file for writing.";
}
buffer_.Append(file_data_); // 添加字符串
int Errno = 0;
ssize_t ret = buffer_.WriteFd(fd, &Errno);
EXPECT_EQ(Errno, 0);
EXPECT_GE(ret, file_data_.size());
close(fd); // 关闭文件描述符
//测试文件的内容是否正确
int fd2 = open("write_fd_test.txt", O_RDONLY);
char buffer[1024];
ret = read(fd2, buffer, 1024);
buffer[ret] = '\0';
EXPECT_STREQ(file_data_.c_str(), buffer);
close(fd2); // 关闭文件描述符
}
TEST_F(BufferTest, ReadData){
// 读取数据
int fd2 = open("write_fd_test.txt", O_RDONLY);
int Errno = 0;
buffer_.ReadFd(fd2,&Errno);
close(fd2); // 关闭文件描述符
EXPECT_STREQ(file_data_.c_str(), buffer_.Peek());
}
};