实现读写无加锁的不定长的数据环形缓冲

在很多情况下需要用到多线程编程，而多线程很多时候要用到生产者、消费者模型，比如处理网络数据，至少需要一个数据接收线程进行网络数据的接收，但一般不会也在该线程中进行数据的处理，因为这处理可能会比较耗时，而网络数据底层缓冲多比较小，处理不及时就会造成数据丢失，这就需要接收线程响应尽可能的快速，一般为接收数据然后将数据放入缓冲中，而数据处理线程则对数据缓冲进行读取数据然后再处理。这就是典型的生产者和消费者模型。这样的场合需要一个数据容量大、效率高、读写互不干扰、数据类型和长度没有限制的环形缓冲实现。以下是参考的PF_RING开源数据捕获项目的思想，实现环形缓冲的一个C++类，一读一写，读写之间没有互斥，是一个高效的多线程数据交互流动解决办法。

RingBuf.h

/*
 * RingBuf.h
 *
 *  Created on: 2013-2-1
 *      Author: Administrator
 *      不定长环形缓存，每个数据包含一个数据描述t_dInfo，
 *      包含数据的一些描述信息，可自定义该结构体内容，
 *      比如hash值，线程同步信息等，数据格式如下
 *     	|   one mesage in buffer  |
 *     	| buf  head |   buf data  |
 *     	|syn|datalen|datainfo|data||||||||||||||||||||
 */

#ifndef RINGBUF_H_
#define RINGBUF_H_

#include <semaphore.h>

typedef unsigned long long  u_int64_t;
typedef unsigned int 		u_int32_t;

#define RING_SYN_FLAG 		0x74


typedef struct _t_bufhdr
{
	unsigned char bySyn;
	unsigned char byReserve[3];
	u_int32_t buflen;//该条缓存长度，为0表示ringbuf翻转
	unsigned char pbuf[0];
}t_bufhdr;

class CRingBuf {
public:
	CRingBuf();
	virtual ~CRingBuf();

	int SetBufSize(u_int32_t unSize);

	void SetHeadLen(u_int32_t unLen);

	void DestroyRing(void);

	int AddDataToRing(char *data,u_int32_t len,void *datahead);

	int ReadDataFromRing(char **pdata,void *datahead);

	//int GetRingState();
	void PrintRingState(void);

private:

	u_int64_t m_ullTotRead;		//已经读出的数据个数
	u_int64_t m_ullTotInsert;	//成功放入的数据的个数

	u_int32_t m_unRemoveOff;	//读扁移量
	u_int32_t m_unInsertOff;	//写扁移量

	u_int64_t m_ullTotPkts;		//总调用放入数据个数
	u_int64_t m_ullTotLost;		//丢失数据个数

	u_int32_t m_unTotMen;		//缓存大小

	u_int32_t m_unSlotHeadLen;	//头长度

	u_int32_t m_unUserHeadLen;	//用户自定义头长度

	char *m_pBuf;				//缓存指针

	int GetNullLen(void);		//获取剩余空间

	 //可加入信号量，实现阻塞
	sem_t m_tHaveData;

};

#endif /* RINGBUF_H_ */

RingBuf.cpp

/*
 * RingBuf.cpp
 *
 *  Created on: 2013-2-1
 *      Author: Administrator
 */

#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include "RingBuf.h"

CRingBuf::CRingBuf() {
	// TODO Auto-generated constructor stub
	m_ullTotRead = 0;		//已经读出的数据个数
	m_ullTotInsert = 0;	//成功放入的数据的个数

	m_unRemoveOff = 0;	//读扁移量
	m_unInsertOff = 0;	//写扁移量

	m_ullTotPkts = 0;		//总调用放入数据个数
	m_ullTotLost = 0;		//丢失数据个数

	m_unTotMen = 0;		//缓存大小

	m_unSlotHeadLen = 0;	//头长度

	m_pBuf = NULL;

	sem_init(&m_tHaveData, 0, 0);
}

CRingBuf::~CRingBuf() {
	// TODO Auto-generated destructor stub

}

int CRingBuf::SetBufSize(u_int32_t unSize)
{
	if(unSize <= sizeof(t_bufhdr))
	{
		printf("setbufsize size %d err\n",unSize);
		return -1;
	}
	m_pBuf = (char *)malloc(unSize);
	if(NULL == m_pBuf)
	{
		printf("malloc failed\n");
		return -2;
	}
	m_unTotMen = unSize;
	printf("Ring buffer size %dMByte\n",m_unTotMen/1024/1024);

	return 0;
}

void CRingBuf::SetHeadLen(u_int32_t unLen)
{
	m_unUserHeadLen = unLen;
	m_unSlotHeadLen = m_unUserHeadLen + sizeof(t_bufhdr);
}
void CRingBuf::DestroyRing(void)
{
	sem_destroy(&m_tHaveData);
	if(m_pBuf != NULL)
	{
		free(m_pBuf);
		m_pBuf = NULL;
	}
	//
}
int CRingBuf::AddDataToRing(char *data,u_int32_t len,void *datahead)
{
	m_ullTotPkts++;

	//是否有足够空间
	u_int32_t unNull = GetNullLen();
	u_int32_t unNeedLen = len + m_unSlotHeadLen;
	if(unNull <= unNeedLen)//没有足够的空间容纳
	{
		m_ullTotLost++;
		return -1;
	}
	//buf翻转情况，数据不能分段
	u_int32_t unBufEndNull = m_unTotMen - m_unInsertOff;
	//	后半段没有足够空间							前半段没有足够空间
	if((unBufEndNull < unNeedLen) && (m_unRemoveOff <= unNeedLen))
	{
		m_ullTotLost++;
		return -1;
	}
	char *write_ptr = NULL;
	t_bufhdr *pHead = NULL;
	//						要保证最后有一个带长度为0 的头
	if(unBufEndNull <= (unNeedLen + sizeof(t_bufhdr)))//后半段空间不足，但是前半段有空间
	{
		write_ptr = m_pBuf + m_unInsertOff;
		pHead = (t_bufhdr *)write_ptr;
		pHead->bySyn = RING_SYN_FLAG;
		pHead->buflen = 0;		//告诉读线程，数据翻转，应该到头读取

		m_unInsertOff = 0;
	}

	write_ptr = m_pBuf + m_unInsertOff;
	pHead = (t_bufhdr *)write_ptr;
	pHead->bySyn = RING_SYN_FLAG;
	pHead->buflen = unNeedLen - sizeof(t_bufhdr);	//该长度为负载长度，不包括自己的头长度
	memcpy(pHead->pbuf,datahead,m_unUserHeadLen);	//先拷贝用户自定义头
	memcpy(pHead->pbuf+m_unUserHeadLen,data,len);	//拷贝实际数据

	m_unInsertOff += unNeedLen;

	m_ullTotInsert++;
	sem_post(&m_tHaveData);
	return 0;
}

int CRingBuf::ReadDataFromRing(char **pdata,void *datahead)
{
	sem_wait(&m_tHaveData);

	if(m_unInsertOff == m_unRemoveOff)//读空
	{
		return 0;
	}
	char *read_ptr = m_pBuf + m_unRemoveOff;
	t_bufhdr * pHead = (t_bufhdr *)read_ptr;
	if(pHead->bySyn != RING_SYN_FLAG)
	{
		printf("err syn in read thread\n");
		return -1;
	}
	if(0 == pHead->buflen)//buffer翻转
	{
		m_unRemoveOff = 0;
		read_ptr = m_pBuf + m_unRemoveOff;
		pHead = (t_bufhdr *)read_ptr;
		if(pHead->bySyn != RING_SYN_FLAG)
		{
			printf("err syn in read thread\n");
			return -2;
		}
	}
	memcpy(datahead,pHead->pbuf,m_unUserHeadLen);//用户自定义数据头

	*pdata = (char *)pHead->pbuf + m_unUserHeadLen;//是否会存在还在访问，但是已经被覆盖？

	u_int32_t unDataLen = pHead->buflen - m_unUserHeadLen;//实际数据大小

	m_unRemoveOff += (pHead->buflen + sizeof(t_bufhdr));

	m_ullTotRead++;

	//printf("data len %d\n",unDataLen);
	return unDataLen;
}

//写线程调用
int CRingBuf::GetNullLen(void)
{
	//if(m_unInsertOff == m_unRemoveOff)//不可能让追赶上
	//m_unRemoveOff在读线程随时都可能改
	u_int32_t unRemoveOff = m_unRemoveOff;

	if(m_unInsertOff < unRemoveOff)
	{
		return (unRemoveOff - m_unInsertOff);
	}
	else if(m_unInsertOff > unRemoveOff)
	{
		return (m_unTotMen -(m_unInsertOff - unRemoveOff));
	}
	else//开始时m_unInsertOff == m_unRemoveOff
	{
		return m_unTotMen;
	}
	return 0;
}

void CRingBuf::PrintRingState(void)
{
	printf("\nTotal Packets:%lld\n",m_ullTotPkts);

	printf("Total Insert:%lld\n",m_ullTotInsert);

	printf("Total Read:%lld\n",m_ullTotRead);

	printf("Total Lost:%lld\n",m_ullTotLost);

	printf("Insert off:%d\n",m_unInsertOff);

	printf("Remove off:%d\n",m_unRemoveOff);

	printf("Total memory size:%d\n",m_unTotMen);

	printf("User head length:%d\n",m_unUserHeadLen);

	printf("Slot head length:%d\n",m_unSlotHeadLen);

}

简单的测试例子

RingBuffer.cpp

//============================================================================
// Name        : RingBuffer.cpp
// Author      : qinjj
// Version     :
// Copyright   : C 2012
// Description : Hello World in C++, Ansi-style
//============================================================================

#include <iostream>
using namespace std;
#include "RingBuf.h"

#include <stdio.h>
#include <stdlib.h>

#include <pthread.h>

typedef struct _t_dataInfo
{
	int index;
	int flag;
	unsigned int unCount;
/*
 *
 *
 * */
}t_dInfo;

class CTestRing
{
public:
	CTestRing()
	{
		m_unWrCount = 0;
		m_unRdCount = 0;
		m_bStop = false;
	};
	~CTestRing()
	{};
	int Init(void)
	{
		if(ringbuf.SetBufSize(1024*1024*50) < 0)//50M
		{
			printf("ring set faild\n");
			exit(1);
		}
		ringbuf.SetHeadLen(sizeof(t_dInfo));
	};

	int Run(void)
	{
		pthread_create(&wrthread, NULL,TH_Write, this);

		pthread_create(&rdthread, NULL,TH_Read, this);

		return 0;
	};

	int Stop()
	{
		m_bStop = true;
	};

	void PrintState(void)
	{
		ringbuf.PrintRingState();
	};

private:
	CRingBuf ringbuf;

	unsigned int m_unWrCount;
	unsigned int m_unRdCount;

	bool m_bStop;

	pthread_t wrthread;
	pthread_t rdthread;
	static void *TH_Write(void *lp)
	{
		CTestRing *pThis = (CTestRing *)lp;
		char buf[4096];
		int iLen = 100;
		t_dInfo info;
		unsigned int *pNum = (unsigned int *)buf;
		while(!pThis->m_bStop)
		{
			*pNum = pThis->m_unWrCount;
			info.unCount = pThis->m_unWrCount;
			iLen = rand()%4096;
			if(iLen < 10)
			{
				iLen = 10;
			}
			buf[iLen-1] = iLen%128;
			//printf("write end data[%d] %d\n",iLen,buf[iLen-1]);

			if(pThis->ringbuf.AddDataToRing(buf,iLen,&info)<0)
			{
				usleep(1000);
				//printf("add faild\n");
				continue;
			}
			pThis->m_unWrCount++;
			usleep(100);
		}


	};

	static void *TH_Read(void *lp)
	{
		CTestRing *pThis = (CTestRing *)lp;
		t_dInfo info;
		char *pData;
		while(!pThis->m_bStop)
		{

			int iLen = pThis->ringbuf.ReadDataFromRing(&pData,&info);
			if(iLen <= 0)
			{
				usleep(10000);
				//printf("read null\n");
				continue;
			}
			if(info.unCount != pThis->m_unRdCount)
			{
				printf("count exp %d,cur %d\n",pThis->m_unRdCount,info.unCount);
				pThis->m_unRdCount = info.unCount;
			}
			unsigned int *pNum = (unsigned int *)pData;
			if(*pNum != pThis->m_unRdCount)
			{
				printf("data err\n");
			}
			pThis->m_unRdCount++;

			if(pData[iLen-1] != iLen%128)
			{
				printf("read end data[%d] %d\n",iLen,pData[iLen-1]);
				printf("end data err\n");
			}
			//if(pThis->m_unRdCount%100 == 0)
				//usleep(1000);
		}
	};




};

int main() {
	cout << "ringbuffer test" << endl; // prints ringbuffer



	CTestRing test;
	test.Init();
	test.Run();

	while(1)
	{
		test.PrintState();
		sleep(1);
	}


	return 0;
}

Makefile

CC=gcc
CXX=g++

CFLAG = -g 

LDFLAG = -pthread

src := $(shell ls *.cpp)
objs := $(patsubst %.cpp,%.o,$(src))

test: $(objs)
	$(CXX) -o $@ $^ $(LDFLAG)
	
%.o:%.cpp
	$(CXX) $(CFLAG) -c -o  $@ $<
	
clean:
	rm -f test *.o