以raft协议为基础的模拟zoomkeep选举唯一主机实现

1.模拟zoomkeep架构实现，以raft协议实现唯一主句推举

2.应用将原有单进程事务分离，在唯一主机做主要业务逻辑，次要业务分离在非主机器运行

3.以tcp长链接建立链接。

4.主机的三种角色leader（唯一主机），candidate（候选者，在主机宕机的时候，追随者转换角色，尝试获取主机权限），follower（追随者，接受leader推送的指令）

5.协议基本过程。机器启动 向所有配置网络主机发送候选者请求。竞选成功的机器转换成leader，并发起所有到follower的链接。失败的candidate自动断开候选请求的链接，转换为follower，等待leader的链接及命令推送

sentinal.h

#ifndef __SENTINAL_H__
#define __SENTINAL_H__

#include <stdio.h>
#include <string.h>
#include <string>
#include <vector>
#include <pthread.h>
#include <time.h>
#include <sys/time.h>

#include <sys/socket.h>
#include <arpa/inet.h>
#include <sys/time.h>
#include <netinet/in.h>
#include <netinet/tcp.h>
#include <sys/types.h>
#include <sys/select.h>
#include <pthread.h>
#include <sys/epoll.h>



enum __ACTOR{
	FOLLOWER,
	CANDIDATE,
	LEADER,
	CANDIDATE1,   //candidate 转换角色 FOLLOW的过度
	CANDIDATE2
};

static char* ACTOR[3] = {"FOLLOWER","CANDIDATE","LEADER"};

/*
RPC PROTOCOL

rpc msg    :1.lear entry,2.candidate entry
rpc type   :1.request   ,2.response
    term   :raft term
    content:raft id

follower   : no genarate request,recv candidate request or leader request
candidate  : genarate candidate request,recv candidate request and response or leader request
leader     : genarate leader request. recv candidate request
*/
struct __rpc_protocol
{
	int  rpctype;    //0.candidate rpc , 1.leader rpc
	int  msgtype;    //request or response  0.request,1.vote response
	int  result;     //response result  0.refuse  , 1.agree
	long term;
	char content[64];

	__rpc_protocol()
	{
		rpctype = 0;
		msgtype = 0;
		result  = 0;
		term    = 0;
		memset(content,0,64);
	}
};

#define MEMSIZE 1024


enum __CONNTYPE
{
	INIT = 0,
	ACTIVE,
	INACTIVE
};

struct __sentinalCli
{
	int  fd;
	int  port;        //本机发起链接，还是远端申请链接
	int  pos;
	__CONNTYPE connType;
	char ip[32];
	//char*memcache;


	__sentinalCli * pre;
	__sentinalCli * next;

	__sentinalCli()
	{
		fd = -1;
		port = 0;
		pos  = 0;
		connType = INIT;
	//	memcache = NULL;
		memset(ip,0,32);

		pre  = NULL;
		next = NULL;
	}
};

class sentinal
{
public:
	~sentinal(){}
	int  Init(const char * path);
	int  InitConnect();              // RPC 监听
	static sentinal * Instance()
	{
		static sentinal instance;
		return &instance;
	}


public:
	__sentinalCli * AddSentinalCli();         //remote connect add
	void AddSentinalCliAuto();   //self  connect add
	bool DealSentinalCli(int fd);
	bool ReadRPC(int fd,__rpc_protocol &rpc);
	__sentinalCli * ConnectSentinalCli(const char * ip_port);
	bool ConnectSentinalCli(__sentinalCli * cli);
	bool checkSentinalCli(__sentinalCli * cli);
	bool SendRPC(int fd,__rpc_protocol rpc);
	void ClearCli();
	void ClearConnect();
	int  start();
	int  svc();
	
public:        // entry  dealing
	bool  on_candidate_entry(int fd,__rpc_protocol &rpc);
	bool  on_leader_entry(int fd,__rpc_protocol &rpc);
public:     //candidate vote -- respose dealing
	bool  on_candidate_response(int fd,__rpc_protocol &rpc);

public:    //request dealing
	bool  on_candidate_request(int fd,__rpc_protocol &rpc);
	bool  on_leader_request(int fd,__rpc_protocol &rpc);
public:
	void setNextTimeOut();
	void timeOutEvent();
	void listAdd(__sentinalCli * cli);
	void listDelete(__sentinalCli * cli);
	void leaderworker();
	void routinue();
	void leaderDetect();
public:
	void AddEpollEvent(int fd);
	void DelEpollEvent(int fd);
	void ClearEpollEvent();

public:
	__ACTOR role()
	{
		return _role;
	}
	std::string LeaderId()
	{
		return _leaderid;
	}
public:
	class sentinalCliManer
	{
	public:
		__sentinalCli *_head;
		__sentinalCli *_tail;

		sentinalCliManer()
		{	
			_head = NULL;
			_tail = NULL;
		}
		~sentinalCliManer()
		{

		}
		bool addCli(__sentinalCli *&cli);
		bool addCli(char * host);
		bool deleteCli(__sentinalCli *&cli);
		bool fetch(__sentinalCli *& cli);
		void clearCli(__sentinalCli *& cli);
		bool clearDupConnect(__sentinalCli *& cli);
	};
private:
	sentinal();
	sentinal(const sentinal&);
	sentinal& operator = (const sentinal&);

	//状态信息
private:

	long          _curterm;
	__ACTOR       _role;
	std::string   _myselfid;      // 各主机id，由ip:port 标示
	std::string   _votefor;       // 已投票主机
	std::string   _leaderid;
	std::vector<std::string> _othid;
	

	//监听信息
	int            _port;
	int            _sfd;

	sentinalCliManer _inactive_cli;
	sentinalCliManer _active_cli;

	// 超时始终设置, 微妙
	struct timeval _nextOverTime;
	struct __candidate_vote
	{
		int agree;
		int refuse;
		int vote;   // vote total
		__candidate_vote()
		{
			agree  = 0;
			refuse = 0; 
			vote   = 0;
		}

		void clear()
		{
			agree  = 0;
			refuse = 0;
			vote   = 0;
		}
	};

	int _epoll_fd;
	__candidate_vote _cvote;
};




#endif

sentinal.cpp

#include "sentinal.h"
#include "DCLogMacro.h"
#include "tinyxml.h"
#include "ByteSwap.h"


#include <signal.h>
#include <unistd.h>
#include <fcntl.h>
#include <errno.h>
#include <string>


sentinal::sentinal()
{
	_curterm = 0;
	_othid.clear();
	_role = FOLLOWER;
	_votefor.clear();
	_nextOverTime.tv_sec = time(NULL);
}

static void sig_deal(int signum)
{
	if(signum == SIGPIPE)
	{
		// do nothing		
	}
}


int sentinal::Init(const char * path)
{
	struct sigaction sa;
	sa.sa_flags = 0;
	sa.sa_handler = sig_deal;
	sigaction(SIGPIPE,&sa,NULL);
	
	if(!path)
	{
		DCBIZLOG(DCLOG_LEVEL_ERROR,-1,"", "path is NULL");
		return -1;
	}

	TiXmlElement * elem;
	TiXmlElement * root;
	TiXmlElement * child = NULL;
		
	TiXmlDocument doc(path);
	doc.LoadFile();
	if(doc.Error())
	{
		DCBIZLOG(DCLOG_LEVEL_ERROR,-1,"", "ERROR|DCConfig::LoagCfg|LoadFile failed: %s, at row[%d], col[%d]", doc.ErrorDesc(), doc.ErrorRow(), doc.ErrorCol());
		return -1;
	}
	root = doc.RootElement();
	if(!root)
	{
		DCBIZLOG(DCLOG_LEVEL_ERROR,-1,"", "ERROR|DCLogConfig::LoagCfg|can not find doc element");
		return -1;
	}

	elem = root->FirstChildElement("rafthost");
	if(!elem)
	{
		DCBIZLOG(DCLOG_LEVEL_ERROR,-1,"", "ERROR|DCLogConfig::LoagCfg|can not find rafthost element");
		return -1;
	}

	TiXmlElement * element;
	for(element = elem->FirstChildElement("peer");element;element = element->NextSiblingElement("peer"))
	{
		if(element->GetText())
		{
			std::string str = element->GetText();
			_othid.push_back(str);
		}
	}

	elem = root->FirstChildElement("raftself");
	if(!elem || !elem->GetText())
	{
		DCBIZLOG(DCLOG_LEVEL_ERROR,-1,"", "ERROR|DCLogConfig::LoagCfg|can not find raftself element");
		return -1;
	}
	_myselfid = elem->GetText();

	/* agent listen port  */
	elem = root->FirstChildElement("listen");
	if(!elem)
	{

		fprintf(stderr,"ERROR|DCLogConfig::LoagCfg|can not find listen\n");
		return -1;
	}
	/*child = elem->FirstChildElement("netport");
	if(!child || !child->GetText())
	{
		fprintf(stderr,"ERROR|DCLogConfig::LoagCfg|can not find listen/netport\n");
		return -1;
	}
	_myselfid += "|";
	_myselfid += child->GetText();
	*/
	return InitConnect();
}

int _setnonblocking(int sock)  
{  
	int opts;  
	opts = fcntl(sock, F_GETFL);  
	if(opts < 0)
	{  
  		DCBIZLOG(DCLOG_LEVEL_ERROR,-1,"", "fcntl(sock,GETFL) failed");  
  		return -1;  
 	}  
  
 	opts = opts | O_NONBLOCK;  
 	if(fcntl(sock, F_SETFL, opts)<0)
 	{  
 		DCBIZLOG(DCLOG_LEVEL_ERROR,-1,"", "fcntl(sock,SETFL,opts)");  
 		return -1;
	} 
	return 0;
} 


int sentinal::InitConnect()
{
	const char * host  = _myselfid.c_str();
	const char * pFind = strchr(host,':');
	if(!pFind)
	{
		DCBIZLOG(DCLOG_LEVEL_ERROR,-1,"","mysefid[%s] illegal .for right example[127.0.0.0:9999]",host);
		return -1;
	}
	DCBIZLOG(DCLOG_LEVEL_DEBUG,0,"","mysefid[%s] ",host);
	_port = atoi(pFind + 1);
	struct sockaddr_in servaddr;
	_sfd = ::socket(AF_INET, SOCK_STREAM, 0);

	if(_sfd < 0)
	{
		DCBIZLOG(DCLOG_LEVEL_ERROR,-1,"","socket init failed");
		return -1;
	}

/// 套接字设置
	int flag = 1;
	int nRet  = ::setsockopt(_sfd, IPPROTO_TCP, TCP_NODELAY, &flag, sizeof(flag));
	if (nRet < 0)
	{
		DCBIZLOG(DCLOG_LEVEL_ERROR,-1,"","setsockopt init TCP_NODELAY failed,nRet[%d]",nRet);
		return -1;
	}
	
	flag = 1;
	nRet = ::setsockopt(_sfd, SOL_SOCKET, SO_REUSEADDR, &flag, sizeof(flag));
	if (nRet < 0)
	{
		DCBIZLOG(DCLOG_LEVEL_ERROR,-1,"","setsockopt init SO_REUSEADDR failed,nRet[%d]",nRet);
		return -1;
	}

	linger  l = {0};   
	l.l_onoff   =   1;
	l.l_linger   =   0;
	nRet = ::setsockopt(_sfd, SOL_SOCKET, SO_LINGER, (const   char*)&l, sizeof(linger));  
	if (nRet)
	{
		DCBIZLOG(DCLOG_LEVEL_ERROR,-1,"","setsockopt init SO_LINGER failed ,nRet[%d]",nRet);
		return -1;
	}	
	
	servaddr.sin_family = AF_INET;
	servaddr.sin_port = htons(_port);
	servaddr.sin_addr.s_addr = htonl(INADDR_ANY);

	nRet = ::bind(_sfd,(struct sockaddr*)&servaddr,sizeof(servaddr));
	if(nRet < 0)
	{
		DCBIZLOG(DCLOG_LEVEL_ERROR,-1,"","socket bind failed nRet[%d],error[%d],info[%s]",nRet,errno,strerror(errno));
		return -1;
	}

	nRet = ::listen(_sfd,1024);
	if(nRet < 0)
	{
		DCBIZLOG(DCLOG_LEVEL_ERROR,-1,"","socket listen failed nRet[%d],error[%d],info[%s]",nRet,errno,strerror(errno));
		return -1;
	}
	
	return _setnonblocking(_sfd);
	//return 0;

}

static void *Deal(void * arg)
{

	sentinal * obj = (sentinal*)arg;
	obj->svc();
	return NULL;
}

int sentinal::start()
{
	pthread_t pid;
	int nRet = pthread_create(&pid,NULL,Deal,(void*)this);
	if(nRet < 0)
	{
		printf("ThreadPro thread create failed nRet[%d]\n",nRet);
		_exit(-1);
		return -1;
	}
	printf("pthread create success\n");
	return 0;
}

#define OPEN_MAX 256

int sentinal::svc()
{
	setNextTimeOut();
	_epoll_fd = epoll_create(OPEN_MAX);
	if(_epoll_fd == -1)
	{
		printf("epoll fd apply failed\n");
		_exit(1);
	}
	AddEpollEvent(_sfd);
	DCBIZLOG(DCLOG_LEVEL_DEBUG,0,"","begin._role---:%s",ACTOR[_role]);
	while(1)
	{
		// 超时事件处理开始
		time_t nTime = time(NULL);
		if(nTime >= _nextOverTime.tv_sec)
		{
			DCBIZLOG(DCLOG_LEVEL_DEBUG,0,"","_role---:%s",ACTOR[_role]);
			timeOutEvent();
			setNextTimeOut();
		}

		int interval = 1;
		struct epoll_event events[OPEN_MAX];
		int nfds = epoll_wait(_epoll_fd,events,OPEN_MAX,1);
		if(nfds > 0)
		{
			for(int i=0;i<nfds;i++)
			{
				if(events[i].data.fd == _sfd)
				{
					AddSentinalCli();
				}
				else 
				{
					if(!DealSentinalCli(events[i].data.fd))
					{
						DelEpollEvent(events[i].data.fd);
					}
					else
					{
						if(_role == CANDIDATE1)
						{
							_role = FOLLOWER;
							ClearConnect();
							break;
						}else if(_role == CANDIDATE2)
						{
							_role = LEADER;
							ClearEpollEvent();
							break;
						}
					}
				}
			}
		}else if(nfds == 0)
		{
		//printf("no net file desript happen\n");
		
		}else
		{
			DCBIZLOG(DCLOG_LEVEL_ERROR,0,"","bad  net file desript happen. error[%d],info[%s]",errno,strerror(errno));	
		}
	}
	return 0;
}

bool sentinal::DealSentinalCli(int fd)
{
	__rpc_protocol rpc;
	if(!ReadRPC(fd,rpc))
	{
		DCBIZLOG(DCLOG_LEVEL_DEBUG,0,"","read msg failed from :fd:%d",fd);
		return false;
	}
	DCBIZLOG(DCLOG_LEVEL_DEBUG,0,"","recv a msg{rpc:%d,type:%d,result:%d,term:%ld,content:%s}.fd:%d",
		rpc.rpctype,rpc.msgtype,rpc.result,rpc.term,rpc.content,fd);
	
	if(rpc.rpctype == 0)
	{
		return on_candidate_entry(fd,rpc);
	}else if(rpc.rpctype == 1)
	{
		return on_leader_entry(fd,rpc);
	}else
	{
		DCBIZLOG(DCLOG_LEVEL_DEBUG,0,"","recv an unknow request %d",rpc.rpctype);
	}
	
	return true;
}

//hear beat request
bool sentinal::on_leader_entry(int fd,__rpc_protocol & rpc)  
{
	DCBIZLOG(DCLOG_LEVEL_DEBUG,0,"","on deal leader request ._role:%s",ACTOR[_role]);
	bool result = true;
	if(_role == LEADER)
	{
		DCBIZLOG(DCLOG_LEVEL_ERROR,-1,"","raft has exit two leader");
		if(rpc.term < _curterm)
		{
			__rpc_protocol response;
			response.rpctype = 1;
			response.msgtype = 0;
			response.term    = _curterm;
			strcpy(response.content,_myselfid.c_str());
			result = SendRPC(fd,response);
		}else
		{
			_role = FOLLOWER;
			_leaderid = rpc.content;
			setNextTimeOut();
		}
	}else if(_role == FOLLOWER)
	{
		_leaderid = rpc.content;
		setNextTimeOut();
	}else if(_role == CANDIDATE)
	{
		_role = CANDIDATE1;
		_leaderid = rpc.content;
		setNextTimeOut();
	}
	_votefor  = "";
	return result;
}

bool sentinal::on_candidate_entry(int fd,__rpc_protocol & rpc)
{
	DCBIZLOG(DCLOG_LEVEL_DEBUG,0,"","on deal candidate request ._role:%s",ACTOR[_role]);
	if(rpc.msgtype == 0)
	{
		return on_candidate_request(fd,rpc); // give a response
	}else if(rpc.msgtype == 1)
	{
		return on_candidate_response(fd,rpc);
	}
	return true;
}

bool sentinal::on_candidate_request(int fd,__rpc_protocol & rpc)
{
	DCBIZLOG(DCLOG_LEVEL_DEBUG,0,"","on deal candidate request ._role:%s",ACTOR[_role]);

	bool result = true ;
	if(_role == FOLLOWER)
	{
		__rpc_protocol response;
		response.rpctype = 0;
		response.msgtype = 1;
		response.term    = _curterm;
		strcpy(response.content,_myselfid.c_str());
		if(_leaderid.size() != 0)
		{
			response.result  = 0;
		}
		else if(rpc.term >= _curterm && _votefor.size() == 0)  
		{
			_votefor = rpc.content;
			response.result  = 1;
		}
		else
		{
			response.result  = 0;
		}
		result = SendRPC(fd,response);
		setNextTimeOut();
	}else if(_role == LEADER)
	{
		__rpc_protocol response;
		response.rpctype = 1;
		response.msgtype = 0;
		response.term    = _curterm;
		strcpy(response.content,_myselfid.c_str());
		response.result  = 0;
		result = SendRPC(fd,response);

		_active_cli.addCli(rpc.content);
		//_followList.push_back(rpc.content);
	}else // candidate
	{
		__rpc_protocol response;
		response.rpctype = 0;
		response.msgtype = 1;
		response.term    = _curterm;
		strcpy(response.content,_myselfid.c_str());
		if(rpc.term > _curterm )
		{		
			response.result  = 1;
			_role = CANDIDATE1;      // 任期比自身高，自降为 follower
			_curterm = rpc.term;
			_votefor = rpc.content;
			setNextTimeOut();
			DCBIZLOG(DCLOG_LEVEL_DEBUG,0,"","switch role to follower");
		}else
		{
			response.result  = 0;
		}
		result = SendRPC(fd,response);
	}
	return result;
}

/*
	计数，并判断是否转入leader
*/
bool sentinal::on_candidate_response(int fd,__rpc_protocol & rpc)
{
	DCBIZLOG(DCLOG_LEVEL_DEBUG,0,"","on deal candidate response ._role:%s",ACTOR[_role]);
	if(_role == FOLLOWER || _role == LEADER)
	{
		//延时回应,忽略
		DCBIZLOG(DCLOG_LEVEL_DEBUG,0,"","recv a later response");
		return true;
	}
	if(rpc.result == 1)
	{
		DCBIZLOG(DCLOG_LEVEL_DEBUG,0,"","recv a vote :agree");
		_cvote.agree++;	
		if(_cvote.agree >= (_cvote.vote+1)/2)
		{
			//超过半数，转入leader
			
			_role = CANDIDATE2;
			_votefor = "";
			_leaderid = "";
			leaderworker();
			DCBIZLOG(DCLOG_LEVEL_DEBUG,0,"","switch role to leader");
		}
	}else if(rpc.result == 0)
	{
	  // candidate term timeout ,cal
	  DCBIZLOG(DCLOG_LEVEL_DEBUG,0,"","recv a vote :refuse");
		_cvote.refuse++;
	}
	else
	{
		DCBIZLOG(DCLOG_LEVEL_DEBUG,0,"","recv an unnormal vote.type:%d,msg:%d,result:%d",rpc.rpctype,rpc.msgtype,rpc.result);
	}
	return true;
}

__sentinalCli * sentinal::AddSentinalCli()
{
	struct sockaddr_in cliaddr;
	char CliAddr[128] = {0};
	#ifdef _HP_UX
		int clilen = 0;
	#else
		socklen_t clilen = 0;
	#endif
	clilen = sizeof(cliaddr);
	int act_fd = ::accept(_sfd,(struct sockaddr *) &cliaddr,&clilen);
	if(act_fd > 0)
	{
		DCBIZLOG(DCLOG_LEVEL_DEBUG,0,"","accept a new connect.fd:%d",act_fd);
		AddEpollEvent(act_fd);
	}

	return NULL;
}

bool sentinal::ReadRPC(int fd,__rpc_protocol & rpc)
{
	static int rpcsize = sizeof(__rpc_protocol);
	char memcache[256] = {0};
	int pos = 0;
	while(1)
	{
		int len = ::read(fd,memcache+pos,rpcsize-pos);
		if(len == -1)
		{
			if(errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR)
			{	
				continue;
			}
			DCBIZLOG(DCLOG_LEVEL_DEBUG,0,"","recv failed error[%d],info[%s],fd:%d",errno,strerror(errno),fd);
			return false;
		}else if(len == 0)
		{
			DCBIZLOG(DCLOG_LEVEL_DEBUG,0,"","recv failed error[%d],info[%s],fd:%d",errno,strerror(errno),fd);
			return false;
		}
		len += pos;
		if(len < rpcsize)
		{
			DCBIZLOG(DCLOG_LEVEL_DEBUG,0,"","not recv all.continue,fd:%d",fd);
			pos = len;
			continue;
		}
		
		memcpy((void*)&rpc,memcache,rpcsize);
		
		rpc.msgtype = ntoh16(rpc.msgtype);
		rpc.rpctype = ntoh16(rpc.rpctype);
		rpc.result  = ntoh16(rpc.result);
		rpc.term    = ntoh32(rpc.term);
		return true;
	}

	return true;
}

bool sentinal::SendRPC(int fd,__rpc_protocol rpc)
{
	DCBIZLOG(DCLOG_LEVEL_DEBUG,0,"","send msg{rpc:%d,type:%d,term:%d,result:%d,content:%s}.fd:%d",
		rpc.rpctype,rpc.msgtype,rpc.term,rpc.result,rpc.content,fd);
	rpc.rpctype = hton16(rpc.rpctype);
	rpc.msgtype = hton16(rpc.msgtype);
	rpc.result  = hton16(rpc.result);
	rpc.term    = hton32(rpc.term);
	int len = ::send(fd,(void*)&rpc,sizeof(__rpc_protocol),0);
	if(len < 0)
	{
		DCBIZLOG(DCLOG_LEVEL_ERROR,-1,"","send RPC failed.fd:%d,error[%d],info[%s]",fd,errno,strerror(errno));
		return false;
	}
	return true;
}

void sentinal::setNextTimeOut()
{
	int interval;
	if(_role == FOLLOWER)
	{
		interval = 5;
	}
	else if(_role == CANDIDATE)
	{
		int randoms=open("/dev/urandom",O_RDONLY);
		int randomn;
		read(randoms,&randomn,sizeof(randomn));
		close(randoms);
		randomn=abs(randomn);
	    interval = randomn%3 + 3;
	}else
	{
		interval = 9999;
	}
	gettimeofday(&_nextOverTime, NULL);
	_nextOverTime.tv_sec += interval;
	//DCBIZLOG(DCLOG_LEVEL_DEBUG,0,"","next time ::__nextOverTime:%ld,interval:%d",_nextOverTime.tv_sec,interval);
	return ;
}

void sentinal::timeOutEvent()
{
	if(_cvote.agree == 1 && _cvote.vote == 1)
	{
		DCBIZLOG(DCLOG_LEVEL_DEBUG,0,"","cluster has just one host.switch to leader");
		_cvote.clear();
		_role = LEADER;
		leaderworker();
		return ;
	}
	
	_cvote.clear();
	_cvote.agree = 1;
	_cvote.vote  = 1;
	if(_role == FOLLOWER)
	{
		DCBIZLOG(DCLOG_LEVEL_DEBUG,0,"","start a vote,follower");
		printf("start vote\n");
		__rpc_protocol request;
		request.rpctype = 0;
		request.msgtype = 0;
		request.term    = ++_curterm;
		strcpy(request.content,_myselfid.c_str());
		
		_role = CANDIDATE;
		_leaderid.clear();
		_votefor.clear();
		std::vector<std::string>::iterator iter;
		for(iter = _othid.begin();iter != _othid.end();iter++)
		{
			__sentinalCli * cli = ConnectSentinalCli(iter->c_str());
			if(cli)
			{
				SendRPC(cli->fd,request);
				_cvote.vote++;
				_inactive_cli.addCli(cli);

				AddEpollEvent(cli->fd);
			}
		}
	}else if(_role == CANDIDATE)   //二次竞选
	{
		DCBIZLOG(DCLOG_LEVEL_DEBUG,0,"","try a vote again,candidate");
		printf("start vote again\n");
		__rpc_protocol request;
		request.rpctype = 0;
		request.msgtype = 0;
		request.term    = ++_curterm;
		strcpy(request.content,_myselfid.c_str());
		_leaderid.clear();
		_votefor.clear();
		__sentinalCli * cli = NULL;
		_inactive_cli.fetch(cli);
		while(cli)
		{
			if(checkSentinalCli(cli))
			{
				SendRPC(cli->fd,request);
				_cvote.vote++;
			}
			_inactive_cli.fetch(cli);
		}
	}
   // leader  do nothing
	return ;
}

bool sentinal::checkSentinalCli(__sentinalCli * cli)
{
	
	return true;
}

bool sentinal::ConnectSentinalCli(__sentinalCli * cli)
{
	struct sockaddr_in serv_addr;
	cli->fd = ::socket(AF_INET,SOCK_STREAM,0);
	if(cli->fd == -1)
	{
		DCBIZLOG(DCLOG_LEVEL_ERROR,0,"","sock init failed,error[%d],info[%s] ",errno,strerror(errno));
		return false;
	}

	int flag = 1;
	int nRet  = ::setsockopt(cli->fd, IPPROTO_TCP, TCP_NODELAY, &flag, sizeof(flag));
	if (nRet < 0)
	{
		::close(cli->fd);
		DCBIZLOG(DCLOG_LEVEL_ERROR,0,"","sock init failed,error[%d],info[%s] ",errno,strerror(errno));
		return false;
	}

	linger	l = {0};   
	l.l_onoff	=	1;
	l.l_linger	 =	 0;
	nRet = ::setsockopt(cli->fd, SOL_SOCKET, SO_LINGER, (const   char*)&l, sizeof(linger));  
	if (nRet)
	{
		::close(cli->fd);
		DCBIZLOG(DCLOG_LEVEL_ERROR,0,"","sock init failed,error[%d],info[%s] ",errno,strerror(errno));
		return false;
	}
	
	serv_addr.sin_family = AF_INET;
	serv_addr.sin_port = htons(cli->port);
	if(inet_pton(AF_INET,cli->ip,&serv_addr.sin_addr) <= 0)
	{
		::close(cli->fd);
		DCBIZLOG(DCLOG_LEVEL_ERROR,0,"","sock init failed,error[%d],info[%s] ",errno,strerror(errno));
		return false;
	}

	nRet = ::connect(cli->fd,(struct sockaddr*)&serv_addr,sizeof(serv_addr));
	if(nRet < 0)
	{
		::close(cli->fd);
		DCBIZLOG(DCLOG_LEVEL_ERROR,0,"","sock init failed,error[%d],info[%s] ",errno,strerror(errno));
		return false;
	}
	return true;
}

__sentinalCli * sentinal::ConnectSentinalCli(const char * ip_port)
{
	const char * pFind = strchr(ip_port,':');
	if(!pFind)
	{
		DCBIZLOG(DCLOG_LEVEL_ERROR,0,"","raft host config[%s] illegal",ip_port);
		return NULL;
	}

	__sentinalCli * cli = new __sentinalCli();
	if(!cli)
	{
		DCBIZLOG(DCLOG_LEVEL_ERROR,0,"","__sentinalCli space apply failed");
		return NULL;
	}
	
	strncpy(cli->ip,ip_port,pFind-ip_port);
	cli->port = atoi(pFind+1);
	cli->connType = ACTIVE;
	
	DCBIZLOG(DCLOG_LEVEL_DEBUG,0,"","connect to ip:%s,port:%d",cli->ip,cli->port);
	if(!ConnectSentinalCli(cli))
	{
		delete cli;
		return NULL;
	}else
	{
		DCBIZLOG(DCLOG_LEVEL_DEBUG,0,"","connect success");
	}
	

	return cli;
}

void sentinal::ClearCli()
{
// 清楚 candidate 发起投票 建立的非leader交互链接
	
	return ;
}

void sentinal::listAdd(__sentinalCli * cli)
{
	return ;
}

void sentinal::listDelete(__sentinalCli * cli)
{
	
	return ;
}

static void * Deall(void*arg)
{
	sentinal * obj = (sentinal*)arg;
	obj->routinue();
	return NULL;
}

void sentinal::leaderworker()
{	
	pthread_t pid;
	int nRet = pthread_create(&pid,NULL,Deall,(void*)this);
	if(nRet < 0)
	{
		printf("ThreadPro thread create failed nRet[%d]\n",nRet);
		_exit(-1);
		return ;
	}
	printf("leader pthread create success\n");
	return;
}

void sentinal::routinue()
{
	DCBIZLOG(DCLOG_LEVEL_DEBUG,0,"","leader  hearbeat");

	__rpc_protocol request;
	request.rpctype = 1;
	request.msgtype = 0;
	request.term    = _curterm;
	strcpy(request.content,_myselfid.c_str());
	request.result  = 0;
	while(1)
	{
		leaderDetect();
		
		struct timeval tm = {0};
		tm.tv_sec = 1;
		tm.tv_usec = 0;
		
		::select(0,NULL,NULL,NULL,&tm);	
		__sentinalCli * cli = NULL;
		_inactive_cli.fetch(cli);
		while(cli)
		{
			//printf("leader  hearbeat.cli:%s.port:%d\n",cli->ip,cli->port);
			DCBIZLOG(DCLOG_LEVEL_DEBUG,0,"","leader  hearbeat.cli:%s.port:%d",cli->ip,cli->port);
			if(!SendRPC(cli->fd,request))
			{
				__sentinalCli * A = cli;
				_inactive_cli.fetch(cli);
				_inactive_cli.clearCli(A);					
			}else
			{
				_inactive_cli.fetch(cli);
			}
			
		}
	}
	
	return ;
}

void sentinal::leaderDetect()

{
	__sentinalCli * cli = NULL;
	_active_cli.fetch(cli);
	while(cli)
	{
		__sentinalCli * A = cli;
		_active_cli.fetch(cli);
		_active_cli.deleteCli(A);
		if(ConnectSentinalCli(A))
		{
			_inactive_cli.addCli(A);
		}
		else
		{
			delete A;
		}		
	}

	return ;
}

bool sentinal::sentinalCliManer::addCli(__sentinalCli *& cli)
{
	DCBIZLOG(DCLOG_LEVEL_DEBUG,0,"","add cli::ip:%s.port:%d.fd:%d",cli->ip,cli->port,cli->fd);
	if(!_head)
	{
		_head = cli;
		_tail = _head;
	}else
	{
		cli->pre = _tail;
		_tail->next = cli;
		_tail = cli;
	}
	return true;
}

bool sentinal::sentinalCliManer::addCli(char * host)
{
	char * pFind = strchr(host,':');
	if(!pFind)
	{
		return false;
	}
	__sentinalCli * cli = new __sentinalCli();
	if(!cli)
	{
		return false;
	}
	strncpy(cli->ip,host,pFind-host);
	cli->port = atoi(pFind + 1);

	addCli(cli);
	return true;
}

bool sentinal::sentinalCliManer::deleteCli(__sentinalCli *& cli)
{
	DCBIZLOG(DCLOG_LEVEL_DEBUG,0,"","delete cli::ip:%s.port:%d.fd:%d",cli->ip,cli->port,cli->fd);
	if(!cli->pre)
	{
		_head = cli->next;
		if(_head)
			_head->pre = NULL;
	}else if(!cli->next)
	{
		_tail = cli->pre;
		_tail->next = NULL;
	}else
	{
		cli->pre->next = cli->next;
		cli->next->pre = cli->pre;
	}
	cli->pre = NULL;
	cli->next = NULL;
	return true;
}

bool sentinal::sentinalCliManer::fetch(__sentinalCli *& cli)
{
	if(!cli)
	{
		cli = _head;
	}
	else
	{
		cli = cli->next;
	}
	return true;
}

void sentinal::sentinalCliManer::clearCli(__sentinalCli *& cli)
{
	DCBIZLOG(DCLOG_LEVEL_DEBUG,0,"","clear cli::ip:%s.port:%d.fd:%d",cli->ip,cli->port,cli->fd);
	deleteCli(cli);
	if(cli->fd)
	{
		::close(cli->fd);
	}
	delete cli;
	cli = NULL;
	return ;
}

bool sentinal::sentinalCliManer::clearDupConnect(__sentinalCli *& cli)
{
	DCBIZLOG(DCLOG_LEVEL_DEBUG,0,"","clearDup cli::ip:%s.port:%d.fd:%d",cli->ip,cli->port,cli->fd);
	__sentinalCli * AA = _head;
	while(AA)
	{
		DCBIZLOG(DCLOG_LEVEL_DEBUG,0,"","AA cli::ip:%s.port:%d.fd:%d",AA->ip,AA->port,AA->fd);
		if(strcmp(cli->ip,AA->ip) == 0 && cli->port == AA->port && cli->fd != AA->fd)
		{
			//clearCli(cli);
			return true;
		}
		fetch(AA);
	}
	return false;
}

void sentinal::AddEpollEvent(int fd)
{
	struct epoll_event event;
	event.data.fd = fd;
	//event.data.u64 = 0;
	event.events = EPOLLIN ;
	epoll_ctl(_epoll_fd,EPOLL_CTL_ADD,fd,&event);
	return ;
}

void sentinal::DelEpollEvent(int fd)
{
	struct epoll_event event;
	event.data.fd = fd;
	event.events = 0;
	epoll_ctl(_epoll_fd,EPOLL_CTL_DEL,fd,&event);
	return ;
}

void sentinal::ClearConnect()
{
	__sentinalCli * cli = NULL;
	_inactive_cli.fetch(cli);
	while(cli)
	{
		DelEpollEvent(cli->fd);
		__sentinalCli *A=cli; 
		_inactive_cli.fetch(cli);
		_inactive_cli.clearCli(A);
	}

	return ;
}

void sentinal::ClearEpollEvent()
{
	__sentinalCli * cli = NULL;
	_inactive_cli.fetch(cli);
	while(cli)
	{
		DelEpollEvent(cli->fd);
		_inactive_cli.fetch(cli);
	}
	return ;
}