1、什么是RabbitMQ工作队列
我们在应用程序使用消息系统时,一般情况下生产者往队列里插入数据时速度是比较快的,但是消费者消费数据往往涉及到一些业务逻辑处理导致速度跟不上生产者生产数据。因此如果一个生产者对应一个消费者的话,很容易导致很多消息堆积在队列里。这时,就得使用工作队列了。一个队列有多个消费者同时消费数据。
下图取自于官方网站(RabbitMQ)的工作队列的图例
P:消息的生产者
C1:消息的消费者1
C2:消息的消费者2
红色:队列
生产者将消息发送到队列,多个消费者同时从队列中获取消息。
工作队列有两种分发数据的方式:轮询分发(Round-robin)和 公平分发(Fair dispatch)。轮询分发:队列给每一个消费者发送数量一样的数据。公平分发:消费者设置每次从队列中取一条数据,并且消费完后手动应答,继续从队列取下一个数据。下面分别是两种分发方式不同的写法。
2、轮询分发(Round-robin)
生产者(Send)生产10条数据,消费者1(Receive1)接收数据并假设处理业务逻辑1s,消费者2(Receive2)接收数据并假设处理业务逻辑2s(生产者先运行,两个消费者同时运行)。
2.1、生产者(Send)代码
public class Send
{
//队列名称
private static final String QUEUE_NAME = "test_work_round_robin_queue";
public static void main(String[] args)
{
try
{
//获取连接
Connection connection = ConnectionUtil.getConnection();
//从连接中获取一个通道
Channel channel = connection.createChannel();
//声明队列
channel.queueDeclare(QUEUE_NAME, false, false, false, null);
for (int i = 0; i < 10; i++)
{
String message = "this is work_round_robin queue message" + i;
System.out.println("[send]:" + message);
//发送消息
channel.basicPublish("", QUEUE_NAME, null, message.getBytes("utf-8"));
Thread.sleep(20 * i);
}
channel.close();
connection.close();
}
catch (IOException | TimeoutException | InterruptedException e)
{
e.printStackTrace();
}
}
}
运行结果:
[send]:this is work_round_robin queue message0
[send]:this is work_round_robin queue message1
[send]:this is work_round_robin queue message2
[send]:this is work_round_robin queue message3
[send]:this is work_round_robin queue message4
[send]:this is work_round_robin queue message5
[send]:this is work_round_robin queue message6
[send]:this is work_round_robin queue message7
[send]:this is work_round_robin queue message8
[send]:this is work_round_robin queue message9
2.2、消费者1(Receive1)代码
public class Receive1
{
//队列名称
private static final String QUEUE_NAME = "test_work_round_robin_queue";
public static void main(String[] args)
{
try
{
//获取连接
Connection connection = ConnectionUtil.getConnection();
//从连接中获取一个通道
Channel channel = connection.createChannel();
//声明队列
channel.queueDeclare(QUEUE_NAME, false, false, false, null);
//定义消费者
DefaultConsumer consumer = new DefaultConsumer(channel)
{
//当消息到达时执行回调方法
@Override
public void handleDelivery(String consumerTag, Envelope envelope, BasicProperties properties,
byte[] body) throws IOException
{
String message = new String(body, "utf-8");
System.out.println("[1] Receive message:" + message);
try
{
//消费者休息1s处理业务
Thread.sleep(1000);
}
catch (InterruptedException e)
{
e.printStackTrace();
}
finally
{
System.out.println("[1] done");
}
}
};
//监听队列
channel.basicConsume(QUEUE_NAME, true, consumer);
}
catch (IOException e)
{
e.printStackTrace();
}
}
}
运行结果:
[1] Receive message:this is work_round_robin queue message0
[1] done
[1] Receive message:this is work_round_robin queue message2
[1] done
[1] Receive message:this is work_round_robin queue message4
[1] done
[1] Receive message:this is work_round_robin queue message6
[1] done
[1] Receive message:this is work_round_robin queue message8
[1] done
2.3、消费者2(Receive2)代码
public class Receive2
{
//队列名称
private static final String QUEUE_NAME = "test_work_round_robin_queue";
public static void main(String[] args)
{
try
{
//获取连接
Connection connection = ConnectionUtil.getConnection();
//从连接中获取一个通道
Channel channel = connection.createChannel();
//声明队列
channel.queueDeclare(QUEUE_NAME, false, false, false, null);
//定义消费者
DefaultConsumer consumer = new DefaultConsumer(channel)
{
//当消息到达时执行回调方法
@Override
public void handleDelivery(String consumerTag, Envelope envelope, BasicProperties properties,
byte[] body) throws IOException
{
String message = new String(body, "utf-8");
System.out.println("[2] Receive message:" + message);
try
{
//消费者休息2s处理业务
Thread.sleep(2000);
}
catch (InterruptedException e)
{
e.printStackTrace();
}
finally
{
System.out.println("[2] done");
}
}
};
//监听队列
channel.basicConsume(QUEUE_NAME, true, consumer);
}
catch (IOException e)
{
e.printStackTrace();
}
}
}
运行结果:
[2] Receive message:this is work_round_robin queue message1
[2] done
[2] Receive message:this is work_round_robin queue message3
[2] done
[2] Receive message:this is work_round_robin queue message5
[2] done
[2] Receive message:this is work_round_robin queue message7
[2] done
[2] Receive message:this is work_round_robin queue message9
[2] done
总结:两个消费者得到的数据量一样的。从运行时可以看到消费者1会先执行完,消费者2会后执行完。并不会因为两个消费者处理数据速度不一样使得两个消费者取得不一样数量的数据。并且当队列数量大的时候通过观察RabbitMQ的管理后台,可以看到管理界面队列中的数据很快就没了,但是这个时候两个消费者其实并没有消费完数据。这种分发方式存在着很大的隐患。
3、公平分发(Fair dispatch)
生产者(Send)生产10条数据,消费者1(Receive1)接收数据并假设处理业务逻辑1s,消费者2(Receive2)接收数据并假设处理业务逻辑2s(生产者先运行,两个消费者同时运行)。
消费者设置每次从队列里取一条数据,并且关闭自动回复机制,每次取完一条数据后,手动回复并继续取下一条数据。
3.1、生产者(Send)代码
public class Send
{
//队列名称
private static final String QUEUE_NAME = "test_work_fair_dispatch_queue";
public static void main(String[] args)
{
try
{
//获取连接
Connection connection = ConnectionUtil.getConnection();
//从连接中获取一个通道
Channel channel = connection.createChannel();
//声明队列
channel.queueDeclare(QUEUE_NAME, false, false, false, null);
for (int i = 0; i < 10; i++)
{
String message = "this is work_fair_dispatch queue message" + i;
System.out.println("[send]:" + message);
//发送消息
channel.basicPublish("", QUEUE_NAME, null, message.getBytes("utf-8"));
Thread.sleep(20 * i);
}
channel.close();
connection.close();
}
catch (IOException | TimeoutException | InterruptedException e)
{
e.printStackTrace();
}
}
}
运行结果:
[send]:this is work_fair_dispatch queue message0
[send]:this is work_fair_dispatch queue message1
[send]:this is work_fair_dispatch queue message2
[send]:this is work_fair_dispatch queue message3
[send]:this is work_fair_dispatch queue message4
[send]:this is work_fair_dispatch queue message5
[send]:this is work_fair_dispatch queue message6
[send]:this is work_fair_dispatch queue message7
[send]:this is work_fair_dispatch queue message8
[send]:this is work_fair_dispatch queue message9
3.2、消费者1(Receive1)代码
public class Receive1
{
//队列名称
private static final String QUEUE_NAME = "test_work_fair_dispatch_queue";
public static void main(String[] args)
{
try
{
//获取连接
Connection connection = ConnectionUtil.getConnection();
//从连接中获取一个通道
final Channel channel = connection.createChannel();
//声明队列
channel.queueDeclare(QUEUE_NAME, false, false, false, null);
//设置每次从队列里取一条数据
int prefetchCount = 1;
channel.basicQos(prefetchCount);
//定义消费者
DefaultConsumer consumer = new DefaultConsumer(channel)
{
//当消息到达时执行回调方法
@Override
public void handleDelivery(String consumerTag, Envelope envelope, BasicProperties properties,
byte[] body) throws IOException
{
String message = new String(body, "utf-8");
System.out.println("[1] Receive message:" + message);
try
{
//消费者休息1s处理业务
Thread.sleep(1000);
}
catch (InterruptedException e)
{
e.printStackTrace();
}
finally
{
System.out.println("[1] done");
//手动应答
channel.basicAck(envelope.getDeliveryTag(), false);
}
}
};
//设置手动应答
boolean autoAck = false;
//监听队列
channel.basicConsume(QUEUE_NAME, autoAck, consumer);
}
catch (IOException e)
{
e.printStackTrace();
}
}
}
运行结果:
[1] Receive message:this is work_fair_dispatch queue message1
[1] done
[1] Receive message:this is work_fair_dispatch queue message2
[1] done
[1] Receive message:this is work_fair_dispatch queue message4
[1] done
[1] Receive message:this is work_fair_dispatch queue message5
[1] done
[1] Receive message:this is work_fair_dispatch queue message7
[1] done
[1] Receive message:this is work_fair_dispatch queue message8
[1] done
3.3、消费者2(Receive2)代码
public class Receive2
{
//队列名称
private static final String QUEUE_NAME = "test_work_fair_dispatch_queue";
public static void main(String[] args)
{
try
{
//获取连接
Connection connection = ConnectionUtil.getConnection();
//从连接中获取一个通道
final Channel channel = connection.createChannel();
//声明队列
channel.queueDeclare(QUEUE_NAME, false, false, false, null);
//保证一次只分发一个
int prefetchCount = 1;
channel.basicQos(prefetchCount);
//定义消费者
DefaultConsumer consumer = new DefaultConsumer(channel)
{
//当消息到达时执行回调方法
@Override
public void handleDelivery(String consumerTag, Envelope envelope, BasicProperties properties,
byte[] body) throws IOException
{
String message = new String(body, "utf-8");
System.out.println("[2] Receive message:" + message);
try
{
//消费者休息2s处理业务
Thread.sleep(2000);
}
catch (InterruptedException e)
{
e.printStackTrace();
}
finally
{
System.out.println("[2] done");
//手动应答
channel.basicAck(envelope.getDeliveryTag(), false);
}
}
};
//设置手动应答
boolean autoAck = false;
//监听队列
channel.basicConsume(QUEUE_NAME, autoAck, consumer);
}
catch (IOException e)
{
e.printStackTrace();
}
}
}
运行结果:
[2] Receive message:this is work_fair_dispatch queue message0
[2] done
[2] Receive message:this is work_fair_dispatch queue message3
[2] done
[2] Receive message:this is work_fair_dispatch queue message6
[2] done
[2] Receive message:this is work_fair_dispatch queue message9
[2] done
总结:消费者1处理了6条数据,消费者2处理了4条数据
与轮询分发不同的是,当每个消费都设置了每次只会从队列取一条数据时,并且关闭自动应答,在每次处理完数据后手动给队列发送确认收到数据。这样队列就会公平给每个消息费者发送数据,消费一条再发第二条,而且可以在管理界面中看到数据是一条条随着消费者消费完从而减少的,并不是一下子全部分发完了。显然公平分发更符合系统设计。
// 创建人: litao;
// 创建时间: 2016-11-1;
#pragma once
#include <map>
#include <list>
#include <vector>
#include <string>
class XmlNode
{
friend class FastXML;
private:
XmlNode();
public:
XmlNode(const char *tagname);
virtual ~XmlNode();
public:
//删除自己;
bool remove();
//增加子节点;
XmlNode* addChild(XmlNode* newChild);
//删除子节点;
void removeChild(XmlNode* nodeChild);
//去掉内存节点,不释放;
void removeAllChildNoDel();
//设置节点值;
void setNodeValue(const char * nodeval);
//获取节点值;
std::string getNodeValue();
//删除属性;
void removeAtrr(const char * attr);
//设置属性值;
void setAttribute(const char * attr,const char * val);
//获取节点名称;
std::string getTagName();
//获取属性值;
std::string getAttribute(const char * attr);
//获得父节点;
XmlNode *getParent();
//获得节点的xml文本,含所有子节点;
std::string getNodeText();
//获取所有节点文本值;
std::string getAllChildNodeText();
//获得所有属性名;
std::vector<std::string> getAttributeNames();
//获得所有属性名和属性值;
void getAttributes(std::vector<std::string>&vcAttr,std::vector<std::string>&vcVal);
//获得所有子节点;
std::vector<XmlNode*> getChildNode();
//根据名称获取第一个节点;
XmlNode* getFirstChildByTagName(std::string tagname);
//获得指定名称的子节点;
std::vector<XmlNode*> getChildNodeByTagName(std::string tagname);
//根据名称查找子孙节点(也包含当前节点);
std::vector<XmlNode*> findChildNodeByTagName(std::string tagname);
//根据名称查找节点;
void findChildNodeByTagName(std::string const& tagname, std::vector<XmlNode*>& pnodes);
//获取叶子节点;
void getAllLeafNode(std::vector<XmlNode*> & vcLeafNode);
//是否有孩子节点;
bool hasChild();
private:
std::vector<std::string>m_vcattr;
std::vector<std::string>m_vcval;
std::string m_tagName;
std::vector<XmlNode*>m_vcChildNode;
std::string m_nodevalue;
XmlNode *m_parent;
std::map<std::string,std::string>m_AttrMap;
bool m_bmap;
private:
void clearAll();
void buildMap();
};
class FastXML
{
public:
FastXML(void);
~FastXML(void);
public:
//从文件加载;
bool load(const char *filePath);
//加载Utf8内存块;
bool loadFromUtf8Buff(const char *pBuff,int nsize);
//加载ansi内存块;
bool loadFromBuff(const char *pBuff);
//保存文件;
bool save(const char *filepath);
//获取文本字符串;
std::string getFileXML();
//获取文件内容(不含头);
std::string getDoc();
//获取根节点;
XmlNode* getRootNode();
//添加一个根节点;
void addRootNode(XmlNode *pNode);
//设置文件头(<?xml version="1.0" encoding="utf-8" standalone="no"?>);
void setXmlHead(std::string strhead);
//获取所有的节点数;
void getNodeSize(XmlNode *pNode,std::string tagname,int &nSize);
public:
void replaceAttr(std::string Attr,std::string oldvalue,std::string newvalue,XmlNode *pNode=NULL);
private:
void decComment(char *&pBuffer);
void decHeadCode(char *&pBuffer);
XmlNode* parserBuffer(char *pBuffer,XmlNode *pParent);
std::string trimRight(const std::string& str);
void inline skipSpace(char *&pBuffer);
std::string utf8ToAnsi(const char * pszUTF8,int nLen,int* pnFailed);
void ansiToUtf8( std::string &strbuff);
int decodeCharUTF8( const char*& pszUTF8 );
void encodeCharUTF8( int nUChar, char* pszUTF8, int& nUTF8Len );
private:
XmlNode m_rootNode;
std::string m_xmlhead;
std::string m_doctype;
std::string m_datatype;
std::vector<std::string>m_vccomment;
};
// 创建人: litao;
// 创建时间: 2016-11-1;
#include "Fastxml.h"
#include<string>
#include <locale>
using namespace std;
string& replace_all_distinct(string& str,const string& old_value,const string& new_value)
{
for(string::size_type pos(0); pos!=string::npos; pos+=new_value.length())
{
if((pos=str.find(old_value,pos))!=string::npos)
{
str.replace(pos,old_value.length(),new_value);
}
else
{
break;
}
}
return str;
}
XmlNode::XmlNode()
{
}
XmlNode::XmlNode(const char *tagname):m_parent(NULL),m_bmap(false)
{
if(tagname)
{
m_tagName = tagname;
}
}
XmlNode::~XmlNode()
{
clearAll();
}
XmlNode* XmlNode::addChild(XmlNode* newChild)
{
newChild->m_parent = this;
m_vcChildNode.push_back(newChild);
return newChild;
}
std::string XmlNode::getTagName()
{
return m_tagName;
}
std::string XmlNode::getAttribute(const char * attr)
{
if(attr)
{
buildMap();
if(m_AttrMap.size() == 0)
return "";
if(m_AttrMap.count(attr))
{
return m_AttrMap[attr];
}
}
return "";
}
XmlNode *XmlNode::getParent()
{
return m_parent;
}
void XmlNode::clearAll()
{
for (size_t i = 0;i < m_vcChildNode.size(); ++i)
{
delete m_vcChildNode[i];
}
m_vcChildNode.clear();
m_AttrMap.clear();
m_vcattr.clear();
m_vcval.clear();
m_parent = NULL;
m_tagName.clear();
m_nodevalue.clear();
}
void XmlNode::setAttribute(const char * attr,const char * val)
{
_ASSERT(attr && val);
if(!attr || !val)
return;
if(strlen(val)==0)
return;
m_bmap = false;
bool bExist = false;
for (size_t i = 0 ;i < m_vcattr.size(); ++i)
{
if(m_vcattr[i] == attr)
{
bExist = true;
m_vcval[i] = val;
}
}
if(!bExist)
{
m_vcattr.push_back(attr);
m_vcval.push_back(val);
}
}
void XmlNode::buildMap()
{
if(!m_bmap)
{
m_bmap = true;
_ASSERT(m_vcattr.size() == m_vcval.size());
m_AttrMap.clear();
for (size_t i = 0;i < m_vcattr.size(); ++i)
{
m_AttrMap.insert(std::pair<std::string,std::string>(m_vcattr[i],m_vcval[i]));
}
}
}
void XmlNode::getAttributes(std::vector<std::string>&vcAttr,std::vector<std::string>&vcVal)
{
vcAttr = m_vcattr;
vcVal = m_vcval;
}
std::vector<XmlNode*> XmlNode::getChildNode()
{
return m_vcChildNode;
}
void XmlNode::setNodeValue(const char * nodeval)
{
if(nodeval)
m_nodevalue = nodeval;
}
std::string XmlNode::getNodeValue()
{
return m_nodevalue;
}
bool XmlNode::remove()
{
XmlNode *pParent = getParent();
if(pParent)
{
pParent->removeChild(this);
return true;
}
return false;
}
void XmlNode::removeChild(XmlNode* nodeChild)
{
for (size_t i = 0; i< m_vcChildNode.size(); ++i)
{
if(m_vcChildNode[i] == nodeChild)
{
m_vcChildNode.erase(m_vcChildNode.begin()+i);
delete nodeChild;
break;
}
}
}
void XmlNode::removeAllChildNoDel()
{
m_vcChildNode.clear();
}
void XmlNode::removeAtrr(const char * attr)
{
for (size_t i = 0;i < m_vcattr.size(); ++i)
{
if(m_vcattr[i] == attr)
{
m_vcattr.erase(m_vcattr.begin()+i);
m_vcval.erase(m_vcval.begin()+i);
m_bmap = false;
break;
}
}
}
std::string XmlNode::getAllChildNodeText()
{
std::string xmltext;
size_t nChildLen = m_vcChildNode.size();
for (size_t i = 0; i < nChildLen; ++i)
{
XmlNode* pNode = m_vcChildNode[i];
std::string elexmltext = pNode->getNodeText();
xmltext += elexmltext;
}
return xmltext;
}
std::string XmlNode::getNodeText()
{
std::string xmltext;
xmltext += "<";
xmltext += m_tagName;
_ASSERT(m_vcattr.size() == m_vcval.size());
size_t nattrsize = m_vcattr.size();
for (size_t i = 0; i < nattrsize; ++i)
{
if(m_vcattr[i].length() > 0)
{
xmltext += " ";
xmltext += m_vcattr[i];
xmltext +="=\"";
xmltext += m_vcval[i];
xmltext += "\"";
}
}
//是否有子节点;
size_t nChildLen = m_vcChildNode.size();
if(nChildLen)
{
xmltext += ">\r\n";
for (size_t i = 0; i < nChildLen; ++i)
{
XmlNode* pNode = m_vcChildNode[i];
std::string elexmltext = pNode->getNodeText();
xmltext += elexmltext;
}
xmltext += m_nodevalue;
xmltext +="</";
xmltext += m_tagName;
xmltext +=">\r\n";
}
else
{
//没子节点就结束;
if(m_nodevalue.length() > 0)
{
xmltext += ">";
xmltext += m_nodevalue;
xmltext +="</";
xmltext += m_tagName;
xmltext +=">\r\n";
}
else
{
xmltext += "/>\r\n";
}
}
return xmltext;
}
std::vector<std::string> XmlNode::getAttributeNames()
{
return m_vcattr;
}
XmlNode* XmlNode::getFirstChildByTagName(std::string tagname)
{
std::vector<XmlNode*> tagNodes = getChildNodeByTagName(tagname);
if(tagNodes.size() >0)
return tagNodes[0];
return NULL;
}
std::vector<XmlNode*> XmlNode::getChildNodeByTagName( std::string tagname)
{
std::vector<XmlNode*> vcChildNode;
for (size_t i = 0; i <m_vcChildNode.size(); ++i)
{
if(m_vcChildNode[i]->m_tagName == tagname)
{
vcChildNode.push_back(m_vcChildNode[i]);
}
}
return vcChildNode;
}
std::vector<XmlNode*> XmlNode::findChildNodeByTagName( std::string tagname)
{
std::vector<XmlNode*> vcChildNode;
if(m_tagName == tagname)
{
vcChildNode.push_back(this);
}
for (size_t i = 0; i <m_vcChildNode.size(); ++i)
{
std::vector<XmlNode*> vcNextChild = m_vcChildNode[i]->findChildNodeByTagName(tagname);
for (size_t j = 0; j < vcNextChild.size(); ++j)
{
vcChildNode.push_back(vcNextChild[j]);
}
}
return vcChildNode;
}
void XmlNode::findChildNodeByTagName( std::string const& tagname, std::vector<XmlNode*>& pnodes)
{
if ( m_tagName == tagname )
{
pnodes.push_back(this);
}
for (std::vector<XmlNode*>::const_iterator Iter = m_vcChildNode.begin();
Iter != m_vcChildNode.end(); ++Iter )
{
(*Iter)->findChildNodeByTagName(tagname, pnodes);
}
}
void XmlNode::getAllLeafNode(std::vector<XmlNode*> & vcLeafNode)
{
std::vector<XmlNode*>vcChild = getChildNode();
if(vcChild.size() == 0)
{
vcLeafNode.push_back(this);
}
else
{
for (size_t i = 0; i < vcChild.size(); ++i)
{
vcChild[i]->getAllLeafNode(vcLeafNode);
}
}
}
bool XmlNode::hasChild()
{
if(m_vcChildNode.size() > 0)
return true;
return false;
}
void FastXML::encodeCharUTF8( int nUChar, char* pszUTF8, int& nUTF8Len)
{
if ( ! (nUChar & ~0x0000007f) ) // < 0x80
{
if ( pszUTF8 )
pszUTF8[nUTF8Len++] = (char)nUChar;
else
++nUTF8Len;
}
else if ( ! (nUChar & ~0x000007ff) ) // < 0x800
{
if ( pszUTF8 )
{
pszUTF8[nUTF8Len++] = (char)(((nUChar&0x7c0)>>6)|0xc0);
pszUTF8[nUTF8Len++] = (char)((nUChar&0x3f)|0x80);
}
else
nUTF8Len += 2;
}
else if ( ! (nUChar & ~0x0000ffff) ) // < 0x10000
{
if ( pszUTF8 )
{
pszUTF8[nUTF8Len++] = (char)(((nUChar&0xf000)>>12)|0xe0);
pszUTF8[nUTF8Len++] = (char)(((nUChar&0xfc0)>>6)|0x80);
pszUTF8[nUTF8Len++] = (char)((nUChar&0x3f)|0x80);
}
else
nUTF8Len += 3;
}
else // < 0x110000
{
if ( pszUTF8 )
{
pszUTF8[nUTF8Len++] = (char)(((nUChar&0x1c0000)>>18)|0xf0);
pszUTF8[nUTF8Len++] = (char)(((nUChar&0x3f000)>>12)|0x80);
pszUTF8[nUTF8Len++] = (char)(((nUChar&0xfc0)>>6)|0x80);
pszUTF8[nUTF8Len++] = (char)((nUChar&0x3f)|0x80);
}
else
nUTF8Len += 4;
}
}
void FastXML::ansiToUtf8( std::string &strbuff)
{
std::string strUTF8;
int nUChar, nCharLen;
wchar_t wcChar;
char szUTF8Char[4];
const char *pANSI = strbuff.c_str();
while ( *pANSI )
{
nCharLen = mbtowc( &wcChar, pANSI, 5 );
if ( nCharLen < 1 )
{
nCharLen = 1;
wcChar = (wchar_t)' ';
}
pANSI += nCharLen;
nUChar = (int)wcChar;
nCharLen = 0;
encodeCharUTF8( nUChar, szUTF8Char, nCharLen );
strUTF8.append(szUTF8Char,nCharLen);
}
strbuff = strUTF8;
}
int FastXML::decodeCharUTF8(const char*& pszUTF8)
{
int nUChar = (unsigned char)*pszUTF8;
++pszUTF8;
if ( nUChar & 0x80 )
{
int nExtraChars;
if ( ! (nUChar & 0x20) )
{
nExtraChars = 1;
nUChar &= 0x1f;
}
else if ( ! (nUChar & 0x10) )
{
nExtraChars = 2;
nUChar &= 0x0f;
}
else if ( ! (nUChar & 0x08) )
{
nExtraChars = 3;
nUChar &= 0x07;
}
else
return -1;
while ( nExtraChars-- )
{
if ( (*pszUTF8 & 0x80) )
{
nUChar = nUChar<<6;
nUChar |= *pszUTF8 & 0x3f;
}
else
return -1;
++pszUTF8;
}
}
return nUChar;
}
std::string FastXML::utf8ToAnsi(const char * pszUTF8,int nLen,int* pnFailed)
{
std::string strANSI;
int nBufferLen = nLen + 4;
strANSI.reserve(nBufferLen);
int nUChar, nCharLen;
char szANSI[2];
if ( pnFailed )
*pnFailed = 0;
const char* pUTF8 = pszUTF8;
while ( *pUTF8 )
{
nUChar = decodeCharUTF8( pUTF8 );
if ( nUChar & ~0xffff )
nCharLen = -1;
else
wctomb_s(&nCharLen,szANSI,MB_CUR_MAX,(wchar_t)nUChar);
if ( nCharLen == -1 )
{
if ( pnFailed )
++(*pnFailed);
}
else
{
strANSI.append(szANSI,nCharLen);
}
}
return strANSI;
}
FastXML::FastXML(void)
{
m_xmlhead = "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"no\"?>";
setlocale(LC_ALL, "");
}
FastXML::~FastXML(void)
{
}
std::string FastXML::trimRight(const std::string& str)
{
if (str.begin() == str.end())
{
return str;
}
std::string t = str;
for (std::string::iterator i = t.end() - 1; i != t.begin(); i--) {
if (!isspace(*i)) {
t.erase(i + 1, t.end());
break;
}
}
return t;
}
void inline FastXML::skipSpace(char *&pBuffer)
{
while(pBuffer && (*pBuffer == ' ' || *pBuffer == '\t' || *pBuffer == '\r' || *pBuffer == '\n'))
{
pBuffer++;
if(*pBuffer==0)
{
pBuffer--;
break;
}
}
}
void FastXML::decComment(char *&pBuffer)
{
skipSpace(pBuffer);
while(strncmp(pBuffer,"<!--",4)==0)
{
//<!-- 注释节点;
const char * pTempText = pBuffer;
pBuffer += 4;
while(*pBuffer)
{
if(*pBuffer == '>')
{
++pBuffer;
char c = *pBuffer;
*pBuffer = 0;
m_vccomment.push_back(pTempText);
*pBuffer = c;
break;
}
++pBuffer;
}
skipSpace(pBuffer);
}
}
void FastXML::decHeadCode(char *&pBuffer)
{
decComment(pBuffer);
skipSpace(pBuffer);
const char *pTempText = NULL;
if(strncmp(pBuffer,"<?",2)==0)
{
//<?xml version="1.0" encoding="utf-8" standalone="no"?>
pTempText = pBuffer;
pBuffer += 2;
while(*pBuffer)
{
if(*pBuffer== '?' && *(pBuffer+1) == '>')
{
pBuffer += 2;
char c = *pBuffer;
*pBuffer = 0;
m_xmlhead = pTempText;
*pBuffer = c;
break;
}
++pBuffer;
}
}
decComment(pBuffer);
skipSpace(pBuffer);
if(strncmp(pBuffer,"<!DOCTYPE",9)==0)
{
//<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
pTempText = pBuffer;
pBuffer += 9;
while(*pBuffer)
{
if(*pBuffer == '>')
{
++pBuffer;
char c = *pBuffer;
*pBuffer = 0;
m_doctype = pTempText;
*pBuffer = c;
break;
}
++pBuffer;
}
}
}
XmlNode *FastXML::parserBuffer(char *pBuffer,XmlNode *pParent)
{
std::string errormsg;
if(!pBuffer)
return NULL;
//确保从<开始;
while(pBuffer && *pBuffer && *pBuffer != '<')
++pBuffer;
decHeadCode(pBuffer);
std::list<XmlNode*>ParentList;
ParentList.push_back(pParent);
XmlNode * pParentElement = pParent;
while (ParentList.size() > 0)
{
if(*pBuffer == 0)
break;
pParentElement = ParentList.back();
std::string tagname = pParentElement->getTagName();
while (*pBuffer)
{
decComment(pBuffer);
if(strncmp(pBuffer,"<![CDATA[",9)==0)
{
//CDATA节点;
const char *pTempText = pBuffer;
pBuffer += 9;
while(*pBuffer)
{
if(*pBuffer == ']' && *(pBuffer+1) == ']' &&*(pBuffer+2)=='>')
{
//跳过]]>;
pBuffer += 3;
char c = *pBuffer;
*pBuffer = 0;
m_datatype = pTempText;
*pBuffer = c;
break;
}
++pBuffer;
}
}
else if(*pBuffer == '<' && *(pBuffer+1) == '/')
{
//节点结束;
pBuffer += 2;
skipSpace(pBuffer);
const char *pNodeName = pBuffer;
pBuffer = strstr(pBuffer,">");
if(!pBuffer)
{
std::string errmsg = "节点"+tagname+"缺少 > \r\n";
errormsg += errmsg;
goto END;
}
else
{
char c = *pBuffer;
*pBuffer = 0;
std::string endtagname = pNodeName;
*pBuffer = c;
endtagname = trimRight(endtagname);
if(endtagname != tagname)
{
//svg格式有错误,输出错误信息;
std::string errmsg = "节点"+tagname+"前后不匹配\r\n";
errormsg += errmsg;
goto END;
}
else
{
ParentList.pop_back();
}
}
break;
}
else if(*pBuffer == '<')
{
++pBuffer;
skipSpace(pBuffer);
const char *pNodeName = pBuffer;
while(*pBuffer != ' ' && *pBuffer != '\t' && *pBuffer!='\r' && *pBuffer!='\n' && *pBuffer !='>' && *pBuffer !='/')
++pBuffer;
if(*pBuffer == ' ' || *pBuffer == '\t' || *pBuffer!='\r' || *pBuffer=='\n' || *pBuffer=='>' || *pBuffer == '/')
{
//提取节点名;
char c = *pBuffer;
*pBuffer = 0;
std::string tempname = pNodeName;
std::string nodename;
for (size_t i = 0; i < tempname.size(); ++i)
{
if(tempname[i] != '/')
nodename += tempname[i];
}
if(nodename.length() == 0)
nodename = "undefine";
XmlNode *newNode = new XmlNode(nodename.c_str());
if(newNode->getTagName()=="otcoItem")
{
int a= 5;
++a;
}
pParentElement->addChild(newNode);
*pBuffer = c;
skipSpace(pBuffer);
const char *pAttrText = pBuffer;
//属性名;
std::string attrname;
//属性值;
std::string attrval;
bool bBreak = false;
while(*pBuffer)
{
//提取属性名,属性值;
if(*pBuffer == '=')
{
//提取属性名;
*pBuffer = 0;
attrname = pAttrText;
++pBuffer;
//查找空格;
skipSpace(pBuffer);
if(*pBuffer == '\"')
{
//左引号;
++pBuffer;
pAttrText = pBuffer;
//查找右引号;
pBuffer = strstr(pBuffer,"\"");
if(pBuffer)
{
*pBuffer = 0;
attrval = pAttrText;
newNode->setAttribute(attrname.c_str(),attrval.c_str());
++pBuffer;
//继续下一步循环;
skipSpace(pBuffer);
pAttrText = pBuffer;
}
else
{
//出错;
std::string errmsg = attrname+"缺乏引号\r\n";
errormsg += errmsg;
goto END;
break;
}
}
}
if(*pBuffer == '>')
{
//判断是否是文本节点,如果不是文本节点,则递归;
++pBuffer;
skipSpace(pBuffer);
if(*pBuffer == '<')
{
//不是文本节点,说明有子节点;
ParentList.push_back(newNode);
pBuffer;
bBreak = true;
break;
}
else
{
//是文本节点;
const char *pTextValue = pBuffer;
pBuffer = strstr(pBuffer,"<");
if(pBuffer)
{
char c = *pBuffer;
*pBuffer = 0;
std::string str = pTextValue;
str = trimRight(str);
newNode->setNodeValue(str.c_str());
*pBuffer = c;
ParentList.push_back(newNode);
bBreak = true;
break;
}
else
{
//找不到<;
errormsg += "文本节点缺乏>\r\n";
goto END;
}
}
}
else if(*pBuffer == '/' && *(pBuffer+1)=='>')
{
//一个节点结束;
pBuffer += 2;
break;
}
++pBuffer;
}
if(bBreak)
break;
}
}
if(*pBuffer !='<')
++pBuffer;
}
}
//判断是否异常,如果异常,返回NULL;
END:
if(errormsg.length() > 0)
{
//抛出文件异常;
}
return pParent;
}
bool FastXML::load( const char *filePath )
{
_ASSERT(filePath);
if(!filePath)
return false;
FILE* fp = NULL;
fopen_s(&fp,filePath, "rb");
if ( !fp )
return false;
fseek( fp, 0, SEEK_END);
int nFileByteLen = ftell(fp);
if(nFileByteLen == 0)
{
fclose(fp);
return false;
}
fseek(fp,0,SEEK_SET);
char *pData = NULL;
pData = new char[nFileByteLen*2+1];
if(!pData)
{
fclose(fp);
return false;
}
memset(pData,0,nFileByteLen*2+1);
pData[nFileByteLen] = 0;
fread(pData, nFileByteLen, 1, fp);
fclose(fp);
int nFailed = 0;
std::string strbuff = utf8ToAnsi(pData,nFileByteLen,&nFailed);
memset(pData,0,nFileByteLen*2+1);
memcpy(pData,strbuff.c_str(),strbuff.length());
loadFromBuff(pData);
delete []pData;
return true;
}
bool FastXML::loadFromUtf8Buff(const char *pBuff,int nsize)
{
char *pData = NULL;
pData = new char[nsize*2+1];
memset(pData,0,nsize*2+1);
if(!pData)
{
return false;
}
pData[nsize+1] = 0;
memcpy(pData,pBuff,nsize);
int nFailed = 0;
std::string strbuff = utf8ToAnsi(pData,nsize,&nFailed);
memset(pData,0,nsize*2+1);
memcpy(pData,strbuff.c_str(),strbuff.length());
loadFromBuff(pData);
delete []pData;
return true;
}
XmlNode * FastXML::getRootNode()
{
std::vector<XmlNode*> vcChildNode = m_rootNode.getChildNode();
if(vcChildNode.size() >= 1)
return vcChildNode[0];
return NULL;
}
bool FastXML::loadFromBuff(const char *pBuff)
{
int nSize = strlen(pBuff);
char *pNewBuff = new char[nSize+1];
if(!pNewBuff)
return false;
pNewBuff[nSize] = 0;
memset(pNewBuff,0,nSize+1);
memcpy(pNewBuff,pBuff,nSize);
m_rootNode.clearAll();
parserBuffer(pNewBuff,&m_rootNode);
delete []pNewBuff;
return true;
}
std::string FastXML::getFileXML()
{
XmlNode *pRootNode = getRootNode();
if(pRootNode)
{
std::string str;
if(m_xmlhead.length() > 0)
str = m_xmlhead + "\r\n";
if(m_doctype.length() > 0)
str += m_doctype+"\r\n";
if(m_vccomment.size() > 0)
{
for (size_t i = 0;i < m_vccomment.size(); ++i)
{
str += m_vccomment[i]+"\r\n";
}
}
str += pRootNode->getNodeText();
return str;
}
return "";
}
void FastXML::setXmlHead( std::string strhead )
{
m_xmlhead = strhead;
}
bool FastXML::save( const char *filepath )
{
_ASSERT(filepath);
if(!filepath)
return false;
std::string strtext = getFileXML();
ansiToUtf8(strtext);
FILE *fp = NULL;
fopen_s(&fp,filepath, "wb");
if(!fp)
return false;
fwrite(strtext.c_str(),strtext.length(),1,fp);
fclose(fp);
return true;
}
void FastXML::addRootNode( XmlNode *pNode )
{
m_rootNode.clearAll();
m_rootNode.addChild(pNode);
}
std::string FastXML::getDoc()
{
XmlNode *pRootNode = getRootNode();
if(pRootNode)
{
std::string str = pRootNode->getNodeText();
return str;
}
return "";
}
void FastXML::getNodeSize(XmlNode *pNode,std::string tagname,int &nSize)
{
if(pNode)
{
if(pNode->getTagName() == tagname)
{
nSize ++;
}
std::vector<XmlNode*>vcChild = pNode->getChildNode();
for (size_t i = 0; i < vcChild.size(); ++i)
{
getNodeSize(vcChild[i],tagname,nSize);
}
}
}
void FastXML::replaceAttr( std::string Attr,std::string oldvalue,std::string newvalue,XmlNode *pNode)
{
if(!pNode)
{
pNode = getRootNode();
}
if(pNode)
{
string strValue = pNode->getAttribute(Attr.c_str());
strValue = replace_all_distinct(strValue,oldvalue,newvalue);
pNode->setAttribute(Attr.c_str(),strValue.c_str());
std::vector<XmlNode*>vcChild = pNode->getChildNode();
for (size_t i = 0;i < vcChild.size(); ++i)
{
replaceAttr(Attr,oldvalue,newvalue,vcChild[i]);
}
}
}
#include "web_frame.h"
namespace socket_web_bus
{
uint64_t ntohl64(uint64_t host)
{
uint64_t ret = 0;
uint32_t high,low;
low = host & 0xFFFFFFFF;
high = (host >> 32) & 0xFFFFFFFF;
low = ntohl(low);
high = ntohl(high);
ret = low;
ret <<= 32;
ret |= high;
return ret;
}
uint64_t hl64ton(uint64_t host)
{
uint64_t ret = 0;
uint32_t high,low;
low = host & 0xFFFFFFFF;
high = (host >> 32) & 0xFFFFFFFF;
low = htonl(low);
high = htonl(high);
ret = low;
ret <<= 32;
ret|= high;
return ret;
}
web_frame::web_frame()
{
}
web_frame::~web_frame()
{
}
void web_frame::decode_web_frame(cl_web_buf& buffer)
{
int pos = 0;
char* message = buffer.get_buf();
fetch_fin(message, pos);
fetch_opcode(message, pos);
fetch_mask(message, pos);
fetch_payload_length(message, pos);
fetch_masking_key(message, pos);
fetch_payload(message, pos);
}
bool web_frame::complete_frame()
{
return fin_ == 1;
}
bool web_frame::zero_len_frame()
{
return (fin_ == 1) && (payload_length_ == 0);
}
bool web_frame::is_close_frame()
{
return opcode_ == 0x8;
}
void web_frame::reset()
{
fin_ = 0;
opcode_ = 0;
mask_ = 0;
memset(masking_key_, 0, sizeof(masking_key_));
payload_length_ = 0;
}
int web_frame::fetch_fin(char *msg, int &pos)
{
fin_ = (unsigned char)msg[pos] >> 7;
return 0;
}
int web_frame::fetch_opcode(char *msg, int &pos)
{
opcode_ = msg[pos] & 0x0f;
pos++;
return 0;
}
int web_frame::fetch_mask(char *msg, int &pos)
{
mask_ = (unsigned char)msg[pos] >> 7;
return 0;
}
int web_frame::fetch_masking_key(char *msg, int &pos)
{
if(mask_ != 1)
return 0;
for(int i = 0; i < 4; i++)
masking_key_[i] = msg[pos + i];
pos += 4;
return 0;
}
int web_frame::fetch_payload_length(char *msg, int &pos)
{
payload_length_ = msg[pos] & 0x7f;
pos++;
if(payload_length_ == 126){
uint16_t length = 0;
memcpy(&length, msg + pos, 2);
pos += 2;
payload_length_ = ntohs(length);
}
else if(payload_length_ == 127){
uint64_t length = 0;
memcpy(&length, msg + pos, 8);
pos += 8;
payload_length_ = ntohl64(length);
}
return 0;
}
int web_frame::fetch_payload(char *msg, int &pos)
{
if(!payload_length_)
return -1;
int nlen = 0;
if(fin_ == 1)
buffer.set_buf_size(payload_length_);
else
{
nlen = buffer.get_buf_size();
buffer.set_buf_size(nlen + payload_length_);
}
char* p_payload = buffer.get_buf(nlen);
if(mask_ != 1)
{
memcpy(p_payload, msg + pos, payload_length_);
}
else
{
for(uint32_t i = 0; i < payload_length_; i++)
{
int j = i % 4;
p_payload[i] = msg[pos + i] ^ masking_key_[j];
}
}
pos += payload_length_;
return 0;
}
cl_web_buf& web_frame::get_msg_buffer()
{
return buffer;
}
};
#ifndef __WEB_FRAME__
#define __WEB_FRAME__
#include <stdint.h>
#include <winsock.h>
#include "data_handler.h"
namespace socket_web_bus
{
class web_frame
{
public:
web_frame();
~web_frame();
bool complete_frame();
bool zero_len_frame();
bool is_close_frame();
void decode_web_frame(cl_web_buf& buffer);
void reset();
cl_web_buf& get_msg_buffer();
private:
int fetch_fin(char *msg, int &pos);
int fetch_opcode(char *msg, int &pos);
int fetch_mask(char *msg, int &pos);
int fetch_masking_key(char *msg, int &pos);
int fetch_payload_length(char *msg, int &pos);
int fetch_payload(char *msg, int &pos);
private:
uint8_t fin_;
uint8_t opcode_;
uint8_t mask_;
uint8_t masking_key_[4];
uint64_t payload_length_;
cl_web_buf buffer;
};
};
#endif
void encode_web_frame(uint8_t fin_,uint8_t opcode_,const char* pPayloadData,uint32_t payLoadLen,sp_web_buf_t& result)
{
if(pPayloadData == NULL || payLoadLen == 0)
return;
//构造第一个字节;
uint8_t fin = 1;//fin_;//默认设置为1,终极帧
uint8_t opcode = 2;//opcode_;//默认设置为2,二进制数据
uint8_t mask = 0;
if(payLoadLen < 126)
{
result->set_buf_size(payLoadLen + 2);
char* buffer = result->get_buf();
buffer[0] = (fin << 7) + opcode;
buffer[1] = (mask <<7) + static_cast<uint8_t>(payLoadLen);
memcpy(&buffer[2],pPayloadData,payLoadLen);
}
else if(payLoadLen < 0xFFFF)
{
result->set_buf_size(payLoadLen + 4);
char* buffer = result->get_buf();
buffer[0] = (fin << 7) + opcode;
buffer[1] = (mask <<7) + 126;
buffer[2] = static_cast<uint8_t>((payLoadLen & 0xFF00) >> 8);
buffer[3] = static_cast<uint8_t>(payLoadLen & 0xFF);
memcpy(&buffer[4],pPayloadData,payLoadLen);
}
else
{
result->set_buf_size(payLoadLen + 10);
char* buffer = result->get_buf();
buffer[0] = (fin << 7) + opcode;
buffer[1] = (mask <<7) + 127;
buffer[2] = 0;
buffer[3] = 0;
buffer[4] = 0;
buffer[5] = 0;
buffer[6] = static_cast<uint8_t>(((payLoadLen & 0xFF000000)>>24));
buffer[7] = static_cast<uint8_t>((payLoadLen & 0xFF0000)>>16);
buffer[8] = static_cast<uint8_t>((payLoadLen & 0xFF00)>>8);
buffer[9] = static_cast<uint8_t>(payLoadLen & 0xFF);
memcpy(&buffer[10],pPayloadData,payLoadLen);
}
}
#ifndef DATA_TYPE_H_
#define DATA_TYPE_H_
#include <stdint.h>
#include <vector>
#include <string>
#include "typeDef.h"
namespace socket_bus
{
using namespace std;
#pragma pack(1)
#pragma warning(disable:4200)
template <typename Elem>
struct st_list
{
uint32_t count;
uint32_t elem_size;
Elem elem[0];
size_t get_size()
{
return (sizeof(count) + sizeof(elem_size) + elem_size * count);
};
static size_t get_size_s(uint32_t count)
{
return (sizeof(st_list) + sizeof(Elem) * count);
};
Elem& operator [](uint32_t index)
{
return *(Elem*)((char*)&elem[0] + elem_size * index);
};
};
#pragma warning(default:4200)
template <typename Info>
struct st_info
{
uint32_t size;
Info info;
uint32_t get_size()
{
return size;
};
static uint32_t get_size_s()
{
return sizeof(st_info);
};
};
template <typename Options, typename Quotes>
struct st_options_quotes
{
st_info<Options> options;
st_list<Quotes> quotes;
st_info<Options>* get_options() const { return (st_info<Options>*)this; };
st_list<Quotes>* get_quotes() const { return (st_list<Quotes>*)((char*)this + get_options()->get_size()); };
size_t get_size() const
{
return (get_options()->get_size() + get_quotes()->get_size());
};
static size_t get_size_s(uint32_t count)
{
return (st_info<Options>::get_size_s() + st_list<Quotes>::get_size_s(count));
};
};
template <uint16_t size>
struct st_string
{
char sz_string[size];
static uint16_t get_size_s() { return size; };
st_string<size>& operator =(const string& str_string)
{
memset(&(sz_string[0]),0,size);
str_string._Copy_s(&sz_string[0], size, size - 1);
return *this;
};
st_string<size>(const char* str_string = "\0")
{
memset(&(sz_string[0]),0,size);
memccpy(&(sz_string[0]),str_string,size,strlen(str_string));
}
operator const char*() const
{
return &sz_string[0];
};
bool operator <(const st_string<size>& str) const
{
return strcmp(this->sz_string,str.sz_string) == -1;
}
void copy_from_buffer(const char*buffer,uint16_t length)
{
memset(&(sz_string[0]),0,size);
memccpy(&(sz_string[0]),buffer,size,length);
}
};
#define DEF_INSTITUTION_ID_STR_MAX_SIZE 8
#define DEF_OPTIONS_CODE_STR_MAX_SIZE 32
#define DEF_SUBSCRIBE_CODE_STR_MAX_SIZE 1024
#define DEF_ACCOUNT_LOGIN_STR_MAX_SIZE 16
#define DEF_PASSWORD_LOGIN_STR_MAX_SIZE 64
typedef st_string<DEF_INSTITUTION_ID_STR_MAX_SIZE> institution_id_t;
typedef st_string<DEF_OPTIONS_CODE_STR_MAX_SIZE> options_code_t;
typedef st_string<DEF_SUBSCRIBE_CODE_STR_MAX_SIZE> subscribe_code_t;
typedef st_string<DEF_ACCOUNT_LOGIN_STR_MAX_SIZE> user_t;
typedef st_string<DEF_PASSWORD_LOGIN_STR_MAX_SIZE> pswd_t;
typedef void (*pfn_ip_port_t)(string, uint16_t);
struct st_ip_port
{
string ip;
uint16_t port;
st_ip_port() {};
st_ip_port(const string& str_ip, uint16_t port_num)
: ip(str_ip), port(port_num)
{};
st_ip_port(const st_ip_port& ip_port)
{
ip = ip_port.ip;
port = ip_port.port;
};
~st_ip_port() {};
st_ip_port& operator =(const st_ip_port& ip_port)
{
if (this != &ip_port)
{
ip = ip_port.ip;
port = ip_port.port;
}
return *this;
};
bool operator ==(const st_ip_port& ip_port)
{
return ((ip == ip_port.ip) && (port == ip_port.port));
};
};
typedef vector<st_ip_port> vtr_ip_port_t;
struct futureItem
{
char code[16]; //合约代码
double trade_day; //交易日
double trade_time; //最后修改时间
double new_price; //最新价
double u_limit_price; //涨停板价
double l_limit_price; //跌停板价
double bid_price1; //申买价一
double ask_price1; //申卖价一
double average_price; //当日均价
};
struct quoteHead
{
int type;
int days;
int warrant_type;
char date[16];
char market[8];
double trade_day;
double trade_time;
double underly_price;
};
struct quoteItem
{
int bid;
int put;
double option_price;
double strike_price;
};
#pragma pack()
};
#endif