CentOS7安装QAT 1.7加密卡

1. 准备材料

1.1 机器

物理PC一台（记作A），安装QAT加密卡硬件和CentOS 7系统。

1.2 Tools

登录A，执行：

# yum install -y kernel kernel-devel openssl-devel pciutils zlib-devel gcc libudev-devel boost-devel

如果内核有更新，重启。

1.3 Code

1.3.1 QAT相关

https://01.org/sites/default/files/downloads/intelr-quickassist-technology/qat1.7.upstream.l.1.0.3-42.tar.gz

https://github.com/intel/QAT_Engine/archive/v0.5.34.tar.gz

1.3.2 OpenSSL

https://github.com/openssl/openssl/archive/OpenSSL_1_1_0g.tar.gz

1.3.3 应用相关

http://nginx.org/download/nginx-1.12.2.tar.gz

http://www.haproxy.org/download/1.8/src/haproxy-1.8.4.tar.gz

只能用浏览器下载。

2. Make && Make install

2.1 QAT driver

登录A，执行：

 

# mkdir /tmp/QAT

# tar zxvf qat1.7.upstream.l.*.tar.gz -C /tmp/QAT

# cd /tmp/QAT

# chmod -R o-rwx *

# ./configure

# make

# make install

2.2 OpenSSL

登录A，执行：

# tar xzf OpenSSL_1_1_0g.tar.gz

# cd openssl-OpenSSL_1_1_0g

# ./config --prefix=/usr/local/ssl

# make depend (ifrecommended by the OpenSSL* build system)

# make

# make install

# export OPENSSL_ENGINES=/usr/local/ssl/lib/engines-1.1

# echo /usr/local/ssl/lib > /etc/ld.so.conf.d/qat.conf

# ldconfig

2.3 QAT Engine

2.3.1 qat_contig_mem

# tar zxf QAT_Engine-0.5.34.tar.gz

# cd QAT_Engine-0.5.34/qat_contig_mem

# make

# make load

# make test

2.3.2 QAT_Engine

# cd QAT_Engine-0.5.34

# ./configure --with-qat_dir=/tmp/QAT\

--with-openssl_dir=/path/to/openssl-OpenSSL_1_1_0g\

--with-openssl_install_dir=/usr/local/ssl\

--enable-upstream_driver \

--enable-usdm

# make

# make install

2.3.3 QAT config files

# cp /path/to/QAT_Engine-0.5.34/qat/config/c6xx/multi_process_event-driven_optimized/*/etc

# service qat_servicestop

# service qat_servicestart

2.4 Nginx

nginx-qat.patch:

diff --git a/src/core/ngx_connection.c b/src/core/ngx_connection.c
index a1baefb..e783b5f 100644
--- a/src/core/ngx_connection.c
+++ b/src/core/ngx_connection.c
@@ -957,6 +957,18 @@ ngx_configure_listening_sockets(ngx_cycle_t *cycle)
     return;
 }

+#if (NGX_SSL)
+void
+ngx_asynch_del_conn(ngx_connection_t *c)
+{
+    if (ngx_del_async_conn) {
+        if (c->num_async_fds) {
+            ngx_del_async_conn(c, NGX_DISABLE_EVENT);
+            c->num_async_fds--;
+        }
+    }
+}
+#endif

 void
 ngx_close_listening_sockets(ngx_cycle_t *cycle)
@@ -986,7 +998,11 @@ ngx_close_listening_sockets(ngx_cycle_t *cycle)
                      * for closed shared listening sockets unless
                      * the events was explicitly deleted
                      */
-
+#if (NGX_SSL)
+                    if (c->asynch) {
+                        ngx_asynch_del_conn(c);
+                    }
+#endif
                     ngx_del_event(c->read, NGX_READ_EVENT, 0);

                 } else {
@@ -1035,6 +1051,9 @@ ngx_get_connection(ngx_socket_t s, ngx_log_t *log)
 {
     ngx_uint_t         instance;
     ngx_event_t       *rev, *wev;
+#if (NGX_SSL)
+    ngx_event_t       *aev;
+#endif
     ngx_connection_t  *c;

     /* disable warning: Win32 SOCKET is u_int while UNIX socket is int */
@@ -1071,11 +1090,17 @@ ngx_get_connection(ngx_socket_t s, ngx_log_t *log)

     rev = c->read;
     wev = c->write;
+#if (NGX_SSL)
+    aev = c->async;
+#endif

     ngx_memzero(c, sizeof(ngx_connection_t));

     c->read = rev;
     c->write = wev;
+#if (NGX_SSL)
+    c->async = aev;
+#endif
     c->fd = s;
     c->log = log;

@@ -1083,17 +1108,32 @@ ngx_get_connection(ngx_socket_t s, ngx_log_t *log)

     ngx_memzero(rev, sizeof(ngx_event_t));
     ngx_memzero(wev, sizeof(ngx_event_t));
+#if (NGX_SSL)
+    ngx_memzero(aev, sizeof(ngx_event_t));
+#endif

     rev->instance = !instance;
     wev->instance = !instance;
+#if (NGX_SSL)
+    aev->instance = !instance;
+#endif

     rev->index = NGX_INVALID_INDEX;
     wev->index = NGX_INVALID_INDEX;
+#if (NGX_SSL)
+    aev->index = NGX_INVALID_INDEX;
+#endif

     rev->data = c;
     wev->data = c;
+#if (NGX_SSL)
+    aev->data = c;
+#endif

     wev->write = 1;
+#if (NGX_SSL)
+    aev->async = 1;
+#endif

     return c;
 }
@@ -1132,11 +1172,26 @@ ngx_close_connection(ngx_connection_t *c)
         ngx_del_timer(c->write);
     }

+#if (NGX_SSL)
+    if (c->async->timer_set) {
+        ngx_del_timer(c->async);
+    }
+
+    if (c->asynch) {
+        ngx_asynch_del_conn(c);
+    }
+#endif
     if (!c->shared) {
         if (ngx_del_conn) {
             ngx_del_conn(c, NGX_CLOSE_EVENT);

         } else {
+#if (NGX_SSL)
+            if (c->asynch) {
+                ngx_asynch_del_conn(c);
+            }
+#endif
+
             if (c->read->active || c->read->disabled) {
                 ngx_del_event(c->read, NGX_READ_EVENT, NGX_CLOSE_EVENT);
             }
@@ -1155,8 +1210,17 @@ ngx_close_connection(ngx_connection_t *c)
         ngx_delete_posted_event(c->write);
     }

+#if (NGX_SSL)
+    if (c->async->posted) {
+        ngx_delete_posted_event(c->async);
+    }
+#endif
+
     c->read->closed = 1;
     c->write->closed = 1;
+#if (NGX_SSL)
+    c->async->closed = 1;
+#endif

     ngx_reusable_connection(c, 0);

diff --git a/src/core/ngx_connection.h b/src/core/ngx_connection.h
index 1d3e3a3..3a481d9 100644
--- a/src/core/ngx_connection.h
+++ b/src/core/ngx_connection.h
@@ -122,8 +122,14 @@ struct ngx_connection_s {
     void               *data;
     ngx_event_t        *read;
     ngx_event_t        *write;
+#if (NGX_SSL)
+    ngx_event_t        *async;
+#endif

     ngx_socket_t        fd;
+#if (NGX_SSL)
+    ngx_socket_t        async_fd;
+#endif

     ngx_recv_pt         recv;
     ngx_send_pt         send;
@@ -149,6 +155,7 @@ struct ngx_connection_s {

 #if (NGX_SSL || NGX_COMPAT)
     ngx_ssl_connection_t  *ssl;
+    ngx_flag_t          asynch;
 #endif

     struct sockaddr    *local_sockaddr;
@@ -181,6 +188,9 @@ struct ngx_connection_s {
     unsigned            tcp_nopush:2;    /* ngx_connection_tcp_nopush_e */

     unsigned            need_last_buf:1;
+#if (NGX_SSL)
+    unsigned char       num_async_fds;
+#endif

 #if (NGX_HAVE_AIO_SENDFILE || NGX_COMPAT)
     unsigned            busy_count:2;
@@ -205,6 +215,7 @@ struct ngx_connection_s {

 ngx_listening_t *ngx_create_listening(ngx_conf_t *cf, struct sockaddr *sockaddr,
     socklen_t socklen);
+void ngx_asynch_del_conn(ngx_connection_t *c);
 ngx_int_t ngx_clone_listening(ngx_conf_t *cf, ngx_listening_t *ls);
 ngx_int_t ngx_set_inherited_sockets(ngx_cycle_t *cycle);
 ngx_int_t ngx_open_listening_sockets(ngx_cycle_t *cycle);
diff --git a/src/core/ngx_cycle.h b/src/core/ngx_cycle.h
index 2b48ccb..f79da12 100644
--- a/src/core/ngx_cycle.h
+++ b/src/core/ngx_cycle.h
@@ -71,7 +71,9 @@ struct ngx_cycle_s {
     ngx_connection_t         *connections;
     ngx_event_t              *read_events;
     ngx_event_t              *write_events;
-
+#if (NGX_SSL)
+    ngx_event_t              *async_events;
+#endif
     ngx_cycle_t              *old_cycle;

     ngx_str_t                 conf_file;
diff --git a/src/event/modules/ngx_epoll_module.c b/src/event/modules/ngx_epoll_module.c
index 76aee08..b45202e 100644
--- a/src/event/modules/ngx_epoll_module.c
+++ b/src/event/modules/ngx_epoll_module.c
@@ -122,6 +122,11 @@ static ngx_int_t ngx_epoll_notify(ngx_event_handler_pt handler);
 #endif
 static ngx_int_t ngx_epoll_process_events(ngx_cycle_t *cycle, ngx_msec_t timer,
     ngx_uint_t flags);
+#if (NGX_SSL)
+static ngx_int_t ngx_epoll_add_async_connection(ngx_connection_t *c);
+static ngx_int_t ngx_epoll_del_async_connection(ngx_connection_t *c,
+        ngx_uint_t flags);
+#endif

 #if (NGX_HAVE_FILE_AIO)
 static void ngx_epoll_eventfd_handler(ngx_event_t *ev);
@@ -196,6 +201,13 @@ static ngx_event_module_t  ngx_epoll_module_ctx = {
         ngx_epoll_process_events,        /* process the events */
         ngx_epoll_init,                  /* init the events */
         ngx_epoll_done,                  /* done the events */
+#if (NGX_SSL)
+        ngx_epoll_add_async_connection,  /* add an async conn */
+        ngx_epoll_del_async_connection   /* del an async conn */
+#else
+        NULL,
+        NULL
+#endif
     }
 };

@@ -758,6 +770,54 @@ ngx_epoll_del_connection(ngx_connection_t *c, ngx_uint_t flags)
     return NGX_OK;
 }

+#if (NGX_SSL)
+static ngx_int_t
+ngx_epoll_add_async_connection(ngx_connection_t *c)
+{
+    struct epoll_event  ee;
+
+    ee.events = EPOLLIN|EPOLLOUT|EPOLLET|EPOLLRDHUP;
+    ee.data.ptr = (void *) ((uintptr_t) c | (c->async->async << 1) | c->async->instance);
+
+    ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "epoll add async connection: fd:%d ev:%08XD", c->async_fd, ee.events);
+    if (epoll_ctl(ep, EPOLL_CTL_ADD, c->async_fd, &ee) == -1) {
+        ngx_log_error(NGX_LOG_ALERT, c->log, ngx_errno,
+                      "async add conn epoll_ctl(EPOLL_CTL_ADD, %d) failed", c->async_fd);
+        return NGX_ERROR;
+    }
+
+    c->async->active = 1;
+
+    return NGX_OK;
+}
+
+
+static ngx_int_t
+ngx_epoll_del_async_connection(ngx_connection_t *c, ngx_uint_t flags)
+{
+    int                 op;
+    struct epoll_event  ee;
+
+    ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
+                   "epoll del async connection: fd:%d", c->async_fd);
+
+    op = EPOLL_CTL_DEL;
+    ee.events = 0;
+    ee.data.ptr = NULL;
+    if (epoll_ctl(ep, op, c->async_fd, &ee)