基于Nginx的TCP服务负载均衡

一、configure配置加上 --with-stream

sudo ./configure --prefix=/usr/local/nginx --with-http_realip_module --with-http_sub_module --with-http_flv_module --with-http_dav_module --with-http_gzip_static_module --with-http_stub_status_module --with-http_addition_module --with-pcre=/home/weijl/download/pcre-8.39 --with-openssl=/home/weijl/download/openssl-1.1.0e --with-http_ssl_module --with-zlib=/home/weijl/download/zlib-1.2.11 --with-stream

二、重新configure、编译、安装nginx程序

（1）configure

sudo ./configure --prefix=/usr/local/nginx --with-http_realip_module --with-http_sub_module --with-http_flv_module --with-http_dav_module --with-http_gzip_static_module --with-http_stub_status_module --with-http_addition_module --with-pcre=/home/weijl/work/pcre-8.39 --with-openssl=/home/weijl/work/openssl-1.1.0e --with-http_ssl_module --with-zlib=/home/weijl/work/zlib-1.2.11 --with-stream

（2）make

（3）make install

三、修改nginx.conf文件

stream{}块为TCP反向代理配置

上游服务器 192.168.0.9:8000---在这只配了一台上游服务器，作负载均衡配多台，轮询或者IP hash

负载均衡服务器192.168.0.6:8000

user  root;
worker_processes  2;

#error_log  logs/error.log;
#error_log  logs/error.log  notice;
#error_log  logs/error.log  info;

#pid        logs/nginx.pid;


events {
    worker_connections  1024;
}

stream {
	upstream tcp_proxy{
		server 192.168.0.9:8000;
#		server 192.168.1.108:3306 weight=1;
	}
        
	server{
		listen 8000;
		proxy_pass tcp_proxy;
	}
}


http {
    include       mime.types;
    default_type  application/octet-stream;

    #log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
    #                  '$status $body_bytes_sent "$http_referer" '
    #                  '"$http_user_agent" "$http_x_forwarded_for"';

    #access_log  logs/access.log  main;

    sendfile        on;
    #tcp_nopush     on;

    #keepalive_timeout  0;
    keepalive_timeout  65;

    #gzip  on;

    server {
        listen      192.168.0.6:8080;
        server_name  www.staticres.com;

        #charset koi8-r;

        #access_log  logs/host.access.log  main;

        location / {
            root   html;
            index  index.html index.htm;
        }

        #error_page  404              /404.html;

        # redirect server error pages to the static page /50x.html
        #
        error_page   500 502 503 504  /50x.html;
        location = /50x.html {
            root   html;
        }

	#静态图片资源  
	location /image/ {  
	    root /home/weijl/work/;  
	    autoindex on;  
	}  

        # proxy the PHP scripts to Apache listening on 127.0.0.1:80
        #
        #location ~ \.php$ {
        #    proxy_pass   http://127.0.0.1;
        #}

        # pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000
        #
        #location ~ \.php$ {
        #    root           html;
        #    fastcgi_pass   127.0.0.1:9000;
        #    fastcgi_index  index.php;
        #    fastcgi_param  SCRIPT_FILENAME  /scripts$fastcgi_script_name;
        #    include        fastcgi_params;
        #}

        # deny access to .htaccess files, if Apache's document root
        # concurs with nginx's one
        #
        #location ~ /\.ht {
        #    deny  all;
        #}
    }


    # another virtual host using mix of IP-, name-, and port-based configuration
    #
    #server {
    #    listen       8000;
    #    listen       somename:8080;
    #    server_name  somename  alias  another.alias;

    #    location / {
    #        root   html;
    #        index  index.html index.htm;
    #    }
    #}


    # HTTPS server
    #
    #server {
    #    listen       443 ssl;
    #    server_name  localhost;

    #    ssl_certificate      cert.pem;
    #    ssl_certificate_key  cert.key;

    #    ssl_session_cache    shared:SSL:1m;
    #    ssl_session_timeout  5m;

    #    ssl_ciphers  HIGH:!aNULL:!MD5;
    #    ssl_prefer_server_ciphers  on;

    #    location / {
    #        root   html;
    #        index  index.html index.htm;
    #    }
    #}

}

 四、上游服务器代码

#include <WinSock2.h>
#include <WS2tcpip.h> 
#pragma comment(lib,"ws2_32.lib")   
#include <stdio.h> 
#include <iostream>

using namespace std;

int main()  
{  
	/// 初始化socket  
	WSADATA wsaData;  
	WORD version = MAKEWORD(2,2);  
	int result = 0;  
	result = WSAStartup(version, &wsaData);  
	if (0 != result)  
	{  
		std::cout << "WSAStartup() error." << std::endl;  
		return -1;  
	}  

	/// 创建侦听socket   
	SOCKET socketListen;  
	socketListen = socket(AF_INET, SOCK_STREAM, 0);  
	if (socketListen == INVALID_SOCKET)  
	{  
		WSACleanup();  
		std::cout << "socket() error." << std::endl;  
		return -1;  
	}  

	/// 服务器地址结构   
	sockaddr_in svrAddress;  
	svrAddress.sin_family = AF_INET;  
	svrAddress.sin_addr.s_addr = INADDR_ANY;  
	svrAddress.sin_port = htons(8000);  

	/// 绑定服务器套接字   
	result = bind(socketListen, (sockaddr*)&svrAddress, sizeof(svrAddress));  
	if (result == SOCKET_ERROR)  
	{  
		closesocket(socketListen);  
		WSACleanup();  
		std::cout << "bind() error." << std::endl;  
		return -1;  
	}  

	/// 开启监听  
	result = listen(socketListen, 5);  
	if (result == SOCKET_ERROR)  
	{  
		closesocket(socketListen);  
		WSACleanup();  
		std::cout << "listen() error." << std::endl;  
		return -1;  
	}  
	std::cout << "服务器启动成功，监听端口：" << ntohs(svrAddress.sin_port) << std::endl;  

	/// select模型   
	fd_set allSockSet;   
	FD_ZERO(&allSockSet);   

	FD_SET(socketListen, &allSockSet); // 将socketListen加入套接字集合中   

	while (true)  
	{  
		fd_set readSet;  
		FD_ZERO(&readSet);                    //初始化可读队列
		readSet = allSockSet;   

		result = select(0, &readSet, NULL, NULL, NULL);  
		if (result == SOCKET_ERROR)  
		{  
			std::cout << "select() error." << std::endl;  
			break;  
		}  

		if (FD_ISSET(socketListen, &readSet))  
		{  
			sockaddr_in clientAddr;  
			int len = sizeof(clientAddr);  

			SOCKET clientSocket = accept(socketListen, (sockaddr*)&clientAddr, &len);         
			if (clientSocket == INVALID_SOCKET)  
			{  
				std::cout << "accept() error." << std::endl;  
				break;  
			}  
			FD_SET(clientSocket, &allSockSet);   /// 将新创建的套接字加入到集合中   

			char ipAddress[16] = { 0 };  
			inet_ntop(AF_INET, &clientAddr, ipAddress, 16);  
			std::cout << "有新的连接[" << ipAddress << ":" << ntohs(clientAddr.sin_port)  
				<< "], 目前客户端的数量为：" << allSockSet.fd_count - 1 << std::endl;       //去掉一个侦听套接字

			continue;  
		}  

		for (u_int i = 0; i < allSockSet.fd_count; ++i)  
		{  
			SOCKET socket = allSockSet.fd_array[i];  

			sockaddr_in clientAddr;  
			int len = sizeof(clientAddr);  
			getpeername(socket, (struct sockaddr *)&clientAddr, &len);      //获得与套接口相连的远程协议地址
			char ipAddress[16] = { 0 };  
			inet_ntop(AF_INET, &clientAddr, ipAddress, 16);  

			/// 可读性监视，可读性指有连接到来、有数据到来、连接已关闭、重置或终止  
			if (FD_ISSET(socket, &readSet))  
			{  
				char bufRecv[100];  
				result = recv(socket, bufRecv, 100, 0);  
				if (result == SOCKET_ERROR)  
				{  
					DWORD err = WSAGetLastError();  
					if (err == WSAECONNRESET)       /// 客户端的socket没有被正常关闭,即没有调用closesocket  
					{  
						std::cout << "客户端[" << ipAddress << ":" << ntohs(clientAddr.sin_port) << "]被强行关闭, ";  
					}  
					else  
					{  
						std::cout << "recv() error，" << std::endl;  
					}  

					closesocket(socket);  
					FD_CLR(socket, &allSockSet);  

					std::cout << "目前客户端的数量为：" << allSockSet.fd_count - 1 << std::endl;  
					break;  
				}  
				else if (result == 0)               /// 客户端的socket调用closesocket正常关闭  
				{  
					closesocket(socket);  
					FD_CLR(socket, &allSockSet);  

					std::cout << "客户端[" << ipAddress << ":" << ntohs(clientAddr.sin_port)   
						<< "]已经退出，目前客户端的数量为：" << allSockSet.fd_count - 1 << std::endl;  
					break;  
				}  

				bufRecv[result] = '\0';  
				std::cout << "来自客户端[" << ipAddress << ":" << ntohs(clientAddr.sin_port)  
					<< "]的消息：" << bufRecv << std::endl;  

				//char szSend[256] = "朕已经收到，你可以滚了！";
				char szSend[4096] = "Welcome to 192.168.0.9\r\n";

				send(socket, szSend, strlen(szSend), 0);
			}  
		}  
	}  

	for (u_int i = 0; i < allSockSet.fd_count; ++i)  
	{  
		SOCKET socket = allSockSet.fd_array[i];  
		closesocket(socket);  
	}  

	WSACleanup();  
	return 0;  
}

 五、浏览器访问结果