任务描述
- upstream的server实例存储在redis中,为openresty设置一个location,来触发加载server的动作
- 访问upstream时,轮询内部的server
- 不包括健康检测,我认为既然你的server都存储在redis里面了,健康检测也应该独立于openresty,发现坏节点,重新加载
技术描述
ngx.balancer ngx.shared.DICT resty.redis
部署环境描述
- http://10.161.4.61:80/ 后端实例
- http://10.161.4.63:80/ 后端实例
- http://10.161.4.63:8000/ openresty
- 10.161.4.63 6379 redis
步骤
部署redis
启动一个redis实例,没有特殊要求,能访问就行
设置openresty
修改配置文件
worker_processes 3;
error_log logs/error.log;
events {
worker_connections 30;
}
http {
lua_shared_dict servers 10m;#全局变量,存储upstream,根据你的后端节点数量,设置大小
lua_shared_dict now_server 1m;#全局变量,用于轮询,记录现在是哪个节点服务
lua_shared_dict lens 1m;#全局变量,记录后端节点的数量
upstream tomcat17 {#这个upstream用于对比测试
server 10.161.4.63:80 weight=1;
server 10.161.4.61:80 weight=1;
keepalive 10;
}
upstream test{#动态加载的upstream
server 192.168.0.18:80 weight=1;#此处必须有一个server,否则会启动失败,可以胡乱写一个
balancer_by_lua_block {
local set_peer = require "set_peer"
set_peer.run_peer()
}
keepalive 10;
}
#init_worker_by_lua_block {
# local set_peer = require "set_peer"
# set_peer.insert_server()
#}
server{
listen 8000;
location ~ ^/api/([-_a-zA-Z0-9/]+) {
access_by_lua_file work/lua/check_access.lua;
content_by_lua_file work/lua/$1.lua;
}
location = /tomcat17 {#用于对比测试的location
proxy_pass http://tomcat17/;
}
location = /insert {#用于加载server的location
content_by_lua_block{
local set_peer = require "set_peer"
set_peer.insert_server()
}
}
location = /test_run {#用于测试的upstream
proxy_pass http://test/;
}
location = /status {
stub_status;
}
}
}
lualib或者你自己的lua目录下创建,set_peer.lua 脚本(刚学lua没多久,写的有点水)
local balancer = require "ngx.balancer"
local now_server = ngx.shared.now_server;
local servers = ngx.shared.servers;
local server_len = ngx.shared.lens;
local redis = require "resty.redis"
local _M = { _VERSION = '1.0' }
local function con_redis()
local red = redis:new()
red:set_timeout(1000)
local ok, err = red:connect("127.0.0.1", 6379)--注意此处
if not ok then
ngx.log(ngx.ERR,"failed to connect: ", err)
return ngx.exit(500)
else
return red
end
end
local function set_peer(host,port)
if not host and not port then
ngx.log(ngx.ERR, " host and port can not be nil")
return ngx.exit(500)
end
local ok, err = balancer.set_current_peer(host, port)
if not ok then
ngx.log(ngx.ERR, "failed to set the current peer: ", err)
return ngx.exit(500)
end
end
local function split( str,reps)
local resultStrList = {}
string.gsub(str,'[^'..reps..']+',function ( w )
table.insert(resultStrList,w)
end)
return resultStrList
end
local function test_server()
local value, err = servers:get(1)
if not value then
ngx.log(ngx.ERR, "no server found: ", err)
ngx.status=ngx.HTTP_INTERNAL_SERVER_ERROR
return ngx.say("no server found")
else
return true
end
end
local function fail_say( fail_str )
-- body
ngx.log(ngx.ERR, fail_str)
ngx.status = ngx.HTTP_INTERNAL_SERVER_ERROR
return ngx.say(fail_str)
end
local function get_lens()
-- body
local len, flags = server_len:get("len")
if not len then
local fail_str = "failed to get len "..flags
fail_say(fail_str)
end
return len
end
local function get_nums()
local value = now_server:get("nums")
if not value then
local fail_str = 'reset nums to 1'
ngx.log(ngx.ERR, fail_str)
local ok, err = now_server:set("nums",1)
if not ok then
fail_str = 'failed to set nums '..err
fail_say(fail_str)
else
return -1
end
end
return value
end
local function round_server()
-- body
local value = get_nums()
if not value then
return
end
--ngx.log(ngx.ERR, '---->','ago',value)
local len = get_lens()
if not len then
return
end
if value+1 >= len then
local ok, err = now_server:set("nums",1)
if not ok then
fail_str = 'failed to set nums '..err
fail_say(fail_str)
else
return 1
end
else
local ok, err = now_server:set( "nums", 2+value )
if not ok then
fail_str = 'failed to set nums '..err
fail_say(fail_str)
else
return 2+value
end
end
end
local function handle_redis(red)
if not red then
return
end
local res, err = red:smembers("servers")
if not res then
ngx.log(ngx.ERR,"failed to smembers servers: ", err)
return ngx.exit(500)
else
return res
end
end
function _M.insert_server()
-- body
local red = con_redis()
if not red then
return
end
local red_result = handle_redis(red)
--ngx.log(ngx.ERR,'-->',red_result[1])
if not red_result then
red:close()
return
end
local nums = 0
for k,v in pairs(red_result) do
local resultStrList = split(v,':')
--ngx.log(ngx.ERR,'-->', resultStrList[1], resultStrList[2])
if resultStrList[1] and resultStrList[2] then
local res, err = servers:set(2*k-1, resultStrList[1])
if not res then
ngx.log(ngx.ERR,"failed to set servers first: ", err)
red:close()
return ngx.exit(500)
end
--ngx.log(ngx.ERR,'-->',2*k-1,'--->',resultStrList[1])
--ngx.log(ngx.ERR,'-->',2*k,'--->',resultStrList[2])
local res1, err1 = servers:set(2*k, resultStrList[2])
if not res1 then
ngx.log(ngx.ERR,"failed to set servers second: ", err1)
local res2, err2 = servers:rpop(k)
if not res2 then
ngx.log(ngx.ERR,"failed to rpop servers first: ", err2)
end
red:close()
return ngx.exit(500)
else
nums = nums+1
--ngx.log(ngx.ERR,'-->', nums)
end
end
end
local res, err = server_len:set('len', nums*2)
if not res then
ngx.log(ngx.ERR,"failed to set servers first: ", err)
return ngx.exit(500)
else
ngx.say('ok,all is ', nums)
return true
end
end
function _M.run_peer()
local ok = test_server()
if not ok then
return
end
local len = round_server()
if not len then
return
end
ngx.log(ngx.ERR,'--->', len)--可以去掉,调试时使用
local host ,err = servers:get(len)
if not host then
ngx.log(ngx.ERR,"failed to get server-host: ", err)
return ngx.exit(500)
end
--ngx.log(ngx.ERR,"get server: ", host, '----', len+1)
local port ,err = servers:get(len+1)
if not port then
ngx.log(ngx.ERR,"failed to get server-port: ", err)
return ngx.exit(500)
end
ngx.log(ngx.ERR, "set peer ", host, ':', port) --可以去掉,调试时使用
set_peer(host, port)
end
return _M
重启openresty
重启一下openresty,启动时监控一下errlog,确保启动成功
在redis内插入upstream的server
SADD servers '10.161.4.63:80' '10.161.4.61:80'
动态加载
curl -X GET http://10.161.4.63:8000/insert
正常的时候会得到 OK,all is 2 的句子
访问测试
连续两次执行,curl -X GET http://10.161.4.63:8000/test_run,查看结果是不是轮询
简单的压力测试
简单测试了一下,觉得性能差不多,仅供参考呀
----------------------------------------------------------------------------------------------------------------------------------
就写到这了,如果有问题,欢迎来qq群 630300475 一起聊聊