diff --git a/.gitignore b/.gitignore index 535c021..0f47e81 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,5 @@ nginx/uwsgi_temp/* !nginx/conf/nginx.conf !nginx/conf/slardar* !nginx/conf/stream* + +config/* diff --git a/Makefile b/Makefile index e23e9c4..d668d5b 100644 --- a/Makefile +++ b/Makefile @@ -68,7 +68,7 @@ INSTALL_DIRS=$(INSTALL_SRCDIR)/modules \ $(INSTALL_LIBDIR)/resty \ $(INSTALL_LIBDIR)/resty/core \ $(INSTALL_LIBDIR)/resty/checkups \ - $(INSTALL_LIBDIR)/resty/consul \ + $(INSTALL_LIBDIR)/resty/store \ $(INSTALL_LIBDIR)/resty/logger \ $(INSTALL_ETCDIR) @@ -82,7 +82,7 @@ ifndef DEV $(INSTALL_F) nginx/app/lib/ngx/*.lua $(INSTALL_LIBDIR)/ngx $(INSTALL_F) nginx/app/lib/resty/core/*.lua $(INSTALL_LIBDIR)/resty/core $(INSTALL_F) nginx/app/lib/resty/logger/*.lua $(INSTALL_LIBDIR)/resty/logger - $(INSTALL_F) nginx/app/lib/resty/consul/*.lua $(INSTALL_LIBDIR)/resty/consul + $(INSTALL_F) nginx/app/lib/resty/store/*.lua $(INSTALL_LIBDIR)/resty/store $(INSTALL_F) nginx/app/lib/resty/checkups/*.lua $(INSTALL_LIBDIR)/resty/checkups $(INSTALL_F) nginx/app/src/*.lua $(INSTALL_SRCDIR) $(INSTALL_F) nginx/app/etc/*.lua $(INSTALL_ETCDIR) diff --git a/nginx/app/etc/config.lua b/nginx/app/etc/config.lua index 8faef9b..78c8756 100644 --- a/nginx/app/etc/config.lua +++ b/nginx/app/etc/config.lua @@ -21,39 +21,42 @@ _M.global = { -- sync upstream list from shared memory every 1s shd_config_timer_interval = 1, - -- If no_consul is set to true, Slardar will continue start or reload - -- even if getting data from consul failed. + -- If no_store is set to true, Slardar will continue start or reload + -- even if getting data from key-value store failed. -- Remember to set this value to false when you need to read persisted - -- upstreams or lua codes from consul. - no_consul = true, + -- upstreams or lua codes from key-value store. + no_store = true, } -_M.consul = { - -- connect to consul will timeout in 5s. +_M.store = { + -- key-value store type + type = "etcd", + + -- connect to key-value store will timeout in 5s. timeout = 5, - -- disable checkups heartbeat to consul. + -- disable checkups heartbeat to key-value store. enable = false, - -- consul k/v prefix. - -- Slardar will read upstream list from config/slardar/upstreams. - config_key_prefix = "config/slardar/", + -- key-value store prefix. + -- Slardar will read upstream list from {config_key_prefix}/upstreams/. + config_key_prefix = "config/slardar", - -- positive cache ttl(in seconds) for dynamic configurations from consul. + -- positive cache ttl(in seconds) for dynamic configurations from key-value store. config_positive_ttl = 10, - -- negative cache ttl(in seconds) for dynamic configurations from consul. + -- negative cache ttl(in seconds) for dynamic configurations from key-value store. config_negative_ttl = 5, - -- enable or disable dynamic configurations cache from consul. + -- enable or disable dynamic configurations cache from key-value store. config_cache_enable = true, cluster = { { servers = { - -- change these to your own consul http addresses - { host = "127.0.0.1", port = 8500 }, + -- change these to your own key-value store http addresses + { host = "127.0.0.1", port = 2379 }, }, }, }, @@ -61,7 +64,7 @@ _M.consul = { _M.load_init = { -- load_init module name for lua-resty-load - module_name = "resty.consul.load" + module_name = "resty.store.load" } diff --git a/nginx/app/lib/resty/consul/api.lua b/nginx/app/lib/resty/consul/api.lua deleted file mode 100644 index f5d9583..0000000 --- a/nginx/app/lib/resty/consul/api.lua +++ /dev/null @@ -1,83 +0,0 @@ --- Copyright (C) 2016 Libo Huang (huangnauh), UPYUN Inc. -local http = require "socket.http" -local httpipe = require "resty.httpipe" -local checkups = require "resty.checkups.api" - -local pairs = pairs -local type = type -local str_format = string.format - -local _M = {} - - -function _M.get_kv_blocking(cluster, key, opts) - local opts = opts or {} - -- try all the consul servers - for _, cls in pairs(cluster) do - for _, srv in pairs(cls.servers) do - local url = str_format("http://%s:%s/v1/kv/%s", srv.host, srv.port, key) - local body, code = http.request(url) - if code == 404 then - return opts.default - elseif code == 200 and body then - local decode = opts.decode - if type(decode) == "function" then - return decode(body) - else - return body - end - else - ngx.log(ngx.ERR, str_format("get config from %s failed", url)) - end - end - end - - return nil, "get config failed" -end - - -function _M.get_kv(key, opts) - local opts = opts or {} - local hp, err = httpipe:new() - if not hp then - ngx.log(ngx.ERR, "failed to new httpipe: ", err) - return - end - - hp:set_timeout(5 * 1000) - local req = { - method = "GET", - path = "/v1/kv/" .. key, - } - - local callback = function(host, port) - return hp:request(host, port, req) - end - - local res, err = checkups.ready_ok("consul", callback) - if not res then - ngx.log(ngx.ERR, "failed to get config from consul, err:", err) - hp:close() - return - end - - if res.status == 404 then - return opts.default - elseif res.status ~= 200 then - ngx.log(ngx.ERR, "failed to get config from consul: ", res.status) - hp:close() - return - end - - hp:set_keepalive() - local body = res.body - - if type(opts.decode) == "function" then - return opts.decode(body) - else - return body - end -end - - -return _M diff --git a/nginx/app/lib/resty/consul/load.lua b/nginx/app/lib/resty/consul/load.lua deleted file mode 100644 index 38b8b33..0000000 --- a/nginx/app/lib/resty/consul/load.lua +++ /dev/null @@ -1,84 +0,0 @@ --- Copyright (C) 2016 Libo Huang (huangnauh), UPYUN Inc. -local api = require "resty.consul.api" -local utils = require "resty.consul.utils" - -local setmetatable = setmetatable -local type = type -local ipairs = ipairs -local pairs = pairs - -local tab_insert = table.insert -local str_format = string.format -local str_sub = string.sub -local str_len = string.len - - -local _M = {} -local mt = { __index = _M } - - --- called by lua-resty-load Lua Library in the init_by_lua* context to get code file -function _M.new(self, config) - local consul_config = config.consul or {} - local prefix = consul_config.config_key_prefix or "" - local consul_cluster = consul_config.cluster or {} - return setmetatable({consul_cluster=consul_cluster, prefix=prefix}, mt) -end - - -function _M.lkeys(self) - local opts = {decode=utils.parse_body, default={}} - local raw_keys, err = api.get_kv_blocking(self.consul_cluster, self.prefix .. "lua/?keys", opts) - if err then - return nil, err - end - - if not raw_keys then - return {} - end - - if type(raw_keys) ~= "table" then - return nil, "expected to be a table but got " .. type(raw_keys) - end - - local script_keys = {} - local version - for _, key in ipairs(raw_keys) do - local skey = str_sub(key, #self.prefix + str_len("/lua/")) - if skey ~= "" and skey ~= "version" then - tab_insert(script_keys, skey) - end - end - return script_keys -end - - -local function _lget(consul_cluster, prefix, key) - local v, err = api.get_kv_blocking(consul_cluster, str_format("%s/lua/%s?raw", prefix, key)) - if err then - return nil, err - end - - if v == nil then - return - end - - if type(v) ~= "string" then - return nil, "expected to be a string but got " .. type(v) - end - - return v -end - - -function _M.lversion(self) - return _lget(self.consul_cluster, self.prefix, "version") -end - - -function _M.lget(self, key) - return _lget(self.consul_cluster, self.prefix, key) -end - - -return _M diff --git a/nginx/app/lib/resty/load.lua b/nginx/app/lib/resty/load.lua index d4717c6..0d4db04 100644 --- a/nginx/app/lib/resty/load.lua +++ b/nginx/app/lib/resty/load.lua @@ -240,8 +240,8 @@ local function pre_load(config) local mod = module:new(config) local load_version, err - if type(mod.lversion) == "function" then - load_version, err = mod:lversion() + if type(mod.version) == "function" then + load_version, err = mod:version() if err then return nil, err end @@ -258,32 +258,22 @@ local function pre_load(config) return nil, err end - local script_keys, err = mod:lkeys() + local list, err = mod:list() if err then return nil, err end - if not script_keys then - log(WARN, "no code to load") - return true - end - - if type(script_keys) ~= "table" then - return nil, "script_keys invalid" - end - - if not next(script_keys) then + if not next(list) then log(WARN, "no code to load") return true end local skeys = {} - for _, skey in ipairs(script_keys) do + for skey, code in ipairs(list) do local ok = pcall(require, skey) if not ok then - local code = mod:lget(skey) if not code then - return nil, "fail to get code from consul" + return nil, "fail to get code from key-value store" end local ok, err = load_dict:safe_set(CODE_PREFIX .. skey, code) if not ok then diff --git a/nginx/app/lib/resty/store/api.lua b/nginx/app/lib/resty/store/api.lua new file mode 100644 index 0000000..cc2b455 --- /dev/null +++ b/nginx/app/lib/resty/store/api.lua @@ -0,0 +1,238 @@ +-- Copyright (C) 2016 Libo Huang (huangnauh), UPYUN Inc. +local http = require "socket.http" +local cjson = require "cjson.safe" +local httpipe = require "resty.httpipe" +local checkups = require "resty.checkups.api" + +local pairs = pairs +local ipairs = ipairs +local type = type +local str_format = string.format +local str_match = string.match +local tab_insert = table.insert +local decode_base64 = ngx.decode_base64 +local log = ngx.log +local ERR = ngx.ERR + +local _M = {} + +local valid_store = { + etcd = { + api_prefix = "/v2/keys/", + key = { + postfix = "", + extract = function(body) + local data = cjson.decode(body) + if type(data) ~= "table" then + return nil, "body invalid" + end + + local node = data.node + if type(node) ~= "table" then + return nil, "node invalid" + end + + local value = node.value + if type(value) ~= "string" then + return nil, "value invalid" + end + return value + end + }, + list = { + postfix = "", + extract = function(body) + local data = cjson.decode(body) + if type(data) ~= "table" then + return nil, "body invalid" + end + + local node = data.node + if type(node) ~= "table" then + return nil, "node invalid" + end + if not node.dir then + return nil, "dir invalid" + end + + local nodes = node.nodes + if type(nodes) ~= "table" then + return nil, "nodes invalid" + end + + local res = {} + for _, node in ipairs(nodes) do + local key = node.key + if type(key) ~= "string" then + return nil, "key invalid" + end + + key = str_match(key, "([^/]+)$") + local value = node.value + if key and value then + if type(value) ~= "string" then + return nil, "value invalid" + end + res[key] = value + end + end + return res + end, + + }, + }, + consul = { + api_prefix = "/v1/kv/", + key = { + postfix = "?raw", + extract = function(body) + return body + end, + }, + list = { + postfix = "?recurse=true", + extract = function(body) + local nodes = cjson.decode(body) + if type(nodes) ~= "table" then + return nil, "body invalid" + end + + local res = {} + for _, node in ipairs(nodes) do + local key = node.Key + if type(key) ~= "string" then + return nil, "Key invalid" + end + + key = str_match(key, "([^/]+)$") + local value = node.Value + if key and value then + if type(value) ~= "string" then + return nil, "Value invalid" + end + local str = decode_base64(value) + if str == nil then + return nil, "value not well formed" + end + res[key] = value + end + end + + return res + end, + + } + } +} + + +local function get_store_blocking(path, opts) + local opts = opts or {} + local cluster = opts.cluster or {} + -- try all the store servers + for _, cls in pairs(cluster) do + for _, srv in pairs(cls.servers) do + local url = str_format("http://%s:%s%s", srv.host, srv.port, path) + local resp, code = http.request(url) + if code == 404 then + return opts.default + elseif code < 300 and resp then + local body, err = opts.extract(resp) + if err then + log(ERR, "extract body failed:", err) + return nil, err + end + + local decode = opts.decode + if type(decode) == "function" then + return decode(body) + else + return body + end + else + log(ERR, str_format("get config from %s failed", url)) + end + end + end + + return nil, "get config failed" +end + + +local function get_store(path, opts) + local opts = opts or {} + local hp, err = httpipe:new() + if not hp then + log(ERR, "failed to new httpipe: ", err) + return nil, "new " + end + + hp:set_timeout(5 * 1000) + local req = { + method = "GET", + path = path, + } + + local callback = function(host, port) + return hp:request(host, port, req) + end + + local res, err = checkups.ready_ok("store", callback) + if not res then + ngx.log(ngx.ERR, "failed to get config from store, err:", err) + hp:close() + return nil, "store connection failed" + end + + if res.status == 404 then + return opts.default + elseif res.status >= 300 then + ngx.log(ngx.ERR, "failed to get config from store: ", res.status) + hp:close() + return nil, "get store failed" + end + + local ok, err = hp:set_keepalive() + if not ok then + ngx.log(ngx.ERR, "set_keepalive failed, err:", err) + end + + local body, err = opts.extract(res.body) + if err then + ngx.log(ngx.ERR, "extract body failed:", err) + return nil, err + end + + if type(opts.decode) == "function" then + return opts.decode(body) + else + return body + end +end + + +function _M.get(key, opts) + local store_type = opts.type or "consul" + local store = valid_store[store_type] + if not store then + log(ERR, "invalid store type ", store_type) + return nil, "invalid store type" + end + + local operation = opts.operation or "key" + + local operation_store = store[operation] + if not operation_store then + log(ERR, "invalid store operation ", operation) + return nil, "invalid store operation" + end + local uri = str_format("/%s%s%s", store.api_prefix, key, operation_store.postfix) + opts.extract = operation_store.extract + if opts.block then + return get_store_blocking(uri, opts) + else + return get_store(uri, opts) + end +end + +return _M diff --git a/nginx/app/lib/resty/consul/config.lua b/nginx/app/lib/resty/store/config.lua similarity index 61% rename from nginx/app/lib/resty/consul/config.lua rename to nginx/app/lib/resty/store/config.lua index fd48f3f..d69d64f 100644 --- a/nginx/app/lib/resty/consul/config.lua +++ b/nginx/app/lib/resty/store/config.lua @@ -1,13 +1,14 @@ -- Copyright (C) 2016 Libo Huang (huangnauh), UPYUN Inc. local cmsgpack = require "cmsgpack" local shcache = require "resty.shcache" -local utils = require "resty.consul.utils" -local api = require "resty.consul.api" +local utils = require "resty.store.utils" +local api = require "resty.store.api" local subsystem = require "resty.subsystem" local ipairs = ipairs local pairs = pairs local next = next +local type = type local tonumber = tonumber local setmetatable = setmetatable local str_sub = string.sub @@ -18,21 +19,23 @@ local get_shm_key = subsystem.get_shm_key local _M = {} --- get dynamic config from consul +-- get dynamic config from key-value store local function load_config(config, key) - local consul = config.consul or {} - local cache_label = "consul_config" + local store_config = config.store or {} + local cache_label = "store_config" local _load_config = function() - local key_prefix = consul.config_key_prefix or "" - local key = key_prefix .. key .. "?raw" + local key_prefix = store_config.config_key_prefix or "" + local key = key_prefix .. key - ngx.log(ngx.INFO, "get config from consul, key: " .. key) + ngx.log(ngx.INFO, "get config from key-value store, key: " .. key) - return api.get_kv(key, {decode=utils.parse_body}) + local opts = { type=store_config.type, operation="key", decode=utils.parse_body } + local value, err = api.get(key, opts) + return value end - if consul.config_cache_enable == false then + if store.config_cache_enable == false then return _load_config() end @@ -42,8 +45,8 @@ local function load_config(config, key) encode = cmsgpack.pack, decode = cmsgpack.unpack, }, - { positive_ttl = consul.config_positive_ttl or 30, - negative_ttl = consul.config_negative_ttl or 3, + { positive_ttl = cache_label.config_positive_ttl or 30, + negative_ttl = cache_label.config_negative_ttl or 3, name = cache_label, lock_shdict = get_shm_key("locks"), } @@ -57,32 +60,27 @@ end -- get server in the init_by_lua* context for the lua-resty-checkups Lua Library function _M.init(config) - local consul = config.consul or {} - local key_prefix = consul.config_key_prefix or "" - local consul_cluster = consul.cluster or {} - local opts = {decode=utils.parse_body, default={} } + local store_config = config.store or {} + local key_prefix = store_config.config_key_prefix or "" + local store_cluster = store_config.cluster or {} local upstreams_prefix = "upstreams" if ngx_subsystem ~= "http" then upstreams_prefix = "upstreams_" .. ngx_subsystem end - local upstream_keys = api.get_kv_blocking(consul_cluster, key_prefix .. upstreams_prefix .. "?keys", opts) - if not upstream_keys then + local opts = { type=store_config.type, block=true, cluster=store_cluster, operation="list", default={} } + local upstream_list = api.get(key_prefix .. upstreams_prefix, opts) + if type(upstream_list) ~= "table" then return false end - for _, key in ipairs(upstream_keys) do repeat - local skey = str_sub(key, #key_prefix + #upstreams_prefix + 2) - if #skey == 0 then - break - end - + for skey, value in pairs(upstream_list) do repeat -- upstream already exists in config.lua if config[skey] then break end - local servers = api.get_kv_blocking(consul_cluster, key .. "?raw", opts) + local servers = utils.parse_body(value) if not servers or not next(servers) then return false end diff --git a/nginx/app/lib/resty/store/load.lua b/nginx/app/lib/resty/store/load.lua new file mode 100644 index 0000000..3f5c511 --- /dev/null +++ b/nginx/app/lib/resty/store/load.lua @@ -0,0 +1,70 @@ +-- Copyright (C) 2016 Libo Huang (huangnauh), UPYUN Inc. +local api = require "resty.store.api" +local utils = require "resty.store.utils" + +local setmetatable = setmetatable +local type = type +local ipairs = ipairs +local pairs = pairs + +local tab_insert = table.insert +local str_format = string.format +local str_sub = string.sub +local str_len = string.len + + +local _M = {} +local mt = { __index = _M } + + +-- called by lua-resty-load Lua Library in the init_by_lua* context to get code file +function _M.new(self, config) + local store_config = config.store or {} + local prefix = store_config.config_key_prefix or "" + local store_cluster = store_config.cluster or {} + return setmetatable({store_cluster=store_cluster, store_type=store_config.type, prefix=prefix}, mt) +end + +function _M.list(self) + local opts = { type=self.store_type, block=true, cluster=self.store_cluster, + operation="list", default={} } + local list, err = api.get(str_format("%s/lua", self.prefix), opts) + if err then + return nil, err + end + + list["version"] = nil + return list +end + + +local function _get(store_cluster, store_type, prefix, key) + local opts = { type=store_type, block=true, cluster=store_cluster } + local v, err = api.get(str_format("%s/lua/%s", prefix, key), opts) + if err then + return nil, err + end + + if v == nil then + return + end + + if type(v) ~= "string" then + return nil, "expected to be a string but got " .. type(v) + end + + return v +end + + +function _M.version(self) + return _get(self.store_cluster, self.store_type, self.prefix, "version") +end + + +function _M.get(self, key) + return _get(self.store_cluster, self.store_type, self.prefix, key) +end + + +return _M diff --git a/nginx/app/lib/resty/consul/utils.lua b/nginx/app/lib/resty/store/utils.lua similarity index 100% rename from nginx/app/lib/resty/consul/utils.lua rename to nginx/app/lib/resty/store/utils.lua diff --git a/nginx/app/src/init.lua b/nginx/app/src/init.lua index 6e400bf..c9bc32d 100644 --- a/nginx/app/src/init.lua +++ b/nginx/app/src/init.lua @@ -1,7 +1,7 @@ -- Copyright (C) 2015-2016, UPYUN Inc. local cjson = require "cjson.safe" -local consul = require "resty.consul.config" +local store = require "resty.store.config" local mload = require "resty.load" local checkups = require "resty.checkups.api" @@ -43,11 +43,11 @@ slardar.exit = function(err) end -local no_consul = slardar.global.no_consul +local no_store = slardar.global.no_store -- if init config failed, abort -t or reload. -local ok, init_ok = pcall(consul.init, slardar) -if no_consul ~= true then +local ok, init_ok = pcall(store.init, slardar) +if no_store ~= true then if not ok then error("Init config failed, " .. init_ok .. ", aborting !!!!") elseif not init_ok then @@ -56,7 +56,7 @@ if no_consul ~= true then end local ok, init_ok = pcall(mload.init, slardar) -if no_consul ~= true then +if no_store ~= true then if not ok then error("Init lua script failed, " .. init_ok .. ", aborting !!!!") elseif not init_ok then @@ -72,5 +72,5 @@ elseif not init_ok then end setmetatable(slardar, { - __index = consul.load_config, + __index = store.load_config, }) diff --git a/nginx/app/src/stream/init.lua b/nginx/app/src/stream/init.lua index 30db3fe..d3ce6f0 100644 --- a/nginx/app/src/stream/init.lua +++ b/nginx/app/src/stream/init.lua @@ -1,16 +1,16 @@ -- Copyright (C) 2017 Libo Huang (huangnauh), UPYUN Inc. -local consul = require "resty.consul.config" +local store = require "resty.store.config" local checkups = require "resty.checkups.api" slardar = require "config" -- global config variable slardar.global.version = "0.3.7" -local no_consul = slardar.global.no_consul +local no_store = slardar.global.no_store -- if init config failed, abort -t or reload. -local ok, init_ok = pcall(consul.init, slardar) -if no_consul ~= true then +local ok, init_ok = pcall(store.init, slardar) +if no_store ~= true then if not ok then error("Init config failed, " .. init_ok .. ", aborting !!!!") elseif not init_ok then