From d0ea08f5fd82a25e786dcc58a2ceca8c180ed242 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Sun, 31 Oct 2021 21:05:36 +0100 Subject: [PATCH 01/65] chore(*) add config files (busted/editor) --- .busted | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .busted diff --git a/.busted b/.busted new file mode 100644 index 0000000..0855a96 --- /dev/null +++ b/.busted @@ -0,0 +1,10 @@ +return { + default = { + ROOT = { "tests/spec" }, + pattern = "%.lua", + lpath = "./?.lua;./?/?.lua;./?/init.lua", + verbose = true, + coverage = false, + output = "gtest", + }, +} From a4d5b8b1ce8dc15594a6452eda4df14ee5868055 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Wed, 3 Nov 2021 16:27:23 +0100 Subject: [PATCH 02/65] feat(topics) add validation and matching --- .editorconfig | 5 + mqtt/init.lua | 177 +++++++++++++++++++++++++ tests/spec/topics.lua | 300 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 482 insertions(+) create mode 100644 tests/spec/topics.lua diff --git a/.editorconfig b/.editorconfig index 1f7c63c..3125997 100644 --- a/.editorconfig +++ b/.editorconfig @@ -7,6 +7,8 @@ root = true [*] end_of_line = lf insert_final_newline = true +trim_trailing_whitespace = true +charset = utf-8 # 4 space tab indentation for lua files [*.lua] @@ -20,3 +22,6 @@ indent_size = 2 [*.yml] indent_style = space indent_size = 2 + +[Makefile] +indent_style = tab diff --git a/mqtt/init.lua b/mqtt/init.lua index c6f597c..fca8353 100644 --- a/mqtt/init.lua +++ b/mqtt/init.lua @@ -81,6 +81,183 @@ function mqtt.run_sync(cl) end end + +--- Validates a topic with wildcards. +-- @param t (string) wildcard topic to validate +-- @return topic, or false+error +function mqtt.validate_subscribe_topic(t) + if type(t) ~= "string" then + return false, "not a string" + end + if #t < 1 then + return false, "minimum topic length is 1" + end + do + local _, count = t:gsub("#", "") + if count > 1 then + return false, "wildcard '#' may only appear once" + end + if count == 1 then + if t ~= "#" and not t:find("/#$") then + return false, "wildcard '#' must be the last character, and be prefixed with '/' (unless the topic is '#')" + end + end + end + do + local t1 = "/"..t.."/" + local i = 1 + while i do + i = t1:find("+", i) + if i then + if t1:sub(i-1, i+1) ~= "/+/" then + return false, "wildcard '+' must be enclosed between '/' (except at start/end)" + end + i = i + 1 + end + end + end + return t +end + +--- Validates a topic without wildcards. +-- @param t (string) topic to validate +-- @return topic, or false+error +function mqtt.validate_publish_topic(t) + if type(t) ~= "string" then + return false, "not a string" + end + if #t < 1 then + return false, "minimum topic length is 1" + end + if t:find("+", nil, true) or t:find("#", nil, true) then + return false, "wildcards '#', and '+' are not allowed when publishing" + end + return t +end + +--- Returns a Lua pattern from topic. +-- Takes a wildcarded-topic and returns a Lua pattern that can be used +-- to validate if a received topic matches the wildcard-topic +-- @param t (string) the wildcard topic +-- @return Lua-pattern (string) or false+err +-- @usage +-- local patt = compile_topic_pattern("homes/+/+/#") +-- +-- local topic = "homes/myhome/living/mainlights/brightness" +-- local homeid, roomid, varargs = topic:match(patt) +function mqtt.compile_topic_pattern(t) + local ok, err = mqtt.validate_subscribe_topic(t) + if not ok then + return ok, err + end + if t == "#" then + t = "(.+)" -- matches anything at least 1 character long + else + t = t:gsub("#","(.-)") -- match anything, can be empty + t = t:gsub("%+","([^/]-)") -- match anything between '/', can be empty + end + return "^"..t.."$" +end + +--- Parses wildcards in a topic into a table. +-- Options include: +-- +-- - `opts.topic`: the wild-carded topic to match against (optional if `opts.pattern` is given) +-- +-- - `opts.pattern`: the compiled pattern for the wild-carded topic (optional if `opts.topic` +-- is given). If not given then topic will be compiled and the result will be +-- stored in this field for future use (cache). +-- +-- - `opts.keys`: (optional) array of field names. The order must be the same as the +-- order of the wildcards in `topic` +-- +-- Returned tables: +-- +-- - `fields` table: the array part will have the values of the wildcards, in +-- the order they appeared. The hash part, will have the field names provided +-- in `opts.keys`, with the values of the corresponding wildcard. If a `#` +-- wildcard was used, that one will be the last in the table. +-- +-- - `varargs` table: will only be returned if the wildcard topic contained the +-- `#` wildcard. The returned table is an array, with all segments that were +-- matched by the `#` wildcard. +-- @param topic (string) incoming topic string (required) +-- @param opts (table) with options (required) +-- @return fields (table) + varargs (table or nil), or false+err on error. +-- @usage +-- local opts = { +-- topic = "homes/+/+/#", +-- keys = { "homeid", "roomid", "varargs"}, +-- } +-- local fields, varargs = topic_match("homes/myhome/living/mainlights/brightness", opts) +-- +-- print(fields[1], fields.homeid) -- "myhome myhome" +-- print(fields[2], fields.roomid) -- "living living" +-- print(fields[3], fields.varargs) -- "mainlights/brightness mainlights/brightness" +-- +-- print(varargs[1]) -- "mainlights" +-- print(varargs[2]) -- "brightness" +function mqtt.topic_match(topic, opts) + if type(topic) ~= "string" then + return false, "expected topic to be a string" + end + if type(opts) ~= "table" then + return false, "expected optionss to be a table" + end + local pattern = opts.pattern + if not pattern then + local ptopic = opts.topic + if not ptopic then + return false, "either 'opts.topic' or 'opts.pattern' must set" + end + local err + pattern, err = mqtt.compile_topic_pattern(ptopic) + if not pattern then + return false, "failed to compile 'opts.topic' into pattern: "..tostring(err) + end + -- store/cache compiled pattern for next time + opts.pattern = pattern + end + local values = { topic:match(pattern) } + if values[1] == nil then + return false, "topic does not match wildcard pattern" + end + local keys = opts.keys + if keys ~= nil then + if type(keys) ~= "table" then + return false, "expected 'opts.keys' to be a table (array)" + end + -- we have a table with keys, copy values to fields + for i, value in ipairs(values) do + local key = keys[i] + if key ~= nil then + values[key] = value + end + end + end + if not pattern:find("%(%.[%-%+]%)%$$") then -- pattern for "#" as last char + -- we're done + return values + end + -- we have a '#' wildcard + local vararg = values[#values] + local varargs = {} + local i = 0 + local ni = 0 + while ni do + ni = vararg:find("/", i, true) + if ni then + varargs[#varargs + 1] = vararg:sub(i, ni-1) + i = ni + 1 + else + varargs[#varargs + 1] = vararg:sub(i, -1) + end + end + + return values, varargs +end + + -- export module table return mqtt diff --git a/tests/spec/topics.lua b/tests/spec/topics.lua new file mode 100644 index 0000000..36070ce --- /dev/null +++ b/tests/spec/topics.lua @@ -0,0 +1,300 @@ +local mqtt = require "mqtt" + +describe("topics", function() + + describe("publish (plain)", function() + it("allows proper topics", function() + local ok, err + ok, err = mqtt.validate_publish_topic("hello/world") + assert.is_nil(err) + assert.is.truthy(ok) + + ok, err = mqtt.validate_publish_topic("hello/world/") + assert.is_nil(err) + assert.is.truthy(ok) + + ok, err = mqtt.validate_publish_topic("/hello/world") + assert.is_nil(err) + assert.is.truthy(ok) + + ok, err = mqtt.validate_publish_topic("/") + assert.is_nil(err) + assert.is.truthy(ok) + + ok, err = mqtt.validate_publish_topic("//////") + assert.is_nil(err) + assert.is.truthy(ok) + + ok, err = mqtt.validate_publish_topic("/") + assert.is_nil(err) + assert.is.truthy(ok) + + end) + + it("returns the topic passed in on success", function() + local ok = mqtt.validate_publish_topic("hello/world") + assert.are.equal("hello/world", ok) + end) + + it("must be a string", function() + local ok, err = mqtt.validate_publish_topic(true) + assert.is_false(ok) + assert.is_string(err) + end) + + it("minimum length 1", function() + local ok, err = mqtt.validate_publish_topic("") + assert.is_false(ok) + assert.is_string(err) + end) + + it("wildcard '#' is not allowed", function() + local ok, err = mqtt.validate_publish_topic("hello/world/#") + assert.is_false(ok) + assert.is_string(err) + end) + + it("wildcard '+' is not allowed", function() + local ok, err = mqtt.validate_publish_topic("hello/+/world") + assert.is_false(ok) + assert.is_string(err) + end) + + end) + + + + describe("subscribe (wildcarded)", function() + + it("allows proper topics", function() + local ok, err + ok, err = mqtt.validate_subscribe_topic("hello/world") + assert.is_nil(err) + assert.is.truthy(ok) + + ok, err = mqtt.validate_subscribe_topic("hello/world/") + assert.is_nil(err) + assert.is.truthy(ok) + + ok, err = mqtt.validate_subscribe_topic("/hello/world") + assert.is_nil(err) + assert.is.truthy(ok) + + ok, err = mqtt.validate_subscribe_topic("/") + assert.is_nil(err) + assert.is.truthy(ok) + + ok, err = mqtt.validate_subscribe_topic("//////") + assert.is_nil(err) + assert.is.truthy(ok) + + ok, err = mqtt.validate_subscribe_topic("#") + assert.is_nil(err) + assert.is.truthy(ok) + + ok, err = mqtt.validate_subscribe_topic("/#") + assert.is_nil(err) + assert.is.truthy(ok) + + ok, err = mqtt.validate_subscribe_topic("+") + assert.is_nil(err) + assert.is.truthy(ok) + + ok, err = mqtt.validate_subscribe_topic("+/hello/#") + assert.is_nil(err) + assert.is.truthy(ok) + + ok, err = mqtt.validate_subscribe_topic("+/+/+/+/+") + assert.is_nil(err) + assert.is.truthy(ok) + end) + + it("returns the topic passed in on success", function() + local ok = mqtt.validate_subscribe_topic("hello/world") + assert.are.equal("hello/world", ok) + end) + + it("must be a string", function() + local ok, err = mqtt.validate_subscribe_topic(true) + assert.is_false(ok) + assert.is_string(err) + end) + + it("minimum length 1", function() + local ok, err = mqtt.validate_subscribe_topic("") + assert.is_false(ok) + assert.is_string(err) + end) + + it("wildcard '#' is only allowed as last segment", function() + local ok, err = mqtt.validate_subscribe_topic("hello/#/world") + assert.is_false(ok) + assert.is_string(err) + end) + + it("wildcard '+' is only allowed as full segment", function() + local ok, err = mqtt.validate_subscribe_topic("hello/+there/world") + assert.is_false(ok) + assert.is_string(err) + end) + + end) + + + + describe("pattern compiler & matcher", function() + + it("basic parsing works", function() + local opts = { + topic = "+/+", + pattern = nil, + keys = { "hello", "world"} + } + local res, err = mqtt.topic_match("hello/world", opts) + assert.is_nil(err) + assert.same(res, { + "hello", "world", + hello = "hello", + world = "world", + }) + -- compiled pattern is now added + assert.not_nil(opts.pattern) + end) + + it("incoming topic is required", function() + local opts = { + topic = "+/+", + pattern = nil, + keys = { "hello", "world"} + } + local ok, err = mqtt.topic_match(nil, opts) + assert.is_false(ok) + assert.is_string(err) + end) + + it("wildcard topic or pattern is required", function() + local opts = { + topic = nil, + pattern = nil, + keys = { "hello", "world"} + } + local ok, err = mqtt.topic_match("hello/world", opts) + assert.is_false(ok) + assert.is_string(err) + end) + + it("pattern must match", function() + local opts = { + topic = "+/+/+", -- one too many + pattern = nil, + keys = { "hello", "world"} + } + local ok, err = mqtt.topic_match("hello/world", opts) + assert.is_false(ok) + assert.is_string(err) + end) + + it("pattern '+' works", function() + local opts = { + topic = "+", + pattern = nil, + keys = { "hello" } + } + -- matches topic + local res, err = mqtt.topic_match("hello", opts) + assert.is_nil(err) + assert.same(res, { + "hello", + hello = "hello", + }) + end) + + it("wildcard '+' matches empty segments", function() + local opts = { + topic = "+/+/+", + pattern = nil, + keys = { "hello", "there", "world"} + } + local res, err = mqtt.topic_match("//", opts) + assert.is_nil(err) + assert.same(res, { + "", "", "", + hello = "", + there = "", + world = "", + }) + end) + + it("pattern '#' matches all segments", function() + local opts = { + topic = "#", + pattern = nil, + keys = nil, + } + local res, var = mqtt.topic_match("hello/there/world", opts) + assert.same(res, { + "hello/there/world" + }) + assert.same(var, { + "hello", + "there", + "world", + }) + end) + + it("pattern '/#' skips first segment", function() + local opts = { + topic = "/#", + pattern = nil, + keys = nil, + } + local res, var = mqtt.topic_match("/hello/world", opts) + assert.same(res, { + "hello/world" + }) + assert.same(var, { + "hello", + "world", + }) + end) + + it("combined wildcards '+/+/#'", function() + local opts = { + topic = "+/+/#", + pattern = nil, + keys = nil, + } + local res, var = mqtt.topic_match("hello/there/my/world", opts) + assert.same(res, { + "hello", + "there", + "my/world" + }) + assert.same(var, { + "my", + "world", + }) + end) + + it("trailing '/' in topic with '#'", function() + local opts = { + topic = "+/+/#", + pattern = nil, + keys = nil, + } + local res, var = mqtt.topic_match("hello/there/world/", opts) + assert.same(res, { + "hello", + "there", + "world/" + }) + assert.same(var, { + "world", + "", + }) + end) + + + end) + +end) From 8fff4313fb0458d5708bf918cc9d951342214ad7 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Tue, 26 Oct 2021 16:17:15 +0200 Subject: [PATCH 03/65] refactor(keepalive) remove keepalive logic from ioloop by refactoring the keepalive logic into its own method, the functionality becomes available to external event loops as well. --- mqtt/client.lua | 39 +++++++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/mqtt/client.lua b/mqtt/client.lua index 3ff03b3..7d43070 100644 --- a/mqtt/client.lua +++ b/mqtt/client.lua @@ -784,6 +784,38 @@ function client_mt:send_connect() return true end +--- Checks last message send, and sends a PINGREQ if necessary. +-- Use this function to send keep-alives when using an external event loop. +-- @return time till next keep_alive, in case of errors (eg. not connected) the second return value is an error string +-- @usage +-- -- example using a Copas event loop to send keep-alives +-- copas.addthread(function() +-- while true do +-- if not my_client then +-- return -- exiting, client was destroyed +-- end +-- copas.sleep(my_client:check_keep_alive()) +-- end +-- end) +function client_mt:check_keep_alive() + local interval = self.args.keep_alive + if not self.connection then + return interval, "network connection is not opened" + end + + local t_now = os_time() + local t_next = self.send_time + interval + + -- send PINGREQ if keep_alive interval is reached + if t_now >= t_next then + local _, err = self:send_pingreq() + return interval, err + end + + return t_next - t_now +end + + -- Internal methods -- Set or rest ioloop for MQTT client @@ -904,12 +936,7 @@ function client_mt:_ioloop_iteration() ok, err = self:_sync_iteration() end - if ok then - -- send PINGREQ if keep_alive interval is reached - if os_time() - self.send_time >= args.keep_alive then - self:send_pingreq() - end - end + self:check_keep_alive() return ok, err else From 6c9c48a6239cf4a1c0e6f98e183e1eb641931ef6 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Wed, 27 Oct 2021 00:36:22 +0200 Subject: [PATCH 04/65] refactor(receive) separate out logic for handling a packet --- mqtt/client.lua | 153 ++++++++++++++++++++++++++---------------------- 1 file changed, 82 insertions(+), 71 deletions(-) diff --git a/mqtt/client.lua b/mqtt/client.lua index 7d43070..54e5b09 100644 --- a/mqtt/client.lua +++ b/mqtt/client.lua @@ -904,6 +904,85 @@ function client_mt:_assign_packet_id(pargs) end end +-- Handle a single received packet +function client_mt:handle_received_packet(packet) + local conn = self.connection + local err + + if not conn.connack then + -- expecting only CONNACK packet here + if packet.type ~= packet_type.CONNACK then + err = "expecting CONNACK but received "..packet.type + self:handle("error", err, self) + self:close_connection("error") + return false, err + end + + -- store connack packet in connection + conn.connack = packet + + -- check CONNACK rc + if packet.rc ~= 0 then + err = str_format("CONNECT failed with CONNACK [rc=%d]: %s", packet.rc, packet:reason_string()) + self:handle("error", err, self, packet) + self:handle("connect", packet, self) + self:close_connection("connection failed") + return false, err + end + + -- fire connect event + self:handle("connect", packet, self) + else + -- connection authorized, so process usual packets + + -- handle packet according its type + local ptype = packet.type + if ptype == packet_type.PINGRESP then -- luacheck: ignore + -- PINGREQ answer, nothing to do + -- TODO: break the connectin in absence of this packet in some timeout + elseif ptype == packet_type.SUBACK then + self:handle("subscribe", packet, self) + elseif ptype == packet_type.UNSUBACK then + self:handle("unsubscribe", packet, self) + elseif ptype == packet_type.PUBLISH then + -- check such packet is not waiting for pubrel acknowledge + self:handle("message", packet, self) + elseif ptype == packet_type.PUBACK then + self:handle("acknowledge", packet, self) + elseif ptype == packet_type.PUBREC then + local packet_id = packet.packet_id + if conn.wait_for_pubrec[packet_id] then + conn.wait_for_pubrec[packet_id] = nil + -- send PUBREL acknowledge + if self:acknowledge_pubrel(packet_id) then + -- and fire acknowledge event + self:handle("acknowledge", packet, self) + end + end + elseif ptype == packet_type.PUBREL then + -- second phase of QoS 2 exchange - check we are already acknowledged such packet by PUBREL + local packet_id = packet.packet_id + if conn.wait_for_pubrel[packet_id] then + -- remove packet from waiting for PUBREL packets table + conn.wait_for_pubrel[packet_id] = nil + -- send PUBCOMP acknowledge + self:acknowledge_pubcomp(packet_id) + end + elseif ptype == packet_type.PUBCOMP then --luacheck: ignore + -- last phase of QoS 2 exchange + -- do nothing here + elseif ptype == packet_type.DISCONNECT then + self:close_connection("disconnect received from broker") + elseif ptype == packet_type.AUTH then + self:handle("auth", packet, self) + -- else + -- print("unhandled packet:", packet) -- debug + end + end + return true +end + + -- Receive packet function in sync mode local function sync_recv(self) return true, self:_receive_packet() @@ -973,8 +1052,6 @@ end -- Performing one IO iteration - receive next packet function client_mt:_io_iteration(recv) - local conn = self.connection - -- first - try to receive packet local ok, packet, err = recv(self) -- print("received packet", ok, packet, err) @@ -1002,75 +1079,9 @@ function client_mt:_io_iteration(recv) -- check some packet received if packet ~= "timeout" and packet ~= "wantread" then - if not conn.connack then - -- expecting only CONNACK packet here - if packet.type ~= packet_type.CONNACK then - err = "expecting CONNACK but received "..packet.type - self:handle("error", err, self) - self:close_connection("error") - return false, err - end - - -- store connack packet in connection - conn.connack = packet - - -- check CONNACK rc - if packet.rc ~= 0 then - err = str_format("CONNECT failed with CONNACK [rc=%d]: %s", packet.rc, packet:reason_string()) - self:handle("error", err, self, packet) - self:handle("connect", packet, self) - self:close_connection("connection failed") - return false, err - end - - -- fire connect event - self:handle("connect", packet, self) - else - -- connection authorized, so process usual packets - - -- handle packet according its type - local ptype = packet.type - if ptype == packet_type.PINGRESP then -- luacheck: ignore - -- PINGREQ answer, nothing to do - -- TODO: break the connectin in absence of this packet in some timeout - elseif ptype == packet_type.SUBACK then - self:handle("subscribe", packet, self) - elseif ptype == packet_type.UNSUBACK then - self:handle("unsubscribe", packet, self) - elseif ptype == packet_type.PUBLISH then - -- check such packet is not waiting for pubrel acknowledge - self:handle("message", packet, self) - elseif ptype == packet_type.PUBACK then - self:handle("acknowledge", packet, self) - elseif ptype == packet_type.PUBREC then - local packet_id = packet.packet_id - if conn.wait_for_pubrec[packet_id] then - conn.wait_for_pubrec[packet_id] = nil - -- send PUBREL acknowledge - if self:acknowledge_pubrel(packet_id) then - -- and fire acknowledge event - self:handle("acknowledge", packet, self) - end - end - elseif ptype == packet_type.PUBREL then - -- second phase of QoS 2 exchange - check we are already acknowledged such packet by PUBREL - local packet_id = packet.packet_id - if conn.wait_for_pubrel[packet_id] then - -- remove packet from waiting for PUBREL packets table - conn.wait_for_pubrel[packet_id] = nil - -- send PUBCOMP acknowledge - self:acknowledge_pubcomp(packet_id) - end - elseif ptype == packet_type.PUBCOMP then --luacheck: ignore - -- last phase of QoS 2 exchange - -- do nothing here - elseif ptype == packet_type.DISCONNECT then - self:close_connection("disconnect received from broker") - elseif ptype == packet_type.AUTH then - self:handle("auth", packet, self) - -- else - -- print("unhandled packet:", packet) -- debug - end + ok, err = self:handle_received_packet(packet) + if not ok then + return ok, err end end From d81f66c27f731befe7cc87bc32bab6d778b8d6ac Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Wed, 27 Oct 2021 00:38:20 +0200 Subject: [PATCH 05/65] fix(error) do not silently discard error --- mqtt/client.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mqtt/client.lua b/mqtt/client.lua index 54e5b09..d7f1fa3 100644 --- a/mqtt/client.lua +++ b/mqtt/client.lua @@ -966,7 +966,7 @@ function client_mt:handle_received_packet(packet) -- remove packet from waiting for PUBREL packets table conn.wait_for_pubrel[packet_id] = nil -- send PUBCOMP acknowledge - self:acknowledge_pubcomp(packet_id) + return self:acknowledge_pubcomp(packet_id) end elseif ptype == packet_type.PUBCOMP then --luacheck: ignore -- last phase of QoS 2 exchange From a5f5d27afbe006fb0f7f99e247e0b41850dd9258 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Wed, 27 Oct 2021 16:18:49 +0200 Subject: [PATCH 06/65] refactor(locals) reduce variable scopes --- mqtt/client.lua | 109 +++++++++++++++++++++++++----------------------- 1 file changed, 57 insertions(+), 52 deletions(-) diff --git a/mqtt/client.lua b/mqtt/client.lua index d7f1fa3..83d6b47 100644 --- a/mqtt/client.lua +++ b/mqtt/client.lua @@ -982,69 +982,74 @@ function client_mt:handle_received_packet(packet) return true end +-- sync-loop +do + -- Receive packet function in sync mode + local function sync_recv(self) + return true, self:_receive_packet() + end --- Receive packet function in sync mode -local function sync_recv(self) - return true, self:_receive_packet() -end - --- Perform one input/output iteration, called by sync receiving loop -function client_mt:_sync_iteration() - return self:_io_iteration(sync_recv) + -- Perform one input/output iteration, called by sync receiving loop + function client_mt:_sync_iteration() + return self:_io_iteration(sync_recv) + end end --- Receive packet function - from ioloop's coroutine -local function ioloop_recv(self) - return coroutine_resume(self.connection.coro) -end +-- ioloop +do + -- Receive packet function - from ioloop's coroutine + local function ioloop_recv(self) + return coroutine_resume(self.connection.coro) + end --- Perform one input/output iteration, called by ioloop -function client_mt:_ioloop_iteration() - -- working according state - local loop = self.ioloop - local args = self.args + -- Perform one input/output iteration, called by ioloop + function client_mt:_ioloop_iteration() + -- working according state + local loop = self.ioloop + local args = self.args - local conn = self.connection - if conn then - -- network connection opened - -- perform packet receiving using ioloop receive function - local ok, err - if loop then - ok, err = self:_io_iteration(ioloop_recv) - else - ok, err = self:_sync_iteration() - end + local conn = self.connection + if conn then + -- network connection opened + -- perform packet receiving using ioloop receive function + local ok, err + if loop then + ok, err = self:_io_iteration(ioloop_recv) + else + ok, err = self:_sync_iteration() + end - self:check_keep_alive() + self:check_keep_alive() - return ok, err - else - -- no connection - first connect, reconnect or remove from ioloop - if self.first_connect then - self.first_connect = false - self:start_connecting() - elseif args.reconnect then - if args.reconnect == true then + return ok, err + else + -- no connection - first connect, reconnect or remove from ioloop + if self.first_connect then + self.first_connect = false self:start_connecting() - else - -- reconnect in specified timeout - if self.reconnect_timer_start then - if os_time() - self.reconnect_timer_start >= args.reconnect then - self.reconnect_timer_start = nil - self:start_connecting() - else - if loop then - loop:can_sleep() + elseif args.reconnect then + if args.reconnect == true then + self:start_connecting() + else + -- reconnect in specified timeout + if self.reconnect_timer_start then + if os_time() - self.reconnect_timer_start >= args.reconnect then + self.reconnect_timer_start = nil + self:start_connecting() + else + if loop then + loop:can_sleep() + end end + else + self.reconnect_timer_start = os_time() end - else - self.reconnect_timer_start = os_time() end - end - else - -- finish working with client - if loop then - loop:remove(self) + else + -- finish working with client + if loop then + loop:remove(self) + end end end end From c2956d664310424a93650c73d7d335e603234a38 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Wed, 27 Oct 2021 17:46:37 +0200 Subject: [PATCH 07/65] feat(ping-timeout) implement ping timeout --- mqtt/client.lua | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/mqtt/client.lua b/mqtt/client.lua index 83d6b47..7f794c5 100644 --- a/mqtt/client.lua +++ b/mqtt/client.lua @@ -702,6 +702,9 @@ function client_mt:send_pingreq() return false, err end + -- set ping timeout; for now 1 ping-request interval + self.ping_expire_time = os_time() + self.args.keep_alive + return true end @@ -735,6 +738,9 @@ function client_mt:open_connection() -- assign connection self.connection = conn + -- reset ping timeout + self.ping_expire_time = nil + -- create receive function local receive = connector.receive self.connection.recv_func = function(size) @@ -785,10 +791,10 @@ function client_mt:send_connect() end --- Checks last message send, and sends a PINGREQ if necessary. --- Use this function to send keep-alives when using an external event loop. +-- Use this function to check and send keep-alives when using an external event loop. -- @return time till next keep_alive, in case of errors (eg. not connected) the second return value is an error string -- @usage --- -- example using a Copas event loop to send keep-alives +-- -- example using a Copas event loop to send and check keep-alives -- copas.addthread(function() -- while true do -- if not my_client then @@ -805,6 +811,16 @@ function client_mt:check_keep_alive() local t_now = os_time() local t_next = self.send_time + interval + local t_timeout = self.ping_expire_time + + -- check last ping request + if t_timeout and t_timeout <= t_now then + -- we timed-out, close and exit + local err = str_format("failed to receive PINGRESP within %d seconds", interval) + self:handle("error", err, self) + self:close_connection("error") + return false, err + end -- send PINGREQ if keep_alive interval is reached if t_now >= t_next then @@ -812,6 +828,10 @@ function client_mt:check_keep_alive() return interval, err end + -- return which ever is earlier, timeout or next ping request + if t_timeout and t_timeout < t_next then + return t_timeout - t_now + end return t_next - t_now end @@ -938,8 +958,8 @@ function client_mt:handle_received_packet(packet) -- handle packet according its type local ptype = packet.type if ptype == packet_type.PINGRESP then -- luacheck: ignore - -- PINGREQ answer, nothing to do - -- TODO: break the connectin in absence of this packet in some timeout + -- PINGREQ answer, clear timeout + self.ping_expire_time = nil elseif ptype == packet_type.SUBACK then self:handle("subscribe", packet, self) elseif ptype == packet_type.UNSUBACK then @@ -1019,7 +1039,9 @@ do ok, err = self:_sync_iteration() end - self:check_keep_alive() + if ok then + ok, err = self:check_keep_alive() + end return ok, err else From af853564435501286363d31339fe0416d50455e3 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Sun, 31 Oct 2021 20:54:44 +0100 Subject: [PATCH 08/65] feat(*) rewrite loop logic --- .busted | 3 + .luacheckrc | 25 +- mqtt/buffered_base.lua | 86 +++ mqtt/client.lua | 610 ++++++++---------- mqtt/init.lua | 15 +- mqtt/ioloop.lua | 137 ++-- mqtt/log.lua | 17 + mqtt/luasocket-copas.lua | 57 +- mqtt/luasocket.lua | 79 ++- mqtt/luasocket_ssl.lua | 61 +- mqtt/ngxsocket.lua | 56 +- mqtt/non_buffered_base.lua | 62 ++ mqtt/protocol5.lua | 4 +- rockspecs/luamqtt-3.4.3-1.rockspec | 3 + ...e-basics.lua => 01-module-basics_spec.lua} | 1 - ...l4-make.lua => 02-protocol4-make_spec.lua} | 1 - ...-parse.lua => 03-protocol4-parse_spec.lua} | 1 - ...l5-make.lua => 04-protocol5-make_spec.lua} | 1 - ...-parse.lua => 05-protocol5-parse_spec.lua} | 1 - ...qtt-client.lua => 06-mqtt-client_spec.lua} | 5 +- tests/spec/{ioloop.lua => 07-ioloop_spec.lua} | 14 +- 21 files changed, 742 insertions(+), 497 deletions(-) create mode 100644 mqtt/buffered_base.lua create mode 100644 mqtt/log.lua create mode 100644 mqtt/non_buffered_base.lua rename tests/spec/{module-basics.lua => 01-module-basics_spec.lua} (99%) rename tests/spec/{protocol4-make.lua => 02-protocol4-make_spec.lua} (99%) rename tests/spec/{protocol4-parse.lua => 03-protocol4-parse_spec.lua} (99%) rename tests/spec/{protocol5-make.lua => 04-protocol5-make_spec.lua} (99%) rename tests/spec/{protocol5-parse.lua => 05-protocol5-parse_spec.lua} (99%) rename tests/spec/{mqtt-client.lua => 06-mqtt-client_spec.lua} (99%) rename tests/spec/{ioloop.lua => 07-ioloop_spec.lua} (82%) diff --git a/.busted b/.busted index 0855a96..600ba91 100644 --- a/.busted +++ b/.busted @@ -1,7 +1,10 @@ return { default = { ROOT = { "tests/spec" }, +<<<<<<< HEAD pattern = "%.lua", +======= +>>>>>>> 0a6e421 (feat(*) rewrite loop logic) lpath = "./?.lua;./?/?.lua;./?/init.lua", verbose = true, coverage = false, diff --git a/.luacheckrc b/.luacheckrc index f4aa5c4..d506403 100644 --- a/.luacheckrc +++ b/.luacheckrc @@ -8,21 +8,20 @@ max_line_length = 200 -std = "min" - -files["mqtt/**"] = { - ignore = { - "113/unpack", - "212/.+_", -- unused argument value_ - } +not_globals = { + "string.len", + "table.getn", } -files["tests/spec/**"] = { - ignore = { - "113/describe", - "113/it", - "143/assert", - } +include_files = { + "**/*.lua", + "*.rockspec", + ".busted", + ".luacheckrc", } +files["tests/spec/**/*.lua"] = { std = "+busted" } +files["examples/openresty/**/*.lua"] = { std = "+ngx_lua" } +files["mqtt/ngxsocket.lua"] = { std = "+ngx_lua" } + -- vim: ts=4 sts=4 sw=4 noet ft=lua diff --git a/mqtt/buffered_base.lua b/mqtt/buffered_base.lua new file mode 100644 index 0000000..a73c68d --- /dev/null +++ b/mqtt/buffered_base.lua @@ -0,0 +1,86 @@ +-- base connector class for buffered reading. +-- +-- Use this base class if the sockets do NOT yield. +-- So LuaSocket for example, when using Copas or OpenResty +-- use the non-buffered base class. +-- +-- This base class derives from `non_buffered_base` it implements the +-- `receive` and `buffer_clear` methods. But adds the `plain_receive` method +-- that must be implemented. +-- +-- NOTE: the `plain_receive` method is supposed to be non-blocking (see its +-- description), but the `send` method has no such facilities, so is `blocking` +-- in this class. Make sure to set the proper timeouts in either method before +-- starting the send/receive. So for example for LuaSocket call `settimeout(0)` +-- before receiving, and `settimeout(30)` before sending. +-- +-- @class mqtt.buffered_base + + +local super = require "mqtt.non_buffered_base" +local buffered = setmetatable({}, super) +buffered.__index = buffered +buffered.super = super +buffered.type = "buffered, blocking i/o" + +-- debug helper function +-- function buffered:buffer_state(msg) +-- print(string.format("buffer: size = %03d last-byte-done = %03d -- %s", +-- #(self.buffer_string or ""), self.buffer_pointer or 0, msg)) +-- end + +-- bytes read were handled, clear those +function buffered:buffer_clear() + -- self:buffer_state("before clearing buffer") + self.buffer_string = nil + self.buffer_pointer = nil +end + +-- read bytes, first from buffer, remaining from function +-- if function returns "idle" then reset read pointer +function buffered:receive(size) + -- self:buffer_state("receive start "..size.." bytes") + + local buf = self.buffer_string or "" + local idx = self.buffer_pointer or 0 + + while size > (#buf - idx) do + -- buffer is lacking bytes, read more... + local data, err = self:plain_receive(#buf - idx + size) + if not data then + if err == self.signal_idle then + -- read timedout, retry entire packet later, reset buffer + self.buffer_pointer = 0 + end + return data, err + end + + -- append received data, and try again + buf = buf .. data + self.buffer_string = buf + -- self:buffer_state("receive added "..#data.." bytes") + end + + self.buffer_pointer = idx + size + local data = buf:sub(idx + 1, idx + size) + -- print("data: ", require("mqtt.tools").hex(data)) + -- self:buffer_state("receive done "..size.." bytes\n") + return data +end + +--- Retrieves the requested number of bytes from the socket, in a non-blocking +-- manner. +-- The implementation MUST read with a timeout that immediately returns if there +-- is no data to read. If there is no data, then it MUST return +-- `nil, self.signal_idle` to indicate it no data was there and we need to retry later. +-- +-- If the receive errors, because of a closed connection it should return +-- `nil, self.signal_closed` to indicate this. Any other errors can be returned +-- as a regular `nil, err`. +-- @tparam size int number of bytes to receive. +-- @return data, or `false, err`, where `err` can be a signal. +function buffered:plain_receive(size) -- luacheck: ignore + error("method 'plain_receive' on buffered connector wasn't implemented") +end + +return buffered diff --git a/mqtt/client.lua b/mqtt/client.lua index 7f794c5..8f66e18 100644 --- a/mqtt/client.lua +++ b/mqtt/client.lua @@ -25,11 +25,6 @@ local str_match = string.match local table = require("table") local table_remove = table.remove -local coroutine = require("coroutine") -local coroutine_create = coroutine.create -local coroutine_resume = coroutine.resume -local coroutine_yield = coroutine.yield - local math = require("math") local math_random = math.random @@ -49,8 +44,7 @@ local protocol5 = require("mqtt.protocol5") local make_packet5 = protocol5.make_packet local parse_packet5 = protocol5.parse_packet -local ioloop = require("mqtt.ioloop") -local ioloop_get = ioloop.get +local log = require "mqtt.log" ------- @@ -60,41 +54,41 @@ local client_mt = {} client_mt.__index = client_mt --- Create and initialize MQTT client instance --- @tparam table args MQTT client creation arguments table --- @tparam string args.uri MQTT broker uri to connect. +-- @tparam table opts MQTT client creation options table +-- @tparam string opts.uri MQTT broker uri to connect. -- Expecting "host:port" or "host" format, in second case the port will be selected automatically: -- 1883 port for plain or 8883 for secure network connections --- @tparam string args.clean clean session start flag --- @tparam[opt=4] number args.version MQTT protocol version to use, either 4 (for MQTT v3.1.1) or 5 (for MQTT v5.0). +-- @tparam string opts.clean clean session start flag +-- @tparam[opt=4] number opts.version MQTT protocol version to use, either 4 (for MQTT v3.1.1) or 5 (for MQTT v5.0). -- Also you may use special values mqtt.v311 or mqtt.v50 for this field. --- @tparam[opt] string args.id MQTT client ID, will be generated by luamqtt library if absent --- @tparam[opt] string args.username username for authorization on MQTT broker --- @tparam[opt] string args.password password for authorization on MQTT broker; not acceptable in absence of username --- @tparam[opt=false] boolean,table args.secure use secure network connection, provided by luasec lua module; +-- @tparam[opt] string opts.id MQTT client ID, will be generated by luamqtt library if absent +-- @tparam[opt] string opts.username username for authorization on MQTT broker +-- @tparam[opt] string opts.password password for authorization on MQTT broker; not acceptable in absence of username +-- @tparam[opt=false] boolean,table opts.secure use secure network connection, provided by luasec lua module; -- set to true to select default params: { mode="client", protocol="tlsv1_2", verify="none", options="all" } -- or set to luasec-compatible table, for example with cafile="...", certificate="...", key="..." --- @tparam[opt] table args.will will message table with required fields { topic="...", payload="..." } +-- @tparam[opt] table opts.will will message table with required fields { topic="...", payload="..." } -- and optional fields { qos=1...3, retain=true/false } --- @tparam[opt=60] number args.keep_alive time interval for client to send PINGREQ packets to the server when network connection is inactive --- @tparam[opt=false] boolean args.reconnect force created MQTT client to reconnect on connection close. +-- @tparam[opt=60] number opts.keep_alive time interval for client to send PINGREQ packets to the server when network connection is inactive +-- @tparam[opt=false] boolean opts.reconnect force created MQTT client to reconnect on connection close. -- Set to number value to provide reconnect timeout in seconds -- It's not recommended to use values < 3 --- @tparam[opt] table args.connector connector table to open and send/receive packets over network connection. +-- @tparam[opt] table opts.connector connector table to open and send/receive packets over network connection. -- default is require("mqtt.luasocket"), or require("mqtt.luasocket_ssl") if secure argument is set --- @tparam[opt="ssl"] string args.ssl_module module name for the luasec-compatible ssl module, default is "ssl" +-- @tparam[opt="ssl"] string opts.ssl_module module name for the luasec-compatible ssl module, default is "ssl" -- may be used in some non-standard lua environments with own luasec-compatible ssl module -- @treturn client_mt MQTT client instance table -function client_mt:__init(args) +function client_mt:__init(opts) if not luamqtt_VERSION then luamqtt_VERSION = require("mqtt")._VERSION end - -- fetch and validate client args - local a = {} -- own client copy of args + -- fetch and validate client opts + local a = {} -- own client copy of opts - for key, value in pairs(args) do + for key, value in pairs(opts) do if type(key) ~= "string" then - error("expecting string key in args, got: "..type(key)) + error("expecting string key in opts, got: "..type(key)) end local value_type = type(value) @@ -141,7 +135,7 @@ function client_mt:__init(args) assert(value_type == "string", "expecting ssl_module to be a string") a.ssl_module = value else - error("unexpected key in client args: "..key.." = "..tostring(value)) + error("unexpected key in client opts: "..key.." = "..tostring(value)) end end @@ -169,6 +163,8 @@ function client_mt:__init(args) assert(type(a.connector.shutdown) == "function", "expecting connector.shutdown to be a function") assert(type(a.connector.send) == "function", "expecting connector.send to be a function") assert(type(a.connector.receive) == "function", "expecting connector.receive to be a function") + assert(a.connector.signal_closed, "missing connector.signal_closed signal value") + assert(a.connector.signal_idle, "missing connector.signal_idle signal value") -- will table content check if a.will then @@ -188,8 +184,8 @@ function client_mt:__init(args) a.keep_alive = 60 end - -- client args - self.args = a + -- client opts + self.opts = a -- event handlers self.handlers = { @@ -221,12 +217,7 @@ function client_mt:__init(args) self._parse_packet = parse_packet5 end - -- automatically add client to default ioloop, if it's available and running, then start connecting - local loop = ioloop_get(false) - if loop and loop.running then - loop:add(self) - self:start_connecting() - end + log:info("MQTT client '%s' created", a.id) end --- Add functions as handlers of given events @@ -239,7 +230,7 @@ function client_mt:on(...) elseif nargs == 1 then events = select(1, ...) else - error("invalid args: expected only one or two arguments") + error("invalid arguments: expected only one or two arguments") end for event, func in pairs(events) do assert(type(event) == "string", "expecting event to be a string") @@ -283,27 +274,27 @@ function client_mt:off(event, func) end --- Subscribe to specified topic. Returns the SUBSCRIBE packet id and calls optional callback when subscription will be created on broker --- @tparam table args subscription arguments --- @tparam string args.topic topic to subscribe --- @tparam[opt=0] number args.qos QoS level for subscription --- @tparam boolean args.no_local for MQTT v5.0 only: no_local flag for subscription --- @tparam boolean args.retain_as_published for MQTT v5.0 only: retain_as_published flag for subscription --- @tparam boolean args.retain_handling for MQTT v5.0 only: retain_handling flag for subscription --- @tparam[opt] table args.properties for MQTT v5.0 only: properties for subscribe operation --- @tparam[opt] table args.user_properties for MQTT v5.0 only: user properties for subscribe operation --- @tparam[opt] function args.callback callback function to be called when subscription will be created +-- @tparam table opts subscription options +-- @tparam string opts.topic topic to subscribe +-- @tparam[opt=0] number opts.qos QoS level for subscription +-- @tparam boolean opts.no_local for MQTT v5.0 only: no_local flag for subscription +-- @tparam boolean opts.retain_as_published for MQTT v5.0 only: retain_as_published flag for subscription +-- @tparam boolean opts.retain_handling for MQTT v5.0 only: retain_handling flag for subscription +-- @tparam[opt] table opts.properties for MQTT v5.0 only: properties for subscribe operation +-- @tparam[opt] table opts.user_properties for MQTT v5.0 only: user properties for subscribe operation +-- @tparam[opt] function opts.callback callback function to be called when subscription will be created -- @return packet id on success or false and error message on failure -function client_mt:subscribe(args) - -- fetch and validate args - assert(type(args) == "table", "expecting args to be a table") - assert(type(args.topic) == "string", "expecting args.topic to be a string") - assert(args.qos == nil or (type(args.qos) == "number" and check_qos(args.qos)), "expecting valid args.qos value") - assert(args.no_local == nil or type(args.no_local) == "boolean", "expecting args.no_local to be a boolean") - assert(args.retain_as_published == nil or type(args.retain_as_published) == "boolean", "expecting args.retain_as_published to be a boolean") - assert(args.retain_handling == nil or type(args.retain_handling) == "boolean", "expecting args.retain_handling to be a boolean") - assert(args.properties == nil or type(args.properties) == "table", "expecting args.properties to be a table") - assert(args.user_properties == nil or type(args.user_properties) == "table", "expecting args.user_properties to be a table") - assert(args.callback == nil or type(args.callback) == "function", "expecting args.callback to be a function") +function client_mt:subscribe(opts) + -- fetch and validate opts + assert(type(opts) == "table", "expecting opts to be a table") + assert(type(opts.topic) == "string", "expecting opts.topic to be a string") + assert(opts.qos == nil or (type(opts.qos) == "number" and check_qos(opts.qos)), "expecting valid opts.qos value") + assert(opts.no_local == nil or type(opts.no_local) == "boolean", "expecting opts.no_local to be a boolean") + assert(opts.retain_as_published == nil or type(opts.retain_as_published) == "boolean", "expecting opts.retain_as_published to be a boolean") + assert(opts.retain_handling == nil or type(opts.retain_handling) == "boolean", "expecting opts.retain_handling to be a boolean") + assert(opts.properties == nil or type(opts.properties) == "table", "expecting opts.properties to be a table") + assert(opts.user_properties == nil or type(opts.user_properties) == "table", "expecting opts.user_properties to be a table") + assert(opts.callback == nil or type(opts.callback) == "function", "expecting opts.callback to be a function") -- check connection is alive if not self.connection then @@ -315,31 +306,34 @@ function client_mt:subscribe(args) type = packet_type.SUBSCRIBE, subscriptions = { { - topic = args.topic, - qos = args.qos, - no_local = args.no_local, - retain_as_published = args.retain_as_published, - retain_handling = args.retain_handling + topic = opts.topic, + qos = opts.qos, + no_local = opts.no_local, + retain_as_published = opts.retain_as_published, + retain_handling = opts.retain_handling }, }, - properties = args.properties, - user_properties = args.user_properties, + properties = opts.properties, + user_properties = opts.user_properties, } self:_assign_packet_id(pargs) local packet_id = pargs.packet_id local subscribe = self._make_packet(pargs) + log:info("subscribing client '%s' to topic '%s' (packet: %d)", self.opts.id, opts.topic, packet_id or -1) + -- send SUBSCRIBE packet local ok, err = self:_send_packet(subscribe) if not ok then err = "failed to send SUBSCRIBE: "..err + log:error("client '%s': %s", self.opts.id, err) self:handle("error", err, self) self:close_connection("error") return false, err end -- add subscribe callback - local callback = args.callback + local callback = opts.callback if callback then local function handler(suback, ...) if suback.packet_id == packet_id then @@ -355,19 +349,19 @@ function client_mt:subscribe(args) end --- Unsubscribe from specified topic, and calls optional callback when subscription will be removed on broker --- @tparam table args subscription arguments --- @tparam string args.topic topic to unsubscribe --- @tparam[opt] table args.properties properties for unsubscribe operation --- @tparam[opt] table args.user_properties user properties for unsubscribe operation --- @tparam[opt] function args.callback callback function to be called when subscription will be removed on broker +-- @tparam table opts subscription options +-- @tparam string opts.topic topic to unsubscribe +-- @tparam[opt] table opts.properties properties for unsubscribe operation +-- @tparam[opt] table opts.user_properties user properties for unsubscribe operation +-- @tparam[opt] function opts.callback callback function to be called when subscription will be removed on broker -- @return packet id on success or false and error message on failure -function client_mt:unsubscribe(args) - -- fetch and validate args - assert(type(args) == "table", "expecting args to be a table") - assert(type(args.topic) == "string", "expecting args.topic to be a string") - assert(args.properties == nil or type(args.properties) == "table", "expecting args.properties to be a table") - assert(args.user_properties == nil or type(args.user_properties) == "table", "expecting args.user_properties to be a table") - assert(args.callback == nil or type(args.callback) == "function", "expecting args.callback to be a function") +function client_mt:unsubscribe(opts) + -- fetch and validate opts + assert(type(opts) == "table", "expecting opts to be a table") + assert(type(opts.topic) == "string", "expecting opts.topic to be a string") + assert(opts.properties == nil or type(opts.properties) == "table", "expecting opts.properties to be a table") + assert(opts.user_properties == nil or type(opts.user_properties) == "table", "expecting opts.user_properties to be a table") + assert(opts.callback == nil or type(opts.callback) == "function", "expecting opts.callback to be a function") -- check connection is alive @@ -378,25 +372,28 @@ function client_mt:unsubscribe(args) -- create UNSUBSCRIBE packet local pargs = { type = packet_type.UNSUBSCRIBE, - subscriptions = {args.topic}, - properties = args.properties, - user_properties = args.user_properties, + subscriptions = {opts.topic}, + properties = opts.properties, + user_properties = opts.user_properties, } self:_assign_packet_id(pargs) local packet_id = pargs.packet_id local unsubscribe = self._make_packet(pargs) + log:info("unsubscribing client '%s' from topic '%s' (packet: %d)", self.opts.id, opts.topic, packet_id or -1) + -- send UNSUBSCRIBE packet local ok, err = self:_send_packet(unsubscribe) if not ok then err = "failed to send UNSUBSCRIBE: "..err + log:error("client '%s' %s", self.opts.id, err) self:handle("error", err, self) self:close_connection("error") return false, err end -- add unsubscribe callback - local callback = args.callback + local callback = opts.callback if callback then local function handler(unsuback, ...) if unsuback.packet_id == packet_id then @@ -412,30 +409,30 @@ function client_mt:unsubscribe(args) end --- Publish message to broker --- @tparam table args publish operation arguments table --- @tparam string args.topic topic to publish message --- @tparam[opt] string args.payload publish message payload --- @tparam[opt=0] number args.qos QoS level for message publication --- @tparam[opt=false] boolean args.retain retain message publication flag --- @tparam[opt=false] boolean args.dup dup message publication flag --- @tparam[opt] table args.properties properties for publishing message --- @tparam[opt] table args.user_properties user properties for publishing message --- @tparam[opt] function args.callback callback to call when published message will be acknowledged +-- @tparam table opts publish operation options table +-- @tparam string opts.topic topic to publish message +-- @tparam[opt] string opts.payload publish message payload +-- @tparam[opt=0] number opts.qos QoS level for message publication +-- @tparam[opt=false] boolean opts.retain retain message publication flag +-- @tparam[opt=false] boolean opts.dup dup message publication flag +-- @tparam[opt] table opts.properties properties for publishing message +-- @tparam[opt] table opts.user_properties user properties for publishing message +-- @tparam[opt] function opts.callback callback to call when publihsed message will be acknowledged -- @return true or packet id on success or false and error message on failure -function client_mt:publish(args) - -- fetch and validate args - assert(type(args) == "table", "expecting args to be a table") - assert(type(args.topic) == "string", "expecting args.topic to be a string") - assert(args.payload == nil or type(args.payload) == "string", "expecting args.payload to be a string") - assert(args.qos == nil or type(args.qos) == "number", "expecting args.qos to be a number") - if args.qos then - assert(check_qos(args.qos), "expecting qos to be a valid QoS value") - end - assert(args.retain == nil or type(args.retain) == "boolean", "expecting args.retain to be a boolean") - assert(args.dup == nil or type(args.dup) == "boolean", "expecting args.dup to be a boolean") - assert(args.properties == nil or type(args.properties) == "table", "expecting args.properties to be a table") - assert(args.user_properties == nil or type(args.user_properties) == "table", "expecting args.user_properties to be a table") - assert(args.callback == nil or type(args.callback) == "function", "expecting args.callback to be a function") +function client_mt:publish(opts) + -- fetch and validate opts + assert(type(opts) == "table", "expecting opts to be a table") + assert(type(opts.topic) == "string", "expecting opts.topic to be a string") + assert(opts.payload == nil or type(opts.payload) == "string", "expecting opts.payload to be a string") + assert(opts.qos == nil or type(opts.qos) == "number", "expecting opts.qos to be a number") + if opts.qos then + assert(check_qos(opts.qos), "expecting qos to be a valid QoS value") + end + assert(opts.retain == nil or type(opts.retain) == "boolean", "expecting opts.retain to be a boolean") + assert(opts.dup == nil or type(opts.dup) == "boolean", "expecting opts.dup to be a boolean") + assert(opts.properties == nil or type(opts.properties) == "table", "expecting opts.properties to be a table") + assert(opts.user_properties == nil or type(opts.user_properties) == "table", "expecting opts.user_properties to be a table") + assert(opts.callback == nil or type(opts.callback) == "function", "expecting opts.callback to be a function") -- check connection is alive local conn = self.connection @@ -444,27 +441,31 @@ function client_mt:publish(args) end -- create PUBLISH packet - args.type = packet_type.PUBLISH - self:_assign_packet_id(args) - local packet_id = args.packet_id - local publish = self._make_packet(args) + opts.type = packet_type.PUBLISH + self:_assign_packet_id(opts) + local packet_id = opts.packet_id + local publish = self._make_packet(opts) + + log:debug("client '%s' publishing to topic '%s' (packet: %d)", self.opts.id, opts.topic, packet_id or -1) + --log:debug("client '%s' publishing to topic '%s' (value '%d')", self.opts.id, opts.topic, opts.payload) -- send PUBLISH packet local ok, err = self:_send_packet(publish) if not ok then err = "failed to send PUBLISH: "..err + log:error("client '%s' %s", self.opts.id, err) self:handle("error", err, self) self:close_connection("error") return false, err end -- record packet id as waited for QoS 2 exchange - if args.qos == 2 then + if opts.qos == 2 then conn.wait_for_pubrec[packet_id] = true end -- add acknowledge callback - local callback = args.callback + local callback = opts.callback if callback then if packet_id then local function handler(ack, ...) @@ -507,6 +508,8 @@ function client_mt:acknowledge(msg, rc, properties, user_properties) return true end + log:debug("client '%s' acknowledging packet %d", self.opts.id, packet_id or -1) + if msg.qos == 1 then -- PUBACK should be sent @@ -523,6 +526,7 @@ function client_mt:acknowledge(msg, rc, properties, user_properties) local ok, err = self:_send_packet(puback) if not ok then err = "failed to send PUBACK: "..err + log:error("client '%s' %s", self.opts.id, err) self:handle("error", err, self) self:close_connection("error") return false, err @@ -543,6 +547,7 @@ function client_mt:acknowledge(msg, rc, properties, user_properties) local ok, err = self:_send_packet(pubrec) if not ok then err = "failed to send PUBREC: "..err + log:error("client '%s' %s", self.opts.id, err) self:handle("error", err, self) self:close_connection("error") return false, err @@ -561,7 +566,7 @@ end -- @tparam[opt] table user_properties user properties for PUBACK/PUBREC packets -- @return true on success or false and error message on failure function client_mt:disconnect(rc, properties, user_properties) - -- validate args + -- validate opts assert(rc == nil or type(rc) == "number", "expecting rc to be a number") assert(properties == nil or type(properties) == "table", "expecting properties to be a table") assert(user_properties == nil or type(user_properties) == "table", "expecting user_properties to be a table") @@ -579,10 +584,13 @@ function client_mt:disconnect(rc, properties, user_properties) user_properties = user_properties, } + log:info("client '%s' disconnecting (rc = %d)", self.opts.id, rc or 0) + -- send DISCONNECT packet local ok, err = self:_send_packet(disconnect) if not ok then err = "failed to send DISCONNECT: "..err + log:error("client '%s' %s", self.opts.id, err) self:handle("error", err, self) self:close_connection("error") return false, err @@ -600,11 +608,11 @@ end -- @tparam[opt] table user_properties user properties for PUBACK/PUBREC packets -- @return true on success or false and error message on failure function client_mt:auth(rc, properties, user_properties) - -- validate args + -- validate opts assert(rc == nil or type(rc) == "number", "expecting rc to be a number") assert(properties == nil or type(properties) == "table", "expecting properties to be a table") assert(user_properties == nil or type(user_properties) == "table", "expecting user_properties to be a table") - assert(self.args.version == 5, "allowed only in MQTT v5.0 protocol") + assert(self.opts.version == 5, "allowed only in MQTT v5.0 protocol") -- check connection is alive if not self.connection then @@ -619,10 +627,13 @@ function client_mt:auth(rc, properties, user_properties) user_properties = user_properties, } + log:info("client '%s' authenticating") + -- send AUTH packet local ok, err = self:_send_packet(auth) if not ok then err = "failed to send AUTH: "..err + log:error("client '%s' %s", self.opts.id, err) self:handle("error", err, self) self:close_connection("error") return false, err @@ -640,28 +651,21 @@ function client_mt:close_connection(reason) return true end - local args = self.args - args.connector.shutdown(conn) - self.connection = nil - conn.close_reason = reason or "unspecified" + reason = reason or "unspecified" - self:handle("close", conn, self) + log:info("client '%s' closing connection (reason: %s)", self.opts.id, reason) - -- check connection is still closed (self.connection may be re-created in "close" handler) - if not self.connection then - -- remove from ioloop - if self.ioloop and not args.reconnect then - self.ioloop:remove(self) - end - end + conn:shutdown() + self.connection = nil + conn.close_reason = reason + self:handle("close", conn, self) return true end --- Start connecting to broker -- @return true on success or false and error message on failure function client_mt:start_connecting() - -- print("start connecting") -- debug -- open network connection local ok, err = self:open_connection() if not ok then @@ -693,17 +697,20 @@ function client_mt:send_pingreq() type = packet_type.PINGREQ, } + log:debug("client '%s' sending PINGREQ", self.opts.id) + -- send PINGREQ packet local ok, err = self:_send_packet(pingreq) if not ok then err = "failed to send PINGREQ: "..err + log:error("client '%s' %s", self.opts.id, err) self:handle("error", err, self) self:close_connection("error") return false, err end -- set ping timeout; for now 1 ping-request interval - self.ping_expire_time = os_time() + self.args.keep_alive + self.ping_expire_time = os_time() + self.opts.keep_alive return true end @@ -715,21 +722,24 @@ function client_mt:open_connection() return true end - local args = self.args - local connector = assert(args.connector, "no connector configured in MQTT client") + local opts = self.opts + local connector = assert(opts.connector, "no connector configured in MQTT client") -- create connection table - local conn = { - uri = args.uri, + local conn = setmetatable({ + uri = opts.uri, wait_for_pubrec = {}, -- a table with packet_id of partially acknowledged sent packets in QoS 2 exchange process wait_for_pubrel = {}, -- a table with packet_id of partially acknowledged received packets in QoS 2 exchange process - } - client_mt._parse_uri(args, conn) - client_mt._apply_secure(args, conn) + }, connector) + client_mt._parse_uri(opts, conn) + client_mt._apply_secure(opts, conn) + + log:info("client '%s' connecting to broker '%s' (using: %s)", self.opts.id, opts.uri, conn.type or "unknown") -- perform connect - local ok, err = connector.connect(conn) + local ok, err = conn:connect() if not ok then + log:error("client '%s' %s", self.opts.id, err) err = "failed to open network connection: "..err self:handle("error", err, self) return false, err @@ -741,14 +751,6 @@ function client_mt:open_connection() -- reset ping timeout self.ping_expire_time = nil - -- create receive function - local receive = connector.receive - self.connection.recv_func = function(size) - return receive(conn, size) - end - - self:_apply_network_timeout() - return true end @@ -760,25 +762,28 @@ function client_mt:send_connect() return false, "network connection is not opened" end - local args = self.args + local opts = self.opts -- create CONNECT packet local connect = self._make_packet{ type = packet_type.CONNECT, - id = args.id, - clean = args.clean, - username = args.username, - password = args.password, - will = args.will, - keep_alive = args.keep_alive, - properties = args.properties, - user_properties = args.user_properties, + id = opts.id, + clean = opts.clean, + username = opts.username, + password = opts.password, + will = opts.will, + keep_alive = opts.keep_alive, + properties = opts.properties, + user_properties = opts.user_properties, } + log:info("client '%s' sending CONNECT (user '%s')", self.opts.id, opts.username or "not specified") + -- send CONNECT packet local ok, err = self:_send_packet(connect) if not ok then err = "failed to send CONNECT: "..err + log:error("client '%s' %s", self.opts.id, err) self:handle("error", err, self) self:close_connection("error") return false, err @@ -804,7 +809,7 @@ end -- end -- end) function client_mt:check_keep_alive() - local interval = self.args.keep_alive + local interval = self.opts.keep_alive if not self.connection then return interval, "network connection is not opened" end @@ -817,6 +822,7 @@ function client_mt:check_keep_alive() if t_timeout and t_timeout <= t_now then -- we timed-out, close and exit local err = str_format("failed to receive PINGRESP within %d seconds", interval) + log:error("client '%s' %s", self.opts.id, err) self:handle("error", err, self) self:close_connection("error") return false, err @@ -838,12 +844,6 @@ end -- Internal methods --- Set or rest ioloop for MQTT client -function client_mt:set_ioloop(loop) - self.ioloop = loop - self:_apply_network_timeout() -end - -- Send PUBREL acknowledge packet - second phase of QoS 2 exchange -- Returns true on success or false and error message on failure function client_mt:acknowledge_pubrel(packet_id) @@ -855,10 +855,13 @@ function client_mt:acknowledge_pubrel(packet_id) -- create PUBREL packet local pubrel = self._make_packet{type=packet_type.PUBREL, packet_id=packet_id, rc=0} + log:debug("client '%s' sending PUBREL (packet: %d)", self.opts.id, packet_id or -1) + -- send PUBREL packet local ok, err = self:_send_packet(pubrel) if not ok then err = "failed to send PUBREL: "..err + log:error("client '%s' %s", self.opts.id, err) self:handle("error", err, self) self:close_connection("error") return false, err @@ -878,10 +881,13 @@ function client_mt:acknowledge_pubcomp(packet_id) -- create PUBCOMP packet local pubcomp = self._make_packet{type=packet_type.PUBCOMP, packet_id=packet_id, rc=0} + log:debug("client '%s' sending PUBCOMP (packet: %d)", self.opts.id, packet_id or -1) + -- send PUBCOMP packet local ok, err = self:_send_packet(pubcomp) if not ok then err = "failed to send PUBCOMP: "..err + log:error("client '%s' %s", self.opts.id, err) self:handle("error", err, self) self:close_connection("error") return false, err @@ -914,7 +920,7 @@ end -- Internal methods --- Assign next packet id for given packet creation args +-- Assign next packet id for given packet creation opts function client_mt:_assign_packet_id(pargs) if not pargs.packet_id then if packet_id_required(pargs) then @@ -929,10 +935,13 @@ function client_mt:handle_received_packet(packet) local conn = self.connection local err + log:debug("client '%s' received '%s' (packet: %s)", self.opts.id, packet.type, tostring(packet.packet_id or "n.a.")) + if not conn.connack then -- expecting only CONNACK packet here if packet.type ~= packet_type.CONNACK then err = "expecting CONNACK but received "..packet.type + log:error("client '%s' %s", self.opts.id, err) self:handle("error", err, self) self:close_connection("error") return false, err @@ -944,12 +953,15 @@ function client_mt:handle_received_packet(packet) -- check CONNACK rc if packet.rc ~= 0 then err = str_format("CONNECT failed with CONNACK [rc=%d]: %s", packet.rc, packet:reason_string()) + log:error("client '%s' %s", self.opts.id, err) self:handle("error", err, self, packet) self:handle("connect", packet, self) self:close_connection("connection failed") return false, err end + log:info("client '%s' connected successfully to '%s'", self.opts.id, conn.uri) + -- fire connect event self:handle("connect", packet, self) else @@ -995,183 +1007,119 @@ function client_mt:handle_received_packet(packet) self:close_connection("disconnect received from broker") elseif ptype == packet_type.AUTH then self:handle("auth", packet, self) - -- else - -- print("unhandled packet:", packet) -- debug + else + log:warn("client '%s' don't know how to handle %s", self.opts.id, ptype) end end return true end --- sync-loop do - -- Receive packet function in sync mode - local function sync_recv(self) - return true, self:_receive_packet() - end - - -- Perform one input/output iteration, called by sync receiving loop - function client_mt:_sync_iteration() - return self:_io_iteration(sync_recv) - end -end - --- ioloop -do - -- Receive packet function - from ioloop's coroutine - local function ioloop_recv(self) - return coroutine_resume(self.connection.coro) - end + -- implict (re)connecting when reading + local function implicit_connect(self) + local opts = self.opts + + if not self.first_connect and not opts.reconnect then + -- this would be a re-connect, but we're not supposed to auto-reconnect + log:debug("client '%s' was disconnected and not set to auto-reconnect", self.opts.id) + return false, "network connection is not opened" + end - -- Perform one input/output iteration, called by ioloop - function client_mt:_ioloop_iteration() - -- working according state - local loop = self.ioloop - local args = self.args + -- should we wait for a timeout between retries? + local t_reconnect = (self.last_connect_time or 0) + (opts.reconnect or 0) + local t_now = os_time() + if t_reconnect > t_now then + -- were delaying before retrying, return remaining delay + return t_reconnect - t_now + end - local conn = self.connection - if conn then - -- network connection opened - -- perform packet receiving using ioloop receive function - local ok, err - if loop then - ok, err = self:_io_iteration(ioloop_recv) - else - ok, err = self:_sync_iteration() - end + self.last_connect_time = t_now - if ok then - ok, err = self:check_keep_alive() + local ok, err = self:start_connecting() + if not ok then + -- we failed to connect + if opts.reconnect then + -- set to auto-reconnect, report the configured retry delay + return opts.reconnect end - + -- not reconnecting, so just report the error return ok, err - else - -- no connection - first connect, reconnect or remove from ioloop - if self.first_connect then - self.first_connect = false - self:start_connecting() - elseif args.reconnect then - if args.reconnect == true then - self:start_connecting() - else - -- reconnect in specified timeout - if self.reconnect_timer_start then - if os_time() - self.reconnect_timer_start >= args.reconnect then - self.reconnect_timer_start = nil - self:start_connecting() - else - if loop then - loop:can_sleep() - end - end - else - self.reconnect_timer_start = os_time() - end - end - else - -- finish working with client - if loop then - loop:remove(self) - end - end end - end -end --- Performing one IO iteration - receive next packet -function client_mt:_io_iteration(recv) - -- first - try to receive packet - local ok, packet, err = recv(self) - -- print("received packet", ok, packet, err) + -- connected succesfully, but don't know how long it took, so return now + -- to be nice to other clients. Return 0 to indicate ready-for-reading. + return 0 + end + + --- Performs a single IO loop step. + -- It will connect if not connected, will re-connect if set to. + -- This should be called repeatedly in a loop. + -- + -- The return value is the time after which this method must be called again. + -- It can be called sooner, but shouldn't be called later. Return values: + -- + -- - `0`; a packet was succesfully handled, so retry immediately, no delays, + -- in case additional data is waiting to be read on the socket. + -- - `>0`; The reconnect timer needs a delay before it can retry (calling + -- sooner is not a problem, it will only reconnect when the delay + -- has actually passed) + -- - `-1`; the socket read timed out, so it is idle. This return code is only + -- returned with buffered connectors (luasocket), never for yielding sockets + -- (Copas or OpenResty) + -- + -- @return time after which to retry or nil+error + function client_mt:step() + local conn = self.connection - -- check coroutine resume status - if not ok then - err = "failed to resume receive packet coroutine: "..tostring(packet) - self:handle("error", err, self) - self:close_connection("error") - return false, err - end + -- try and connect if not connected yet + if not conn then + return implicit_connect(self) + end - -- check for communication error - if packet == false then - if err == "closed" then - self:close_connection("connection closed by broker") - return false, err - else - err = "failed to receive next packet: "..err - self:handle("error", err, self) - self:close_connection("error") - return false, err + local packet, err = self:_receive_packet() + if not packet then + if err == conn.signal_idle then + -- connection was idle, nothing happened + return -1 + elseif err == conn.signal_closed then + self:close_connection("connection closed by broker") + return false, err + else + err = "failed to receive next packet: "..tostring(err) + log:error("client '%s' %s", self.opts.id, err) + self:handle("error", err, self) + self:close_connection("error") + return false, err + end end - end - -- check some packet received - if packet ~= "timeout" and packet ~= "wantread" then + local ok ok, err = self:handle_received_packet(packet) if not ok then - return ok, err - end - end - - return true -end - --- Apply ioloop network timeout to already established connection (if any) -function client_mt:_apply_network_timeout() - local conn = self.connection - if conn then - local loop = self.ioloop - if loop then - -- apply connection timeout - self.args.connector.settimeout(conn, loop.args.timeout) - - -- connection packets receive loop coroutine - conn.coro = coroutine_create(function() - while true do - local packet, err = self:_receive_packet() - if not packet then - return false, err - else - coroutine_yield(packet) - end - end - end) - - -- replace connection recv_func with coroutine-based version - local sync_recv_func = conn.recv_func - conn.recv_func = function(...) - while true do - local data, err = sync_recv_func(...) - if not data and (err == "timeout" or err == "wantread") then - loop.timeouted = true - coroutine_yield(err) - else - return data, err - end - end - end - conn.sync_recv_func = sync_recv_func - else - -- disable connection timeout - self.args.connector.settimeout(conn, nil) - - -- replace back usual (blocking) connection recv_func - if conn.sync_recv_func then - conn.recv_func = conn.sync_recv_func - conn.sync_recv_func = nil + if not self.opts.reconnect then + -- no auto-reconnect, so return error + return ok, err + else + -- reconnect, return 0 to retry asap, the reconnect code + -- will then do the right-thing TM + return 0 end end + + -- succesfully handled packed, maybe there is more, so retry asap + return 0 end end --- Fill given connection table with host and port according given args -function client_mt._parse_uri(args, conn) - local host, port = str_match(args.uri, "^([^%s]+):(%d+)$") +-- Fill given connection table with host and port according given opts +function client_mt._parse_uri(opts, conn) + local host, port = str_match(opts.uri, "^([^%s]+):(%d+)$") if not host then -- trying pattern without port host = assert(str_match(conn.uri, "^([^%s]+)$"), "invalid uri format: expecting at least host/ip in .uri") end if not port then - if args.secure then + if opts.secure then port = 8883 -- default MQTT secure connection port else port = 1883 -- default MQTT connection port @@ -1182,9 +1130,9 @@ function client_mt._parse_uri(args, conn) conn.host, conn.port = host, port end --- Creates the conn.secure_params table and its content according client creation args -function client_mt._apply_secure(args, conn) - local secure = args.secure +-- Creates the conn.secure_params table and its content according client creation opts +function client_mt._apply_secure(opts, conn) + local secure = opts.secure if secure then conn.secure = true if type(secure) == "table" then @@ -1192,12 +1140,12 @@ function client_mt._apply_secure(args, conn) else conn.secure_params = { mode = "client", - protocol = "tlsv1_2", + protocol = "any", verify = "none", - options = "all", + options = {"all", "no_sslv2", "no_sslv3", "no_tlsv1"}, } end - conn.ssl_module = args.ssl_module or "ssl" + conn.ssl_module = opts.ssl_module or "ssl" end end @@ -1213,13 +1161,9 @@ function client_mt:_send_packet(packet) return false, "sending empty packet" end -- and send binary packet to network connection - local i, err = 1 - local send = self.args.connector.send - while i < len do - i, err = send(conn, data, i) - if not i then - return false, "connector.send failed: "..err - end + local ok, err = conn:send(data) + if not ok then + return false, "connector.send failed: "..err end self.send_time = os_time() return true @@ -1231,17 +1175,33 @@ function client_mt:_receive_packet() if not conn then return false, "network connection is not opened" end - -- parse packet - local packet, err = self._parse_packet(conn.recv_func) - if not packet then + -- read & parse packet + local packet, err = self._parse_packet( + function(size) + return conn:receive(size) + end + ) + if packet then + -- succesful packet, clear handled data and return it + conn:buffer_clear() + return packet + end + + -- check if we need more data, if not, clear the buffer because were done with + -- the data in that case + if err == conn.signal_idle then + -- we need more data, so do not clear buffer, just return the error return false, err end - return packet + + -- some other error, can't use buffered data, dispose of it + conn:buffer_clear() + return false, err end -- Represent MQTT client as string function client_mt:__tostring() - return str_format("mqtt.client{id=%q}", tostring(self.args.id)) + return str_format("mqtt.client{id=%q}", tostring(self.opts.id)) end -- Garbage collection handler @@ -1256,7 +1216,7 @@ end -- @section exported --- Create, initialize and return new MQTT client instance --- @param ... see arguments of client_mt:__init(args) +-- @param ... see arguments of client_mt:__init(opts) -- @see client_mt:__init -- @treturn client_mt MQTT client instance function client.create(...) diff --git a/mqtt/init.lua b/mqtt/init.lua index fca8353..67d97a3 100644 --- a/mqtt/init.lua +++ b/mqtt/init.lua @@ -28,14 +28,16 @@ for key, value in pairs(const) do end -- load required stuff -local type = type +local log = require "mqtt.log" + local select = select local require = require local client = require("mqtt.client") local client_create = client.create -local ioloop_get = require("mqtt.ioloop").get +local ioloop = require("mqtt.ioloop") +local ioloop_get = ioloop.get --- Create new MQTT client instance -- @param ... Same as for mqtt.client.create(...) @@ -49,17 +51,18 @@ end mqtt.get_ioloop = ioloop_get --- Run default ioloop for given MQTT clients or functions --- @param ... MQTT clients or lopp functions to add to ioloop +-- @param ... MQTT clients or loop functions to add to ioloop -- @see mqtt.ioloop.get -- @see mqtt.ioloop.run_until_clients function mqtt.run_ioloop(...) + log:info("starting default ioloop instance") local loop = ioloop_get() for i = 1, select("#", ...) do local cl = select(i, ...) loop:add(cl) - if type(cl) ~= "function" then - cl:start_connecting() - end + -- if type(cl) ~= "function" then -- TODO: remove + -- cl:start_connecting() + -- end end return loop:run_until_clients() end diff --git a/mqtt/ioloop.lua b/mqtt/ioloop.lua index b903c9a..575f799 100644 --- a/mqtt/ioloop.lua +++ b/mqtt/ioloop.lua @@ -30,6 +30,7 @@ local ioloop = {} -- load required stuff +local log = require "mqtt.log" local next = next local type = type local ipairs = ipairs @@ -39,106 +40,151 @@ local setmetatable = setmetatable local table = require("table") local tbl_remove = table.remove +local math = require("math") +local math_min = math.min + --- ioloop instances metatable -- @type ioloop_mt local ioloop_mt = {} ioloop_mt.__index = ioloop_mt --- Initialize ioloop instance --- @tparam table args ioloop creation arguments table --- @tparam[opt=0.005] number args.timeout network operations timeout in seconds --- @tparam[opt=0] number args.sleep sleep interval after each iteration --- @tparam[opt] function args.sleep_function custom sleep function to call after each iteration +-- @tparam table opts ioloop creation options table +-- @tparam[opt=0.005] number opts.timeout network operations timeout in seconds +-- @tparam[opt=0] number opts.sleep sleep interval after each iteration +-- @tparam[opt] function opts.sleep_function custom sleep function to call after each iteration -- @treturn ioloop_mt ioloop instance -function ioloop_mt:__init(args) - args = args or {} - args.timeout = args.timeout or 0.005 - args.sleep = args.sleep or 0 - args.sleep_function = args.sleep_function or require("socket").sleep - self.args = args +function ioloop_mt:__init(opts) + log:debug("initializing ioloop instance '%s'", tostring(self)) + opts = opts or {} + opts.timeout = opts.timeout or 0.005 + opts.sleep = opts.sleep or 0 + opts.sleep_function = opts.sleep_function or require("socket").sleep + self.opts = opts self.clients = {} self.running = false --ioloop running flag, used by MQTT clients which are adding after this ioloop started to run end --- Add MQTT client or a loop function to the ioloop instance --- @tparam client_mt|function client MQTT client or a loop function to add to ioloop +-- @tparam client_mt|function client MQTT client or a loop function to add to ioloop -- @return true on success or false and error message on failure function ioloop_mt:add(client) local clients = self.clients if clients[client] then - return false, "such MQTT client or loop function is already added to this ioloop" + if type(client) == "table" then + log:warn("MQTT client '%s' was already added to ioloop '%s'", client.opts.id, tostring(self)) + return false, "MQTT client was already added to this ioloop" + else + log:warn("MQTT loop function '%s' was already added to this ioloop '%s'", tostring(client), tostring(self)) + return false, "MQTT loop function was already added to this ioloop" + end end clients[#clients + 1] = client clients[client] = true - -- associate ioloop with adding MQTT client - if type(client) ~= "function" then - client:set_ioloop(self) + if type(client) == "table" then + log:info("adding client '%s' to ioloop '%s'", client.opts.id, tostring(self)) + -- create and add function for PINGREQ + local function f() + if not clients[client] then + -- the client were supposed to do keepalive for is gone, remove ourselves + self:remove(f) + end + return client:check_keep_alive() + end + -- add it to start doing keepalive checks + self:add(f) + else + log:info("adding function '%s' to ioloop '%s'", tostring(client), tostring(self)) end return true end --- Remove MQTT client or a loop function from the ioloop instance --- @tparam client_mt|function client MQTT client or a loop function to remove from ioloop +-- @tparam client_mt|function client MQTT client or a loop function to remove from ioloop -- @return true on success or false and error message on failure function ioloop_mt:remove(client) local clients = self.clients if not clients[client] then - return false, "no such MQTT client or loop function was added to ioloop" + if type(client) == "table" then + log:warn("MQTT client not found '%s' in ioloop '%s'", client.opts.id, tostring(self)) + return false, "MQTT client not found" + else + log:warn("MQTT loop function not found '%s' in ioloop '%s'", tostring(client), tostring(self)) + return false, "MQTT loop function not found" + end end - clients[client] = nil -- search an index of client to remove for i, item in ipairs(clients) do if item == client then + -- found it, remove tbl_remove(clients, i) + clients[client] = nil break end end - -- unlink ioloop from MQTT client - if type(client) ~= "function" then - client:set_ioloop(nil) + if type(client) == "table" then + log:info("removed client '%s' from ioloop '%s'", client.opts.id, tostring(self)) + else + log:info("removed loop function '%s' from ioloop '%s'", tostring(client), tostring(self)) end return true end ---- Perform one ioloop iteration +--- Perform one ioloop iteration. +-- TODO: make this smarter do not wake-up functions or clients returned a longer +-- sleep delay. Currently it's pretty much a busy loop. function ioloop_mt:iteration() - self.timeouted = false + local opts = self.opts + local sleep = opts.sleep + for _, client in ipairs(self.clients) do + local t, err + -- read data and handle events if type(client) ~= "function" then - client:_ioloop_iteration() + t, err = client:step() + if t == -1 then + -- no data read, client is idle + t = nil + elseif not t then + if not client.opts.reconnect then + -- error and not reconnecting, remove the client + log:error("client '%s' failed with '%s', will not re-connect", client.opts.id, err) + self:remove(client) + t = nil + else + -- error, but will reconnect + log:error("client '%s' failed with '%s', will try re-connecting", client.opts.id, err) + t = 0 -- try immediately + end + end else - client() + t = client() end + t = t or opts.sleep + sleep = math_min(sleep, t) end -- sleep a bit - local args = self.args - local sleep = args.sleep if sleep > 0 then - args.sleep_function(sleep) + opts.sleep_function(sleep) end end ---- Perform sleep if no one of the network operation in current iteration was not timeouted -function ioloop_mt:can_sleep() - if not self.timeouted then - local args = self.args - args.sleep_function(args.timeout) - self.timeouted = true - end -end - ---- Run ioloop until at least one client are in ioloop +--- Run ioloop while there is at least one client/function in the ioloop function ioloop_mt:run_until_clients() + log:info("ioloop started with %d clients/functions", #self.clients) + self.running = true while next(self.clients) do self:iteration() end self.running = false + + log:info("ioloop finished with %d clients/functions", #self.clients) end ------- @@ -146,9 +192,9 @@ end --- Create IO loop instance with given options -- @see ioloop_mt:__init -- @treturn ioloop_mt ioloop instance -local function ioloop_create(args) +local function ioloop_create(opts) local inst = setmetatable({}, ioloop_mt) - inst:__init(args) + inst:__init(opts) return inst end ioloop.create = ioloop_create @@ -158,16 +204,15 @@ local ioloop_instance --- Returns default ioloop instance -- @tparam[opt=true] boolean autocreate Automatically create ioloop instance --- @tparam[opt] table args Arguments for creating ioloop instance +-- @tparam[opt] table opts Arguments for creating ioloop instance -- @treturn ioloop_mt ioloop instance -function ioloop.get(autocreate, args) +function ioloop.get(autocreate, opts) if autocreate == nil then autocreate = true end - if autocreate then - if not ioloop_instance then - ioloop_instance = ioloop_create(args) - end + if autocreate and not ioloop_instance then + log:info("auto-creating default ioloop instance") + ioloop_instance = ioloop_create(opts) end return ioloop_instance end diff --git a/mqtt/log.lua b/mqtt/log.lua new file mode 100644 index 0000000..42a6964 --- /dev/null +++ b/mqtt/log.lua @@ -0,0 +1,17 @@ +-- logging + +-- returns a LuaLogging compatible logger object if LuaLogging was already loaded +-- otherwise returns a stub + +local ll = package.loaded.logging +if ll and type(ll) == "table" and ll.defaultLogger and + tostring(ll._VERSION):find("LuaLogging") then + -- default LuaLogging logger is available + return ll.defaultLogger() +else + -- just use a stub logger with only no-op functions + local nop = function() end + return setmetatable({}, { + __index = function(self, key) self[key] = nop return nop end + }) +end diff --git a/mqtt/luasocket-copas.lua b/mqtt/luasocket-copas.lua index 069229e..7c604d1 100644 --- a/mqtt/luasocket-copas.lua +++ b/mqtt/luasocket-copas.lua @@ -2,7 +2,10 @@ -- NOTE: you will need to install copas like this: luarocks install copas -- module table -local connector = {} +local super = require "mqtt.non_buffered_base" +local connector = setmetatable({}, super) +connector.__index = connector +connector.super = super local socket = require("socket") local copas = require("copas") @@ -10,36 +13,52 @@ local copas = require("copas") -- Open network connection to .host and .port in conn table -- Store opened socket to conn table -- Returns true on success, or false and error text on failure -function connector.connect(conn) - local sock, err = socket.connect(conn.host, conn.port) - if not sock then - return false, "socket.connect failed: "..err +function connector:connect() + local sock = copas.wrap(socket.tcp(), self.secure_params) + sock:settimeout(self.timeout) + + local ok, err = socket:connect(self.host, self.port) + if not ok then + return false, "copas.connect failed: "..err end - conn.sock = sock + self.sock = sock return true end -- Shutdown network connection -function connector.shutdown(conn) - conn.sock:shutdown() +function connector:shutdown() + self.sock:shutdown() end -- Send data to network connection -function connector.send(conn, data, i, j) - local ok, err = copas.send(conn.sock, data, i, j) - return ok, err +function connector:send(data) + local i = 1 + local err + while i < #data do + i, err = self.sock:send(data, i) + if not i then + return false, err + end + end + return true end -- Receive given amount of data from network connection -function connector.receive(conn, size) - local ok, err = copas.receive(conn.sock, size) - return ok, err -end +function connector:receive(size) + local sock = self.sock + local data, err = sock:receive(size) + if data then + return data + end --- Set connection's socket to non-blocking mode and set a timeout for it -function connector.settimeout(conn, timeout) - conn.timeout = timeout - conn.sock:settimeout(0) + -- note: signal_idle is not needed here since Copas takes care + -- of that. The read is non blocking, so a timeout is a real error and not + -- a signal to retry. + if err == "closed" then + return false, self.signal_closed + else + return false, err + end end -- export module table diff --git a/mqtt/luasocket.lua b/mqtt/luasocket.lua index 5b8de2b..7a36b99 100644 --- a/mqtt/luasocket.lua +++ b/mqtt/luasocket.lua @@ -1,51 +1,80 @@ -- DOC: http://w3.impa.br/~diego/software/luasocket/tcp.html -- module table -local luasocket = {} +local super = require "mqtt.buffered_base" +local luasocket = setmetatable({}, super) +luasocket.__index = luasocket +luasocket.super = super local socket = require("socket") +-- table with error messages that indicate a read timeout +luasocket.timeout_errors = { + timeout = true, +} + -- Open network connection to .host and .port in conn table -- Store opened socket to conn table -- Returns true on success, or false and error text on failure -function luasocket.connect(conn) - local sock, err = socket.connect(conn.host, conn.port) - if not sock then - return false, "socket.connect failed: "..err +function luasocket:connect() + self:buffer_clear() -- sanity + local sock = socket.tcp() + sock:settimeout(self.timeout) + + local ok, err = sock:connect(self.host, self.port) + if not ok then + return false, "socket.connect failed to connect to '"..tostring(self.host)..":"..tostring(self.port).."': "..err end - conn.sock = sock + + self.sock = sock return true end -- Shutdown network connection -function luasocket.shutdown(conn) - conn.sock:shutdown() +function luasocket:shutdown() + self.sock:shutdown() end -- Send data to network connection -function luasocket.send(conn, data, i, j) - local ok, err = conn.sock:send(data, i, j) - -- print(" luasocket.send:", ok, err, require("mqtt.tools").hex(data)) - return ok, err +function luasocket:send(data) + local sock = self.sock + local i = 0 + local err + + sock:settimeout(self.timeout) + + while i < #data do + i, err = sock:send(data, i + 1) + if not i then + return false, err + end + end + + return true end -- Receive given amount of data from network connection -function luasocket.receive(conn, size) - local ok, err = conn.sock:receive(size) - -- if ok then - -- print(" luasocket.receive:", size, require("mqtt.tools").hex(ok)) - -- elseif err ~= "timeout" then - -- print(" luasocket.receive:", ok, err) - -- end - return ok, err -end +function luasocket:plain_receive(size) + local sock = self.sock --- Set connection's socket to non-blocking mode and set a timeout for it -function luasocket.settimeout(conn, timeout) - conn.timeout = timeout - conn.sock:settimeout(timeout, "t") + sock:settimeout(0.010) -- TODO: setting to 0 fails??? it shouldn't + local data, err = sock:receive(size) + + if data then + return data + end + + -- convert error to signal if required + if self.timeout_errors[err or -1] then + return false, self.signal_idle + elseif err == "closed" then + return false, self.signal_closed + else + return false, err + end end + -- export module table return luasocket diff --git a/mqtt/luasocket_ssl.lua b/mqtt/luasocket_ssl.lua index 15b31cb..53cb92d 100644 --- a/mqtt/luasocket_ssl.lua +++ b/mqtt/luasocket_ssl.lua @@ -1,55 +1,66 @@ -- DOC: http://w3.impa.br/~diego/software/luasocket/tcp.html -- module table -local luasocket_ssl = {} +local super = require "mqtt.luasocket" +local luasocket_ssl = setmetatable({}, super) +luasocket_ssl.__index = luasocket_ssl +luasocket_ssl.super = super local type = type local assert = assert -local luasocket = require("mqtt.luasocket") + +-- table with error messages that indicate a read timeout +-- luasec has 2 extra timeout messages over luasocket +luasocket_ssl.timeout_errors = { + wantread = true, + wantwrite = true, +} +for k,v in pairs(super.timeout_errors) do luasocket_ssl.timeout_errors[k] = v end -- Open network connection to .host and .port in conn table -- Store opened socket to conn table -- Returns true on success, or false and error text on failure -function luasocket_ssl.connect(conn) - assert(type(conn.secure_params) == "table", "expecting .secure_params to be a table") +function luasocket_ssl:connect() + assert(type(self.secure_params) == "table", "expecting .secure_params to be a table") -- open usual TCP connection - local ok, err = luasocket.connect(conn) + local ok, err = super.connect(self) if not ok then - return false, "luasocket connect failed: "..err + return false, "luasocket connect failed: "..tostring(err) end - local wrapped -- load right ssl module - local ssl = require(conn.ssl_module or "ssl") + local ssl = require(self.ssl_module or "ssl") + + -- Wrap socket in TLS one + do + local wrapped + wrapped, err = ssl.wrap(self.sock, self.secure_params) + if not wrapped then + super.shutdown(self) + return false, "ssl.wrap() failed: "..tostring(err) + end - -- TLS/SSL initialization - wrapped, err = ssl.wrap(conn.sock, conn.secure_params) - if not wrapped then - conn.sock:shutdown() - return false, "ssl.wrap() failed: "..err + -- replace sock in connection table with wrapped secure socket + self.sock = wrapped end - ok = wrapped:dohandshake() + + -- do TLS/SSL initialization/handshake + self.sock:settimeout(self.timeout) + ok, err = self.sock:dohandshake() if not ok then - conn.sock:shutdown() - return false, "ssl dohandshake failed" + self:shutdown() + return false, "ssl dohandshake failed: "..tostring(err) end - -- replace sock in connection table with wrapped secure socket - conn.sock = wrapped return true end -- Shutdown network connection -function luasocket_ssl.shutdown(conn) - conn.sock:close() +function luasocket_ssl:shutdown() + self.sock:close() -- why does ssl use 'close' where luasocket uses 'shutdown'?? end --- Copy original methods from mqtt.luasocket module -luasocket_ssl.send = luasocket.send -luasocket_ssl.receive = luasocket.receive -luasocket_ssl.settimeout = luasocket.settimeout - -- export module table return luasocket_ssl diff --git a/mqtt/ngxsocket.lua b/mqtt/ngxsocket.lua index 349780c..5a16d48 100644 --- a/mqtt/ngxsocket.lua +++ b/mqtt/ngxsocket.lua @@ -1,53 +1,55 @@ -- module table -- thanks to @irimiab: https://github.com/xHasKx/luamqtt/issues/13 -local ngxsocket = {} +local super = require "mqtt.non_buffered_base" +local ngxsocket = setmetatable({}, super) +ngxsocket.__index = ngxsocket +ngxsocket.super = super -- load required stuff -local string_sub = string.sub -local ngx_socket_tcp = ngx.socket.tcp -- luacheck: ignore +local ngx_socket_tcp = ngx.socket.tcp -- Open network connection to .host and .port in conn table -- Store opened socket to conn table -- Returns true on success, or false and error text on failure -function ngxsocket.connect(conn) - local socket = ngx_socket_tcp() - socket:settimeout(0x7FFFFFFF) - local sock, err = socket:connect(conn.host, conn.port) - if not sock then +function ngxsocket:connect() + local sock = ngx_socket_tcp() + sock:settimeout(self.timeout * 1000) -- millisecs + local ok, err = sock:connect(self.host, self.port) + if not ok then return false, "socket:connect failed: "..err end - if conn.secure then - socket:sslhandshake() + if self.secure then + sock:sslhandshake() end - conn.sock = socket + self.sock = sock return true end -- Shutdown network connection -function ngxsocket.shutdown(conn) - conn.sock:close() +function ngxsocket:shutdown() + self.sock:close() end -- Send data to network connection -function ngxsocket.send(conn, data, i, j) - if i then - return conn.sock:send(string_sub(data, i, j)) - else - return conn.sock:send(data) - end +function ngxsocket:send(data) + return self.sock:send(data) end -- Receive given amount of data from network connection -function ngxsocket.receive(conn, size) - return conn.sock:receive(size) -end +function ngxsocket:receive(size) + local sock = self.sock + local data, err = sock:receive(size) + if data then + return data + end --- Set connection's socket to non-blocking mode and set a timeout for it -function ngxsocket.settimeout(conn, timeout) - if not timeout then - conn.sock:settimeout(0x7FFFFFFF) + -- note: signal_idle is not needed here since OpenResty takes care + -- of that. The read is non blocking, so a timeout is a real error and not + -- a signal to retry. + if err == "closed" then + return false, self.signal_closed else - conn.sock:settimeout(timeout * 1000) + return false, err end end diff --git a/mqtt/non_buffered_base.lua b/mqtt/non_buffered_base.lua new file mode 100644 index 0000000..77f1e4b --- /dev/null +++ b/mqtt/non_buffered_base.lua @@ -0,0 +1,62 @@ +-- base connector class for non-buffered reading. +-- +-- Use this base class if the sockets DO yield. +-- So Copas or OpenResty for example, when using LuaSocket +-- use the buffered base class. +-- +-- NOTE: when the send operation can also yield (as is the case with Copas and +-- OpenResty) you should wrap the `send` handler in a lock to prevent a half-send +-- message from being interleaved by another message send from another thread. +-- +-- @class mqtt.non_buffered_base + + +local non_buffered = { + type = "non-buffered, yielding i/o", + timeout = 30 -- default timeout +} +non_buffered.__index = non_buffered + +-- we need to specify signals for these conditions such that the client +-- doesn't have to rely on magic strings like "timeout", "wantread", etc. +-- the connector is responsible for translating those connector specific +-- messages to a generic signal +non_buffered.signal_idle = {} -- read timeout occured, so we're idle need to come back later and try again +non_buffered.signal_closed = {} -- remote closed the connection + +--- Clears consumed bytes. +-- Called by the mqtt client when the consumed bytes from the buffer are handled +-- and can be cleared from the buffer. +-- A no-op for the non-buffered classes, since the sockets yield when incomplete. +function non_buffered.buffer_clear() +end + +--- Retrieves the requested number of bytes from the socket. +-- If the receive errors, because of a closed connection it should return +-- `nil, self.signal_closed` to indicate this. Any other errors can be returned +-- as a regular `nil, err`. +-- @tparam size int number of retrieve to return. +-- @return data, or `false, err`, where `err` can be a signal. +function non_buffered:receive(size) -- luacheck: ignore + error("method 'receive' on non-buffered connector wasn't implemented") +end + +--- Open network connection to `self.host` and `self.port`. +-- @return `true` on success, or `false, err` on failure +function non_buffered:connect() -- luacheck: ignore + error("method 'connect' on connector wasn't implemented") +end + +--- Shutdown the network connection. +function non_buffered:shutdown() -- luacheck: ignore + error("method 'shutdown' on connector wasn't implemented") +end + +--- Shutdown the network connection. +-- @tparam data string data to send +-- @return `true` on success, or `false, err` on failure +function non_buffered:send(data) -- luacheck: ignore + error("method 'send' on connector wasn't implemented") +end + +return non_buffered diff --git a/mqtt/protocol5.lua b/mqtt/protocol5.lua index 2e9088d..17842c6 100644 --- a/mqtt/protocol5.lua +++ b/mqtt/protocol5.lua @@ -189,8 +189,8 @@ local property_pairs = { make = make_uint8_0_or_1, parse = parse_uint8_0_or_1, }, { 0x26, "user_property", -- NOTE: not implemented intentionally - make = function(value_) error("not implemented") end, - parse = function(read_func_) error("not implemented") end, }, + make = function(value_) error("not implemented") end, -- luacheck: ignore + parse = function(read_func_) error("not implemented") end, }, -- luacheck: ignore { 0x27, "maximum_packet_size", make = make_uint32, parse = parse_uint32, }, diff --git a/rockspecs/luamqtt-3.4.3-1.rockspec b/rockspecs/luamqtt-3.4.3-1.rockspec index 45a658f..d06219c 100644 --- a/rockspecs/luamqtt-3.4.3-1.rockspec +++ b/rockspecs/luamqtt-3.4.3-1.rockspec @@ -34,5 +34,8 @@ build = { ["mqtt.protocol4"] = "mqtt/protocol4.lua", ["mqtt.protocol5"] = "mqtt/protocol5.lua", ["mqtt.tools"] = "mqtt/tools.lua", + ["mqtt.log"] = "mqtt/log.lua", + ["mqtt.buffered_base"] = "mqtt/buffered_base.lua", + ["mqtt.non_buffered_base"] = "mqtt/non_buffered_base.lua", }, } diff --git a/tests/spec/module-basics.lua b/tests/spec/01-module-basics_spec.lua similarity index 99% rename from tests/spec/module-basics.lua rename to tests/spec/01-module-basics_spec.lua index c9aa56c..6b4ef25 100644 --- a/tests/spec/module-basics.lua +++ b/tests/spec/01-module-basics_spec.lua @@ -1,4 +1,3 @@ --- busted -e 'package.path="./?/init.lua;./?.lua;"..package.path' tests/spec/module-basics.lua -- DOC v3.1.1: http://docs.oasis-open.org/mqtt/mqtt/v3.1.1/errata01/os/mqtt-v3.1.1-errata01-os-complete.html -- DOC v5.0: http://docs.oasis-open.org/mqtt/mqtt/v5.0/cos01/mqtt-v5.0-cos01.html diff --git a/tests/spec/protocol4-make.lua b/tests/spec/02-protocol4-make_spec.lua similarity index 99% rename from tests/spec/protocol4-make.lua rename to tests/spec/02-protocol4-make_spec.lua index 73b856c..3de5239 100644 --- a/tests/spec/protocol4-make.lua +++ b/tests/spec/02-protocol4-make_spec.lua @@ -1,4 +1,3 @@ --- busted -e 'package.path="./?/init.lua;./?.lua;"..package.path' tests/spec/protocol4-make.lua -- DOC: http://docs.oasis-open.org/mqtt/mqtt/v3.1.1/errata01/os/mqtt-v3.1.1-errata01-os-complete.html describe("MQTT v3.1.1 protocol: making packets", function() diff --git a/tests/spec/protocol4-parse.lua b/tests/spec/03-protocol4-parse_spec.lua similarity index 99% rename from tests/spec/protocol4-parse.lua rename to tests/spec/03-protocol4-parse_spec.lua index d2a417c..46725c5 100644 --- a/tests/spec/protocol4-parse.lua +++ b/tests/spec/03-protocol4-parse_spec.lua @@ -1,4 +1,3 @@ --- busted -e 'package.path="./?/init.lua;./?.lua;"..package.path' tests/spec/protocol4-parse.lua -- DOC: http://docs.oasis-open.org/mqtt/mqtt/v3.1.1/errata01/os/mqtt-v3.1.1-errata01-os-complete.html describe("MQTT v3.1.1 protocol: parsing packets", function() diff --git a/tests/spec/protocol5-make.lua b/tests/spec/04-protocol5-make_spec.lua similarity index 99% rename from tests/spec/protocol5-make.lua rename to tests/spec/04-protocol5-make_spec.lua index 40b1581..bc7e937 100644 --- a/tests/spec/protocol5-make.lua +++ b/tests/spec/04-protocol5-make_spec.lua @@ -1,4 +1,3 @@ --- busted -e 'package.path="./?/init.lua;./?.lua;"..package.path' tests/spec/protocol5-make.lua -- DOC: https://docs.oasis-open.org/mqtt/mqtt/v5.0/cos02/mqtt-v5.0-cos02.html describe("MQTT v5.0 protocol: making packets: CONNECT[1]", function() diff --git a/tests/spec/protocol5-parse.lua b/tests/spec/05-protocol5-parse_spec.lua similarity index 99% rename from tests/spec/protocol5-parse.lua rename to tests/spec/05-protocol5-parse_spec.lua index 99d3a5e..aa15448 100644 --- a/tests/spec/protocol5-parse.lua +++ b/tests/spec/05-protocol5-parse_spec.lua @@ -1,4 +1,3 @@ --- busted -e 'package.path="./?/init.lua;./?.lua;"..package.path' tests/spec/protocol5-parse.lua -- DOC: https://docs.oasis-open.org/mqtt/mqtt/v5.0/cos02/mqtt-v5.0-cos02.html -- returns read_func-compatible function diff --git a/tests/spec/mqtt-client.lua b/tests/spec/06-mqtt-client_spec.lua similarity index 99% rename from tests/spec/mqtt-client.lua rename to tests/spec/06-mqtt-client_spec.lua index 8be205a..f829373 100644 --- a/tests/spec/mqtt-client.lua +++ b/tests/spec/06-mqtt-client_spec.lua @@ -1,4 +1,3 @@ --- busted -e 'package.path="./?/init.lua;"..package.path;' spec/*.lua -- DOC: http://docs.oasis-open.org/mqtt/mqtt/v3.1.1/errata01/os/mqtt-v3.1.1-errata01-os-complete.html -- DOC v5.0: http://docs.oasis-open.org/mqtt/mqtt/v5.0/cos01/mqtt-v5.0-cos01.html @@ -16,7 +15,7 @@ describe("invalid arguments to mqtt.client constructor", function() local mqtt = require("mqtt") it("argument table key is not a string", function() - assert.has_error(function() mqtt.client{1} end, "expecting string key in args, got: number") + assert.has_error(function() mqtt.client{1} end, "expecting string key in opts, got: number") end) it("id is not a string", function() @@ -48,7 +47,7 @@ describe("invalid arguments to mqtt.client constructor", function() end) it("unexpected key", function() - assert.has_error(function() mqtt.client{unexpected=true} end, "unexpected key in client args: unexpected = true") + assert.has_error(function() mqtt.client{unexpected=true} end, "unexpected key in client opts: unexpected = true") end) end) diff --git a/tests/spec/ioloop.lua b/tests/spec/07-ioloop_spec.lua similarity index 82% rename from tests/spec/ioloop.lua rename to tests/spec/07-ioloop_spec.lua index dbba470..9c181ff 100644 --- a/tests/spec/ioloop.lua +++ b/tests/spec/07-ioloop_spec.lua @@ -1,8 +1,14 @@ --- busted -e 'package.path="./?/init.lua;./?.lua;"..package.path' tests/spec/module-basics.lua -- DOC v3.1.1: http://docs.oasis-open.org/mqtt/mqtt/v3.1.1/errata01/os/mqtt-v3.1.1-errata01-os-complete.html -- DOC v5.0: http://docs.oasis-open.org/mqtt/mqtt/v5.0/cos01/mqtt-v5.0-cos01.html describe("ioloop", function() + do -- configure logging + local lualogging = require("logging") + lualogging.defaultLevel(lualogging.DEBUG) + lualogging.defaultLogPattern("%date [%level] %message (%source)\n") + lualogging.defaultTimestampPattern("%y/%m/%d %H:%M:%S") + end + -- load MQTT lua library local mqtt = require("mqtt") @@ -22,9 +28,14 @@ describe("ioloop", function() -- configure MQTT client handlers client:on{ + error = function(err) + print("[error] ", err) + end, connect = function() + --print "connected" -- subscribe, then send signal message assert(client:subscribe{topic=prefix.."/ioloop/signal", callback=function() + --print "subscribed" assert(client:publish{ topic = prefix.."/ioloop/signal", payload = "signal", @@ -44,6 +55,7 @@ describe("ioloop", function() if signal then -- disconnect MQTT client, thus it will be removed from ioloop client:disconnect() + mqtt.get_ioloop():remove(client) -- and remove this function from ioloop to stop it (no more clients left) mqtt.get_ioloop():remove(loop_func) From 136f3cecb03a7917cebe358840a19cb836d8350b Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Mon, 1 Nov 2021 01:21:30 +0100 Subject: [PATCH 09/65] feat(ioloop) add incremental back-off for idle clients --- mqtt/ioloop.lua | 55 ++++++++++++++++------------ mqtt/luasocket.lua | 4 +- tests/spec/01-module-basics_spec.lua | 1 - 3 files changed, 34 insertions(+), 26 deletions(-) diff --git a/mqtt/ioloop.lua b/mqtt/ioloop.lua index 575f799..b6d8bbe 100644 --- a/mqtt/ioloop.lua +++ b/mqtt/ioloop.lua @@ -50,18 +50,21 @@ ioloop_mt.__index = ioloop_mt --- Initialize ioloop instance -- @tparam table opts ioloop creation options table --- @tparam[opt=0.005] number opts.timeout network operations timeout in seconds --- @tparam[opt=0] number opts.sleep sleep interval after each iteration +-- @tparam[opt=0] number opts.sleep_min min sleep interval after each iteration +-- @tparam[opt=0.002] number opts.sleep_step increase in sleep after every idle iteration +-- @tparam[opt=0.030] number opts.sleep_max max sleep interval after each iteration -- @tparam[opt] function opts.sleep_function custom sleep function to call after each iteration -- @treturn ioloop_mt ioloop instance function ioloop_mt:__init(opts) log:debug("initializing ioloop instance '%s'", tostring(self)) opts = opts or {} - opts.timeout = opts.timeout or 0.005 - opts.sleep = opts.sleep or 0 + opts.sleep_min = opts.sleep_min or 0 + opts.sleep_step = opts.sleep_step or 0.002 + opts.sleep_max = opts.sleep_max or 0.030 opts.sleep_function = opts.sleep_function or require("socket").sleep self.opts = opts self.clients = {} + self.timeouts = setmetatable({}, { __mode = "v" }) self.running = false --ioloop running flag, used by MQTT clients which are adding after this ioloop started to run end @@ -81,6 +84,7 @@ function ioloop_mt:add(client) end clients[#clients + 1] = client clients[client] = true + self.timeouts[client] = self.opts.sleep_min if type(client) == "table" then log:info("adding client '%s' to ioloop '%s'", client.opts.id, tostring(self)) @@ -136,36 +140,41 @@ function ioloop_mt:remove(client) end --- Perform one ioloop iteration. --- TODO: make this smarter do not wake-up functions or clients returned a longer --- sleep delay. Currently it's pretty much a busy loop. +-- TODO: make this smarter do not wake-up functions or clients returning a longer +-- sleep delay. Currently they will be tried earlier if another returns a smaller delay. function ioloop_mt:iteration() local opts = self.opts - local sleep = opts.sleep + local sleep = opts.sleep_max for _, client in ipairs(self.clients) do local t, err -- read data and handle events if type(client) ~= "function" then t, err = client:step() - if t == -1 then - -- no data read, client is idle - t = nil - elseif not t then - if not client.opts.reconnect then - -- error and not reconnecting, remove the client - log:error("client '%s' failed with '%s', will not re-connect", client.opts.id, err) - self:remove(client) - t = nil - else - -- error, but will reconnect - log:error("client '%s' failed with '%s', will try re-connecting", client.opts.id, err) - t = 0 -- try immediately - end + else + t = client() or opts.sleep_max + end + if t == -1 then + -- no data read, client is idle, step up timeout + t = math_min(self.timeouts[client] + opts.sleep_step, opts.sleep_max) + self.timeouts[client] = t + elseif not t then + -- an error from a client was returned + if not client.opts.reconnect then + -- error and not reconnecting, remove the client + log:fatal("client '%s' failed with '%s', will not re-connect", client.opts.id, err) + self:remove(client) + t = opts.sleep_max + else + -- error, but will reconnect + log:error("client '%s' failed with '%s', will try re-connecting", client.opts.id, err) + t = opts.sleep_min -- try asap end else - t = client() + -- a number of seconds was returned + t = math_min(t, opts.sleep_max) + self.timeouts[client] = opts.sleep_min end - t = t or opts.sleep sleep = math_min(sleep, t) end -- sleep a bit diff --git a/mqtt/luasocket.lua b/mqtt/luasocket.lua index 7a36b99..4cd24f6 100644 --- a/mqtt/luasocket.lua +++ b/mqtt/luasocket.lua @@ -57,9 +57,9 @@ end function luasocket:plain_receive(size) local sock = self.sock - sock:settimeout(0.010) -- TODO: setting to 0 fails??? it shouldn't - local data, err = sock:receive(size) + sock:settimeout(0) + local data, err = sock:receive(size) if data then return data end diff --git a/tests/spec/01-module-basics_spec.lua b/tests/spec/01-module-basics_spec.lua index 6b4ef25..3fa3e2e 100644 --- a/tests/spec/01-module-basics_spec.lua +++ b/tests/spec/01-module-basics_spec.lua @@ -289,7 +289,6 @@ describe("MQTT lua library component test:", function() assert.are.equal(2097152, protocol.parse_var_length_nonzero(make_read_func("80808001"))) assert.are.equal(268435455, protocol.parse_var_length_nonzero(make_read_func("FFFFFF7F"))) assert.is_false(protocol.parse_var_length_nonzero(make_read_func("FFFFFFFF"))) - end) it("protocol.next_packet_id", function() From 40bcd43d3fa871f84a81f508822aa87a20b97a75 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Thu, 4 Nov 2021 14:48:30 +0100 Subject: [PATCH 10/65] fix(mqtt5) pass errors on as is (like mqtt4) since errors can be 'signals' they cannot be concatenated --- .busted | 4 ---- mqtt/protocol.lua | 8 ++++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/.busted b/.busted index 600ba91..0d0a67d 100644 --- a/.busted +++ b/.busted @@ -1,10 +1,6 @@ return { default = { ROOT = { "tests/spec" }, -<<<<<<< HEAD - pattern = "%.lua", -======= ->>>>>>> 0a6e421 (feat(*) rewrite loop logic) lpath = "./?.lua;./?/?.lua;./?/init.lua", verbose = true, coverage = false, diff --git a/mqtt/protocol.lua b/mqtt/protocol.lua index 3e6233c..ae34c43 100644 --- a/mqtt/protocol.lua +++ b/mqtt/protocol.lua @@ -556,14 +556,14 @@ function protocol.start_parse_packet(read_func) -- DOC[v5.0]: https://docs.oasis-open.org/mqtt/mqtt/v5.0/os/mqtt-v5.0-os.html#_Toc3901020 byte1, err = read_func(1) if not byte1 then - return false, "failed to read first byte: "..err + return false, err end byte1 = str_byte(byte1, 1, 1) local ptype = rshift(byte1, 4) local flags = band(byte1, 0xF) len, err = parse_var_length(read_func) if not len then - return false, "failed to parse remaining length: "..err + return false, err end -- create packet parser instance (aka input) @@ -574,14 +574,14 @@ function protocol.start_parse_packet(read_func) data = "" end if not data then - return false, "failed to read packet data: "..err + return false, err end input.available = data:len() -- read data function for the input instance input.read_func = function(size) if size > input.available then - return false, "not enough data to read size: "..size + return false, size end local off = input[1] local res = str_sub(data, off, off + size - 1) From 8f8773a19c9e47a9ae9c8b5aa416a2bd7188fc3b Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Wed, 3 Nov 2021 14:56:59 +0100 Subject: [PATCH 11/65] wip --- .busted | 1 + mqtt/client.lua | 34 +++++++++++++++++++++++++++--- mqtt/init.lua | 2 +- mqtt/ioloop.lua | 2 +- tests/configure-log.lua | 17 +++++++++++++++ tests/run-for-all-lua-versions.sh | 4 ++-- tests/spec/06-mqtt-client_spec.lua | 30 +++++++++++++++++++++++++- tests/spec/07-ioloop_spec.lua | 6 ------ 8 files changed, 82 insertions(+), 14 deletions(-) create mode 100644 tests/configure-log.lua diff --git a/.busted b/.busted index 0d0a67d..b8c1e0b 100644 --- a/.busted +++ b/.busted @@ -2,6 +2,7 @@ return { default = { ROOT = { "tests/spec" }, lpath = "./?.lua;./?/?.lua;./?/init.lua", + helper = "./tests/configure-log.lua", verbose = true, coverage = false, output = "gtest", diff --git a/mqtt/client.lua b/mqtt/client.lua index 8f66e18..06e2af2 100644 --- a/mqtt/client.lua +++ b/mqtt/client.lua @@ -3,7 +3,35 @@ -- @alias client local client = {} --- TODO: list event names +-- event names: + +-- "error": function(errmsg, client_object, [packet]) +-- on errors +-- optional packet: only if received CONNACK.rc ~= 0 when connecting + +-- "close": function(connection_object, client_object) +-- upon closing the connection +-- connection object will have .close_reason (string) + +-- "connect": function(packet, client_object) +-- upon a succesful connect, after receiving the CONNACK packet from the broker +-- ???? => on a refused connect; if received CONNACK.rc ~= 0 when connecting + +-- "subscribe": function(packet, client_object) +-- upon a succesful subscription, after receiving the SUBACK packet from the broker + +-- "unsubscribe": function(packet, client_object) +-- upon a succesful unsubscription, after receiving the UNSUBACK packet from the broker + +-- "message": function(packet, client_object) +-- upon receiving a PUBLISH packet from the broker + +-- "acknowledge": function(packet, client_object) +-- upon receiving PUBACK +-- upon receiving PUBREC (event fires after sending PUBREL) + +-- "auth": function(packet, client_object) +-- upon receiving an AUTH packet ------- @@ -282,7 +310,7 @@ end -- @tparam boolean opts.retain_handling for MQTT v5.0 only: retain_handling flag for subscription -- @tparam[opt] table opts.properties for MQTT v5.0 only: properties for subscribe operation -- @tparam[opt] table opts.user_properties for MQTT v5.0 only: user properties for subscribe operation --- @tparam[opt] function opts.callback callback function to be called when subscription will be created +-- @tparam[opt] function opts.callback callback function to be called when subscription is acknowledged by broker -- @return packet id on success or false and error message on failure function client_mt:subscribe(opts) -- fetch and validate opts @@ -417,7 +445,7 @@ end -- @tparam[opt=false] boolean opts.dup dup message publication flag -- @tparam[opt] table opts.properties properties for publishing message -- @tparam[opt] table opts.user_properties user properties for publishing message --- @tparam[opt] function opts.callback callback to call when publihsed message will be acknowledged +-- @tparam[opt] function opts.callback callback to call when publihsed message has been acknowledged by the broker -- @return true or packet id on success or false and error message on failure function client_mt:publish(opts) -- fetch and validate opts diff --git a/mqtt/init.lua b/mqtt/init.lua index 67d97a3..f47f462 100644 --- a/mqtt/init.lua +++ b/mqtt/init.lua @@ -50,7 +50,7 @@ end -- @function mqtt.get_ioloop mqtt.get_ioloop = ioloop_get ---- Run default ioloop for given MQTT clients or functions +--- Run default ioloop for given MQTT clients or functions. -- @param ... MQTT clients or loop functions to add to ioloop -- @see mqtt.ioloop.get -- @see mqtt.ioloop.run_until_clients diff --git a/mqtt/ioloop.lua b/mqtt/ioloop.lua index b6d8bbe..8a328af 100644 --- a/mqtt/ioloop.lua +++ b/mqtt/ioloop.lua @@ -53,7 +53,7 @@ ioloop_mt.__index = ioloop_mt -- @tparam[opt=0] number opts.sleep_min min sleep interval after each iteration -- @tparam[opt=0.002] number opts.sleep_step increase in sleep after every idle iteration -- @tparam[opt=0.030] number opts.sleep_max max sleep interval after each iteration --- @tparam[opt] function opts.sleep_function custom sleep function to call after each iteration +-- @tparam[opt=luasocket.sleep] function opts.sleep_function custom sleep function to call after each iteration -- @treturn ioloop_mt ioloop instance function ioloop_mt:__init(opts) log:debug("initializing ioloop instance '%s'", tostring(self)) diff --git a/tests/configure-log.lua b/tests/configure-log.lua new file mode 100644 index 0000000..acfd980 --- /dev/null +++ b/tests/configure-log.lua @@ -0,0 +1,17 @@ +local ansicolors = require("ansicolors") -- https://github.com/kikito/ansicolors.lua +local ll = require("logging") +require "logging.console" + +-- configure the default logger used when testing +ll.defaultLogger(ll.console { + logLevel = ll.DEBUG, + destination = "stderr", + timestampPattern = "%y-%m-%d %H:%M:%S", + logPatterns = { + [ll.DEBUG] = ansicolors("%date%{cyan} %level %message %{reset}(%source)\n"), + [ll.INFO] = ansicolors("%date %level %message\n"), + [ll.WARN] = ansicolors("%date%{yellow} %level %message\n"), + [ll.ERROR] = ansicolors("%date%{red bright} %level %message %{reset}(%source)\n"), + [ll.FATAL] = ansicolors("%date%{magenta bright} %level %message %{reset}(%source)\n"), + } +}) diff --git a/tests/run-for-all-lua-versions.sh b/tests/run-for-all-lua-versions.sh index 7bcf489..66bb405 100755 --- a/tests/run-for-all-lua-versions.sh +++ b/tests/run-for-all-lua-versions.sh @@ -56,10 +56,10 @@ for ver in -l5.1 -l5.2 -l5.3 -l5.4 -j2.0 -j2.1; do echo "installing coveralls lib for $ver" luarocks install luacov-coveralls echo "running tests and collecting coverage for $ver" - busted -e 'package.path="./?/init.lua;./?.lua;"..package.path;require("luacov.runner")(".luacov")' $BFLAGS tests/spec/*.lua + busted -o utfTerminal --coverage 2> /dev/null else echo "running tests for $ver" - busted -e 'package.path="./?/init.lua;./?.lua;"..package.path' $BFLAGS tests/spec/*.lua + busted -o utfTerminal 2> /dev/null fi done diff --git a/tests/spec/06-mqtt-client_spec.lua b/tests/spec/06-mqtt-client_spec.lua index f829373..84508b2 100644 --- a/tests/spec/06-mqtt-client_spec.lua +++ b/tests/spec/06-mqtt-client_spec.lua @@ -1,6 +1,8 @@ -- DOC: http://docs.oasis-open.org/mqtt/mqtt/v3.1.1/errata01/os/mqtt-v3.1.1-errata01-os-complete.html -- DOC v5.0: http://docs.oasis-open.org/mqtt/mqtt/v5.0/cos01/mqtt-v5.0-cos01.html +local log = require("logging").defaultLogger() + describe("MQTT lua library", function() -- load MQTT lua library local mqtt = require("mqtt") @@ -233,9 +235,14 @@ describe("MQTT client", function() local prefix = "luamqtt/"..tostring(math.floor(math.random()*1e13)) -- set on-connect handler - client:on("connect", function() + client:on("connect", function(packet) + log:warn("connect: %d %s", packet.rc, packet:reason_string()) + assert(packet.rc == 0, packet:reason_string()) assert(client:send_pingreq()) -- NOTE: not required, it's here only to improve code coverage + log:warn("subscribing to '.../0/test'") assert(client:subscribe{topic=prefix.."/0/test", callback=function() + log:warn("subscription to '.../0/test' confirmed") + log:warn("now publishing 'initial' to '.../0/test'") assert(client:publish{ topic = prefix.."/0/test", payload = "initial", @@ -249,8 +256,14 @@ describe("MQTT client", function() if msg.topic == prefix.."/0/test" then -- re-subscribe test + log:warn("received message on '.../0/test', payload: %s", msg.payload) + log:warn("unsubscribing from '.../0/test'") assert(client:unsubscribe{topic=prefix.."/0/test", callback=function() + log:warn("unsubscribe from '.../0/test' confirmed") + log:warn("subscribing to '.../#'") assert(client:subscribe{topic=prefix.."/#", qos=2, callback=function() + log:warn("subscription to '.../#' confirmed") + log:warn("now publishing 'testing QoS 1' to '.../1/test'") assert(client:publish{ topic = prefix.."/1/test", payload = "testing QoS 1", @@ -258,32 +271,45 @@ describe("MQTT client", function() properties = properties, user_properties = user_properties, callback = function() + log:warn("publishing 'testing QoS 1' to '.../1/test' confirmed") acknowledge = true if acknowledge and test_msg_2 then -- done + log:warn("both `acknowledge` (by me) and `test_msg_2` are set, disconnecting now") assert(client:disconnect()) + else + log:warn("only `acknowledge` (by me) is set, not `test_msg_2`. So not disconnecting yet") end end, }) end}) end}) elseif msg.topic == prefix.."/1/test" then + log:warn("received message on '.../1/test', payload: %s", msg.payload) if case.args.version == mqtt.v50 then assert(type(msg.properties) == "table") assert.are.same(properties.message_expiry_interval, msg.properties.message_expiry_interval) assert(type(msg.user_properties) == "table") assert.are.same(user_properties.hello, msg.user_properties.hello) end + log:warn("now publishing 'testing QoS 2' to '.../2/test'") assert(client:publish{ topic = prefix.."/2/test", payload = "testing QoS 2", qos = 2, + callback = function() + log:warn("publishing 'testing QoS 2' to '.../2/test' confirmed") + end, }) elseif msg.topic == prefix.."/2/test" then + log:warn("received message on '.../2/test', payload: %s", msg.payload) test_msg_2 = true if acknowledge and test_msg_2 then -- done + log:warn("both `test_msg_2` (by me) and `acknowledge` (by me) are set, disconnecting now") assert(client:disconnect()) + else + log:warn("only `test_msg_2` (by me) is set, not `acknowledge`. So not disconnecting yet") end end end, @@ -294,6 +320,8 @@ describe("MQTT client", function() close = function(conn) close_reason = conn.close_reason + -- remove our client from the loop to make it exit. + require("mqtt.ioloop").get():remove(client) end, } diff --git a/tests/spec/07-ioloop_spec.lua b/tests/spec/07-ioloop_spec.lua index 9c181ff..a2ae799 100644 --- a/tests/spec/07-ioloop_spec.lua +++ b/tests/spec/07-ioloop_spec.lua @@ -2,12 +2,6 @@ -- DOC v5.0: http://docs.oasis-open.org/mqtt/mqtt/v5.0/cos01/mqtt-v5.0-cos01.html describe("ioloop", function() - do -- configure logging - local lualogging = require("logging") - lualogging.defaultLevel(lualogging.DEBUG) - lualogging.defaultLogPattern("%date [%level] %message (%source)\n") - lualogging.defaultTimestampPattern("%y/%m/%d %H:%M:%S") - end -- load MQTT lua library local mqtt = require("mqtt") From eda35db187119ec614be5fce5807ab12493e310c Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Thu, 4 Nov 2021 16:03:31 +0100 Subject: [PATCH 12/65] fix(client) first_connect was not properly updated --- mqtt/client.lua | 8 +++++++- mqtt/ioloop.lua | 2 +- tests/spec/06-mqtt-client_spec.lua | 10 ++++++++++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/mqtt/client.lua b/mqtt/client.lua index 06e2af2..e4a3099 100644 --- a/mqtt/client.lua +++ b/mqtt/client.lua @@ -230,8 +230,9 @@ function client_mt:__init(opts) self._to_remove_handlers = {} -- state - self.first_connect = true -- contains true to perform one network connection attempt after client creation self.send_time = 0 -- time of the last network send from client side + self.first_connect = true -- contains true to perform one network connection attempt after client creation + -- Note: remains true, during the connect process. False after succes or failure. -- packet creation/parse functions according version if not a.version then @@ -697,12 +698,14 @@ function client_mt:start_connecting() -- open network connection local ok, err = self:open_connection() if not ok then + self.first_connect = false return false, err end -- send CONNECT packet ok, err = self:send_connect() if not ok then + self.first_connect = false return false, err end @@ -972,6 +975,7 @@ function client_mt:handle_received_packet(packet) log:error("client '%s' %s", self.opts.id, err) self:handle("error", err, self) self:close_connection("error") + self.first_connect = false return false, err end @@ -985,6 +989,7 @@ function client_mt:handle_received_packet(packet) self:handle("error", err, self, packet) self:handle("connect", packet, self) self:close_connection("connection failed") + self.first_connect = false return false, err end @@ -992,6 +997,7 @@ function client_mt:handle_received_packet(packet) -- fire connect event self:handle("connect", packet, self) + self.first_connect = false else -- connection authorized, so process usual packets diff --git a/mqtt/ioloop.lua b/mqtt/ioloop.lua index 8a328af..5a11148 100644 --- a/mqtt/ioloop.lua +++ b/mqtt/ioloop.lua @@ -162,7 +162,7 @@ function ioloop_mt:iteration() -- an error from a client was returned if not client.opts.reconnect then -- error and not reconnecting, remove the client - log:fatal("client '%s' failed with '%s', will not re-connect", client.opts.id, err) + log:info("client '%s' returned '%s', no re-connect set, removing client", client.opts.id, err) self:remove(client) t = opts.sleep_max else diff --git a/tests/spec/06-mqtt-client_spec.lua b/tests/spec/06-mqtt-client_spec.lua index 84508b2..a4a3638 100644 --- a/tests/spec/06-mqtt-client_spec.lua +++ b/tests/spec/06-mqtt-client_spec.lua @@ -367,8 +367,10 @@ describe("last will message", function() local function send_self_destroy() if not client1_ready or not client2_ready then + log:warn("not self destroying, clients not ready") return end + log:warn("client1 publishing 'self-destructing-message' to '.../stop' topic") assert(client1:publish{ topic = prefix.."/stop", payload = "self-destructing-message", @@ -378,13 +380,17 @@ describe("last will message", function() client1:on{ connect = function() -- subscribe, then send self-destructing message + log:warn("client1 is now connected") + log:warn("client1 subscribing to '.../stop' topic") assert(client1:subscribe{topic=prefix.."/stop", callback=function() client1_ready = true + log:warn("client1 subscription to '.../stop' topic confirmed, client 1 is ready for self destruction") send_self_destroy() end}) end, message = function() -- break connection with broker on any message + log:warn("client1 received a message and is now closing its connection") client1:close_connection("self-destructed") end, } @@ -394,13 +400,17 @@ describe("last will message", function() client2:on{ connect = function() -- subscribe to will-message topic + log:warn("client2 is now connected") + log:warn("client2 subscribing to will-topic: '.../willtest' topic") assert(client2:subscribe{topic=will_topic, callback=function() client2_ready = true + log:warn("client2 subscription to will-topic '.../willtest' confirmed, client 2 is ready for self destruction") send_self_destroy() end}) end, message = function(msg) will_received = msg.topic == will_topic + log:warn("client2 received a message, topic is: '%s', client 2 is now closing its connection",tostring(msg.topic)) client2:disconnect() end, } From 567ee31c02aab280d17dee2247f5b6e43a96a26a Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Thu, 4 Nov 2021 19:27:39 +0100 Subject: [PATCH 13/65] fix(tests) fix more tests, ioloop related --- mqtt/client.lua | 17 ++++---- tests/spec/06-mqtt-client_spec.lua | 63 ++++++++++++++++++++++++------ 2 files changed, 60 insertions(+), 20 deletions(-) diff --git a/mqtt/client.lua b/mqtt/client.lua index e4a3099..a2041cb 100644 --- a/mqtt/client.lua +++ b/mqtt/client.lua @@ -349,7 +349,7 @@ function client_mt:subscribe(opts) local packet_id = pargs.packet_id local subscribe = self._make_packet(pargs) - log:info("subscribing client '%s' to topic '%s' (packet: %d)", self.opts.id, opts.topic, packet_id or -1) + log:info("subscribing client '%s' to topic '%s' (packet: %s)", self.opts.id, opts.topic, packet_id or "n.a.") -- send SUBSCRIBE packet local ok, err = self:_send_packet(subscribe) @@ -409,7 +409,7 @@ function client_mt:unsubscribe(opts) local packet_id = pargs.packet_id local unsubscribe = self._make_packet(pargs) - log:info("unsubscribing client '%s' from topic '%s' (packet: %d)", self.opts.id, opts.topic, packet_id or -1) + log:info("unsubscribing client '%s' from topic '%s' (packet: %s)", self.opts.id, opts.topic, packet_id or "n.a.") -- send UNSUBSCRIBE packet local ok, err = self:_send_packet(unsubscribe) @@ -475,8 +475,7 @@ function client_mt:publish(opts) local packet_id = opts.packet_id local publish = self._make_packet(opts) - log:debug("client '%s' publishing to topic '%s' (packet: %d)", self.opts.id, opts.topic, packet_id or -1) - --log:debug("client '%s' publishing to topic '%s' (value '%d')", self.opts.id, opts.topic, opts.payload) + log:debug("client '%s' publishing to topic '%s' (packet: %s)", self.opts.id, opts.topic, packet_id or "n.a.") -- send PUBLISH packet local ok, err = self:_send_packet(publish) @@ -537,7 +536,7 @@ function client_mt:acknowledge(msg, rc, properties, user_properties) return true end - log:debug("client '%s' acknowledging packet %d", self.opts.id, packet_id or -1) + log:debug("client '%s' acknowledging packet %s", self.opts.id, packet_id or "n.a.") if msg.qos == 1 then -- PUBACK should be sent @@ -886,7 +885,7 @@ function client_mt:acknowledge_pubrel(packet_id) -- create PUBREL packet local pubrel = self._make_packet{type=packet_type.PUBREL, packet_id=packet_id, rc=0} - log:debug("client '%s' sending PUBREL (packet: %d)", self.opts.id, packet_id or -1) + log:debug("client '%s' sending PUBREL (packet: %s)", self.opts.id, packet_id or "n.a.") -- send PUBREL packet local ok, err = self:_send_packet(pubrel) @@ -912,7 +911,7 @@ function client_mt:acknowledge_pubcomp(packet_id) -- create PUBCOMP packet local pubcomp = self._make_packet{type=packet_type.PUBCOMP, packet_id=packet_id, rc=0} - log:debug("client '%s' sending PUBCOMP (packet: %d)", self.opts.id, packet_id or -1) + log:debug("client '%s' sending PUBCOMP (packet: %s)", self.opts.id, packet_id or "n.a.") -- send PUBCOMP packet local ok, err = self:_send_packet(pubcomp) @@ -966,7 +965,7 @@ function client_mt:handle_received_packet(packet) local conn = self.connection local err - log:debug("client '%s' received '%s' (packet: %s)", self.opts.id, packet.type, tostring(packet.packet_id or "n.a.")) + log:debug("client '%s' received '%s' (packet: %s)", self.opts.id, packet_type[packet.type], packet.packet_id or "n.a.") if not conn.connack then -- expecting only CONNACK packet here @@ -993,7 +992,7 @@ function client_mt:handle_received_packet(packet) return false, err end - log:info("client '%s' connected successfully to '%s'", self.opts.id, conn.uri) + log:info("client '%s' connected successfully to '%s:%s'", self.opts.id, conn.host, conn.port) -- fire connect event self:handle("connect", packet, self) diff --git a/tests/spec/06-mqtt-client_spec.lua b/tests/spec/06-mqtt-client_spec.lua index a4a3638..6f36968 100644 --- a/tests/spec/06-mqtt-client_spec.lua +++ b/tests/spec/06-mqtt-client_spec.lua @@ -2,6 +2,7 @@ -- DOC v5.0: http://docs.oasis-open.org/mqtt/mqtt/v5.0/cos01/mqtt-v5.0-cos01.html local log = require("logging").defaultLogger() +local socket = require("socket") describe("MQTT lua library", function() -- load MQTT lua library @@ -325,12 +326,8 @@ describe("MQTT client", function() end, } - -- and wait for connection to broker is closed - if case.sync then - mqtt.run_sync(client) - else - mqtt.run_ioloop(client) - end + -- and wait for connection to broker to be closed + mqtt.run_ioloop(client) assert.are.same({}, errors) assert.is_true(acknowledge) @@ -363,7 +360,7 @@ describe("last will message", function() username = "stPwSVV73Eqw5LSv0iMXbc4EguS7JyuZR9lxU5uLxI5tiNM8ToTVqNpu85pFtJv9", } - local client1_ready, client2_ready + local client1_ready, client2_ready, clients_done local function send_self_destroy() if not client1_ready or not client2_ready then @@ -392,6 +389,7 @@ describe("last will message", function() -- break connection with broker on any message log:warn("client1 received a message and is now closing its connection") client1:close_connection("self-destructed") + clients_done = (clients_done or 0)+1 end, } @@ -412,17 +410,28 @@ describe("last will message", function() will_received = msg.topic == will_topic log:warn("client2 received a message, topic is: '%s', client 2 is now closing its connection",tostring(msg.topic)) client2:disconnect() + clients_done = (clients_done or 0)+1 end, } - mqtt.run_ioloop(client1, client2) + local timer do + local timeout = socket.gettime() + 30 + function timer() + if clients_done == 2 then + require("mqtt.ioloop").get():remove(timer) + end + assert(socket.gettime() < timeout, "test failed due to timeout") + end + end + + mqtt.run_ioloop(client1, client2, timer) assert.is_true(will_received) end) end) -describe("no_local flag for subscription: ", function() +describe("no_local flag for subscription:", function() local mqtt = require("mqtt") local prefix = "luamqtt/" .. tostring(math.floor(math.random()*1e13)) local no_local_topic = prefix .. "/no_local_test" @@ -437,7 +446,7 @@ describe("no_local flag for subscription: ", function() version = mqtt.v50 } - it("msg should not be received", function() + it("msg should not be received #only", function() local c1 = mqtt.client(conn_args) local c2 = mqtt.client(conn_args) @@ -460,26 +469,34 @@ describe("no_local flag for subscription: ", function() local function send() if not s1.subscribed or not s2.subscribed then + log:warn("not sending because clients are not both subscribed") return end + log:warn("both clients are subscribed, now sending...") + socket.sleep(0.2) -- shouldn't be necessary, but test is flaky otherwise + log:warn("client1: publishing 'message' to topic '.../no_local_test'") assert(c1:publish{ topic = no_local_topic, payload = "message", callback = function() s1.published = s1.published + 1 + log:warn("client1: publishing to topic '.../no_local_test' confirmed, count: %d", s1.published) end }) end c1:on{ connect = function() + log:warn("client1: is now connected") s1.connected = true send() + log:warn("client1: subscribing to topic '.../#', with 'no_local'") assert(c1:subscribe{ topic = prefix .. "/#", no_local = true, callback = function() s1.subscribed = true + log:warn("client1: subscription to topic '.../#' with 'no_local' confirmed") send() end }) @@ -487,13 +504,18 @@ describe("no_local flag for subscription: ", function() message = function(msg) s1.messages[#s1.messages + 1] = msg.payload if msg.payload == "stop" then + log:warn("client1: received message, with payload 'stop'. Will now disconnect.") assert(c1:disconnect()) + else + log:warn("client1: received message, with payload '%s' (but waiting for 'stop')", msg.payload) end end, error = function(err) s1.errors[#s1.errors + 1] = err + log:warn("client1: received error: '%s'", tostring(err)) end, close = function(conn) + log:warn("client1: closed, reason: %s", conn.close_reason) s1.close_reason = conn.close_reason end } @@ -501,11 +523,14 @@ describe("no_local flag for subscription: ", function() c2:on{ connect = function() s2.connected = true + log:warn("client2: is now connected") + log:warn("client2: subscribing to topic '.../#', without 'no_local'") assert(c2:subscribe{ topic = no_local_topic, no_local = false, callback = function() s2.subscribed = true + log:warn("client2: subscription to topic '.../#' without 'no_local' confirmed") send() end }) @@ -513,15 +538,21 @@ describe("no_local flag for subscription: ", function() message = function(msg) s2.messages[#s2.messages + 1] = msg.payload if msg.payload == "message" then + log:warn("client2: received message, with payload 'message'") + log:warn("client2: publishing to topic '.../no_local_test'', with payload 'stop'") assert(c2:publish{ topic = no_local_topic, payload = "stop", callback = function() s2.published = s2.published + 1 + log:warn("client2: publishing to topic '.../no_local_test' confirmed, count: %d", s2.published) end }) elseif msg.payload == "stop" then + log:warn("client2: received message, with payload 'stop'. Will now disconnect.") assert(c2:disconnect()) + else + log:warn("client2: received message, with payload '%s' (but waiting for 'stop' or 'message')", msg.payload) end end, error = function(err) @@ -532,7 +563,17 @@ describe("no_local flag for subscription: ", function() end } - mqtt.run_ioloop(c1, c2) + local timer do + local timeout = socket.gettime() + 30 + function timer() + if s1.close_reason and s2.close_reason then + require("mqtt.ioloop").get():remove(timer) + end + assert(socket.gettime() < timeout, "test failed due to timeout") + end + end + + mqtt.run_ioloop(c1, c2, timer) assert.is_true(s1.connected, "client 1 is not connected") assert.is_true(s2.connected, "client 2 is not connected") From 18cc55b22263cc217f9936cf4164ae87f0a416bc Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Thu, 4 Nov 2021 20:06:43 +0100 Subject: [PATCH 14/65] fix(copas) fix copas connector and test --- mqtt/luasocket-copas.lua | 2 +- tests/spec/06-mqtt-client_spec.lua | 41 +++++++++++++++--------------- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/mqtt/luasocket-copas.lua b/mqtt/luasocket-copas.lua index 7c604d1..57632dd 100644 --- a/mqtt/luasocket-copas.lua +++ b/mqtt/luasocket-copas.lua @@ -17,7 +17,7 @@ function connector:connect() local sock = copas.wrap(socket.tcp(), self.secure_params) sock:settimeout(self.timeout) - local ok, err = socket:connect(self.host, self.port) + local ok, err = sock:connect(self.host, self.port) if not ok then return false, "copas.connect failed: "..err end diff --git a/tests/spec/06-mqtt-client_spec.lua b/tests/spec/06-mqtt-client_spec.lua index 6f36968..8e0bd98 100644 --- a/tests/spec/06-mqtt-client_spec.lua +++ b/tests/spec/06-mqtt-client_spec.lua @@ -446,7 +446,7 @@ describe("no_local flag for subscription:", function() version = mqtt.v50 } - it("msg should not be received #only", function() + it("msg should not be received", function() local c1 = mqtt.client(conn_args) local c2 = mqtt.client(conn_args) @@ -607,12 +607,16 @@ describe("#copas connector", function() connector = require("mqtt.luasocket-copas"), } + local test_finished = false + client:on{ connect = function() - -- print("client connected") + log:warn("client is now connected") + log:warn("client subscribing to topic '.../copas'") assert(client:subscribe{topic=prefix.."/copas", qos=1, callback=function() - -- print("subscribed") - copas.sleep(2) + log:warn("client subscription to topic '.../copas' confirmed") + --copas.sleep(2) -- TODO: remove??? + log:warn("client publishing 'copas test' to topic '.../copas' confirmed") assert(client:publish{ topic = prefix.."/copas", payload = "copas test", @@ -623,34 +627,31 @@ describe("#copas connector", function() message = function(msg) assert(client:acknowledge(msg)) - -- print("received", msg) if msg.topic == prefix.."/copas" and msg.payload == "copas test" then - -- print("disconnect") + log:warn("client received '%s' to topic '.../copas' confirmed", msg.payload) assert(client:disconnect()) + log:warn("disconnected now") + test_finished = true end end } - local ticks = 0 - local mqtt_finished = false - copas.addthread(function() - mqtt.run_sync(client) - mqtt_finished = true - assert.is_true(ticks > 1, "expecting mqtt client run takes at least one tick") - end) - - copas.addthread(function() - for _ = 1, 3 do - copas.sleep(1) - ticks = ticks + 1 + while true do + local timeout = client:step() + if not timeout then + -- exited + return + end + if timeout > 0 then + copas.sleep(timeout) + end end end) copas.loop() - assert.is_true(mqtt_finished, "expecting mqtt client to finish its work") - assert.is_true(ticks == 3, "expecting 3 ticks") + assert.is_true(test_finished, "expecting mqtt client to finish its work") end) end) From 13b43d473a881dbfa8f22e10b2d06b481de8b105 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Thu, 4 Nov 2021 20:51:45 +0100 Subject: [PATCH 15/65] chore(ci) add ansicolors+lualogging for testing --- tests/run-for-all-lua-versions.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/run-for-all-lua-versions.sh b/tests/run-for-all-lua-versions.sh index 66bb405..948c9c1 100755 --- a/tests/run-for-all-lua-versions.sh +++ b/tests/run-for-all-lua-versions.sh @@ -37,6 +37,9 @@ for ver in -l5.1 -l5.2 -l5.3 -l5.4 -j2.0 -j2.1; do fi luarocks install busted > /dev/null luarocks install copas > /dev/null + luarocks install ansicolors > /dev/null + # using --dev since the 'defaulLogger' feature hasn't been released yet + luarocks install lualogging > /dev/null if [ -d /usr/lib/x86_64-linux-gnu ]; then # debian-based OS [ -f /etc/lsb-release ] && . /etc/lsb-release From 57ff13468fd18905b4a7671b1d383730fd66f926 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Fri, 5 Nov 2021 07:17:33 +0100 Subject: [PATCH 16/65] fix(keep-alive) should always return an interval as first result --- mqtt/client.lua | 2 +- tests/spec/06-mqtt-client_spec.lua | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/mqtt/client.lua b/mqtt/client.lua index a2041cb..4980c22 100644 --- a/mqtt/client.lua +++ b/mqtt/client.lua @@ -855,7 +855,7 @@ function client_mt:check_keep_alive() log:error("client '%s' %s", self.opts.id, err) self:handle("error", err, self) self:close_connection("error") - return false, err + return interval, err end -- send PINGREQ if keep_alive interval is reached diff --git a/tests/spec/06-mqtt-client_spec.lua b/tests/spec/06-mqtt-client_spec.lua index 8e0bd98..88a8d95 100644 --- a/tests/spec/06-mqtt-client_spec.lua +++ b/tests/spec/06-mqtt-client_spec.lua @@ -615,7 +615,6 @@ describe("#copas connector", function() log:warn("client subscribing to topic '.../copas'") assert(client:subscribe{topic=prefix.."/copas", qos=1, callback=function() log:warn("client subscription to topic '.../copas' confirmed") - --copas.sleep(2) -- TODO: remove??? log:warn("client publishing 'copas test' to topic '.../copas' confirmed") assert(client:publish{ topic = prefix.."/copas", From f6f6131fca44162f48d1584e59c4e7e9eb741fd9 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Fri, 5 Nov 2021 09:05:33 +0100 Subject: [PATCH 17/65] chore(examples) fix copas examples, remove sync example --- examples/copas-example.lua | 69 +++++++++++++++++++++++++------------- examples/copas.lua | 37 ++++++++++++-------- examples/sync.lua | 59 -------------------------------- 3 files changed, 68 insertions(+), 97 deletions(-) delete mode 100644 examples/sync.lua diff --git a/examples/copas-example.lua b/examples/copas-example.lua index c22e2af..0c85dd9 100644 --- a/examples/copas-example.lua +++ b/examples/copas-example.lua @@ -2,10 +2,9 @@ local mqtt = require("mqtt") local copas = require("copas") -local mqtt_ioloop = require("mqtt.ioloop") local num_pings = 10 -- total number of ping-pongs -local timeout = 1 -- timeout between ping-pongs +local delay = 1 -- delay between ping-pongs local suffix = tostring(math.random(1000000)) -- mqtt topic suffix to distinct simultaneous running of this script -- NOTE: more about flespi tokens: https://flespi.com/kb/tokens-access-keys-to-flespi-platform @@ -16,6 +15,8 @@ local ping = mqtt.client{ username = token, clean = true, version = mqtt.v50, + -- NOTE: copas connector + connector = require("mqtt.luasocket-copas"), } local pong = mqtt.client{ @@ -23,6 +24,8 @@ local pong = mqtt.client{ username = token, clean = true, version = mqtt.v50, + -- NOTE: copas connector + connector = require("mqtt.luasocket-copas"), } ping:on{ @@ -30,17 +33,21 @@ ping:on{ assert(connack.rc == 0) print("ping connected") - for i = 1, num_pings do - copas.sleep(timeout) - print("ping", i) - assert(ping:publish{ topic = "luamqtt/copas-ping/"..suffix, payload = "ping"..i, qos = 1 }) - end - - copas.sleep(timeout) - - print("ping done") - assert(ping:publish{ topic = "luamqtt/copas-ping/"..suffix, payload = "done", qos = 1 }) - ping:disconnect() + -- adding another thread; copas handlers should return quickly, anything + -- that can wait should be off-loaded from the handler to a thread. + -- Especially anything that yields; socket reads/writes and sleeps, and the + -- code below does both, sleeping, and writing (implicit in 'publish') + copas.addthread(function() + for i = 1, num_pings do + copas.sleep(delay) + print("ping", i) + assert(ping:publish{ topic = "luamqtt/copas-ping/"..suffix, payload = "ping"..i, qos = 1 }) + end + + print("ping done") + assert(ping:publish{ topic = "luamqtt/copas-ping/"..suffix, payload = "done", qos = 1 }) + ping:disconnect() + end) end, error = function(err) print("ping MQTT client error:", err) @@ -72,19 +79,33 @@ pong:on{ end, } +local function add_client(cl) + -- add keep-alive timer + local timer = copas.addthread(function() + while cl do + copas.sleep(cl:check_keep_alive()) + end + end) + -- add client to connect and listen + copas.addthread(function() + while cl do + local timeout = cl:step() + if not timeout then + cl = nil -- exiting, inform keep-alive timer + copas.wakeup(timer) + else + if timeout > 0 then + copas.sleep(timeout) + end + end + end + end) +end + print("running copas loop...") -copas.addthread(function() - local ioloop = mqtt_ioloop.create{ sleep = 0.01, sleep_function = copas.sleep } - ioloop:add(ping) - ioloop:run_until_clients() -end) - -copas.addthread(function() - local ioloop = mqtt_ioloop.create{ sleep = 0.01, sleep_function = copas.sleep } - ioloop:add(pong) - ioloop:run_until_clients() -end) +add_client(ping) +add_client(pong) copas.loop() diff --git a/examples/copas.lua b/examples/copas.lua index 199e49a..6eb6c5a 100644 --- a/examples/copas.lua +++ b/examples/copas.lua @@ -55,22 +55,31 @@ client:on{ end } --- run io loop for client until connection close -copas.addthread(function() - print("running client in separated copas thread #1...") - mqtt.run_sync(client) - -- NOTE: in sync mode no automatic reconnect is working, but you may just wrap "mqtt.run_sync(client)" call in a loop like this: - -- while true do - -- mqtt.run_sync(client) - -- end -end) +local function add_client(cl) + -- add keep-alive timer + local timer = copas.addthread(function() + while cl do + copas.sleep(cl:check_keep_alive()) + end + end) + -- add client to connect and listen + copas.addthread(function() + while cl do + local timeout = cl:step() + if not timeout then + cl = nil -- exiting + copas.wakeup(timer) + else + if timeout > 0 then + copas.sleep(timeout) + end + end + end + end) +end -copas.addthread(function() - print("execution of separated copas thread #2...") - copas.sleep(0.1) - print("thread #2 stopped") -end) +add_client(client) copas.loop() print("done, copas loop is stopped") diff --git a/examples/sync.lua b/examples/sync.lua deleted file mode 100644 index b415b07..0000000 --- a/examples/sync.lua +++ /dev/null @@ -1,59 +0,0 @@ --- load mqtt module -local mqtt = require("mqtt") - --- create mqtt client -local client = mqtt.client{ - -- NOTE: this broker is not working sometimes; comment username = "..." below if you still want to use it - -- uri = "test.mosquitto.org", - uri = "mqtt.flespi.io", - -- NOTE: more about flespi tokens: https://flespi.com/kb/tokens-access-keys-to-flespi-platform - username = "stPwSVV73Eqw5LSv0iMXbc4EguS7JyuZR9lxU5uLxI5tiNM8ToTVqNpu85pFtJv9", - clean = true, -} -print("created MQTT client", client) - -client:on{ - connect = function(connack) - if connack.rc ~= 0 then - print("connection to broker failed:", connack:reason_string(), connack) - return - end - print("connected:", connack) -- successful connection - - -- subscribe to test topic and publish message after it - assert(client:subscribe{ topic="luamqtt/#", qos=1, callback=function(suback) - print("subscribed:", suback) - - -- publish test message - print('publishing test message "hello" to "luamqtt/simpletest" topic...') - assert(client:publish{ - topic = "luamqtt/simpletest", - payload = "hello", - qos = 1 - }) - end}) - end, - - message = function(msg) - assert(client:acknowledge(msg)) - - print("received:", msg) - print("disconnecting...") - assert(client:disconnect()) - end, - - error = function(err) - print("MQTT client error:", err) - end, - - close = function() - print("MQTT conn closed") - end -} - --- run io loop for client until connection close --- please note that in sync mode background PINGREQ's are not available, and automatic reconnects too -print("running client in synchronous input/output loop") -mqtt.run_sync(client) - -print("done, synchronous input/output loop is stopped") \ No newline at end of file From 6115148eafef3c4e1318fab16eee21e3865d5890 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Fri, 5 Nov 2021 15:25:55 +0100 Subject: [PATCH 18/65] fix(docs) update ssl defaults to match code --- mqtt/client.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mqtt/client.lua b/mqtt/client.lua index 4980c22..3caac72 100644 --- a/mqtt/client.lua +++ b/mqtt/client.lua @@ -93,7 +93,7 @@ client_mt.__index = client_mt -- @tparam[opt] string opts.username username for authorization on MQTT broker -- @tparam[opt] string opts.password password for authorization on MQTT broker; not acceptable in absence of username -- @tparam[opt=false] boolean,table opts.secure use secure network connection, provided by luasec lua module; --- set to true to select default params: { mode="client", protocol="tlsv1_2", verify="none", options="all" } +-- set to true to select default params: { mode="client", protocol="any", verify="none", options={ "all","no_sslv2","no_sslv3","no_tlsv1" } -- or set to luasec-compatible table, for example with cafile="...", certificate="...", key="..." -- @tparam[opt] table opts.will will message table with required fields { topic="...", payload="..." } -- and optional fields { qos=1...3, retain=true/false } From 7ff684a0dba7fe918269c45a4bf2e474fcdb1e66 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Sat, 6 Nov 2021 13:52:06 +0100 Subject: [PATCH 19/65] feat(options) allow full uri to be specified --- mqtt/client.lua | 127 +++++++++++++++++++++----- tests/spec/06-mqtt-client_spec.lua | 141 +++++++++++++++++++++++++++++ 2 files changed, 246 insertions(+), 22 deletions(-) diff --git a/mqtt/client.lua b/mqtt/client.lua index 3caac72..c9e9237 100644 --- a/mqtt/client.lua +++ b/mqtt/client.lua @@ -48,7 +48,6 @@ local os_time = os.time local string = require("string") local str_format = string.format local str_gsub = string.gsub -local str_match = string.match local table = require("table") local table_remove = table.remove @@ -84,14 +83,17 @@ client_mt.__index = client_mt --- Create and initialize MQTT client instance -- @tparam table opts MQTT client creation options table -- @tparam string opts.uri MQTT broker uri to connect. --- Expecting "host:port" or "host" format, in second case the port will be selected automatically: --- 1883 port for plain or 8883 for secure network connections +-- Expecting '[mqtt[s]://][username[:password]@]hostname[:port]' format. Any option specifically added to the options +-- table will take precedence over the option specified in this uri. +-- @tparam[opt] string opts.protocol either 'mqtt' or 'mqtts' +-- @tparam[opt] string opts.username username for authorization on MQTT broker +-- @tparam[opt] string opts.password password for authorization on MQTT broker; not acceptable in absence of username +-- @tparam[opt] string opts.hostnamne hostname of the MQTT broker to connect to +-- @tparam[opt] int opts.port port number to connect to on the MQTT broker, defaults to 1883 port for plain or 8883 for secure network connections -- @tparam string opts.clean clean session start flag -- @tparam[opt=4] number opts.version MQTT protocol version to use, either 4 (for MQTT v3.1.1) or 5 (for MQTT v5.0). -- Also you may use special values mqtt.v311 or mqtt.v50 for this field. -- @tparam[opt] string opts.id MQTT client ID, will be generated by luamqtt library if absent --- @tparam[opt] string opts.username username for authorization on MQTT broker --- @tparam[opt] string opts.password password for authorization on MQTT broker; not acceptable in absence of username -- @tparam[opt=false] boolean,table opts.secure use secure network connection, provided by luasec lua module; -- set to true to select default params: { mode="client", protocol="any", verify="none", options={ "all","no_sslv2","no_sslv3","no_tlsv1" } -- or set to luasec-compatible table, for example with cafile="...", certificate="...", key="..." @@ -170,7 +172,6 @@ function client_mt:__init(opts) -- check required arguments assert(a.uri, 'expecting uri="..." to create MQTT client') assert(a.clean ~= nil, "expecting clean=true or clean=false to create MQTT client") - assert(not a.password or a.username, "password is not accepted in absence of username") if not a.id then -- generate random client id @@ -194,6 +195,9 @@ function client_mt:__init(opts) assert(a.connector.signal_closed, "missing connector.signal_closed signal value") assert(a.connector.signal_idle, "missing connector.signal_idle signal value") + -- validate connection properties + client_mt._parse_connection_opts(a, { uri = opts.uri }) + -- will table content check if a.will then assert(type(a.will.topic) == "string", "expecting will.topic to be a string") @@ -761,8 +765,7 @@ function client_mt:open_connection() wait_for_pubrec = {}, -- a table with packet_id of partially acknowledged sent packets in QoS 2 exchange process wait_for_pubrel = {}, -- a table with packet_id of partially acknowledged received packets in QoS 2 exchange process }, connector) - client_mt._parse_uri(opts, conn) - client_mt._apply_secure(opts, conn) + client_mt._parse_connection_opts(opts, conn) log:info("client '%s' connecting to broker '%s' (using: %s)", self.opts.id, opts.uri, conn.type or "unknown") @@ -1145,32 +1148,101 @@ do end -- Fill given connection table with host and port according given opts -function client_mt._parse_uri(opts, conn) - local host, port = str_match(opts.uri, "^([^%s]+):(%d+)$") - if not host then - -- trying pattern without port - host = assert(str_match(conn.uri, "^([^%s]+)$"), "invalid uri format: expecting at least host/ip in .uri") - end - if not port then +-- uri: mqtt[s]://[username][:password]@host.domain[:port] +function client_mt._parse_connection_opts(opts, conn) + local uri = assert(conn.uri) + + -- protocol + local uriprotocol = uri:match("^([%a%d%-]-)://") + if uriprotocol then + uriprotocol = uriprotocol:lower() + uri = uri:gsub("^[%a%d%-]-://","") + + if uriprotocol == "mqtts" and opts.secure == nil then + opts.secure = true -- NOTE: goes into client 'opts' table, not in 'conn' + + elseif uriprotocol == "mqtt" and opts.secure == nil then + opts.secure = false -- NOTE: goes into client 'opts' table, not in 'conn' + + elseif uriprotocol == "mqtts" and opts.secure == false then + error("cannot use protocol 'mqtts' with 'secure=false'") + + elseif uriprotocol == "mqtt" and opts.secure then + error("cannot use protocol 'mqtt' with 'secure=true|table'") + end + else + -- no protocol info found in uri if opts.secure then - port = 8883 -- default MQTT secure connection port + uriprotocol = "mqtts" else - port = 1883 -- default MQTT connection port + uriprotocol = "mqtt" end + end + + conn.protocol = opts.protocol or uriprotocol + assert(type(conn.protocol) == "string", "expected protocol to be a string") + assert(conn.protocol:match("mqtts?"), "only 'mqtt(s)' protocol is supported in the uri, got '"..tostring(conn.protocol).."'") + -- print("protocol: ", uriprotocol) + + -- creds, host/port + local creds, host_port + if uri:find("@") then + host_port = uri:match("@(.-)$"):lower() + creds = uri:gsub("@.-$", "") else - port = tonumber(port) + host_port = uri end - conn.host, conn.port = host, port -end + -- print("creds: ", creds) + -- print("host_port:", host_port) + + -- host-port + local host, port = host_port:match("^([^:]+):?([^:]*)$") + if port and #port > 0 then + port = assert(tonumber(port), "port in uri must be numeric, got: '"..port.."'") + else + port = nil + end + -- print("port: ", port) + -- print("host: ", host) + conn.host = opts.host or host + assert(type(conn.host) == "string", "expected host to be a string") + -- default port + conn.port = opts.port or port + if not conn.port then + if opts.secure then + conn.port = 8883 -- default MQTT secure connection port + else + conn.port = 1883 -- default MQTT connection port + end + end + assert(type(conn.port) == "number", "expected port to be a number") + + + -- username-password + local username, password + if creds then + username, password = creds:match("^([^:]+):?([^:]*)$") + if password and #password == 0 then + password = nil + end + end + -- NOTE: these go into client 'opts' table, not in 'conn' + opts.username = opts.username or username + assert(opts.username == nil or type(opts.username) == "string", "expected username to be a string") + opts.password = opts.password or password + assert(opts.password == nil or type(opts.password) == "string", "expected password to be a string") + assert(not conn.password or conn.username, "password is not accepted in absence of username") + -- print("username: ", username) + -- print("password: ", password) + --- Creates the conn.secure_params table and its content according client creation opts -function client_mt._apply_secure(opts, conn) local secure = opts.secure if secure then conn.secure = true if type(secure) == "table" then conn.secure_params = secure else + -- TODO: these defaults should go in the connectors, as they can differ per environment (eg. Nginx) conn.secure_params = { mode = "client", protocol = "any", @@ -1179,6 +1251,12 @@ function client_mt._apply_secure(opts, conn) } end conn.ssl_module = opts.ssl_module or "ssl" + assert(conn.ssl_module == nil or type(conn.ssl_module) == "string", "expected ssl_module to be a string") + else + -- sanity + conn.secure = false + conn.secure_params = nil + conn.ssl_module = nil end end @@ -1260,6 +1338,11 @@ end ------- +if _G._TEST then + -- export functions for test purposes (different name!) + client.__parse_connection_opts = client_mt._parse_connection_opts +end + -- export module table return client diff --git a/tests/spec/06-mqtt-client_spec.lua b/tests/spec/06-mqtt-client_spec.lua index 88a8d95..060685c 100644 --- a/tests/spec/06-mqtt-client_spec.lua +++ b/tests/spec/06-mqtt-client_spec.lua @@ -13,6 +13,147 @@ describe("MQTT lua library", function() end) end) + + +describe("uri parsing:", function() + + -- @param opts uri string, or options table + -- @param expected_conn expected connection table after parsing + -- @param expected_opts (optional) if given expected options table after parsing + local function try(opts, expected_conn, expected_opts) + -- reload client in test mode + _G._TEST = true + package.loaded["mqtt.client"] = nil + local client = require("mqtt.client") + + if type(opts) == "string" then + opts = { uri = opts } + end + local conn = { + uri = opts.uri + } + + client.__parse_connection_opts(opts, conn) + + expected_conn.uri = opts.uri -- must remain the same anyway, so add here + conn.secure_params = nil -- not validating those + assert.same(expected_conn, conn) + + if expected_opts then + expected_opts.uri = opts.uri -- must remain the same anyway, so add here + assert.same(expected_opts, opts) + end + return conn, opts + end + + + describe("valid uri strings", function() + + it("protocol+user+password+host+port", function() + try("mqtts://usr:pwd@host.com:123", { + -- expected conn + host = "host.com", + port = 123, + protocol = "mqtts", + secure = true, + ssl_module = "ssl", + }, { + -- expected opts + password = "pwd", + secure = true, -- was set because of protocol + username = "usr", + }) + end) + + it("user+password+host+port", function() + try("usr:pwd@host.com:123", { + -- expected conn + host = "host.com", + port = 123, + protocol = "mqtt", + secure = false, + ssl_module = nil, + }, { + -- expected opts + password = "pwd", + secure = nil, + username = "usr", + }) + end) + + it("protocol+host+port", function() + try("mqtts://host.com:123", { + -- expected conn + host = "host.com", + port = 123, + protocol = "mqtts", + secure = true, + ssl_module = "ssl", + }, { + -- expected opts + secure = true, -- was set because of protocol + }) + end) + + it("host+port", function() + try("host.com:123", { + -- expected conn + host = "host.com", + port = 123, + protocol = "mqtt", + secure = false, + ssl_module = nil, + }, { + -- expected opts + }) + end) + + it("host only", function() + try("host.com", { + -- expected conn + host = "host.com", + port = 1883, -- default port + protocol = "mqtt", + secure = false, + ssl_module = nil, + }, { + -- expected opts + }) + end) + + end) + + + it("uri properties are overridden by specific properties", function() + try({ + uri = "mqtt://usr:pwd@host.com:123", + host = "another.com", + port = 456, + protocol = "mqtt", + password = "king", + username = "arthur", + }, { + -- expected conn + host = "another.com", + port = 456, + protocol = "mqtt", + secure = false, + ssl_module = nil, + }, { + -- expected opts + host = "another.com", + port = 456, + protocol = "mqtt", + password = "king", + secure = false, + username = "arthur", + }) + end) + +end) + + + describe("invalid arguments to mqtt.client constructor", function() -- load MQTT lua library local mqtt = require("mqtt") From 30c3efcda382d04a92c5722422f306e0d7124802 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Fri, 5 Nov 2021 12:43:22 +0100 Subject: [PATCH 20/65] chore(connectors) move files in subdir --- mqtt/client.lua | 6 +++--- mqtt/{ => connector/base}/buffered_base.lua | 4 ++-- mqtt/{ => connector/base}/non_buffered_base.lua | 2 +- mqtt/{luasocket-copas.lua => connector/copas.lua} | 2 +- mqtt/{ => connector}/luasocket.lua | 2 +- mqtt/{ => connector}/luasocket_ssl.lua | 2 +- mqtt/{ngxsocket.lua => connector/nginx.lua} | 2 +- rockspecs/luamqtt-3.4.3-1.rockspec | 12 ++++++------ tests/spec/01-module-basics_spec.lua | 6 ++++-- tests/spec/06-mqtt-client_spec.lua | 4 ++-- 10 files changed, 22 insertions(+), 20 deletions(-) rename mqtt/{ => connector/base}/buffered_base.lua (96%) rename mqtt/{ => connector/base}/non_buffered_base.lua (98%) rename mqtt/{luasocket-copas.lua => connector/copas.lua} (96%) rename mqtt/{ => connector}/luasocket.lua (96%) rename mqtt/{ => connector}/luasocket_ssl.lua (97%) rename mqtt/{ngxsocket.lua => connector/nginx.lua} (95%) diff --git a/mqtt/client.lua b/mqtt/client.lua index c9e9237..fb679ec 100644 --- a/mqtt/client.lua +++ b/mqtt/client.lua @@ -104,7 +104,7 @@ client_mt.__index = client_mt -- Set to number value to provide reconnect timeout in seconds -- It's not recommended to use values < 3 -- @tparam[opt] table opts.connector connector table to open and send/receive packets over network connection. --- default is require("mqtt.luasocket"), or require("mqtt.luasocket_ssl") if secure argument is set +-- default is require("mqtt.connector") which tries to auto-detect. -- @tparam[opt="ssl"] string opts.ssl_module module name for the luasec-compatible ssl module, default is "ssl" -- may be used in some non-standard lua environments with own luasec-compatible ssl module -- @treturn client_mt MQTT client instance table @@ -181,9 +181,9 @@ function client_mt:__init(opts) -- default connector if a.connector == nil then if a.secure then - a.connector = require("mqtt.luasocket_ssl") + a.connector = require("mqtt.connector.luasocket_ssl") else - a.connector = require("mqtt.luasocket") + a.connector = require("mqtt.connector.luasocket") end end -- validate connector content diff --git a/mqtt/buffered_base.lua b/mqtt/connector/base/buffered_base.lua similarity index 96% rename from mqtt/buffered_base.lua rename to mqtt/connector/base/buffered_base.lua index a73c68d..034dc6f 100644 --- a/mqtt/buffered_base.lua +++ b/mqtt/connector/base/buffered_base.lua @@ -14,10 +14,10 @@ -- starting the send/receive. So for example for LuaSocket call `settimeout(0)` -- before receiving, and `settimeout(30)` before sending. -- --- @class mqtt.buffered_base +-- @class mqtt.connector.base.buffered_base -local super = require "mqtt.non_buffered_base" +local super = require "mqtt.connector.base.non_buffered_base" local buffered = setmetatable({}, super) buffered.__index = buffered buffered.super = super diff --git a/mqtt/non_buffered_base.lua b/mqtt/connector/base/non_buffered_base.lua similarity index 98% rename from mqtt/non_buffered_base.lua rename to mqtt/connector/base/non_buffered_base.lua index 77f1e4b..5c1996f 100644 --- a/mqtt/non_buffered_base.lua +++ b/mqtt/connector/base/non_buffered_base.lua @@ -8,7 +8,7 @@ -- OpenResty) you should wrap the `send` handler in a lock to prevent a half-send -- message from being interleaved by another message send from another thread. -- --- @class mqtt.non_buffered_base +-- @class mqtt.connector.base.non_buffered_base local non_buffered = { diff --git a/mqtt/luasocket-copas.lua b/mqtt/connector/copas.lua similarity index 96% rename from mqtt/luasocket-copas.lua rename to mqtt/connector/copas.lua index 57632dd..ab07dc4 100644 --- a/mqtt/luasocket-copas.lua +++ b/mqtt/connector/copas.lua @@ -2,7 +2,7 @@ -- NOTE: you will need to install copas like this: luarocks install copas -- module table -local super = require "mqtt.non_buffered_base" +local super = require "mqtt.connector.base.non_buffered_base" local connector = setmetatable({}, super) connector.__index = connector connector.super = super diff --git a/mqtt/luasocket.lua b/mqtt/connector/luasocket.lua similarity index 96% rename from mqtt/luasocket.lua rename to mqtt/connector/luasocket.lua index 4cd24f6..4b90774 100644 --- a/mqtt/luasocket.lua +++ b/mqtt/connector/luasocket.lua @@ -1,7 +1,7 @@ -- DOC: http://w3.impa.br/~diego/software/luasocket/tcp.html -- module table -local super = require "mqtt.buffered_base" +local super = require "mqtt.connector.base.buffered_base" local luasocket = setmetatable({}, super) luasocket.__index = luasocket luasocket.super = super diff --git a/mqtt/luasocket_ssl.lua b/mqtt/connector/luasocket_ssl.lua similarity index 97% rename from mqtt/luasocket_ssl.lua rename to mqtt/connector/luasocket_ssl.lua index 53cb92d..4dcf349 100644 --- a/mqtt/luasocket_ssl.lua +++ b/mqtt/connector/luasocket_ssl.lua @@ -1,7 +1,7 @@ -- DOC: http://w3.impa.br/~diego/software/luasocket/tcp.html -- module table -local super = require "mqtt.luasocket" +local super = require "mqtt.connector.luasocket" local luasocket_ssl = setmetatable({}, super) luasocket_ssl.__index = luasocket_ssl luasocket_ssl.super = super diff --git a/mqtt/ngxsocket.lua b/mqtt/connector/nginx.lua similarity index 95% rename from mqtt/ngxsocket.lua rename to mqtt/connector/nginx.lua index 5a16d48..70ae5c0 100644 --- a/mqtt/ngxsocket.lua +++ b/mqtt/connector/nginx.lua @@ -1,6 +1,6 @@ -- module table -- thanks to @irimiab: https://github.com/xHasKx/luamqtt/issues/13 -local super = require "mqtt.non_buffered_base" +local super = require "mqtt.connector.base.non_buffered_base" local ngxsocket = setmetatable({}, super) ngxsocket.__index = ngxsocket ngxsocket.super = super diff --git a/rockspecs/luamqtt-3.4.3-1.rockspec b/rockspecs/luamqtt-3.4.3-1.rockspec index d06219c..df86c0e 100644 --- a/rockspecs/luamqtt-3.4.3-1.rockspec +++ b/rockspecs/luamqtt-3.4.3-1.rockspec @@ -26,16 +26,16 @@ build = { ["mqtt.ioloop"] = "mqtt/ioloop.lua", ["mqtt.bit53"] = "mqtt/bit53.lua", ["mqtt.bitwrap"] = "mqtt/bitwrap.lua", - ["mqtt.luasocket"] = "mqtt/luasocket.lua", - ["mqtt.luasocket_ssl"] = "mqtt/luasocket_ssl.lua", - ["mqtt.luasocket-copas"] = "mqtt/luasocket-copas.lua", - ["mqtt.ngxsocket"] = "mqtt/ngxsocket.lua", ["mqtt.protocol"] = "mqtt/protocol.lua", ["mqtt.protocol4"] = "mqtt/protocol4.lua", ["mqtt.protocol5"] = "mqtt/protocol5.lua", ["mqtt.tools"] = "mqtt/tools.lua", ["mqtt.log"] = "mqtt/log.lua", - ["mqtt.buffered_base"] = "mqtt/buffered_base.lua", - ["mqtt.non_buffered_base"] = "mqtt/non_buffered_base.lua", + ["mqtt.connector.base.buffered_base"] = "mqtt/connector/base/buffered_base.lua", + ["mqtt.connector.base.non_buffered_base"] = "mqtt/connector/base/non_buffered_base.lua", + ["mqtt.connector.luasocket"] = "mqtt/connector/luasocket.lua", + ["mqtt.connector.luasocket_ssl"] = "mqtt/connector/luasocket_ssl.lua", + ["mqtt.connector.copas"] = "mqtt/connector/copas.lua", + ["mqtt.connector.nginx"] = "mqtt/connector/nginx.lua", }, } diff --git a/tests/spec/01-module-basics_spec.lua b/tests/spec/01-module-basics_spec.lua index 3fa3e2e..ea866b1 100644 --- a/tests/spec/01-module-basics_spec.lua +++ b/tests/spec/01-module-basics_spec.lua @@ -17,8 +17,10 @@ describe("MQTT lua library component test:", function() require("mqtt.const") require("mqtt.client") require("mqtt.ioloop") - require("mqtt.luasocket") - require("mqtt.luasocket_ssl") + require("mqtt.connector.luasocket") + require("mqtt.connector.luasocket_ssl") + require("mqtt.connector.copas") + -- require("mqtt.connector.nginx") -- cannot load this one without nginx require("mqtt.protocol4") require("mqtt.protocol5") end) diff --git a/tests/spec/06-mqtt-client_spec.lua b/tests/spec/06-mqtt-client_spec.lua index 060685c..b6cb47e 100644 --- a/tests/spec/06-mqtt-client_spec.lua +++ b/tests/spec/06-mqtt-client_spec.lua @@ -218,7 +218,7 @@ describe("correct arguments to mqtt.client constructor", function() }, user_properties = {a="b", c="d"}, reconnect = 5, - connector = require("mqtt.luasocket"), + connector = require("mqtt.connector.luasocket"), ssl_module = "ssl", } end) @@ -745,7 +745,7 @@ describe("#copas connector", function() username = flespi_token, version = mqtt.v50, - connector = require("mqtt.luasocket-copas"), + connector = require("mqtt.connector.copas"), } local test_finished = false From ab736fedae087112c67d83fab8c88e654a78b4df Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Fri, 5 Nov 2021 13:01:40 +0100 Subject: [PATCH 21/65] chore(luasocket) integrate luasocket and luasocket-ssl --- mqtt/client.lua | 9 +--- mqtt/connector/luasocket.lua | 34 +++++++++++++- mqtt/connector/luasocket_ssl.lua | 67 ---------------------------- rockspecs/luamqtt-3.4.3-1.rockspec | 1 - tests/spec/01-module-basics_spec.lua | 1 - 5 files changed, 34 insertions(+), 78 deletions(-) delete mode 100644 mqtt/connector/luasocket_ssl.lua diff --git a/mqtt/client.lua b/mqtt/client.lua index fb679ec..50f2d4f 100644 --- a/mqtt/client.lua +++ b/mqtt/client.lua @@ -179,13 +179,8 @@ function client_mt:__init(opts) end -- default connector - if a.connector == nil then - if a.secure then - a.connector = require("mqtt.connector.luasocket_ssl") - else - a.connector = require("mqtt.connector.luasocket") - end - end + a.connector = a.connector or require("mqtt.connector.luasocket") + -- validate connector content assert(type(a.connector) == "table", "expecting connector to be a table") assert(type(a.connector.connect) == "function", "expecting connector.connect to be a function") diff --git a/mqtt/connector/luasocket.lua b/mqtt/connector/luasocket.lua index 4b90774..fcc7b7f 100644 --- a/mqtt/connector/luasocket.lua +++ b/mqtt/connector/luasocket.lua @@ -7,16 +7,24 @@ luasocket.__index = luasocket luasocket.super = super local socket = require("socket") +local _, ssl = pcall(require, "ssl") -- table with error messages that indicate a read timeout luasocket.timeout_errors = { - timeout = true, + timeout = true, -- luasocket + wantread = true, -- luasec + wantwrite = true, -- luasec } -- Open network connection to .host and .port in conn table -- Store opened socket to conn table -- Returns true on success, or false and error text on failure function luasocket:connect() + if self.secure_params then + assert(ssl, "LuaSec ssl module not found, secure connections unavailable") + assert(type(self.secure_params) == "table", "expecting .secure_params to be a table if given") + end + self:buffer_clear() -- sanity local sock = socket.tcp() sock:settimeout(self.timeout) @@ -26,13 +34,35 @@ function luasocket:connect() return false, "socket.connect failed to connect to '"..tostring(self.host)..":"..tostring(self.port).."': "..err end + if self.secure_params then + -- Wrap socket in TLS one + do + local wrapped + wrapped, err = ssl.wrap(sock, self.secure_params) + if not wrapped then + sock:close(self) -- close TCP level + return false, "ssl.wrap() failed: "..tostring(err) + end + -- replace sock with wrapped secure socket + sock = wrapped + end + + -- do TLS/SSL initialization/handshake + sock:settimeout(self.timeout) -- sanity; again since its now a luasec socket + ok, err = sock:dohandshake() + if not ok then + sock:close() + return false, "ssl dohandshake failed: "..tostring(err) + end + end + self.sock = sock return true end -- Shutdown network connection function luasocket:shutdown() - self.sock:shutdown() + self.sock:close() end -- Send data to network connection diff --git a/mqtt/connector/luasocket_ssl.lua b/mqtt/connector/luasocket_ssl.lua deleted file mode 100644 index 4dcf349..0000000 --- a/mqtt/connector/luasocket_ssl.lua +++ /dev/null @@ -1,67 +0,0 @@ --- DOC: http://w3.impa.br/~diego/software/luasocket/tcp.html - --- module table -local super = require "mqtt.connector.luasocket" -local luasocket_ssl = setmetatable({}, super) -luasocket_ssl.__index = luasocket_ssl -luasocket_ssl.super = super - -local type = type -local assert = assert - --- table with error messages that indicate a read timeout --- luasec has 2 extra timeout messages over luasocket -luasocket_ssl.timeout_errors = { - wantread = true, - wantwrite = true, -} -for k,v in pairs(super.timeout_errors) do luasocket_ssl.timeout_errors[k] = v end - --- Open network connection to .host and .port in conn table --- Store opened socket to conn table --- Returns true on success, or false and error text on failure -function luasocket_ssl:connect() - assert(type(self.secure_params) == "table", "expecting .secure_params to be a table") - - -- open usual TCP connection - local ok, err = super.connect(self) - if not ok then - return false, "luasocket connect failed: "..tostring(err) - end - - -- load right ssl module - local ssl = require(self.ssl_module or "ssl") - - -- Wrap socket in TLS one - do - local wrapped - wrapped, err = ssl.wrap(self.sock, self.secure_params) - if not wrapped then - super.shutdown(self) - return false, "ssl.wrap() failed: "..tostring(err) - end - - -- replace sock in connection table with wrapped secure socket - self.sock = wrapped - end - - -- do TLS/SSL initialization/handshake - self.sock:settimeout(self.timeout) - ok, err = self.sock:dohandshake() - if not ok then - self:shutdown() - return false, "ssl dohandshake failed: "..tostring(err) - end - - return true -end - --- Shutdown network connection -function luasocket_ssl:shutdown() - self.sock:close() -- why does ssl use 'close' where luasocket uses 'shutdown'?? -end - --- export module table -return luasocket_ssl - --- vim: ts=4 sts=4 sw=4 noet ft=lua diff --git a/rockspecs/luamqtt-3.4.3-1.rockspec b/rockspecs/luamqtt-3.4.3-1.rockspec index df86c0e..e81c593 100644 --- a/rockspecs/luamqtt-3.4.3-1.rockspec +++ b/rockspecs/luamqtt-3.4.3-1.rockspec @@ -34,7 +34,6 @@ build = { ["mqtt.connector.base.buffered_base"] = "mqtt/connector/base/buffered_base.lua", ["mqtt.connector.base.non_buffered_base"] = "mqtt/connector/base/non_buffered_base.lua", ["mqtt.connector.luasocket"] = "mqtt/connector/luasocket.lua", - ["mqtt.connector.luasocket_ssl"] = "mqtt/connector/luasocket_ssl.lua", ["mqtt.connector.copas"] = "mqtt/connector/copas.lua", ["mqtt.connector.nginx"] = "mqtt/connector/nginx.lua", }, diff --git a/tests/spec/01-module-basics_spec.lua b/tests/spec/01-module-basics_spec.lua index ea866b1..9138eb0 100644 --- a/tests/spec/01-module-basics_spec.lua +++ b/tests/spec/01-module-basics_spec.lua @@ -18,7 +18,6 @@ describe("MQTT lua library component test:", function() require("mqtt.client") require("mqtt.ioloop") require("mqtt.connector.luasocket") - require("mqtt.connector.luasocket_ssl") require("mqtt.connector.copas") -- require("mqtt.connector.nginx") -- cannot load this one without nginx require("mqtt.protocol4") From e34562fa85292f5c7fe5a0502ba2ed2dfae71379 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Fri, 5 Nov 2021 13:38:56 +0100 Subject: [PATCH 22/65] feat(connector) add auto-detection for connectors --- mqtt/client.lua | 2 +- mqtt/connector/init.lua | 22 ++++++++++ tests/spec/06-mqtt-client_spec.lua | 66 ---------------------------- tests/spec/08-copas_spec.lua | 69 ++++++++++++++++++++++++++++++ 4 files changed, 92 insertions(+), 67 deletions(-) create mode 100644 mqtt/connector/init.lua create mode 100644 tests/spec/08-copas_spec.lua diff --git a/mqtt/client.lua b/mqtt/client.lua index 50f2d4f..84be8b7 100644 --- a/mqtt/client.lua +++ b/mqtt/client.lua @@ -179,7 +179,7 @@ function client_mt:__init(opts) end -- default connector - a.connector = a.connector or require("mqtt.connector.luasocket") + a.connector = a.connector or require("mqtt.connector") -- validate connector content assert(type(a.connector) == "table", "expecting connector to be a table") diff --git a/mqtt/connector/init.lua b/mqtt/connector/init.lua new file mode 100644 index 0000000..0e0199e --- /dev/null +++ b/mqtt/connector/init.lua @@ -0,0 +1,22 @@ +--- auto detect the connector to use. +-- This is based on a.o. libraries already loaded, so 'require' this +-- module as late as possible (after the other modules) +local log = require "mqtt.log" + +if type(ngx) == "table" then + -- there is a global 'ngx' table, so we're running OpenResty + log:info("LuaMQTT auto-detected Nginx as the runtime environment") + return require("mqtt.connector.nginx") + +elseif package.loaded.copas then + -- 'copas' was already loaded + log:info("LuaMQTT auto-detected Copas as the io-loop in use") + return require("mqtt.connector.copas") + +elseif pcall(require, "socket") and tostring(require("socket")._VERSION):find("LuaSocket") then + -- LuaSocket is available + log:info("LuaMQTT auto-detected LuaSocket as the socket library to use") + return require("mqtt.connector.luasocket") +end + +error("connector auto-detection failed, please specify one explicitly") diff --git a/tests/spec/06-mqtt-client_spec.lua b/tests/spec/06-mqtt-client_spec.lua index b6cb47e..a773701 100644 --- a/tests/spec/06-mqtt-client_spec.lua +++ b/tests/spec/06-mqtt-client_spec.lua @@ -729,70 +729,4 @@ describe("no_local flag for subscription:", function() end) end) -describe("#copas connector", function() - local mqtt = require("mqtt") - local copas = require("copas") - local prefix = "luamqtt/" .. tostring(math.floor(math.random()*1e13)) - - it("test", function() - -- NOTE: more about flespi tokens: - -- https://flespi.com/kb/tokens-access-keys-to-flespi-platform - local flespi_token = "stPwSVV73Eqw5LSv0iMXbc4EguS7JyuZR9lxU5uLxI5tiNM8ToTVqNpu85pFtJv9" - - local client = mqtt.client{ - uri = "mqtt.flespi.io", - clean = true, - username = flespi_token, - version = mqtt.v50, - - connector = require("mqtt.connector.copas"), - } - - local test_finished = false - - client:on{ - connect = function() - log:warn("client is now connected") - log:warn("client subscribing to topic '.../copas'") - assert(client:subscribe{topic=prefix.."/copas", qos=1, callback=function() - log:warn("client subscription to topic '.../copas' confirmed") - log:warn("client publishing 'copas test' to topic '.../copas' confirmed") - assert(client:publish{ - topic = prefix.."/copas", - payload = "copas test", - qos = 1, - }) - end}) - end, - - message = function(msg) - assert(client:acknowledge(msg)) - if msg.topic == prefix.."/copas" and msg.payload == "copas test" then - log:warn("client received '%s' to topic '.../copas' confirmed", msg.payload) - assert(client:disconnect()) - log:warn("disconnected now") - test_finished = true - end - end - } - - copas.addthread(function() - while true do - local timeout = client:step() - if not timeout then - -- exited - return - end - if timeout > 0 then - copas.sleep(timeout) - end - end - end) - - copas.loop() - - assert.is_true(test_finished, "expecting mqtt client to finish its work") - end) -end) - -- vim: ts=4 sts=4 sw=4 noet ft=lua diff --git a/tests/spec/08-copas_spec.lua b/tests/spec/08-copas_spec.lua new file mode 100644 index 0000000..2efac72 --- /dev/null +++ b/tests/spec/08-copas_spec.lua @@ -0,0 +1,69 @@ +local log = require("logging").defaultLogger() + +describe("copas connector", function() + local mqtt = require("mqtt") + local copas = require("copas") + local prefix = "luamqtt/" .. tostring(math.floor(math.random()*1e13)) + + it("test", function() + -- NOTE: more about flespi tokens: + -- https://flespi.com/kb/tokens-access-keys-to-flespi-platform + local flespi_token = "stPwSVV73Eqw5LSv0iMXbc4EguS7JyuZR9lxU5uLxI5tiNM8ToTVqNpu85pFtJv9" + + local client = mqtt.client{ + uri = "mqtt.flespi.io", + clean = true, + username = flespi_token, + version = mqtt.v50, + + -- connector = require("mqtt.connector.copas"), -- will be auto-detected + } + + local test_finished = false + + client:on{ + connect = function() + log:warn("client is now connected") + log:warn("client subscribing to topic '.../copas'") + assert(client:subscribe{topic=prefix.."/copas", qos=1, callback=function() + log:warn("client subscription to topic '.../copas' confirmed") + log:warn("client publishing 'copas test' to topic '.../copas' confirmed") + assert(client:publish{ + topic = prefix.."/copas", + payload = "copas test", + qos = 1, + }) + end}) + end, + + message = function(msg) + assert(client:acknowledge(msg)) + if msg.topic == prefix.."/copas" and msg.payload == "copas test" then + log:warn("client received '%s' to topic '.../copas' confirmed", msg.payload) + assert(client:disconnect()) + log:warn("disconnected now") + test_finished = true + end + end + } + + copas.addthread(function() + while true do + local timeout = client:step() + if not timeout then + -- exited + return + end + if timeout > 0 then + copas.sleep(timeout) + end + end + end) + + copas.loop() + + assert.is_true(test_finished, "expecting mqtt client to finish its work") + end) +end) + +-- vim: ts=4 sts=4 sw=4 noet ft=lua From a600bd7b60a540ab36ed08aa32625d733cb6438b Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Fri, 5 Nov 2021 13:48:51 +0100 Subject: [PATCH 23/65] chore(test) added secure test for copas --- mqtt/connector/copas.lua | 8 ++- tests/spec/08-copas_spec.lua | 102 ++++++++++++++++++----------------- 2 files changed, 60 insertions(+), 50 deletions(-) diff --git a/mqtt/connector/copas.lua b/mqtt/connector/copas.lua index ab07dc4..bcfa63e 100644 --- a/mqtt/connector/copas.lua +++ b/mqtt/connector/copas.lua @@ -9,11 +9,17 @@ connector.super = super local socket = require("socket") local copas = require("copas") +local _, ssl = pcall(require, "ssl") -- Open network connection to .host and .port in conn table -- Store opened socket to conn table -- Returns true on success, or false and error text on failure function connector:connect() + if self.secure_params then + assert(ssl, "LuaSec ssl module not found, secure connections unavailable") + assert(type(self.secure_params) == "table", "expecting .secure_params to be a table if given") + end + local sock = copas.wrap(socket.tcp(), self.secure_params) sock:settimeout(self.timeout) @@ -27,7 +33,7 @@ end -- Shutdown network connection function connector:shutdown() - self.sock:shutdown() + self.sock:close() end -- Send data to network connection diff --git a/tests/spec/08-copas_spec.lua b/tests/spec/08-copas_spec.lua index 2efac72..11ccb6f 100644 --- a/tests/spec/08-copas_spec.lua +++ b/tests/spec/08-copas_spec.lua @@ -5,65 +5,69 @@ describe("copas connector", function() local copas = require("copas") local prefix = "luamqtt/" .. tostring(math.floor(math.random()*1e13)) - it("test", function() - -- NOTE: more about flespi tokens: - -- https://flespi.com/kb/tokens-access-keys-to-flespi-platform - local flespi_token = "stPwSVV73Eqw5LSv0iMXbc4EguS7JyuZR9lxU5uLxI5tiNM8ToTVqNpu85pFtJv9" + for _, secure in ipairs { false, true } do - local client = mqtt.client{ - uri = "mqtt.flespi.io", - clean = true, - username = flespi_token, - version = mqtt.v50, + it("test 'secure = "..tostring(secure).."'", function() + -- NOTE: more about flespi tokens: + -- https://flespi.com/kb/tokens-access-keys-to-flespi-platform + local flespi_token = "stPwSVV73Eqw5LSv0iMXbc4EguS7JyuZR9lxU5uLxI5tiNM8ToTVqNpu85pFtJv9" - -- connector = require("mqtt.connector.copas"), -- will be auto-detected - } + local client = mqtt.client{ + uri = "mqtt.flespi.io", + clean = true, + secure = secure, + username = flespi_token, + version = mqtt.v50, - local test_finished = false + -- connector = require("mqtt.connector.copas"), -- will be auto-detected + } - client:on{ - connect = function() - log:warn("client is now connected") - log:warn("client subscribing to topic '.../copas'") - assert(client:subscribe{topic=prefix.."/copas", qos=1, callback=function() - log:warn("client subscription to topic '.../copas' confirmed") - log:warn("client publishing 'copas test' to topic '.../copas' confirmed") - assert(client:publish{ - topic = prefix.."/copas", - payload = "copas test", - qos = 1, - }) - end}) - end, + local test_finished = false - message = function(msg) - assert(client:acknowledge(msg)) - if msg.topic == prefix.."/copas" and msg.payload == "copas test" then - log:warn("client received '%s' to topic '.../copas' confirmed", msg.payload) - assert(client:disconnect()) - log:warn("disconnected now") - test_finished = true - end - end - } + client:on{ + connect = function() + log:warn("client is now connected") + log:warn("client subscribing to topic '.../copas'") + assert(client:subscribe{topic=prefix.."/copas", qos=1, callback=function() + log:warn("client subscription to topic '.../copas' confirmed") + log:warn("client publishing 'copas test' to topic '.../copas' confirmed") + assert(client:publish{ + topic = prefix.."/copas", + payload = "copas test", + qos = 1, + }) + end}) + end, - copas.addthread(function() - while true do - local timeout = client:step() - if not timeout then - -- exited - return + message = function(msg) + assert(client:acknowledge(msg)) + if msg.topic == prefix.."/copas" and msg.payload == "copas test" then + log:warn("client received '%s' to topic '.../copas' confirmed", msg.payload) + assert(client:disconnect()) + log:warn("disconnected now") + test_finished = true + end end - if timeout > 0 then - copas.sleep(timeout) + } + + copas.addthread(function() + while true do + local timeout = client:step() + if not timeout then + -- exited + return + end + if timeout > 0 then + copas.sleep(timeout) + end end - end - end) + end) - copas.loop() + copas.loop() - assert.is_true(test_finished, "expecting mqtt client to finish its work") - end) + assert.is_true(test_finished, "expecting mqtt client to finish its work") + end) + end end) -- vim: ts=4 sts=4 sw=4 noet ft=lua From fe062f66aeac7cf751638503c3183d5f1739c516 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Fri, 5 Nov 2021 15:05:52 +0100 Subject: [PATCH 24/65] fix(ssl) restore 'ssl_module' option and add proper checks --- .luacheckrc | 3 ++- mqtt/connector/copas.lua | 5 +++-- mqtt/connector/luasocket.lua | 5 +++-- mqtt/connector/nginx.lua | 2 ++ 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.luacheckrc b/.luacheckrc index d506403..49306e6 100644 --- a/.luacheckrc +++ b/.luacheckrc @@ -22,6 +22,7 @@ include_files = { files["tests/spec/**/*.lua"] = { std = "+busted" } files["examples/openresty/**/*.lua"] = { std = "+ngx_lua" } -files["mqtt/ngxsocket.lua"] = { std = "+ngx_lua" } +files["mqtt/connector/init.lua"] = { std = "+ngx_lua" } +files["mqtt/connector/nginx.lua"] = { std = "+ngx_lua" } -- vim: ts=4 sts=4 sw=4 noet ft=lua diff --git a/mqtt/connector/copas.lua b/mqtt/connector/copas.lua index bcfa63e..c224ec4 100644 --- a/mqtt/connector/copas.lua +++ b/mqtt/connector/copas.lua @@ -9,15 +9,16 @@ connector.super = super local socket = require("socket") local copas = require("copas") -local _, ssl = pcall(require, "ssl") -- Open network connection to .host and .port in conn table -- Store opened socket to conn table -- Returns true on success, or false and error text on failure function connector:connect() if self.secure_params then - assert(ssl, "LuaSec ssl module not found, secure connections unavailable") assert(type(self.secure_params) == "table", "expecting .secure_params to be a table if given") + assert(self.ssl_module == "ssl", "Copas connector only supports 'ssl_module' called 'ssl'") + local _, ssl = pcall(require, "ssl") + assert(ssl, "ssl_module 'ssl' not found, secure connections unavailable") end local sock = copas.wrap(socket.tcp(), self.secure_params) diff --git a/mqtt/connector/luasocket.lua b/mqtt/connector/luasocket.lua index fcc7b7f..049f454 100644 --- a/mqtt/connector/luasocket.lua +++ b/mqtt/connector/luasocket.lua @@ -7,7 +7,6 @@ luasocket.__index = luasocket luasocket.super = super local socket = require("socket") -local _, ssl = pcall(require, "ssl") -- table with error messages that indicate a read timeout luasocket.timeout_errors = { @@ -20,9 +19,11 @@ luasocket.timeout_errors = { -- Store opened socket to conn table -- Returns true on success, or false and error text on failure function luasocket:connect() + local ssl, _ if self.secure_params then - assert(ssl, "LuaSec ssl module not found, secure connections unavailable") assert(type(self.secure_params) == "table", "expecting .secure_params to be a table if given") + _, ssl = pcall(require, self.ssl_module) + assert(ssl, "ssl_module '"..tostring(self.ssl_module).."' not found, secure connections unavailable") end self:buffer_clear() -- sanity diff --git a/mqtt/connector/nginx.lua b/mqtt/connector/nginx.lua index 70ae5c0..1d916f0 100644 --- a/mqtt/connector/nginx.lua +++ b/mqtt/connector/nginx.lua @@ -12,6 +12,8 @@ local ngx_socket_tcp = ngx.socket.tcp -- Store opened socket to conn table -- Returns true on success, or false and error text on failure function ngxsocket:connect() + assert(self.ssl_module == "ssl", "specifying custom ssl module when using Nginx connector is not supported") + local sock = ngx_socket_tcp() sock:settimeout(self.timeout * 1000) -- millisecs local ok, err = sock:connect(self.host, self.port) From 2db6e72e3def5dcb7f6a5578e25644c191b4a8e1 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Sun, 7 Nov 2021 15:27:30 +0100 Subject: [PATCH 25/65] chore(validation) move secure conn validation+defaults to conn --- mqtt/client.lua | 17 ++++--------- mqtt/connector/base/luasec.lua | 29 +++++++++++++++++++++++ mqtt/connector/base/non_buffered_base.lua | 5 ++++ mqtt/connector/copas.lua | 24 +++++++++++-------- mqtt/connector/luasocket.lua | 19 +++++++++++---- mqtt/connector/nginx.lua | 20 +++++++++++----- rockspecs/luamqtt-3.4.3-1.rockspec | 1 + 7 files changed, 82 insertions(+), 33 deletions(-) create mode 100644 mqtt/connector/base/luasec.lua diff --git a/mqtt/client.lua b/mqtt/client.lua index 84be8b7..3d73cb4 100644 --- a/mqtt/client.lua +++ b/mqtt/client.lua @@ -183,6 +183,7 @@ function client_mt:__init(opts) -- validate connector content assert(type(a.connector) == "table", "expecting connector to be a table") + assert(type(a.connector.validate) == "function", "expecting connector.validate to be a function") assert(type(a.connector.connect) == "function", "expecting connector.connect to be a function") assert(type(a.connector.shutdown) == "function", "expecting connector.shutdown to be a function") assert(type(a.connector.send) == "function", "expecting connector.send to be a function") @@ -191,7 +192,9 @@ function client_mt:__init(opts) assert(a.connector.signal_idle, "missing connector.signal_idle signal value") -- validate connection properties - client_mt._parse_connection_opts(a, { uri = opts.uri }) + local test_conn = setmetatable({ uri = opts.uri }, a.connector) + client_mt._parse_connection_opts(a, test_conn) + test_conn:validate() -- will table content check if a.will then @@ -1234,17 +1237,7 @@ function client_mt._parse_connection_opts(opts, conn) local secure = opts.secure if secure then conn.secure = true - if type(secure) == "table" then - conn.secure_params = secure - else - -- TODO: these defaults should go in the connectors, as they can differ per environment (eg. Nginx) - conn.secure_params = { - mode = "client", - protocol = "any", - verify = "none", - options = {"all", "no_sslv2", "no_sslv3", "no_tlsv1"}, - } - end + conn.secure_params = secure ~= true and secure or nil conn.ssl_module = opts.ssl_module or "ssl" assert(conn.ssl_module == nil or type(conn.ssl_module) == "string", "expected ssl_module to be a string") else diff --git a/mqtt/connector/base/luasec.lua b/mqtt/connector/base/luasec.lua new file mode 100644 index 0000000..660dfec --- /dev/null +++ b/mqtt/connector/base/luasec.lua @@ -0,0 +1,29 @@ +-- validates the LuaSec options, and applies defaults +return function(conn) + if conn.secure then + local params = conn.secure_params + if not params then + -- set default LuaSec options + conn.secure_params = { + mode = "client", + protocol = "any", + verify = "none", + options = {"all", "no_sslv2", "no_sslv3", "no_tlsv1"}, + } + return + end + + local ok, ssl = pcall(require, conn.ssl_module) + assert(ok, "ssl_module '"..tostring(conn.ssl_module).."' not found, secure connections unavailable") + + assert(type(params) == "table", "expecting .secure_params to be a table, got: "..type(params)) + + params.mode = params.mode or "client" + assert(params.mode == "client", "secure parameter 'mode' must be set to 'client' if given, got: "..tostring(params.mode)) + + local ctx, err = ssl.newcontext(params) + if not ctx then + error("Couldn't create secure context: "..tostring(err)) + end + end +end diff --git a/mqtt/connector/base/non_buffered_base.lua b/mqtt/connector/base/non_buffered_base.lua index 5c1996f..eb9e0f8 100644 --- a/mqtt/connector/base/non_buffered_base.lua +++ b/mqtt/connector/base/non_buffered_base.lua @@ -24,6 +24,11 @@ non_buffered.__index = non_buffered non_buffered.signal_idle = {} -- read timeout occured, so we're idle need to come back later and try again non_buffered.signal_closed = {} -- remote closed the connection +--- Validate connection options. +function non_buffered:shutdown() -- luacheck: ignore + error("method 'validate' on connector wasn't implemented") +end + --- Clears consumed bytes. -- Called by the mqtt client when the consumed bytes from the buffer are handled -- and can be cleared from the buffer. diff --git a/mqtt/connector/copas.lua b/mqtt/connector/copas.lua index c224ec4..1369f1a 100644 --- a/mqtt/connector/copas.lua +++ b/mqtt/connector/copas.lua @@ -9,18 +9,23 @@ connector.super = super local socket = require("socket") local copas = require("copas") +local validate_luasec = require("mqtt.connector.base.luasec") + + +-- validate connection options +function connector:validate() + if self.secure then + assert(self.ssl_module == "ssl" or self.ssl_module == nil, "Copas connector only supports 'ssl' as 'ssl_module'") + + validate_luasec(self) + end +end -- Open network connection to .host and .port in conn table -- Store opened socket to conn table -- Returns true on success, or false and error text on failure function connector:connect() - if self.secure_params then - assert(type(self.secure_params) == "table", "expecting .secure_params to be a table if given") - assert(self.ssl_module == "ssl", "Copas connector only supports 'ssl_module' called 'ssl'") - local _, ssl = pcall(require, "ssl") - assert(ssl, "ssl_module 'ssl' not found, secure connections unavailable") - end - + self:validate() local sock = copas.wrap(socket.tcp(), self.secure_params) sock:settimeout(self.timeout) @@ -58,11 +63,10 @@ function connector:receive(size) return data end - -- note: signal_idle is not needed here since Copas takes care - -- of that. The read is non blocking, so a timeout is a real error and not - -- a signal to retry. if err == "closed" then return false, self.signal_closed + elseif err == "timout" then + return false, self.signal_idle else return false, err end diff --git a/mqtt/connector/luasocket.lua b/mqtt/connector/luasocket.lua index 049f454..7f27702 100644 --- a/mqtt/connector/luasocket.lua +++ b/mqtt/connector/luasocket.lua @@ -7,6 +7,8 @@ luasocket.__index = luasocket luasocket.super = super local socket = require("socket") +local validate_luasec = require("mqtt.connector.base.luasec") + -- table with error messages that indicate a read timeout luasocket.timeout_errors = { @@ -15,15 +17,22 @@ luasocket.timeout_errors = { wantwrite = true, -- luasec } +-- validate connection options +function luasocket:validate() + if self.secure then + validate_luasec(self) + end +end + -- Open network connection to .host and .port in conn table -- Store opened socket to conn table -- Returns true on success, or false and error text on failure function luasocket:connect() - local ssl, _ - if self.secure_params then - assert(type(self.secure_params) == "table", "expecting .secure_params to be a table if given") - _, ssl = pcall(require, self.ssl_module) - assert(ssl, "ssl_module '"..tostring(self.ssl_module).."' not found, secure connections unavailable") + self:validate() + + local ssl + if self.secure then + ssl = require(self.ssl_module) end self:buffer_clear() -- sanity diff --git a/mqtt/connector/nginx.lua b/mqtt/connector/nginx.lua index 1d916f0..dbb9c7a 100644 --- a/mqtt/connector/nginx.lua +++ b/mqtt/connector/nginx.lua @@ -8,14 +8,23 @@ ngxsocket.super = super -- load required stuff local ngx_socket_tcp = ngx.socket.tcp + +-- validate connection options +function ngxsocket:validate() + if self.secure then + assert(self.ssl_module == "ssl", "specifying custom ssl module when using Nginx connector is not supported") + assert(type(self.secure_params) == "table", "expecting .secure_params to be a table") + -- TODO: validate nginx stuff + end +end + -- Open network connection to .host and .port in conn table -- Store opened socket to conn table -- Returns true on success, or false and error text on failure function ngxsocket:connect() - assert(self.ssl_module == "ssl", "specifying custom ssl module when using Nginx connector is not supported") - local sock = ngx_socket_tcp() - sock:settimeout(self.timeout * 1000) -- millisecs + -- set read-timeout to 'nil' to not timeout at all + assert(sock:settimeouts(self.timeout * 1000, self.timeout * 1000, nil)) -- millisecs local ok, err = sock:connect(self.host, self.port) if not ok then return false, "socket:connect failed: "..err @@ -45,11 +54,10 @@ function ngxsocket:receive(size) return data end - -- note: signal_idle is not needed here since OpenResty takes care - -- of that. The read is non blocking, so a timeout is a real error and not - -- a signal to retry. if err == "closed" then return false, self.signal_closed + elseif err == "timout" then + return false, self.signal_idle else return false, err end diff --git a/rockspecs/luamqtt-3.4.3-1.rockspec b/rockspecs/luamqtt-3.4.3-1.rockspec index e81c593..3f30b18 100644 --- a/rockspecs/luamqtt-3.4.3-1.rockspec +++ b/rockspecs/luamqtt-3.4.3-1.rockspec @@ -33,6 +33,7 @@ build = { ["mqtt.log"] = "mqtt/log.lua", ["mqtt.connector.base.buffered_base"] = "mqtt/connector/base/buffered_base.lua", ["mqtt.connector.base.non_buffered_base"] = "mqtt/connector/base/non_buffered_base.lua", + ["mqtt.connector.base.luasec"] = "mqtt/connector/base/luasec.lua", ["mqtt.connector.luasocket"] = "mqtt/connector/luasocket.lua", ["mqtt.connector.copas"] = "mqtt/connector/copas.lua", ["mqtt.connector.nginx"] = "mqtt/connector/nginx.lua", From c87bbbf472bcd7eb2426b75e27fbf373a6ddf766 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Mon, 8 Nov 2021 09:23:03 +0100 Subject: [PATCH 26/65] fix(reconnect) fix reconnect logic --- mqtt/client.lua | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/mqtt/client.lua b/mqtt/client.lua index 3d73cb4..62407f5 100644 --- a/mqtt/client.lua +++ b/mqtt/client.lua @@ -1051,16 +1051,16 @@ end do -- implict (re)connecting when reading local function implicit_connect(self) - local opts = self.opts + local reconnect = self.opts.reconnect - if not self.first_connect and not opts.reconnect then + if not self.first_connect and not reconnect then -- this would be a re-connect, but we're not supposed to auto-reconnect log:debug("client '%s' was disconnected and not set to auto-reconnect", self.opts.id) return false, "network connection is not opened" end -- should we wait for a timeout between retries? - local t_reconnect = (self.last_connect_time or 0) + (opts.reconnect or 0) + local t_reconnect = (self.last_connect_time or 0) + (reconnect or 0) local t_now = os_time() if t_reconnect > t_now then -- were delaying before retrying, return remaining delay @@ -1072,12 +1072,7 @@ do local ok, err = self:start_connecting() if not ok then -- we failed to connect - if opts.reconnect then - -- set to auto-reconnect, report the configured retry delay - return opts.reconnect - end - -- not reconnecting, so just report the error - return ok, err + return reconnect, err end -- connected succesfully, but don't know how long it took, so return now @@ -1104,6 +1099,7 @@ do -- @return time after which to retry or nil+error function client_mt:step() local conn = self.connection + local reconnect = self.opts.reconnect -- try and connect if not connected yet if not conn then @@ -1117,27 +1113,20 @@ do return -1 elseif err == conn.signal_closed then self:close_connection("connection closed by broker") - return false, err + return reconnect and 0, err else err = "failed to receive next packet: "..tostring(err) log:error("client '%s' %s", self.opts.id, err) self:handle("error", err, self) self:close_connection("error") - return false, err + return reconnect and 0, err end end local ok ok, err = self:handle_received_packet(packet) if not ok then - if not self.opts.reconnect then - -- no auto-reconnect, so return error - return ok, err - else - -- reconnect, return 0 to retry asap, the reconnect code - -- will then do the right-thing TM - return 0 - end + return reconnect and 0, err end -- succesfully handled packed, maybe there is more, so retry asap From 587b121fa32b1731684ea4148dc8f5017b59ffa6 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Mon, 8 Nov 2021 09:23:49 +0100 Subject: [PATCH 27/65] feat(client) pass event handlers on init --- mqtt/client.lua | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/mqtt/client.lua b/mqtt/client.lua index 62407f5..f053016 100644 --- a/mqtt/client.lua +++ b/mqtt/client.lua @@ -164,6 +164,9 @@ function client_mt:__init(opts) elseif key == "ssl_module" then assert(value_type == "string", "expecting ssl_module to be a string") a.ssl_module = value + elseif key == "on" then + assert(value_type == "table", "expecting 'on' to be a table with events and callbacks") + a.on = value else error("unexpected key in client opts: "..key.." = "..tostring(value)) end @@ -248,6 +251,11 @@ function client_mt:__init(opts) self._parse_packet = parse_packet5 end + -- register event handlers + if a.on then + self:on(self.opts.on) + end + log:info("MQTT client '%s' created", a.id) end From ca6f39b5b4bf43196fd545044bab085c9d3137db Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Mon, 8 Nov 2021 09:57:19 +0100 Subject: [PATCH 28/65] feat(shutdown) adds a client:shutdown method and event To enable disctintion between error type closing and client side closing. Shutdown will also disble reconnects, whereas just closing or disconnecting would initiate reconnects. --- mqtt/client.lua | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/mqtt/client.lua b/mqtt/client.lua index f053016..3ecb50c 100644 --- a/mqtt/client.lua +++ b/mqtt/client.lua @@ -13,6 +13,9 @@ local client = {} -- upon closing the connection -- connection object will have .close_reason (string) +-- "shutdown": function(client_object) +-- upon shutting down the client (diconnecting an no more reconnects) + -- "connect": function(packet, client_object) -- upon a succesful connect, after receiving the CONNACK packet from the broker -- ???? => on a refused connect; if received CONNACK.rc ~= 0 when connecting @@ -101,8 +104,8 @@ client_mt.__index = client_mt -- and optional fields { qos=1...3, retain=true/false } -- @tparam[opt=60] number opts.keep_alive time interval for client to send PINGREQ packets to the server when network connection is inactive -- @tparam[opt=false] boolean opts.reconnect force created MQTT client to reconnect on connection close. --- Set to number value to provide reconnect timeout in seconds --- It's not recommended to use values < 3 +-- Set to number value to provide reconnect timeout in seconds. +-- It's not recommended to use values < 3. See also `client_mt:shutdown`. -- @tparam[opt] table opts.connector connector table to open and send/receive packets over network connection. -- default is require("mqtt.connector") which tries to auto-detect. -- @tparam[opt="ssl"] string opts.ssl_module module name for the luasec-compatible ssl module, default is "ssl" @@ -230,6 +233,7 @@ function client_mt:__init(opts) error = {}, close = {}, auth = {}, + shutdown = {}, } self._handling = {} self._to_remove_handlers = {} @@ -640,6 +644,19 @@ function client_mt:disconnect(rc, properties, user_properties) return true end +--- Shutsdown the client. +-- Disconnects if still connected, and disables reconnecting. +-- Raises the "shutdown" event +-- @param see `client_mt:disconnect`. +function client_mt:shutdown(rc, properties, user_properties) + log:debug("client '%s' shutting down", self.opts.id) + self.first_connect = false + self.opts.reconnect = false + self:disconnect(rc, properties, user_properties) + self:handle("shutdown", self) + return true +end + --- Send AUTH packet to authenticate client on broker, in MQTT v5.0 protocol -- @tparam[opt=0] number rc Authenticate Reason Code -- @tparam[opt] table properties properties for PUBACK/PUBREC packets From c02e3d24c9fd4ebdf552410ba07850931f23a55e Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Mon, 8 Nov 2021 10:42:58 +0100 Subject: [PATCH 29/65] fix(rockspec) add 'connector.init' module to rockspec --- rockspecs/luamqtt-3.4.3-1.rockspec | 1 + 1 file changed, 1 insertion(+) diff --git a/rockspecs/luamqtt-3.4.3-1.rockspec b/rockspecs/luamqtt-3.4.3-1.rockspec index 3f30b18..65dcd16 100644 --- a/rockspecs/luamqtt-3.4.3-1.rockspec +++ b/rockspecs/luamqtt-3.4.3-1.rockspec @@ -31,6 +31,7 @@ build = { ["mqtt.protocol5"] = "mqtt/protocol5.lua", ["mqtt.tools"] = "mqtt/tools.lua", ["mqtt.log"] = "mqtt/log.lua", + ["mqtt.connector.init"] = "mqtt/connector/init.lua", ["mqtt.connector.base.buffered_base"] = "mqtt/connector/base/buffered_base.lua", ["mqtt.connector.base.non_buffered_base"] = "mqtt/connector/base/non_buffered_base.lua", ["mqtt.connector.base.luasec"] = "mqtt/connector/base/luasec.lua", From 0e15eaa4b4b86f93e717792f3a518532ab09373f Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Mon, 8 Nov 2021 11:18:53 +0100 Subject: [PATCH 30/65] chore(examples) update to use 'opts.on' and auto-connector select --- examples/copas-example.lua | 108 ++++++++++++++++---------------- examples/copas.lua | 72 +++++++++++---------- examples/last-will/client-1.lua | 48 +++++++------- examples/last-will/client-2.lua | 72 ++++++++++----------- examples/mqtt5-simple.lua | 88 +++++++++++++------------- examples/simple.lua | 73 ++++++++++----------- 6 files changed, 230 insertions(+), 231 deletions(-) diff --git a/examples/copas-example.lua b/examples/copas-example.lua index 0c85dd9..cadf852 100644 --- a/examples/copas-example.lua +++ b/examples/copas-example.lua @@ -11,72 +11,70 @@ local suffix = tostring(math.random(1000000)) -- mqtt topic suffix to distinct s local token = "stPwSVV73Eqw5LSv0iMXbc4EguS7JyuZR9lxU5uLxI5tiNM8ToTVqNpu85pFtJv9" local ping = mqtt.client{ - uri = "mqtt.flespi.io", + uri = "mqtt://mqtt.flespi.io", username = token, clean = true, version = mqtt.v50, - -- NOTE: copas connector - connector = require("mqtt.luasocket-copas"), + + -- create event handlers + on = { + connect = function(connack, self) + assert(connack.rc == 0) + print("ping connected") + + -- adding another thread; copas handlers should return quickly, anything + -- that can wait should be off-loaded from the handler to a thread. + -- Especially anything that yields; socket reads/writes and sleeps, and the + -- code below does both, sleeping, and writing (implicit in 'publish') + copas.addthread(function() + for i = 1, num_pings do + copas.sleep(delay) + print("ping", i) + assert(self:publish{ topic = "luamqtt/copas-ping/"..suffix, payload = "ping"..i, qos = 1 }) + end + + print("ping done") + assert(self:publish{ topic = "luamqtt/copas-ping/"..suffix, payload = "done", qos = 1 }) + self:disconnect() + end) + end, + error = function(err) + print("ping MQTT client error:", err) + end, + }, -- close 'on', event handlers } local pong = mqtt.client{ - uri = "mqtt.flespi.io", + uri = "mqtt://mqtt.flespi.io", username = token, clean = true, version = mqtt.v50, - -- NOTE: copas connector - connector = require("mqtt.luasocket-copas"), -} -ping:on{ - connect = function(connack) - assert(connack.rc == 0) - print("ping connected") - - -- adding another thread; copas handlers should return quickly, anything - -- that can wait should be off-loaded from the handler to a thread. - -- Especially anything that yields; socket reads/writes and sleeps, and the - -- code below does both, sleeping, and writing (implicit in 'publish') - copas.addthread(function() - for i = 1, num_pings do - copas.sleep(delay) - print("ping", i) - assert(ping:publish{ topic = "luamqtt/copas-ping/"..suffix, payload = "ping"..i, qos = 1 }) + -- create event handlers + on = { + connect = function(connack, self) + assert(connack.rc == 0) + print("pong connected") + + assert(self:subscribe{ topic="luamqtt/copas-ping/"..suffix, qos=1, callback=function(suback) + assert(suback.rc[1] > 0) + print("pong subscribed") + end }) + end, + + message = function(msg, self) + print("pong: received", msg.payload) + assert(self:acknowledge(msg)) + + if msg.payload == "done" then + print("pong done") + self:disconnect() end - - print("ping done") - assert(ping:publish{ topic = "luamqtt/copas-ping/"..suffix, payload = "done", qos = 1 }) - ping:disconnect() - end) - end, - error = function(err) - print("ping MQTT client error:", err) - end, -} - -pong:on{ - connect = function(connack) - assert(connack.rc == 0) - print("pong connected") - - assert(pong:subscribe{ topic="luamqtt/copas-ping/"..suffix, qos=1, callback=function(suback) - assert(suback.rc[1] > 0) - print("pong subscribed") - end }) - end, - - message = function(msg) - print("pong: received", msg.payload) - assert(pong:acknowledge(msg)) - - if msg.payload == "done" then - print("pong done") - pong:disconnect() - end - end, - error = function(err) - print("pong MQTT client error:", err) - end, + end, + error = function(err) + print("pong MQTT client error:", err) + end, + }, -- close 'on', event handlers } local function add_client(cl) diff --git a/examples/copas.lua b/examples/copas.lua index 6eb6c5a..aa12b5f 100644 --- a/examples/copas.lua +++ b/examples/copas.lua @@ -6,55 +6,53 @@ local copas = require("copas") local client = mqtt.client{ -- NOTE: this broker is not working sometimes; comment username = "..." below if you still want to use it -- uri = "test.mosquitto.org", - uri = "mqtt.flespi.io", + uri = "mqtt://mqtt.flespi.io", -- NOTE: more about flespi tokens: https://flespi.com/kb/tokens-access-keys-to-flespi-platform username = "stPwSVV73Eqw5LSv0iMXbc4EguS7JyuZR9lxU5uLxI5tiNM8ToTVqNpu85pFtJv9", clean = true, - -- NOTE: copas connector - connector = require("mqtt.luasocket-copas"), -} -print("created MQTT client", client) - -client:on{ - connect = function(connack) - if connack.rc ~= 0 then - print("connection to broker failed:", connack:reason_string(), connack) - return - end - print("connected:", connack) -- successful connection + -- create event handlers + on = { + connect = function(connack, self) + if connack.rc ~= 0 then + print("connection to broker failed:", connack:reason_string(), connack) + return + end + print("connected:", connack) -- successful connection - -- subscribe to test topic and publish message after it - assert(client:subscribe{ topic="luamqtt/#", qos=1, callback=function(suback) - print("subscribed:", suback) + -- subscribe to test topic and publish message after it + assert(self:subscribe{ topic="luamqtt/#", qos=1, callback=function(suback) + print("subscribed:", suback) - -- publish test message - print('publishing test message "hello" to "luamqtt/simpletest" topic...') - assert(client:publish{ - topic = "luamqtt/simpletest", - payload = "hello", - qos = 1 - }) - end}) - end, + -- publish test message + print('publishing test message "hello" to "luamqtt/simpletest" topic...') + assert(self:publish{ + topic = "luamqtt/simpletest", + payload = "hello", + qos = 1 + }) + end}) + end, - message = function(msg) - assert(client:acknowledge(msg)) + message = function(msg, self) + assert(self:acknowledge(msg)) - print("received:", msg) - print("disconnecting...") - assert(client:disconnect()) - end, + print("received:", msg) + print("disconnecting...") + assert(self:disconnect()) + end, - error = function(err) - print("MQTT client error:", err) - end, + error = function(err) + print("MQTT client error:", err) + end, - close = function() - print("MQTT conn closed") - end + close = function() + print("MQTT conn closed") + end + }, -- close 'on', event handlers } +print("created MQTT client", client) local function add_client(cl) -- add keep-alive timer diff --git a/examples/last-will/client-1.lua b/examples/last-will/client-1.lua index 586d426..aef30c5 100644 --- a/examples/last-will/client-1.lua +++ b/examples/last-will/client-1.lua @@ -2,46 +2,46 @@ local mqtt = require("mqtt") -- create mqtt client local client = mqtt.client{ - id = "luamqtt-example-will-1", + id = "mqtts://luamqtt-example-will-1", -- NOTE: this broker is not working sometimes; comment username = "..." below if you still want to use it -- uri = "test.mosquitto.org", uri = "mqtt.flespi.io", -- NOTE: more about flespi tokens: https://flespi.com/kb/tokens-access-keys-to-flespi-platform username = "stPwSVV73Eqw5LSv0iMXbc4EguS7JyuZR9lxU5uLxI5tiNM8ToTVqNpu85pFtJv9", clean = true, - secure = true, -- specifying last will message will = { topic = "luamqtt/lost", payload = "client-1 connection lost last will message", }, -} -client:on{ - connect = function(connack) - if connack.rc ~= 0 then - print("connection to broker failed:", connack:reason_string(), connack) - return - end - print("connected:", connack) + -- event handlers + on = { + connect = function(connack, self) + if connack.rc ~= 0 then + print("connection to broker failed:", connack:reason_string(), connack) + return + end + print("connected:", connack) - -- subscribe to topic when we are expecting connection close command from client-2 - assert(client:subscribe{ topic="luamqtt/close", qos=1, callback=function() - print("subscribed to luamqtt/close, waiting for connection close command from client-2") - end}) - end, + -- subscribe to topic when we are expecting connection close command from client-2 + assert(self:subscribe{ topic="luamqtt/close", qos=1, callback=function() + print("subscribed to luamqtt/close, waiting for connection close command from client-2") + end}) + end, - message = function(msg) - assert(client:acknowledge(msg)) + message = function(msg, self) + assert(self:acknowledge(msg)) - print("received:", msg) - print("closing connection without DISCONNECT and stopping client-1") - client:close_connection() -- will message should be sent - end, + print("received:", msg) + print("closing connection without DISCONNECT and stopping client-1") + self:close_connection() -- will message should be sent + end, - error = function(err) - print("MQTT client error:", err) - end, + error = function(err) + print("MQTT client error:", err) + end, + } } -- start receive loop diff --git a/examples/last-will/client-2.lua b/examples/last-will/client-2.lua index 81e29b3..2b917ca 100644 --- a/examples/last-will/client-2.lua +++ b/examples/last-will/client-2.lua @@ -2,49 +2,49 @@ local mqtt = require("mqtt") -- create mqtt client local client = mqtt.client{ - id = "luamqtt-example-will-2", + id = "mqtts://luamqtt-example-will-2", -- NOTE: this broker is not working sometimes; comment username = "..." below if you still want to use it -- uri = "test.mosquitto.org", uri = "mqtt.flespi.io", -- NOTE: more about flespi tokens: https://flespi.com/kb/tokens-access-keys-to-flespi-platform username = "stPwSVV73Eqw5LSv0iMXbc4EguS7JyuZR9lxU5uLxI5tiNM8ToTVqNpu85pFtJv9", clean = true, - secure = true, -} -client:on{ - connect = function(connack) - if connack.rc ~= 0 then - print("connection to broker failed:", connack:reason_string(), connack) - return - end - print("connected:", connack) - - -- subscribe to topic when we are expecting last-will message from client-1 - assert(client:subscribe{ topic="luamqtt/lost", qos=1, callback=function() - print("subscribed to luamqtt/lost") - - -- publish close command to client-1 - assert(client:publish{ - topic = "luamqtt/close", - payload = "Dear client-1, please close your connection", - qos = 1, - }) - print("published close command") - end}) - end, - - message = function(msg) - assert(client:acknowledge(msg)) - - print("received:", msg) - print("disconnecting and stopping client-2") - client:disconnect() - end, - - error = function(err) - print("MQTT client error:", err) - end, + -- event handlers + on = { + connect = function(connack, self) + if connack.rc ~= 0 then + print("connection to broker failed:", connack:reason_string(), connack) + return + end + print("connected:", connack) + + -- subscribe to topic when we are expecting last-will message from client-1 + assert(self:subscribe{ topic="luamqtt/lost", qos=1, callback=function() + print("subscribed to luamqtt/lost") + + -- publish close command to client-1 + assert(self:publish{ + topic = "luamqtt/close", + payload = "Dear client-1, please close your connection", + qos = 1, + }) + print("published close command") + end}) + end, + + message = function(msg, self) + assert(self:acknowledge(msg)) + + print("received:", msg) + print("disconnecting and stopping client-2") + self:disconnect() + end, + + error = function(err) + print("MQTT client error:", err) + end, + } } -- start receive loop diff --git a/examples/mqtt5-simple.lua b/examples/mqtt5-simple.lua index 2c85b98..9cd4434 100644 --- a/examples/mqtt5-simple.lua +++ b/examples/mqtt5-simple.lua @@ -2,56 +2,58 @@ local mqtt = require("mqtt") -- create mqtt client local client = mqtt.client{ - uri = "mqtt.flespi.io", + uri = "mqtt://mqtt.flespi.io", -- NOTE: more about flespi tokens: https://flespi.com/kb/tokens-access-keys-to-flespi-platform username = "stPwSVV73Eqw5LSv0iMXbc4EguS7JyuZR9lxU5uLxI5tiNM8ToTVqNpu85pFtJv9", clean = true, version = mqtt.v50, -} -print("created MQTT v5.0 client:", client) -client:on{ - connect = function(connack) - if connack.rc ~= 0 then - print("connection to broker failed:", connack:reason_string(), connack) - return - end - print("connected:", connack) -- successful connection - - -- subscribe to test topic and publish message after it - assert(client:subscribe{ topic="luamqtt/#", qos=1, callback=function(suback) - print("subscribed:", suback) - - -- publish test message - print('publishing test message "hello" to "luamqtt/simpletest" topic...') - assert(client:publish{ - topic = "luamqtt/simpletest", - payload = "hello", - qos = 1, - properties = { - payload_format_indicator = 1, - content_type = "text/plain", - }, - user_properties = { - hello = "world", - }, - }) - end}) - end, - - message = function(msg) - assert(client:acknowledge(msg)) - - print("received:", msg) - print("disconnecting...") - assert(client:disconnect()) - end, - - error = function(err) - print("MQTT client error:", err) - end, + -- create event handlers + on = { + connect = function(connack, self) + if connack.rc ~= 0 then + print("connection to broker failed:", connack:reason_string(), connack) + return + end + print("connected:", connack) -- successful connection + + -- subscribe to test topic and publish message after it + assert(self:subscribe{ topic="luamqtt/#", qos=1, callback=function(suback) + print("subscribed:", suback) + + -- publish test message + print('publishing test message "hello" to "luamqtt/simpletest" topic...') + assert(self:publish{ + topic = "luamqtt/simpletest", + payload = "hello", + qos = 1, + properties = { + payload_format_indicator = 1, + content_type = "text/plain", + }, + user_properties = { + hello = "world", + }, + }) + end}) + end, + + message = function(msg, self) + assert(self:acknowledge(msg)) + + print("received:", msg) + print("disconnecting...") + assert(self:disconnect()) + end, + + error = function(err) + print("MQTT client error:", err) + end, + }, -- close 'on', event handlers } + +print("created MQTT v5.0 client:", client) print("running ioloop for it") mqtt.run_ioloop(client) diff --git a/examples/simple.lua b/examples/simple.lua index 6364183..d181b1b 100644 --- a/examples/simple.lua +++ b/examples/simple.lua @@ -5,48 +5,49 @@ local mqtt = require("mqtt") local client = mqtt.client{ -- NOTE: this broker is not working sometimes; comment username = "..." below if you still want to use it -- uri = "test.mosquitto.org", - uri = "mqtt.flespi.io", + uri = "mqtt://mqtt.flespi.io", -- NOTE: more about flespi tokens: https://flespi.com/kb/tokens-access-keys-to-flespi-platform username = "stPwSVV73Eqw5LSv0iMXbc4EguS7JyuZR9lxU5uLxI5tiNM8ToTVqNpu85pFtJv9", clean = true, -} -print("created MQTT client", client) -client:on{ - connect = function(connack) - if connack.rc ~= 0 then - print("connection to broker failed:", connack:reason_string(), connack) - return - end - print("connected:", connack) -- successful connection - - -- subscribe to test topic and publish message after it - assert(client:subscribe{ topic="luamqtt/#", qos=1, callback=function(suback) - print("subscribed:", suback) - - -- publish test message - print('publishing test message "hello" to "luamqtt/simpletest" topic...') - assert(client:publish{ - topic = "luamqtt/simpletest", - payload = "hello", - qos = 1 - }) - end}) - end, - - message = function(msg) - assert(client:acknowledge(msg)) - - print("received:", msg) - print("disconnecting...") - assert(client:disconnect()) - end, - - error = function(err) - print("MQTT client error:", err) - end, + -- create event handlers + on = { + connect = function(connack, self) + if connack.rc ~= 0 then + print("connection to broker failed:", connack:reason_string(), connack) + return + end + print("connected:", connack) -- successful connection + + -- subscribe to test topic and publish message after it + assert(self:subscribe{ topic="luamqtt/#", qos=1, callback=function(suback) + print("subscribed:", suback) + + -- publish test message + print('publishing test message "hello" to "luamqtt/simpletest" topic...') + assert(self:publish{ + topic = "luamqtt/simpletest", + payload = "hello", + qos = 1 + }) + end}) + end, + + message = function(msg, self) + assert(self:acknowledge(msg)) + + print("received:", msg) + print("disconnecting...") + assert(self:disconnect()) + end, + + error = function(err) + print("MQTT client error:", err) + end, + }, -- close 'on', event handlers } +print("created MQTT client", client) print("running ioloop for it") mqtt.run_ioloop(client) From 1fc585909e45e82d937f2701bc8e65ebf8638437 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Mon, 8 Nov 2021 16:32:30 +0100 Subject: [PATCH 31/65] fix(openresty) fix/rewrite the openresty examples --- .gitignore | 1 + examples/openresty/README.md | 39 +++++---- examples/openresty/app/luamqtt-example.lua | 94 +++++++++++++++++++++ examples/openresty/app/main-ioloop.lua | 96 ---------------------- examples/openresty/app/main-sync.lua | 89 -------------------- examples/openresty/conf/nginx.conf | 16 +++- examples/openresty/quit.sh | 7 ++ examples/openresty/start.sh | 5 +- examples/openresty/stop.sh | 3 +- mqtt/connector/nginx.lua | 4 +- 10 files changed, 142 insertions(+), 212 deletions(-) create mode 100644 examples/openresty/app/luamqtt-example.lua delete mode 100644 examples/openresty/app/main-ioloop.lua delete mode 100644 examples/openresty/app/main-sync.lua create mode 100755 examples/openresty/quit.sh diff --git a/.gitignore b/.gitignore index 1bc6e22..43f6a90 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ local/ luacov.stats.out luacov.report.out +examples/openresty/logs/*.log *.swp diff --git a/examples/openresty/README.md b/examples/openresty/README.md index 9f8bd92..1d8e659 100644 --- a/examples/openresty/README.md +++ b/examples/openresty/README.md @@ -1,29 +1,28 @@ # openresty example -Provided example is based on [the official Getting Started article](https://openresty.org/en/getting-started.html). +Openresty is primarily a server, and accepts incoming connections. +This means that running an MQTT client inside OpenResty will require +some tricks. -There is a two ways to run MQTT client in openresty: +Since OpenResty sockets cannot pass a context boundary (without being +closed), and we need a background task listening on the socket, we're +creating a timer context, and then handle everything from within that +context. -* in synchronous mode -* in ioloop mode +In the timer we'll spawn a thread that will do the listening, and the +timer itself will go in an endless loop to do the keepalives. -# synchronous mode +**Caveats** -Started MQTT client is connecting, subscribing and waiting for incoming MQTT publications as you code it, without any magic asynchronous work. - -**Caveats**: The keep_alive feature will not work as there is no way for MQTT client to break its receive() operation in keep_alive interval and send PINGREQ packet to MQTT broker to maintain connection. It may lead to disconnects from MQTT broker side in absence of traffic in opened MQTT connection. After disconnecting from broker there is a way to reconnect using openresty's timer. - -# ioloop mode - -Started MQTT client is connecting, subscribing and waiting for incoming MQTT publications as you code it, maintaining established connection using PINGREQ packets to broker in configured keep_alive interval. - -**Caveats**: own luamqtt's ioloop is based on the ability of sockets to timeout its receive() operation, allowing MQTT client to awake in some configured interval and send PINGREQ packet to broker to maintain opened connection, but on every timeout the openresty is writing such in its error.log: - - stream lua tcp socket read timed out, context: ngx.timer +* Due to the socket limitation we cannot Publish anything from another + context. If you run into "bad request" errors on socket operations, you + are probably accessing a socket from another context. +* In the long run, timers do leak memory, since timer contexts are + supposed to be short-lived. Consider implementing a secondary mechanism + to restart the timer-context and restart the client. # Files -* [conf/nginx.conf](conf/nginx.conf): configuration for the nginx daemon to run lua script -* [app/main-sync.lua](app/main-sync.lua): example lua script maintaining connection to some MQTT broker, in synchronous mode -* [app/main-ioloop.lua](app/main-ioloop.lua): example lua script maintaining connection to some MQTT broker, in ioloop mode -* start.sh, stop.sh, restart.sh: optional scripts to manage openresty instance +* [conf/nginx.conf](conf/nginx.conf): configuration for the nginx daemon to run lua scripts +* [app/luamqtt-example.lua](app/luamqtt-example.lua): example lua script maintaining connection +* `start.sh`, `stop.sh`, `quit.sh`, `restart.sh`: optional scripts to manage the OpenResty instance diff --git a/examples/openresty/app/luamqtt-example.lua b/examples/openresty/app/luamqtt-example.lua new file mode 100644 index 0000000..25c1f55 --- /dev/null +++ b/examples/openresty/app/luamqtt-example.lua @@ -0,0 +1,94 @@ +-- runs in init_worker_by_lua phase + +-- IMPORTANT: set up logging before loading MQTT lib +if pcall(require, "logging.nginx") then + -- LuaLogging nginx forwarder is available + ngx.log(ngx.INFO, "forwarding LuaMQTT logs to nginx log, using LuaLogging 'nginx' logger") + local ll = require("logging") + ll.defaultLogger(ll.nginx()) -- forward logs to nginx logs +else + ngx.log(ngx.WARN, "LuaLogging module 'logging.nginx' not found, it is strongly recommended to install that module. ", + "See https://github.com/lunarmodules/lualogging.") +end + + +local mqtt = require "mqtt" + + +local function client_add(client) + local ok, err = ngx.timer.at(0, function() + -- spawn a thread to listen on the socket + local coro = ngx.thread.spawn(function() + while true do + local sleeptime = client:step() + if not sleeptime then + ngx.log(ngx.INFO, "MQTT client '", client.opts.id, "' exited, stopping client-thread") + return + else + if sleeptime > 0 then + ngx.sleep(sleeptime * 1000) + end + end + end + end) + + -- endless keep-alive loop + while not ngx.worker.exiting() do + ngx.sleep((client:check_keep_alive())) -- double (()) to trim to 1 argument + end + + -- exiting + ngx.log(ngx.DEBUG, "MQTT client '", client.opts.id, "' keep-alive loop exited") + client:disconnect() + ngx.thread.wait(coro) + ngx.log(ngx.DEBUG, "MQTT client '", client.opts.id, "' exit complete") + end) + + if not ok then + ngx.log(ngx.CRIT, "Failed to start timer-context for device '", client.id,"': ", err) + end +end + + + +-- create mqtt client +local client = mqtt.client{ + -- NOTE: this broker is not working sometimes; comment username = "..." below if you still want to use it + -- uri = "test.mosquitto.org", + uri = "mqtts://mqtt.flespi.io", + -- NOTE: more about flespi tokens: https://flespi.com/kb/tokens-access-keys-to-flespi-platform + username = "stPwSVV73Eqw5LSv0iMXbc4EguS7JyuZR9lxU5uLxI5tiNM8ToTVqNpu85pFtJv9", + clean = true, + + -- event handlers + on = { + connect = function(connack, self) + if connack.rc ~= 0 then + return + end + + -- subscribe to test topic and publish message after it + assert(self:subscribe{ topic="luamqtt/#", qos=1, callback=function() + -- publish test message + assert(self:publish{ + topic = "luamqtt/simpletest", + payload = "hello", + qos = 1 + }) + end}) + end, + + message = function(msg, self) + assert(self:acknowledge(msg)) + + ngx.log(ngx.INFO, "received:", msg) + end, + + close = function(conn) + ngx.log(ngx.INFO, "MQTT conn closed:", conn.close_reason) + end + } +} + +-- start the client +client_add(client) diff --git a/examples/openresty/app/main-ioloop.lua b/examples/openresty/app/main-ioloop.lua deleted file mode 100644 index 86f4552..0000000 --- a/examples/openresty/app/main-ioloop.lua +++ /dev/null @@ -1,96 +0,0 @@ --- luacheck: globals ngx -local log = ngx.log -local timer_at = ngx.timer.at -local ERR = ngx.ERR -local tbl_concat = table.concat - -local function trace(...) - local line = {} - for i = 1, select("#", ...) do - line[i] = tostring(select(i, ...)) - end - log(ERR, tbl_concat(line, " ")) -end - -trace("main.lua started") - -local start_timer - -local function on_timer(...) - trace("on_timer: ", ...) - - local mqtt = require("mqtt") - local ioloop = require("mqtt.ioloop") - - -- create mqtt client - local client = mqtt.client{ - -- NOTE: this broker is not working sometimes; comment username = "..." below if you still want to use it - -- uri = "test.mosquitto.org", - uri = "mqtt.flespi.io", - -- NOTE: more about flespi tokens: https://flespi.com/kb/tokens-access-keys-to-flespi-platform - username = "stPwSVV73Eqw5LSv0iMXbc4EguS7JyuZR9lxU5uLxI5tiNM8ToTVqNpu85pFtJv9", - clean = true, - connector = require("mqtt.ngxsocket"), - secure = true, -- optional - } - trace("created MQTT client", client) - - client:on{ - connect = function(connack) - if connack.rc ~= 0 then - trace("connection to broker failed:", connack) - return - end - trace("connected:", connack) -- successful connection - - -- subscribe to test topic and publish message after it - assert(client:subscribe{ topic="luamqtt/#", qos=1, callback=function(suback) - trace("subscribed:", suback) - - -- publish test message - trace('publishing test message "hello" to "luamqtt/simpletest" topic...') - assert(client:publish{ - topic = "luamqtt/simpletest", - payload = "hello", - qos = 1 - }) - end}) - end, - - message = function(msg) - assert(client:acknowledge(msg)) - - trace("received:", msg) - end, - - error = function(err) - trace("MQTT client error:", err) - end, - - close = function(conn) - trace("MQTT conn closed:", conn.close_reason) - end - } - - trace("begin ioloop") - local loop = ioloop.create{ - timeout = client.args.keep_alive, - sleep_function = ngx.sleep, - } - loop:add(client) - client:start_connecting() - loop:run_until_clients() - trace("done ioloop") - - -- to reconnect - start_timer() -end - -start_timer = function() - local ok, err = timer_at(1, on_timer) - if not ok then - trace("failed to start timer:", err) - end -end - -start_timer() diff --git a/examples/openresty/app/main-sync.lua b/examples/openresty/app/main-sync.lua deleted file mode 100644 index 09e4158..0000000 --- a/examples/openresty/app/main-sync.lua +++ /dev/null @@ -1,89 +0,0 @@ --- luacheck: globals ngx -local log = ngx.log -local timer_at = ngx.timer.at -local ERR = ngx.ERR -local tbl_concat = table.concat - -local function trace(...) - local line = {} - for i = 1, select("#", ...) do - line[i] = tostring(select(i, ...)) - end - log(ERR, tbl_concat(line, " ")) -end - -trace("main.lua started") - -local start_timer - -local function on_timer(...) - trace("on_timer: ", ...) - - local mqtt = require("mqtt") - - -- create mqtt client - local client = mqtt.client{ - -- NOTE: this broker is not working sometimes; comment username = "..." below if you still want to use it - -- uri = "test.mosquitto.org", - uri = "mqtt.flespi.io", - -- NOTE: more about flespi tokens: https://flespi.com/kb/tokens-access-keys-to-flespi-platform - username = "stPwSVV73Eqw5LSv0iMXbc4EguS7JyuZR9lxU5uLxI5tiNM8ToTVqNpu85pFtJv9", - clean = true, - connector = require("mqtt.ngxsocket"), - secure = true, -- optional - } - trace("created MQTT client", client) - - client:on{ - connect = function(connack) - if connack.rc ~= 0 then - trace("connection to broker failed:", connack) - return - end - trace("connected:", connack) -- successful connection - - -- subscribe to test topic and publish message after it - assert(client:subscribe{ topic="luamqtt/#", qos=1, callback=function(suback) - trace("subscribed:", suback) - - -- publish test message - trace('publishing test message "hello" to "luamqtt/simpletest" topic...') - assert(client:publish{ - topic = "luamqtt/simpletest", - payload = "hello", - qos = 1 - }) - end}) - end, - - message = function(msg) - assert(client:acknowledge(msg)) - - trace("received:", msg) - end, - - error = function(err) - trace("MQTT client error:", err) - end, - - close = function(conn) - trace("MQTT conn closed:", conn.close_reason) - end - } - - trace("running client in synchronous input/output loop") - mqtt.run_sync(client) - trace("done, synchronous input/output loop is stopped") - - -- to reconnect - start_timer() -end - -start_timer = function() - local ok, err = timer_at(1, on_timer) - if not ok then - trace("failed to start timer:", err) - end -end - -start_timer() diff --git a/examples/openresty/conf/nginx.conf b/examples/openresty/conf/nginx.conf index ad673c6..7b4dfdb 100644 --- a/examples/openresty/conf/nginx.conf +++ b/examples/openresty/conf/nginx.conf @@ -1,5 +1,16 @@ +# the code block below runs in "init-worker", meaning that each worker +# will create it's own client. By default the number of worker processes +# will equal the number of CPU-cores. +# Since we want only a single client, we set the number of workers to 1. worker_processes 1; -error_log logs/error.log; + + +# Set the log destination, and the log level. This is for Nginx logs only. +# LuaMQTT uses LuaLogging for logs, it is strongly recommended to install it. +# The example code will configure LuaLogging to automatically forward the logs +# to the nginx log-file specified here, once LuaLogging nginx logger is found. +error_log logs/error.log debug; + events { worker_connections 1024; @@ -12,6 +23,7 @@ stream { resolver 8.8.8.8; - init_worker_by_lua_file "app/main-sync.lua"; + # the code file to execute + init_worker_by_lua_file "app/luamqtt-example.lua"; } diff --git a/examples/openresty/quit.sh b/examples/openresty/quit.sh new file mode 100755 index 0000000..57b15d7 --- /dev/null +++ b/examples/openresty/quit.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -e + +PATH=/opt/openresty/nginx/sbin:$PATH +export PATH +nginx -p "$(pwd)" -s quit diff --git a/examples/openresty/start.sh b/examples/openresty/start.sh index 93734fb..50bf45b 100755 --- a/examples/openresty/start.sh +++ b/examples/openresty/start.sh @@ -4,5 +4,8 @@ set -e PATH=/opt/openresty/nginx/sbin:$PATH export PATH -nginx -p `pwd`/ -c conf/nginx.conf +nginx -p "$(pwd)" -c conf/nginx.conf +# since this is an example, start tailing the logs +touch logs/error.log +tail -F logs/error.log diff --git a/examples/openresty/stop.sh b/examples/openresty/stop.sh index 87ce494..f7307de 100755 --- a/examples/openresty/stop.sh +++ b/examples/openresty/stop.sh @@ -4,5 +4,4 @@ set -e PATH=/opt/openresty/nginx/sbin:$PATH export PATH -nginx -p `pwd`/ -s stop - +nginx -p "$(pwd)" -s stop diff --git a/mqtt/connector/nginx.lua b/mqtt/connector/nginx.lua index dbb9c7a..61912fa 100644 --- a/mqtt/connector/nginx.lua +++ b/mqtt/connector/nginx.lua @@ -13,7 +13,7 @@ local ngx_socket_tcp = ngx.socket.tcp function ngxsocket:validate() if self.secure then assert(self.ssl_module == "ssl", "specifying custom ssl module when using Nginx connector is not supported") - assert(type(self.secure_params) == "table", "expecting .secure_params to be a table") + assert(self.secure_params == nil or type(self.secure_params) == "table", "expecting .secure_params to be a table if given") -- TODO: validate nginx stuff end end @@ -24,7 +24,7 @@ end function ngxsocket:connect() local sock = ngx_socket_tcp() -- set read-timeout to 'nil' to not timeout at all - assert(sock:settimeouts(self.timeout * 1000, self.timeout * 1000, nil)) -- millisecs + sock:settimeouts(self.timeout * 1000, self.timeout * 1000, 24*60*60*1000) -- millisecs local ok, err = sock:connect(self.host, self.port) if not ok then return false, "socket:connect failed: "..err From dc1c26187162f0a1c3547473dced9cea53b1b989 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Mon, 8 Nov 2021 19:41:13 +0100 Subject: [PATCH 32/65] feat(*) add nginx/copas specific code for adding clients It uses the same auto-detection as the connectors, and has the same signatures. So client code can be implemented quickly independent of the environment/ioloop in use --- .luacheckrc | 3 +- examples/copas-example.lua | 24 +-------- examples/copas.lua | 26 +--------- examples/openresty/app/luamqtt-example.lua | 39 +-------------- mqtt/connector/init.lua | 31 +++++------- mqtt/init.lua | 3 -- mqtt/loop/copas.lua | 50 +++++++++++++++++++ mqtt/loop/detect.lua | 30 ++++++++++++ mqtt/loop/init.lua | 15 ++++++ mqtt/loop/ioloop.lua | 14 ++++++ mqtt/loop/nginx.lua | 57 ++++++++++++++++++++++ rockspecs/luamqtt-3.4.3-1.rockspec | 5 ++ 12 files changed, 190 insertions(+), 107 deletions(-) create mode 100644 mqtt/loop/copas.lua create mode 100644 mqtt/loop/detect.lua create mode 100644 mqtt/loop/init.lua create mode 100644 mqtt/loop/ioloop.lua create mode 100644 mqtt/loop/nginx.lua diff --git a/.luacheckrc b/.luacheckrc index 49306e6..4a1bf9e 100644 --- a/.luacheckrc +++ b/.luacheckrc @@ -22,7 +22,8 @@ include_files = { files["tests/spec/**/*.lua"] = { std = "+busted" } files["examples/openresty/**/*.lua"] = { std = "+ngx_lua" } -files["mqtt/connector/init.lua"] = { std = "+ngx_lua" } +files["mqtt/loop/detect.lua"] = { std = "+ngx_lua" } +files["mqtt/loop/nginx.lua"] = { std = "+ngx_lua" } files["mqtt/connector/nginx.lua"] = { std = "+ngx_lua" } -- vim: ts=4 sts=4 sw=4 noet ft=lua diff --git a/examples/copas-example.lua b/examples/copas-example.lua index cadf852..9e3368b 100644 --- a/examples/copas-example.lua +++ b/examples/copas-example.lua @@ -2,6 +2,7 @@ local mqtt = require("mqtt") local copas = require("copas") +local add_client = require("mqtt.loop").add local num_pings = 10 -- total number of ping-pongs local delay = 1 -- delay between ping-pongs @@ -77,29 +78,6 @@ local pong = mqtt.client{ }, -- close 'on', event handlers } -local function add_client(cl) - -- add keep-alive timer - local timer = copas.addthread(function() - while cl do - copas.sleep(cl:check_keep_alive()) - end - end) - -- add client to connect and listen - copas.addthread(function() - while cl do - local timeout = cl:step() - if not timeout then - cl = nil -- exiting, inform keep-alive timer - copas.wakeup(timer) - else - if timeout > 0 then - copas.sleep(timeout) - end - end - end - end) -end - print("running copas loop...") add_client(ping) diff --git a/examples/copas.lua b/examples/copas.lua index aa12b5f..14aea68 100644 --- a/examples/copas.lua +++ b/examples/copas.lua @@ -1,6 +1,7 @@ -- load mqtt module local mqtt = require("mqtt") local copas = require("copas") +local add_client = require("mqtt.loop").add -- create mqtt client local client = mqtt.client{ @@ -54,30 +55,7 @@ local client = mqtt.client{ print("created MQTT client", client) -local function add_client(cl) - -- add keep-alive timer - local timer = copas.addthread(function() - while cl do - copas.sleep(cl:check_keep_alive()) - end - end) - -- add client to connect and listen - copas.addthread(function() - while cl do - local timeout = cl:step() - if not timeout then - cl = nil -- exiting - copas.wakeup(timer) - else - if timeout > 0 then - copas.sleep(timeout) - end - end - end - end) -end - - add_client(client) copas.loop() + print("done, copas loop is stopped") diff --git a/examples/openresty/app/luamqtt-example.lua b/examples/openresty/app/luamqtt-example.lua index 25c1f55..8138e6c 100644 --- a/examples/openresty/app/luamqtt-example.lua +++ b/examples/openresty/app/luamqtt-example.lua @@ -13,42 +13,7 @@ end local mqtt = require "mqtt" - - -local function client_add(client) - local ok, err = ngx.timer.at(0, function() - -- spawn a thread to listen on the socket - local coro = ngx.thread.spawn(function() - while true do - local sleeptime = client:step() - if not sleeptime then - ngx.log(ngx.INFO, "MQTT client '", client.opts.id, "' exited, stopping client-thread") - return - else - if sleeptime > 0 then - ngx.sleep(sleeptime * 1000) - end - end - end - end) - - -- endless keep-alive loop - while not ngx.worker.exiting() do - ngx.sleep((client:check_keep_alive())) -- double (()) to trim to 1 argument - end - - -- exiting - ngx.log(ngx.DEBUG, "MQTT client '", client.opts.id, "' keep-alive loop exited") - client:disconnect() - ngx.thread.wait(coro) - ngx.log(ngx.DEBUG, "MQTT client '", client.opts.id, "' exit complete") - end) - - if not ok then - ngx.log(ngx.CRIT, "Failed to start timer-context for device '", client.id,"': ", err) - end -end - +local add_client = require("mqtt.loop").add -- create mqtt client @@ -91,4 +56,4 @@ local client = mqtt.client{ } -- start the client -client_add(client) +add_client(client) diff --git a/mqtt/connector/init.lua b/mqtt/connector/init.lua index 0e0199e..826e587 100644 --- a/mqtt/connector/init.lua +++ b/mqtt/connector/init.lua @@ -1,22 +1,15 @@ --- auto detect the connector to use. -- This is based on a.o. libraries already loaded, so 'require' this -- module as late as possible (after the other modules) -local log = require "mqtt.log" - -if type(ngx) == "table" then - -- there is a global 'ngx' table, so we're running OpenResty - log:info("LuaMQTT auto-detected Nginx as the runtime environment") - return require("mqtt.connector.nginx") - -elseif package.loaded.copas then - -- 'copas' was already loaded - log:info("LuaMQTT auto-detected Copas as the io-loop in use") - return require("mqtt.connector.copas") - -elseif pcall(require, "socket") and tostring(require("socket")._VERSION):find("LuaSocket") then - -- LuaSocket is available - log:info("LuaMQTT auto-detected LuaSocket as the socket library to use") - return require("mqtt.connector.luasocket") -end - -error("connector auto-detection failed, please specify one explicitly") +local loops = setmetatable({ + copas = "mqtt.connector.copas", + nginx = "mqtt.connector.nginx", + ioloop = "mqtt.connector.luasocket" +}, { + __index = function() + error("failed to auto-detect connector to use, please set one explicitly", 2) + end +}) +local loop = require("mqtt.loop.detect")() + +return require(loops[loop]) diff --git a/mqtt/init.lua b/mqtt/init.lua index f47f462..7e968f9 100644 --- a/mqtt/init.lua +++ b/mqtt/init.lua @@ -60,9 +60,6 @@ function mqtt.run_ioloop(...) for i = 1, select("#", ...) do local cl = select(i, ...) loop:add(cl) - -- if type(cl) ~= "function" then -- TODO: remove - -- cl:start_connecting() - -- end end return loop:run_until_clients() end diff --git a/mqtt/loop/copas.lua b/mqtt/loop/copas.lua new file mode 100644 index 0000000..9b58661 --- /dev/null +++ b/mqtt/loop/copas.lua @@ -0,0 +1,50 @@ +local copas = require "copas" +local log = require "mqtt.log" + +local client_registry = {} + +local _M = {} + + +--- Add MQTT client to the Copas scheduler. +-- The client will automatically be removed after it exits. +-- @tparam cl client to add to the Copas scheduler +-- @return true on success or false and error message on failure +function _M.add(cl) + if client_registry[cl] then + log:warn("MQTT client '%s' was already added to Copas", cl.opts.id) + return false, "MQTT client was already added to Copas" + end + client_registry[cl] = true + + -- add keep-alive timer + local timer = copas.addthread(function() + while client_registry[cl] do + copas.sleep(cl:check_keep_alive()) + end + end) + + -- add client to connect and listen + copas.addthread(function() + while client_registry[cl] do + local timeout = cl:step() + if not timeout then + client_registry[cl] = nil -- exiting + log:debug("MQTT client '%s' exited, removed from Copas", cl.opts.id) + copas.wakeup(timer) + else + if timeout > 0 then + copas.sleep(timeout) + end + end + end + end) + + return true +end + +return setmetatable(_M, { + __call = function(self, ...) + return self.add(...) + end, +}) diff --git a/mqtt/loop/detect.lua b/mqtt/loop/detect.lua new file mode 100644 index 0000000..4fc15b6 --- /dev/null +++ b/mqtt/loop/detect.lua @@ -0,0 +1,30 @@ +--- Module returns a single function to detect the io-loop in use. +-- Either 'copas', 'nginx', or 'ioloop', or nil+error +local log = require "mqtt.log" + +local loop +return function() + if loop then return loop end + if type(ngx) == "table" then + -- there is a global 'ngx' table, so we're running OpenResty + log:info("LuaMQTT auto-detected Nginx as the runtime environment") + loop = "nginx" + return loop + + elseif package.loaded.copas then + -- 'copas' was already loaded + log:info("LuaMQTT auto-detected Copas as the io-loop in use") + loop = "copas" + return loop + + elseif pcall(require, "socket") and tostring(require("socket")._VERSION):find("LuaSocket") then + -- LuaSocket is available + log:info("LuaMQTT auto-detected LuaSocket as the socket library to use with mqtt-ioloop") + loop = "ioloop" + return loop + + else + -- unknown + return nil, "LuaMQTT io-loop/connector auto-detection failed, please specify one explicitly" + end +end diff --git a/mqtt/loop/init.lua b/mqtt/loop/init.lua new file mode 100644 index 0000000..e8976de --- /dev/null +++ b/mqtt/loop/init.lua @@ -0,0 +1,15 @@ +--- auto detect the connector to use. +-- This is based on a.o. libraries already loaded, so 'require' this +-- module as late as possible (after the other modules) +local loops = setmetatable({ + copas = "mqtt.loop.copas", + nginx = "mqtt.loop.nginx", + ioloop = "mqtt.loop.ioloop" +}, { + __index = function() + error("failed to auto-detect connector to use, please set one explicitly", 2) + end +}) +local loop = require("mqtt.loop.detect")() + +return require(loops[loop]) diff --git a/mqtt/loop/ioloop.lua b/mqtt/loop/ioloop.lua new file mode 100644 index 0000000..4b83038 --- /dev/null +++ b/mqtt/loop/ioloop.lua @@ -0,0 +1,14 @@ +local _M = {} + +local mqtt = require "mqtt" + +function _M.add(client) + local default_loop = mqtt.get_ioloop() + return default_loop:add(client) +end + +return setmetatable(_M, { + __call = function(self, ...) + return self.add(...) + end, +}) diff --git a/mqtt/loop/nginx.lua b/mqtt/loop/nginx.lua new file mode 100644 index 0000000..bcea387 --- /dev/null +++ b/mqtt/loop/nginx.lua @@ -0,0 +1,57 @@ + +local client_registry = {} + +local _M = {} + + +--- Add MQTT client to the Nginx environment. +-- The client will automatically be removed after it exits. +-- @tparam cl client to add +-- @return true on success or false and error message on failure +function _M.add(client) + if client_registry[client] then + ngx.log(ngx.WARN, "MQTT client '%s' was already added to Nginx", client.opts.id) + return false, "MQTT client was already added to Nginx" + end + + local ok, err = ngx.timer.at(0, function() + -- spawn a thread to listen on the socket + local coro = ngx.thread.spawn(function() + while true do + local sleeptime = client:step() + if not sleeptime then + ngx.log(ngx.INFO, "MQTT client '", client.opts.id, "' exited, stopping client-thread") + client_registry[client] = nil + return + else + if sleeptime > 0 then + ngx.sleep(sleeptime * 1000) + end + end + end + end) + + -- endless keep-alive loop + while not ngx.worker.exiting() do + ngx.sleep((client:check_keep_alive())) -- double (()) to trim to 1 argument + end + + -- exiting + client_registry[client] = nil + ngx.log(ngx.DEBUG, "MQTT client '", client.opts.id, "' keep-alive loop exited") + client:disconnect() + ngx.thread.wait(coro) + ngx.log(ngx.DEBUG, "MQTT client '", client.opts.id, "' exit complete") + end) + + if not ok then + ngx.log(ngx.CRIT, "Failed to start timer-context for device '", client.id,"': ", err) + end +end + + +return setmetatable(_M, { + __call = function(self, ...) + return self.add(...) + end, +}) diff --git a/rockspecs/luamqtt-3.4.3-1.rockspec b/rockspecs/luamqtt-3.4.3-1.rockspec index 65dcd16..a48cfa9 100644 --- a/rockspecs/luamqtt-3.4.3-1.rockspec +++ b/rockspecs/luamqtt-3.4.3-1.rockspec @@ -38,5 +38,10 @@ build = { ["mqtt.connector.luasocket"] = "mqtt/connector/luasocket.lua", ["mqtt.connector.copas"] = "mqtt/connector/copas.lua", ["mqtt.connector.nginx"] = "mqtt/connector/nginx.lua", + ["mqtt.loop.init"] = "mqtt/loop/init.lua", + ["mqtt.loop.detect"] = "mqtt/loop/detect.lua", + ["mqtt.loop.ioloop"] = "mqtt/loop/ioloop.lua", + ["mqtt.loop.copas"] = "mqtt/loop/copas.lua", + ["mqtt.loop.nginx"] = "mqtt/loop/nginx.lua", }, } From 2d10112b5f3b3516bca82f7b7b1ffe4a61a7d244 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Mon, 8 Nov 2021 20:15:09 +0100 Subject: [PATCH 33/65] fix(example) typos --- examples/last-will/client-1.lua | 4 ++-- examples/last-will/client-2.lua | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/last-will/client-1.lua b/examples/last-will/client-1.lua index aef30c5..d8c607b 100644 --- a/examples/last-will/client-1.lua +++ b/examples/last-will/client-1.lua @@ -2,10 +2,10 @@ local mqtt = require("mqtt") -- create mqtt client local client = mqtt.client{ - id = "mqtts://luamqtt-example-will-1", + id = "luamqtt-example-will-1", -- NOTE: this broker is not working sometimes; comment username = "..." below if you still want to use it -- uri = "test.mosquitto.org", - uri = "mqtt.flespi.io", + uri = "mqtts://mqtt.flespi.io", -- NOTE: more about flespi tokens: https://flespi.com/kb/tokens-access-keys-to-flespi-platform username = "stPwSVV73Eqw5LSv0iMXbc4EguS7JyuZR9lxU5uLxI5tiNM8ToTVqNpu85pFtJv9", clean = true, diff --git a/examples/last-will/client-2.lua b/examples/last-will/client-2.lua index 2b917ca..7c9988c 100644 --- a/examples/last-will/client-2.lua +++ b/examples/last-will/client-2.lua @@ -2,10 +2,10 @@ local mqtt = require("mqtt") -- create mqtt client local client = mqtt.client{ - id = "mqtts://luamqtt-example-will-2", + id = "luamqtt-example-will-2", -- NOTE: this broker is not working sometimes; comment username = "..." below if you still want to use it -- uri = "test.mosquitto.org", - uri = "mqtt.flespi.io", + uri = "mqtts://mqtt.flespi.io", -- NOTE: more about flespi tokens: https://flespi.com/kb/tokens-access-keys-to-flespi-platform username = "stPwSVV73Eqw5LSv0iMXbc4EguS7JyuZR9lxU5uLxI5tiNM8ToTVqNpu85pFtJv9", clean = true, From b5d03b8c27202a347be7e49f65adec19c69b81f8 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Tue, 9 Nov 2021 12:32:57 +0100 Subject: [PATCH 34/65] chore(docs) update docs --- LICENSE | 4 +- README.md | 144 ++------- docs/config.ld | 27 +- docs_topics/01-installation.md | 15 + docs_topics/02-dependencies.md | 31 ++ docs_topics/03-lua_versions.md | 9 + docs_topics/04-mqtt_versions.md | 8 + docs_topics/05-connectors.md | 30 ++ examples/openresty/README.md | 3 +- .../{luamqtt-example.lua => openresty.lua} | 20 +- examples/openresty/conf/nginx.conf | 2 +- mqtt/client.lua | 281 ++++++++++-------- mqtt/connector/copas.lua | 26 +- mqtt/connector/init.lua | 25 +- mqtt/connector/luasocket.lua | 23 +- mqtt/connector/nginx.lua | 27 +- mqtt/init.lua | 67 ++--- mqtt/ioloop.lua | 122 ++++---- mqtt/loop/copas.lua | 12 +- mqtt/loop/init.lua | 28 +- mqtt/loop/ioloop.lua | 10 + mqtt/loop/nginx.lua | 14 +- 22 files changed, 552 insertions(+), 376 deletions(-) create mode 100644 docs_topics/01-installation.md create mode 100644 docs_topics/02-dependencies.md create mode 100644 docs_topics/03-lua_versions.md create mode 100644 docs_topics/04-mqtt_versions.md create mode 100644 docs_topics/05-connectors.md rename examples/openresty/app/{luamqtt-example.lua => openresty.lua} (85%) diff --git a/LICENSE b/LICENSE index 1bd5a40..0233293 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ -MIT License +# MIT License -Copyright (c) 2018 Alexander Kiranov +Copyright (c) 2018-2021 Alexander Kiranov Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index 215a884..deddeae 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,6 @@ -# luamqtt - Pure-lua MQTT v3.1.1 and v5.0 client +# luamqtt + +Pure-lua MQTT v3.1.1 and v5.0 client ![luamqtt logo](./logo.svg) @@ -17,7 +19,7 @@ This library is written in **pure-lua** to provide maximum portability. * Full MQTT v3.1.1 client-side support * Full MQTT v5.0 client-side support -* Several long-living MQTT clients in one script thanks to ioloop +* Support for Copas, OpenResty/Nginx, and an included lightweight ioloop. # Documentation @@ -31,127 +33,6 @@ See [flespi forum thread](https://forum.flespi.com/d/97-luamqtt-mqtt-client-writ [https://github.com/xHasKx/luamqtt](https://github.com/xHasKx/luamqtt) -# Dependencies - -The only main dependency is a [**luasocket**](https://luarocks.org/modules/luasocket/luasocket) to establishing TCP connection to the MQTT broker. Install it like this: - -```sh -luarocks install luasocket -``` - -On Lua 5.1 it also depends on [**LuaBitOp**](http://bitop.luajit.org/) (**bit**) library to perform bitwise operations. -It's not listed in package dependencies, please install it manually like this: - -```sh -luarocks install luabitop -``` - -## luasec (SSL/TLS) - -To establish secure network connection (SSL/TSL) to MQTT broker -you also need [**luasec**](https://github.com/brunoos/luasec) module, please install it manually like this: - -```sh -luarocks install luasec -``` - -This stage is optional and may be skipped if you don't need the secure network connection (e.g. broker is located in your local network). - -# Lua versions - -It's tested to work on Debian 9 GNU/Linux with Lua versions: - -* Lua 5.1 ... Lua 5.4 (**i.e. any modern Lua version**) -* LuaJIT 2.0.0 ... LuaJIT 2.1.0 beta3 -* It may also work on other Lua versions without any guarantees - -Also I've successfully run it under **Windows** and it was ok, but installing luarock-modules may be a non-trivial task on this OS. - -# Installation - -As the luamqtt is almost zero-dependency you have to install required Lua libraries by yourself, before using the luamqtt library: - -```sh -luarocks install luasocket # optional if you will use your own connectors (see below) -luarocks install luabitop # you don't need this for lua 5.3 and above -luarocks install luasec # you don't need this if you don't want to use SSL connections -``` - -Then you may install the luamqtt library itself: - -```sh -luarocks install luamqtt -``` - -Or for development purposes; - -```sh -# development branch: -luarocks install luamqtt --dev - -# or from the cloned repo: -luarocks make -``` - -[LuaRocks page](http://luarocks.org/modules/xhaskx/luamqtt) - -# Examples - -Here is a short version of [`examples/simple.lua`](examples/simple.lua): - -```lua --- load mqtt library -local mqtt = require("mqtt") - --- create MQTT client, flespi tokens info: https://flespi.com/kb/tokens-access-keys-to-flespi-platform -local client = mqtt.client{ uri = "mqtt.flespi.io", username = os.getenv("FLESPI_TOKEN"), clean = true } - --- assign MQTT client event handlers -client:on{ - connect = function(connack) - if connack.rc ~= 0 then - print("connection to broker failed:", connack:reason_string(), connack) - return - end - - -- connection established, now subscribe to test topic and publish a message after - assert(client:subscribe{ topic="luamqtt/#", qos=1, callback=function() - assert(client:publish{ topic = "luamqtt/simpletest", payload = "hello" }) - end}) - end, - - message = function(msg) - assert(client:acknowledge(msg)) - - -- receive one message and disconnect - print("received message", msg) - client:disconnect() - end, -} - --- run ioloop for client -mqtt.run_ioloop(client) -``` - -More examples placed in [`examples/`](examples/) directory. Also checkout tests in [`tests/spec/mqtt-client.lua`](tests/spec/mqtt-client.lua) - -Also you can learn MQTT protocol by reading [`tests/spec/protocol4-make.lua`](tests/spec/protocol4-make.lua) and [`tests/spec/protocol4-parse.lua`](tests/spec/protocol4-parse.lua) tests - -# Connectors - -Connector is a network connection layer for luamqtt. There is a three standard connectors included: - -* [`luasocket`](mqtt/luasocket.lua) -* [`luasocket_ssl`](mqtt/luasocket_ssl.lua) -* [`ngxsocket`](mqtt/ngxsocket.lua) - for using in [openresty environment](examples/openresty) - -The `luasocket` or `luasocket_ssl` connector will be used by default, if not specified, according `secure=true/false` option per MQTT client. - -In simple terms, connector is a set of functions to establish a network stream (TCP connection usually) and send/receive data through it. -Every MQTT client instance may have their own connector. - -And it's very simple to implement your own connector to make luamqtt works in your environment. - # Bugs & contributing Please [file a GitHub issue](https://github.com/xHasKx/luamqtt/issues) if you found any bug. @@ -160,10 +41,21 @@ And of course, any contribution are welcome! # Tests -To run tests in this git repo you need [**busted**](https://luarocks.org/modules/olivine-labs/busted): +To run tests in this git repo you need [**busted**](https://luarocks.org/modules/olivine-labs/busted) as well as some dependencies: + +Prepare: +```sh +luarocks install busted +luarocks install luacov +luarocks install luasocket +luarocks install luasec +luarocks install copas +luarocks install lualogging +``` +Running the tests: ```sh -busted -e 'package.path="./?/init.lua;./?.lua;"..package.path' tests/spec/*.lua +busted ``` There is a script to run all tests for all supported lua versions, using [hererocks](https://github.com/mpeterv/hererocks): @@ -180,7 +72,7 @@ To collect code coverage stats - install luacov using luarocks and then execute: ```sh # collect stats during tests -busted -v -e 'package.path="./?/init.lua;./?.lua;"..package.path;require("luacov.runner")(".luacov")' tests/spec/*.lua +busted --coverage # generate report into luacov.report.out file luacov diff --git a/docs/config.ld b/docs/config.ld index 19827bf..8d97a61 100644 --- a/docs/config.ld +++ b/docs/config.ld @@ -1,7 +1,21 @@ -- usage: -- execute `ldoc .` in this docs directory -file = {"../mqtt/init.lua", "../mqtt/const.lua", "../mqtt/client.lua", "../mqtt/ioloop.lua", "../mqtt/protocol.lua"} +file = { + "../mqtt/init.lua", + "../mqtt/client.lua", + "../mqtt/ioloop.lua", + + "../mqtt/loop/init.lua", + "../mqtt/loop/copas.lua", + "../mqtt/loop/nginx.lua", + "../mqtt/loop/ioloop.lua", + + "../mqtt/connector/init.lua", + "../mqtt/connector/copas.lua", + "../mqtt/connector/nginx.lua", + "../mqtt/connector/luasocket.lua", +} project = "luamqtt" package = "mqtt" dir = "." @@ -12,12 +26,17 @@ full_description = "Source code: https://github.com/xHasKx/luamqtt" examples = { "../examples/simple.lua", - "../examples/sync.lua", "../examples/mqtt5-simple.lua", - "../examples/copas-example.lua", + "../examples/copas.lua", + "../examples/openresty/app/openresty.lua", } -topics = {"../README.md", "../LICENSE"} +use_markdown_titles = true +topics = { + "../README.md", + "../LICENSE", + "../docs_topics/", +} format = "markdown" plain = true diff --git a/docs_topics/01-installation.md b/docs_topics/01-installation.md new file mode 100644 index 0000000..94f0cd9 --- /dev/null +++ b/docs_topics/01-installation.md @@ -0,0 +1,15 @@ +# Installation + +As luamqtt is almost zero-dependency you have to install any optional Lua libraries by +yourself, before using the luamqtt library. + +When installing using [LuaRocks](http://luarocks.org/modules/xhaskx/luamqtt), the +LuaSocket dependency will automatically be installed as well, as it is a listed dependency +in the rockspec. + + luarocks install luamqtt + +To install from source clone the repo and make sure the `./mqtt/` folder is in your +Lua search path. + +Check the [dependencies](./02-dependencies.md.html) on how (and when) to install those. diff --git a/docs_topics/02-dependencies.md b/docs_topics/02-dependencies.md new file mode 100644 index 0000000..aede65e --- /dev/null +++ b/docs_topics/02-dependencies.md @@ -0,0 +1,31 @@ +# Dependencies + +The dependencies differ slightly based on the environment you use, and the requirements you have: + +* [**luasocket**](https://luarocks.org/modules/luasocket/luasocket) to establish TCP connections to the MQTT broker. + This is a listed dependency in the luamqtt rockspec, so it will automatically be installed if you use LuaRocks to + install luamqtt. To install it manually: + luarocks install luasocket + +* [**copas**](https://github.com/keplerproject/copas) module for asynchoneous IO. Copas is an advanced co-routine + scheduler with far more features than the included `ioloop`. For anything more than a few devices, or for devices which + require network IO beyond mqtt alone, Copas is the better alternative. Copas is also pure-Lua, but has parallel network + IO (as opposed to sequential network IO in `ioloop`), and has features like; threads, timers, locks, semaphores, and + non-blocking clients for http(s), (s)ftp, and smtp. + luarocks install copas + +* [**luasec**](https://github.com/brunoos/luasec) module for SSL/TLS based connections. This is optional and may be + skipped if you don't need secure network connections (e.g. broker is located in your local network). It's not listed + in package dependencies, please install it manually like this: + luarocks install luasec + +* [**LuaBitOp**](http://bitop.luajit.org/) library to perform bitwise operations, which is required only on + Lua 5.1. It's not listed in package dependencies, please install it manually like this: + luarocks install luabitop + +* [**LuaLogging**](https://github.com/lunarmodules/lualogging/) to enable logging by the MQTT client. This is optional + but highly recommended for long running clients. This is a great debugging aid when developing your clients. Also when + using OpenResty as your runtime, you'll definitely want to use this, see + [openresty.lua](https://xhaskx.github.io/luamqtt/examples/openresty.lua.html) for an example. + It's not listed in package dependencies, please install it manually like this: + luarocks install lualogging diff --git a/docs_topics/03-lua_versions.md b/docs_topics/03-lua_versions.md new file mode 100644 index 0000000..3674f37 --- /dev/null +++ b/docs_topics/03-lua_versions.md @@ -0,0 +1,9 @@ +# Lua versions + +It's tested to work on Debian 9 GNU/Linux with Lua versions: + +* Lua 5.1 ... Lua 5.3 (**i.e. any modern Lua version**) +* LuaJIT 2.0.0 ... LuaJIT 2.1.0 beta3 +* It may also work on other Lua versions without any guarantees + +Also has run under **Windows** and it was ok, but installing luarocks-modules may be a non-trivial task on this OS. diff --git a/docs_topics/04-mqtt_versions.md b/docs_topics/04-mqtt_versions.md new file mode 100644 index 0000000..db10c1b --- /dev/null +++ b/docs_topics/04-mqtt_versions.md @@ -0,0 +1,8 @@ +# MQTT versions + +Currently supported versions: + +* [MQTT v3.1.1 protocol](http://docs.oasis-open.org/mqtt/mqtt/v3.1.1/errata01/os/mqtt-v3.1.1-errata01-os-complete.html) version. +* [MQTT v5.0 protocol](http://docs.oasis-open.org/mqtt/mqtt/v5.0/mqtt-v5.0.html) version. + +Both protocols have full control packets support. diff --git a/docs_topics/05-connectors.md b/docs_topics/05-connectors.md new file mode 100644 index 0000000..9023962 --- /dev/null +++ b/docs_topics/05-connectors.md @@ -0,0 +1,30 @@ +# Connectors + +A connector is a network connection layer for luamqtt. This ensures clean separation between the socket +implementation and the client/protocol implementation. + +By default luamqtt ships with connectors for `ioloop`, `copas`, and `nginx`. It will auto-detect which +one to use using the `mqtt.loop` module. + +## building your own + +If you have a different socket implementation you can write your own connector. + +There are 2 base-classes `mqtt.connector.base.buffered-base` and `mqtt.connector.base.non-buffered-base` +to build on, which to pick depends on the environment. + +The main question is what event/io loop mechanism does your implementation have? + +* a single main (co)routione that runs, and doesn't yield when doing network IO. In this case + you should use the `buffered_base` and read on sockets with a `0` timeout. Check the + `mqtt.connector.luasocket` implementation for an example (this is what `ioloop` uses). + +* multiple co-routines that run within a scheduler, and doing non-blocking network IO (receive/send + will implicitly yield control to the scheduler so it will run other tasks until the socket is ready). + This is what Copas and Nginx do, and it requires the `non_buffered_base`. + +The main thing to look for when checking out the existing implementations is the network timeout settings, +and the returned `signals`. + + + diff --git a/examples/openresty/README.md b/examples/openresty/README.md index 1d8e659..6479473 100644 --- a/examples/openresty/README.md +++ b/examples/openresty/README.md @@ -24,5 +24,6 @@ timer itself will go in an endless loop to do the keepalives. # Files * [conf/nginx.conf](conf/nginx.conf): configuration for the nginx daemon to run lua scripts -* [app/luamqtt-example.lua](app/luamqtt-example.lua): example lua script maintaining connection +* [app/openresty.lua](app/openresty.lua): example lua script maintaining connection +* [mqtt/loop/nginx.lua](mqtt/loop/nginx.lua): how to add a client in an Nginx environment * `start.sh`, `stop.sh`, `quit.sh`, `restart.sh`: optional scripts to manage the OpenResty instance diff --git a/examples/openresty/app/luamqtt-example.lua b/examples/openresty/app/openresty.lua similarity index 85% rename from examples/openresty/app/luamqtt-example.lua rename to examples/openresty/app/openresty.lua index 8138e6c..baec0f4 100644 --- a/examples/openresty/app/luamqtt-example.lua +++ b/examples/openresty/app/openresty.lua @@ -33,14 +33,18 @@ local client = mqtt.client{ end -- subscribe to test topic and publish message after it - assert(self:subscribe{ topic="luamqtt/#", qos=1, callback=function() - -- publish test message - assert(self:publish{ - topic = "luamqtt/simpletest", - payload = "hello", - qos = 1 - }) - end}) + assert(self:subscribe { + topic = "luamqtt/#", + qos = 1, + callback = function() + -- publish test message + assert(self:publish{ + topic = "luamqtt/simpletest", + payload = "hello", + qos = 1 + }) + end + }) end, message = function(msg, self) diff --git a/examples/openresty/conf/nginx.conf b/examples/openresty/conf/nginx.conf index 7b4dfdb..6dc6fdd 100644 --- a/examples/openresty/conf/nginx.conf +++ b/examples/openresty/conf/nginx.conf @@ -24,6 +24,6 @@ stream { resolver 8.8.8.8; # the code file to execute - init_worker_by_lua_file "app/luamqtt-example.lua"; + init_worker_by_lua_file "app/openresty.lua"; } diff --git a/mqtt/client.lua b/mqtt/client.lua index 3ecb50c..e81a245 100644 --- a/mqtt/client.lua +++ b/mqtt/client.lua @@ -1,40 +1,6 @@ ---- MQTT client module --- @module mqtt.client --- @alias client -local client = {} - --- event names: - --- "error": function(errmsg, client_object, [packet]) --- on errors --- optional packet: only if received CONNACK.rc ~= 0 when connecting - --- "close": function(connection_object, client_object) --- upon closing the connection --- connection object will have .close_reason (string) - --- "shutdown": function(client_object) --- upon shutting down the client (diconnecting an no more reconnects) - --- "connect": function(packet, client_object) --- upon a succesful connect, after receiving the CONNACK packet from the broker --- ???? => on a refused connect; if received CONNACK.rc ~= 0 when connecting - --- "subscribe": function(packet, client_object) --- upon a succesful subscription, after receiving the SUBACK packet from the broker - --- "unsubscribe": function(packet, client_object) --- upon a succesful unsubscription, after receiving the UNSUBACK packet from the broker - --- "message": function(packet, client_object) --- upon receiving a PUBLISH packet from the broker - --- "acknowledge": function(packet, client_object) --- upon receiving PUBACK --- upon receiving PUBREC (event fires after sending PUBREL) - --- "auth": function(packet, client_object) --- upon receiving an AUTH packet +--- This class contains the MQTT client implementation. +-- @classmod Client +local _M = {} ------- @@ -79,39 +45,48 @@ local log = require "mqtt.log" ------- --- MQTT client instance metatable --- @type client_mt -local client_mt = {} -client_mt.__index = client_mt +local Client = {} +Client.__index = Client ---- Create and initialize MQTT client instance +--- Create and initialize MQTT client instance. Typically this is not called directly, +-- but through `Client.create`. -- @tparam table opts MQTT client creation options table --- @tparam string opts.uri MQTT broker uri to connect. --- Expecting '[mqtt[s]://][username[:password]@]hostname[:port]' format. Any option specifically added to the options +-- @tparam string opts.uri MQTT broker uri to connect. Expected format: +--
`[mqtt[s]://][username[:password]@]hostname[:port]` +--
Any option specifically added to the options -- table will take precedence over the option specified in this uri. --- @tparam[opt] string opts.protocol either 'mqtt' or 'mqtts' +-- @tparam boolean opts.clean clean session start flag +-- @tparam[opt] string opts.protocol either `"mqtt"` or `"mqtts"` -- @tparam[opt] string opts.username username for authorization on MQTT broker -- @tparam[opt] string opts.password password for authorization on MQTT broker; not acceptable in absence of username --- @tparam[opt] string opts.hostnamne hostname of the MQTT broker to connect to --- @tparam[opt] int opts.port port number to connect to on the MQTT broker, defaults to 1883 port for plain or 8883 for secure network connections --- @tparam string opts.clean clean session start flag --- @tparam[opt=4] number opts.version MQTT protocol version to use, either 4 (for MQTT v3.1.1) or 5 (for MQTT v5.0). --- Also you may use special values mqtt.v311 or mqtt.v50 for this field. +-- @tparam[opt] string opts.host hostname of the MQTT broker to connect to +-- @tparam[opt] int opts.port port number to connect to on the MQTT broker, defaults to `1883` port for plain or `8883` for secure network connections +-- @tparam[opt=4] number opts.version MQTT protocol version to use, either `4` (for MQTT v3.1.1) or `5` (for MQTT v5.0). +-- Also you may use special values `mqtt.v311` or `mqtt.v50` for this field. -- @tparam[opt] string opts.id MQTT client ID, will be generated by luamqtt library if absent --- @tparam[opt=false] boolean,table opts.secure use secure network connection, provided by luasec lua module; --- set to true to select default params: { mode="client", protocol="any", verify="none", options={ "all","no_sslv2","no_sslv3","no_tlsv1" } --- or set to luasec-compatible table, for example with cafile="...", certificate="...", key="..." --- @tparam[opt] table opts.will will message table with required fields { topic="...", payload="..." } --- and optional fields { qos=1...3, retain=true/false } --- @tparam[opt=60] number opts.keep_alive time interval for client to send PINGREQ packets to the server when network connection is inactive +-- @tparam[opt=false] boolean,table opts.secure use secure network connection, provided by the lua module set in `opts.ssl_module`. +-- Set to true to select default parameters, check individual `mqtt.connectors` for supported options. +-- @tparam[opt] table opts.will will message table with required fields `{ topic="...", payload="..." }` +-- and optional fields `{ qos=0...2, retain=true/false }` +-- @tparam[opt=60] number opts.keep_alive time interval (in seconds) for client to send PINGREQ packets to the server when network connection is inactive -- @tparam[opt=false] boolean opts.reconnect force created MQTT client to reconnect on connection close. -- Set to number value to provide reconnect timeout in seconds. --- It's not recommended to use values < 3. See also `client_mt:shutdown`. +-- It's not recommended to use values `< 3`. See also `Client:shutdown`. -- @tparam[opt] table opts.connector connector table to open and send/receive packets over network connection. --- default is require("mqtt.connector") which tries to auto-detect. --- @tparam[opt="ssl"] string opts.ssl_module module name for the luasec-compatible ssl module, default is "ssl" +-- default is `require("mqtt.connector")` which tries to auto-detect. See `mqtt.connector`. +-- @tparam[opt="ssl"] string opts.ssl_module module name for the luasec-compatible ssl module, default is `"ssl"` -- may be used in some non-standard lua environments with own luasec-compatible ssl module --- @treturn client_mt MQTT client instance table -function client_mt:__init(opts) +-- @tparam[opt] table opts.on List of event-handlers. See `Client:on` for the format. +-- @treturn Client MQTT client instance table +-- @usage +-- local Client = require "mqtt.client" +-- +-- local my_client = Client.create { +-- uri = "mqtts://broker.host.com", +-- clean = true, +-- version = mqtt.v50, +-- } +function Client:__init(opts) if not luamqtt_VERSION then luamqtt_VERSION = require("mqtt")._VERSION end @@ -199,7 +174,7 @@ function client_mt:__init(opts) -- validate connection properties local test_conn = setmetatable({ uri = opts.uri }, a.connector) - client_mt._parse_connection_opts(a, test_conn) + Client._parse_connection_opts(a, test_conn) test_conn:validate() -- will table content check @@ -263,9 +238,47 @@ function client_mt:__init(opts) log:info("MQTT client '%s' created", a.id) end ---- Add functions as handlers of given events --- @param ... (event_name, function) or { event1 = func1, event2 = func2 } table -function client_mt:on(...) +--- Add functions as handlers of given events. +-- @tparam table events MQTT client creation options table +-- @tparam function events.connect `function(connack_packet, client_obj)`
+-- After a connect attempt, after receiving the CONNACK packet from the broker. +-- check `connack_packet.rc == 0` for a succesful connect. +-- @tparam functon events.error `function(errmsg, client_obj [, packet])`
+-- on errors, optional `packet` is only provided if the +-- received `CONNACK.rc ~= 0` when connecting. +-- @tparam functon events.close `function(connection_obj, client_obj)`
+-- upon closing the connection. `connection_obj.close_reason` +-- (string) will hold the close reason. +-- @tparam functon events.shutdown `function(client_obj)`
+-- upon shutting down the client (diconnecting an no more reconnects). +-- @tparam functon events.subscribe `function(suback_packet, client_obj)`
+-- upon a succesful subscription, after receiving the SUBACK packet from the broker +-- @tparam functon events.unsubscribe `function(unsuback_packet, client_obj)`
+-- upon a succesful unsubscription, after receiving the UNSUBACK packet from the broker +-- @tparam functon events.message `function(publish_packet, client_obj)`
+-- upon receiving a PUBLISH packet from the broker +-- @tparam functon events.acknowledge `function(ack_packet, client_obj)`
+-- upon receiving a PUBACK or PUBREC packet from the broker +-- @tparam functon events.auth `function(auth_packet, client_obj)`
+-- upon receiving an AUTH packet +-- @usage +-- client:on { +-- connect = function(pck, self) +-- if pck.rc ~= 0 then +-- return -- connection failed +-- end +-- -- succesfully connected +-- end, +-- message = function(pck, self) +-- -- handle received message +-- end, +-- } +-- +-- -- an alternative way to add individual handlers; +-- client:on("message", function(pck, self) +-- -- handle received message +-- end) +function Client:on(...) local nargs = select("#", ...) local events if nargs == 2 then @@ -296,10 +309,22 @@ local function remove_item(list, item) end end ---- Remove given function handler for specified event +--- Remove given function handler for specified event. -- @tparam string event event name to remove handler -- @tparam function func handler function to remove -function client_mt:off(event, func) +-- @usage +-- local handler = function(pck, self) +-- -- handle received message +-- end +-- +-- -- add event handler +-- client:on { +-- message = handler +-- } +-- +-- -- remove it again +-- client:off("message", handler) +function Client:off(event, func) local handlers = self.handlers[event] if not handlers then error("invalid event '"..tostring(event).."' to handle") @@ -327,7 +352,7 @@ end -- @tparam[opt] table opts.user_properties for MQTT v5.0 only: user properties for subscribe operation -- @tparam[opt] function opts.callback callback function to be called when subscription is acknowledged by broker -- @return packet id on success or false and error message on failure -function client_mt:subscribe(opts) +function Client:subscribe(opts) -- fetch and validate opts assert(type(opts) == "table", "expecting opts to be a table") assert(type(opts.topic) == "string", "expecting opts.topic to be a string") @@ -396,9 +421,9 @@ end -- @tparam string opts.topic topic to unsubscribe -- @tparam[opt] table opts.properties properties for unsubscribe operation -- @tparam[opt] table opts.user_properties user properties for unsubscribe operation --- @tparam[opt] function opts.callback callback function to be called when subscription will be removed on broker +-- @tparam[opt] function opts.callback callback function to be called when the unsubscre is acknowledged by the broker -- @return packet id on success or false and error message on failure -function client_mt:unsubscribe(opts) +function Client:unsubscribe(opts) -- fetch and validate opts assert(type(opts) == "table", "expecting opts to be a table") assert(type(opts.topic) == "string", "expecting opts.topic to be a string") @@ -460,9 +485,9 @@ end -- @tparam[opt=false] boolean opts.dup dup message publication flag -- @tparam[opt] table opts.properties properties for publishing message -- @tparam[opt] table opts.user_properties user properties for publishing message --- @tparam[opt] function opts.callback callback to call when publihsed message has been acknowledged by the broker +-- @tparam[opt] function opts.callback callback to call when published message has been acknowledged by the broker -- @return true or packet id on success or false and error message on failure -function client_mt:publish(opts) +function Client:publish(opts) -- fetch and validate opts assert(type(opts) == "table", "expecting opts to be a table") assert(type(opts.topic) == "string", "expecting opts.topic to be a string") @@ -532,7 +557,7 @@ end -- @tparam[opt] table properties properties for PUBACK/PUBREC packets -- @tparam[opt] table user_properties user properties for PUBACK/PUBREC packets -- @return true on success or false and error message on failure -function client_mt:acknowledge(msg, rc, properties, user_properties) +function Client:acknowledge(msg, rc, properties, user_properties) assert(type(msg) == "table" and msg.type == packet_type.PUBLISH, "expecting msg to be a publish packet") assert(rc == nil or type(rc) == "number", "expecting rc to be a number") assert(properties == nil or type(properties) == "table", "expecting properties to be a table") @@ -602,12 +627,14 @@ function client_mt:acknowledge(msg, rc, properties, user_properties) return true end ---- Send DISCONNECT packet to the broker and close the connection +--- Send DISCONNECT packet to the broker and close the connection. +-- Note: if the client is set to automatically reconnect, it will do so. If you +-- want to disconnect and NOT reconnect, use `Client:shutdown`. -- @tparam[opt=0] number rc The Disconnect Reason Code value from MQTT v5.0 protocol -- @tparam[opt] table properties properties for PUBACK/PUBREC packets -- @tparam[opt] table user_properties user properties for PUBACK/PUBREC packets -- @return true on success or false and error message on failure -function client_mt:disconnect(rc, properties, user_properties) +function Client:disconnect(rc, properties, user_properties) -- validate opts assert(rc == nil or type(rc) == "number", "expecting rc to be a number") assert(properties == nil or type(properties) == "table", "expecting properties to be a table") @@ -644,15 +671,17 @@ function client_mt:disconnect(rc, properties, user_properties) return true end ---- Shutsdown the client. --- Disconnects if still connected, and disables reconnecting. --- Raises the "shutdown" event --- @param see `client_mt:disconnect`. -function client_mt:shutdown(rc, properties, user_properties) +--- Shuts the client down. +-- Disconnects if still connected, and disables reconnecting. If the client is +-- added to an ioloop, this will prevent an automatic reconnect. +-- Raises the "shutdown" event. +-- @param ... see `Client:disconnect` +-- @return `true` +function Client:shutdown(...) log:debug("client '%s' shutting down", self.opts.id) self.first_connect = false self.opts.reconnect = false - self:disconnect(rc, properties, user_properties) + self:disconnect(...) self:handle("shutdown", self) return true end @@ -662,7 +691,7 @@ end -- @tparam[opt] table properties properties for PUBACK/PUBREC packets -- @tparam[opt] table user_properties user properties for PUBACK/PUBREC packets -- @return true on success or false and error message on failure -function client_mt:auth(rc, properties, user_properties) +function Client:auth(rc, properties, user_properties) -- validate opts assert(rc == nil or type(rc) == "number", "expecting rc to be a number") assert(properties == nil or type(properties) == "table", "expecting properties to be a table") @@ -699,7 +728,7 @@ end --- Immediately close established network connection, without graceful session finishing with DISCONNECT packet -- @tparam[opt] string reason the reasong string of connection close -function client_mt:close_connection(reason) +function Client:close_connection(reason) assert(not reason or type(reason) == "string", "expecting reason to be a string") local conn = self.connection if not conn then @@ -720,7 +749,7 @@ end --- Start connecting to broker -- @return true on success or false and error message on failure -function client_mt:start_connecting() +function Client:start_connecting() -- open network connection local ok, err = self:open_connection() if not ok then @@ -743,7 +772,7 @@ end --- Send PINGREQ packet -- @return true on success or false and error message on failure -function client_mt:send_pingreq() +function Client:send_pingreq() -- check connection is alive if not self.connection then return false, "network connection is not opened" @@ -774,7 +803,7 @@ end --- Open network connection to the broker -- @return true on success or false and error message on failure -function client_mt:open_connection() +function Client:open_connection() if self.connection then return true end @@ -788,7 +817,7 @@ function client_mt:open_connection() wait_for_pubrec = {}, -- a table with packet_id of partially acknowledged sent packets in QoS 2 exchange process wait_for_pubrel = {}, -- a table with packet_id of partially acknowledged received packets in QoS 2 exchange process }, connector) - client_mt._parse_connection_opts(opts, conn) + Client._parse_connection_opts(opts, conn) log:info("client '%s' connecting to broker '%s' (using: %s)", self.opts.id, opts.uri, conn.type or "unknown") @@ -812,7 +841,7 @@ end --- Send CONNECT packet into opened network connection -- @return true on success or false and error message on failure -function client_mt:send_connect() +function Client:send_connect() -- check connection is alive if not self.connection then return false, "network connection is not opened" @@ -852,8 +881,11 @@ function client_mt:send_connect() end --- Checks last message send, and sends a PINGREQ if necessary. --- Use this function to check and send keep-alives when using an external event loop. --- @return time till next keep_alive, in case of errors (eg. not connected) the second return value is an error string +-- Use this function to check and send keep-alives when using an external event loop. When using the +-- included modules to add clients (see `mqtt.loop`), this will be taken care of automatically. +-- @treturn[1] number time till next keep_alive (in seconds) +-- @treturn[2] number time till next keep_alive (in seconds) +-- @treturn[2] string in case of errors (eg. not connected) the second return value is an error string -- @usage -- -- example using a Copas event loop to send and check keep-alives -- copas.addthread(function() @@ -864,7 +896,7 @@ end -- copas.sleep(my_client:check_keep_alive()) -- end -- end) -function client_mt:check_keep_alive() +function Client:check_keep_alive() local interval = self.opts.keep_alive if not self.connection then return interval, "network connection is not opened" @@ -902,7 +934,7 @@ end -- Send PUBREL acknowledge packet - second phase of QoS 2 exchange -- Returns true on success or false and error message on failure -function client_mt:acknowledge_pubrel(packet_id) +function Client:acknowledge_pubrel(packet_id) -- check connection is alive if not self.connection then return false, "network connection is not opened" @@ -928,7 +960,7 @@ end -- Send PUBCOMP acknowledge packet - last phase of QoS 2 exchange -- Returns true on success or false and error message on failure -function client_mt:acknowledge_pubcomp(packet_id) +function Client:acknowledge_pubcomp(packet_id) -- check connection is alive if not self.connection then return false, "network connection is not opened" @@ -953,18 +985,18 @@ function client_mt:acknowledge_pubcomp(packet_id) end -- Call specified event handlers -function client_mt:handle(event, ...) +function Client:handle(event, ...) local handlers = self.handlers[event] if not handlers then error("invalid event '"..tostring(event).."' to handle") end - self._handling[event] = true -- protecting self.handlers[event] table from modifications by client_mt:off() when iterating + self._handling[event] = true -- protecting self.handlers[event] table from modifications by Client:off() when iterating for _, handler in ipairs(handlers) do handler(...) end self._handling[event] = nil - -- process handlers removing, scheduled by client_mt:off() + -- process handlers removing, scheduled by Client:off() local to_remove = self._to_remove_handlers[event] if to_remove then for _, func in ipairs(to_remove) do @@ -977,7 +1009,7 @@ end -- Internal methods -- Assign next packet id for given packet creation opts -function client_mt:_assign_packet_id(pargs) +function Client:_assign_packet_id(pargs) if not pargs.packet_id then if packet_id_required(pargs) then self._last_packet_id = next_packet_id(self._last_packet_id) @@ -987,7 +1019,7 @@ function client_mt:_assign_packet_id(pargs) end -- Handle a single received packet -function client_mt:handle_received_packet(packet) +function Client:handle_received_packet(packet) local conn = self.connection local err @@ -1107,22 +1139,22 @@ do --- Performs a single IO loop step. -- It will connect if not connected, will re-connect if set to. - -- This should be called repeatedly in a loop. + -- This should be called repeatedly in a loop. When using the included modules to + -- add clients (see `mqtt.loop`), this will be taken care of automatically. -- -- The return value is the time after which this method must be called again. - -- It can be called sooner, but shouldn't be called later. Return values: - -- - -- - `0`; a packet was succesfully handled, so retry immediately, no delays, - -- in case additional data is waiting to be read on the socket. - -- - `>0`; The reconnect timer needs a delay before it can retry (calling - -- sooner is not a problem, it will only reconnect when the delay - -- has actually passed) - -- - `-1`; the socket read timed out, so it is idle. This return code is only - -- returned with buffered connectors (luasocket), never for yielding sockets - -- (Copas or OpenResty) - -- - -- @return time after which to retry or nil+error - function client_mt:step() + -- It can be called sooner, but shouldn't be called later. + -- @return[1] `-1`: the socket read timed out, so it is idle. This return code is only + -- returned with buffered connectors (luasocket), never for yielding sockets + -- (Copas or OpenResty) + -- @return[2] `0`: a packet was succesfully handled, so retry immediately, no delays, + -- in case additional data is waiting to be read on the socket. + -- @return[3] `>0`: The reconnect timer needs a delay before it can retry (calling + -- sooner is not a problem, it will only reconnect when the delay + -- has actually passed) + -- @return[4] nil + -- @return[4] error message + function Client:step() local conn = self.connection local reconnect = self.opts.reconnect @@ -1161,7 +1193,7 @@ end -- Fill given connection table with host and port according given opts -- uri: mqtt[s]://[username][:password]@host.domain[:port] -function client_mt._parse_connection_opts(opts, conn) +function Client._parse_connection_opts(opts, conn) local uri = assert(conn.uri) -- protocol @@ -1263,7 +1295,7 @@ function client_mt._parse_connection_opts(opts, conn) end -- Send given packet to opened network connection -function client_mt:_send_packet(packet) +function Client:_send_packet(packet) local conn = self.connection if not conn then return false, "network connection is not opened" @@ -1283,7 +1315,7 @@ function client_mt:_send_packet(packet) end -- Receive one packet from established network connection -function client_mt:_receive_packet() +function Client:_receive_packet() local conn = self.connection if not conn then return false, "network connection is not opened" @@ -1313,12 +1345,12 @@ function client_mt:_receive_packet() end -- Represent MQTT client as string -function client_mt:__tostring() +function Client:__tostring() return str_format("mqtt.client{id=%q}", tostring(self.opts.id)) end -- Garbage collection handler -function client_mt:__gc() +function Client:__gc() -- close network connection if it's available, without sending DISCONNECT packet if self.connection then self:close_connection("garbage") @@ -1329,12 +1361,13 @@ end -- @section exported --- Create, initialize and return new MQTT client instance --- @param ... see arguments of client_mt:__init(opts) --- @see client_mt:__init --- @treturn client_mt MQTT client instance -function client.create(...) - local cl = setmetatable({}, client_mt) - cl:__init(...) +-- @name client.create +-- @param ... see arguments of `Client:__init` +-- @see Client:__init +-- @treturn Client MQTT client instance +function _M.create(opts) + local cl = setmetatable({}, Client) + cl:__init(opts) return cl end @@ -1342,10 +1375,10 @@ end if _G._TEST then -- export functions for test purposes (different name!) - client.__parse_connection_opts = client_mt._parse_connection_opts + _M.__parse_connection_opts = Client._parse_connection_opts end -- export module table -return client +return _M -- vim: ts=4 sts=4 sw=4 noet ft=lua diff --git a/mqtt/connector/copas.lua b/mqtt/connector/copas.lua index 1369f1a..83453af 100644 --- a/mqtt/connector/copas.lua +++ b/mqtt/connector/copas.lua @@ -1,7 +1,27 @@ --- DOC: https://keplerproject.github.io/copas/ --- NOTE: you will need to install copas like this: luarocks install copas +--- Copas based connector. +-- +-- Copas is an advanced coroutine scheduler in pure-Lua. It uses LuaSocket +-- under the hood, but in a non-blocking way. It also uses LuaSec for TLS +-- based connections (like the `mqtt.connector.luasocket` one). And hence uses +-- the same defaults for the `secure` option when creating the `client`. +-- +-- Caveats: +-- +-- * the `client` option `ssl_module` is not supported by the Copas connector, +-- It will always use the module named `ssl`. +-- +-- * multiple threads cannot send simultaneously (simple scenarios will just +-- work) +-- +-- * since the client creates a long lived connection for reading, it returns +-- upon receiving a packet, to call an event handler. The handler must return +-- swiftly, since while the handler runs the socket will not be reading. +-- Any task that might take longer than a few milliseconds should be off +-- loaded to another thread. +-- +-- NOTE: you will need to install copas like this: `luarocks install copas`. +-- @module mqtt.connector.copas --- module table local super = require "mqtt.connector.base.non_buffered_base" local connector = setmetatable({}, super) connector.__index = connector diff --git a/mqtt/connector/init.lua b/mqtt/connector/init.lua index 826e587..20db5fa 100644 --- a/mqtt/connector/init.lua +++ b/mqtt/connector/init.lua @@ -1,6 +1,25 @@ ---- auto detect the connector to use. --- This is based on a.o. libraries already loaded, so 'require' this --- module as late as possible (after the other modules) +--- Auto detect the connector to use. +-- The different environments require different socket implementations to work +-- properly. The 'connectors' are an abstraction to facilitate that without +-- having to modify the client itself. +-- +-- This module is will auto-detect the environment and return the proper +-- module from; +-- +-- * `mqtt.connector.nginx` for using the non-blocking OpenResty co-socket apis +-- +-- * `mqtt.connector.copas` for the non-blocking Copas wrapped sockets +-- +-- * `mqtt.connector.luasocket` for LuaSocket based sockets (blocking) +-- +-- Since the selection is based on a.o. packages loaded, make sure that in case +-- of using the `copas` scheduler, you require it before the `mqtt` modules. +-- +-- Since the `client` defaults to this module (`mqtt.connector`) there typically +-- is no need to use this directly. When implementing your own connectors, +-- the included connectors provide good examples of what to look out for. +-- @module mqtt.connector + local loops = setmetatable({ copas = "mqtt.connector.copas", nginx = "mqtt.connector.nginx", diff --git a/mqtt/connector/luasocket.lua b/mqtt/connector/luasocket.lua index 7f27702..6a25431 100644 --- a/mqtt/connector/luasocket.lua +++ b/mqtt/connector/luasocket.lua @@ -1,6 +1,25 @@ --- DOC: http://w3.impa.br/~diego/software/luasocket/tcp.html +--- LuaSocket (and LuaSec) based connector. +-- +-- This connector works with the blocking LuaSocket sockets. This connector uses +-- `LuaSec` for TLS connections. This is the connector used for the included +-- `mqtt.ioloop` scheduler. +-- +-- When using TLS / MQTTS connections, the `secure` option passed to the `client` +-- when creating it, can be the standard table of options as used by LuaSec +-- for creating a context. When omitted the defaults will be; +-- `{ mode="client", protocol="any", verify="none", +-- options={ "all", "no_sslv2", "no_sslv3", "no_tlsv1" } }` +-- +-- Caveats: +-- +-- * since the client creates a long lived connection for reading, it returns +-- upon receiving a packet, to call an event handler. The handler must return +-- swiftly, since while the handler runs the socket will not be reading. +-- Any task that might take longer than a few milliseconds should be off +-- loaded to another task. +-- +-- @module mqtt.connector.luasocket --- module table local super = require "mqtt.connector.base.buffered_base" local luasocket = setmetatable({}, super) luasocket.__index = luasocket diff --git a/mqtt/connector/nginx.lua b/mqtt/connector/nginx.lua index 61912fa..00bdadb 100644 --- a/mqtt/connector/nginx.lua +++ b/mqtt/connector/nginx.lua @@ -1,5 +1,30 @@ --- module table +--- Nginx OpenResty co-sockets based connector. +-- +-- This connector works with the non-blocking openresty sockets. Note that the +-- secure setting haven't been implemented yet. It will simply use defaults +-- when doing a TLS handshake. +-- +-- Caveats: +-- +-- * sockets cannot cross phase/context boundaries. So all client interaction +-- must be done from the timer context in which the client threads run. +-- +-- * multiple threads cannot send simultaneously (simple scenarios will just +-- work) +-- +-- * since the client creates a long lived connection for reading, it returns +-- upon receiving a packet, to call an event handler. The handler must return +-- swiftly, since while the handler runs the socket will not be reading. +-- Any task that might take longer than a few milliseconds should be off +-- loaded to another thread. +-- +-- * Nginx timers should be short lived because memory is only released after +-- the context is destroyed. In this case we're using the fro prolonged periods +-- of time, so be aware of this and implement client restarts if required. +-- -- thanks to @irimiab: https://github.com/xHasKx/luamqtt/issues/13 +-- @module mqtt.connector.nginx + local super = require "mqtt.connector.base.non_buffered_base" local ngxsocket = setmetatable({}, super) ngxsocket.__index = ngxsocket diff --git a/mqtt/init.lua b/mqtt/init.lua index 7e968f9..c97333e 100644 --- a/mqtt/init.lua +++ b/mqtt/init.lua @@ -19,6 +19,12 @@ CONVENTIONS: -- @tfield string _VERSION luamqtt library version string -- @table mqtt -- @see mqtt.const +-- @usage +-- local client = mqtt.client { +-- uri = "mqtts://aladdin:soopersecret@mqttbroker.com", +-- clean = true, +-- version = mqtt.v50, -- specify constant for MQTT version +-- } local mqtt = {} -- copy all values from const module @@ -40,20 +46,24 @@ local ioloop = require("mqtt.ioloop") local ioloop_get = ioloop.get --- Create new MQTT client instance --- @param ... Same as for mqtt.client.create(...) --- @see mqtt.client.client_mt:__init +-- @param ... Same as for `Client.create`(...) +-- @see Client:__init function mqtt.client(...) return client_create(...) end ---- Returns default ioloop instance +--- Returns default `ioloop` instance. Shortcut to `Ioloop.get`. -- @function mqtt.get_ioloop +-- @see Ioloop.get mqtt.get_ioloop = ioloop_get ---- Run default ioloop for given MQTT clients or functions. --- @param ... MQTT clients or loop functions to add to ioloop --- @see mqtt.ioloop.get --- @see mqtt.ioloop.run_until_clients +--- Run default `ioloop` for given MQTT clients or functions. +-- Will not return until all clients/functions have exited. +-- @param ... MQTT clients or loop functions to add to ioloop, see `Ioloop:add` for details on functions. +-- @see Ioloop.get +-- @see Ioloop.run_until_clients +-- @usage +-- mqtt.run_ioloop(client1, client2, func1) function mqtt.run_ioloop(...) log:info("starting default ioloop instance") local loop = ioloop_get() @@ -64,27 +74,11 @@ function mqtt.run_ioloop(...) return loop:run_until_clients() end ---- Run synchronous input/output loop for only one given MQTT client. --- Provided client's connection will be opened. --- Client reconnect feature will not work, and keep_alive too. --- @param cl MQTT client instance to run -function mqtt.run_sync(cl) - local ok, err = cl:start_connecting() - if not ok then - return false, err - end - while cl.connection do - ok, err = cl:_sync_iteration() - if not ok then - return false, err - end - end -end - --- Validates a topic with wildcards. -- @param t (string) wildcard topic to validate -- @return topic, or false+error +-- @usage local t = assert(mqtt.validate_subscribe_topic("base/+/thermostat/#")) function mqtt.validate_subscribe_topic(t) if type(t) ~= "string" then return false, "not a string" @@ -122,6 +116,7 @@ end --- Validates a topic without wildcards. -- @param t (string) topic to validate -- @return topic, or false+error +-- @usage local t = assert(mqtt.validate_publish_topic("base/living/thermostat/setpoint")) function mqtt.validate_publish_topic(t) if type(t) ~= "string" then return false, "not a string" @@ -160,30 +155,22 @@ function mqtt.compile_topic_pattern(t) end --- Parses wildcards in a topic into a table. --- Options include: --- --- - `opts.topic`: the wild-carded topic to match against (optional if `opts.pattern` is given) --- --- - `opts.pattern`: the compiled pattern for the wild-carded topic (optional if `opts.topic` +-- @tparam topic string incoming topic string +-- @tparam table opts parsing options table +-- @tparam string opts.topic the wild-carded topic to match against (optional if `opts.pattern` is given) +-- @tparam string opts.pattern the compiled pattern for the wild-carded topic (optional if `opts.topic` -- is given). If not given then topic will be compiled and the result will be -- stored in this field for future use (cache). --- --- - `opts.keys`: (optional) array of field names. The order must be the same as the +-- @tparam array opts.keys array of field names. The order must be the same as the -- order of the wildcards in `topic` --- --- Returned tables: --- --- - `fields` table: the array part will have the values of the wildcards, in +-- @return[1] `fields` table: the array part will have the values of the wildcards, in -- the order they appeared. The hash part, will have the field names provided -- in `opts.keys`, with the values of the corresponding wildcard. If a `#` -- wildcard was used, that one will be the last in the table. --- --- - `varargs` table: will only be returned if the wildcard topic contained the +-- @return[1] `varargs` table: will only be returned if the wildcard topic contained the -- `#` wildcard. The returned table is an array, with all segments that were -- matched by the `#` wildcard. --- @param topic (string) incoming topic string (required) --- @param opts (table) with options (required) --- @return fields (table) + varargs (table or nil), or false+err on error. +-- @return[2] false+err on error, eg. topic didn't match or pattern was invalid. -- @usage -- local opts = { -- topic = "homes/+/+/#", diff --git a/mqtt/ioloop.lua b/mqtt/ioloop.lua index 5a11148..94b895f 100644 --- a/mqtt/ioloop.lua +++ b/mqtt/ioloop.lua @@ -1,33 +1,25 @@ ---- ioloop module --- @module mqtt.ioloop --- @alias ioloop - ---[[ - ioloop module - - In short: allowing you to work with several MQTT clients in one script, and allowing them to maintain - a long-term connection to broker, using PINGs. - - NOTE: this module will work only with MQTT clients using standard luasocket/luasocket_ssl connectors. - - In long: - Providing an IO loop instance dealing with efficient (as much as possible in limited lua IO) network communication - for several MQTT clients in the same OS thread. - The main idea is that you are creating an ioloop instance, then adding created and connected MQTT clients to it. - The ioloop instance is setting a non-blocking mode for sockets in MQTT clients and setting a small timeout - for their receive/send operations. Then ioloop is starting an endless loop trying to receive/send data for all added MQTT clients. - You may add more or remove some MQTT clients from the ioloop after it's created and started. - - Using that ioloop is allowing you to run a MQTT client for long time, through sending PINGREQ packets to broker - in keepAlive interval to maintain long-term connection. - - Also, any function can be added to the ioloop instance, and it will be called in the same endless loop over and over - alongside with added MQTT clients to provide you a piece of processor time to run your own logic (like running your own - network communications or any other thing good working in an io-loop) -]] - --- module table -local ioloop = {} +--- This class contains the ioloop implementation. +-- +-- In short: allowing you to work with several MQTT clients in one script, and allowing them to maintain +-- a long-term connection to broker, using PINGs. This is the bundled alternative to Copas and Nginx. +-- +-- NOTE: this module will work only with MQTT clients using the `connector.luasocket` connector. +-- +-- Providing an IO loop instance dealing with efficient (as much as possible in limited lua IO) network communication +-- for several MQTT clients in the same OS thread. +-- The main idea is that you are creating an ioloop instance, then adding MQTT clients to it. +-- Then ioloop is starting an endless loop trying to receive/send data for all added MQTT clients. +-- You may add more or remove some MQTT clients to/from the ioloop after it has been created and started. +-- +-- Using an ioloop is allowing you to run a MQTT client for long time, through sending PINGREQ packets to broker +-- in keepAlive interval to maintain long-term connection. +-- +-- Also, any function can be added to the ioloop instance, and it will be called in the same endless loop over and over +-- alongside with added MQTT clients to provide you a piece of processor time to run your own logic (like running your own +-- network communications or any other thing good working in an io-loop) +-- @classmod Ioloop + +local _M = {} -- load required stuff local log = require "mqtt.log" @@ -44,18 +36,17 @@ local math = require("math") local math_min = math.min --- ioloop instances metatable --- @type ioloop_mt -local ioloop_mt = {} -ioloop_mt.__index = ioloop_mt +local Ioloop = {} +Ioloop.__index = Ioloop ---- Initialize ioloop instance +--- Initialize ioloop instance. -- @tparam table opts ioloop creation options table -- @tparam[opt=0] number opts.sleep_min min sleep interval after each iteration -- @tparam[opt=0.002] number opts.sleep_step increase in sleep after every idle iteration -- @tparam[opt=0.030] number opts.sleep_max max sleep interval after each iteration -- @tparam[opt=luasocket.sleep] function opts.sleep_function custom sleep function to call after each iteration --- @treturn ioloop_mt ioloop instance -function ioloop_mt:__init(opts) +-- @treturn Ioloop ioloop instance +function Ioloop:__init(opts) log:debug("initializing ioloop instance '%s'", tostring(self)) opts = opts or {} opts.sleep_min = opts.sleep_min or 0 @@ -68,10 +59,32 @@ function ioloop_mt:__init(opts) self.running = false --ioloop running flag, used by MQTT clients which are adding after this ioloop started to run end ---- Add MQTT client or a loop function to the ioloop instance +--- Add MQTT client or a loop function to the ioloop instance. +-- When adding a function, the function should on each call return the time (in seconds) it wishes to sleep. The ioloop +-- will sleep after each iteration based on what clients/functions returned. So the function may be called sooner than +-- the requested time, but will not be called later. -- @tparam client_mt|function client MQTT client or a loop function to add to ioloop -- @return true on success or false and error message on failure -function ioloop_mt:add(client) +-- @usage +-- -- create a timer on a 1 second interval +-- local timer do +-- local interval = 1 +-- local next_call = socket.gettime() + interval +-- timer = function() +-- if next_call >= socket.gettime() then +-- +-- -- do stuff here +-- +-- next_call = socket.gettime() + interval +-- return interval +-- else +-- return next_call - socket.gettime() +-- end +-- end +-- end +-- +-- loop:add(timer) +function Ioloop:add(client) local clients = self.clients if clients[client] then if type(client) == "table" then @@ -108,7 +121,7 @@ end --- Remove MQTT client or a loop function from the ioloop instance -- @tparam client_mt|function client MQTT client or a loop function to remove from ioloop -- @return true on success or false and error message on failure -function ioloop_mt:remove(client) +function Ioloop:remove(client) local clients = self.clients if not clients[client] then if type(client) == "table" then @@ -142,7 +155,7 @@ end --- Perform one ioloop iteration. -- TODO: make this smarter do not wake-up functions or clients returning a longer -- sleep delay. Currently they will be tried earlier if another returns a smaller delay. -function ioloop_mt:iteration() +function Ioloop:iteration() local opts = self.opts local sleep = opts.sleep_max @@ -183,8 +196,10 @@ function ioloop_mt:iteration() end end ---- Run ioloop while there is at least one client/function in the ioloop -function ioloop_mt:run_until_clients() +--- Run the ioloop. +-- While there is at least one client/function in the ioloop it will continue +-- iterating. After all clients/functions are gone, it will return. +function Ioloop:run_until_clients() log:info("ioloop started with %d clients/functions", #self.clients) self.running = true @@ -196,32 +211,35 @@ function ioloop_mt:run_until_clients() log:info("ioloop finished with %d clients/functions", #self.clients) end -------- +--- Exported functions +-- @section exported + --- Create IO loop instance with given options --- @see ioloop_mt:__init --- @treturn ioloop_mt ioloop instance -local function ioloop_create(opts) - local inst = setmetatable({}, ioloop_mt) +-- @name ioloop.create +-- @see Ioloop:__init +-- @treturn Ioloop ioloop instance +function _M.create(opts) + local inst = setmetatable({}, Ioloop) inst:__init(opts) return inst end -ioloop.create = ioloop_create -- Default ioloop instance local ioloop_instance --- Returns default ioloop instance +-- @name ioloop.get -- @tparam[opt=true] boolean autocreate Automatically create ioloop instance -- @tparam[opt] table opts Arguments for creating ioloop instance --- @treturn ioloop_mt ioloop instance -function ioloop.get(autocreate, opts) +-- @treturn Ioloop ioloop instance +function _M.get(autocreate, opts) if autocreate == nil then autocreate = true end if autocreate and not ioloop_instance then log:info("auto-creating default ioloop instance") - ioloop_instance = ioloop_create(opts) + ioloop_instance = _M.create(opts) end return ioloop_instance end @@ -229,6 +247,6 @@ end ------- -- export module table -return ioloop +return _M -- vim: ts=4 sts=4 sw=4 noet ft=lua diff --git a/mqtt/loop/copas.lua b/mqtt/loop/copas.lua index 9b58661..3491528 100644 --- a/mqtt/loop/copas.lua +++ b/mqtt/loop/copas.lua @@ -1,3 +1,8 @@ +--- Copas specific client handling module. +-- Typically this module is not used directly, but through `mqtt.loop` when +-- auto-detecting the environment. +-- @module mqtt.loop.copas + local copas = require "copas" local log = require "mqtt.log" @@ -7,9 +12,10 @@ local _M = {} --- Add MQTT client to the Copas scheduler. --- The client will automatically be removed after it exits. --- @tparam cl client to add to the Copas scheduler --- @return true on success or false and error message on failure +-- The client will automatically be removed after it exits. It will set up a +-- thread to call `Client:check_keep_alive`. +-- @param cl mqtt-client to add to the Copas scheduler +-- @return `true` on success or `false` and error message on failure function _M.add(cl) if client_registry[cl] then log:warn("MQTT client '%s' was already added to Copas", cl.opts.id) diff --git a/mqtt/loop/init.lua b/mqtt/loop/init.lua index e8976de..f7a4c3d 100644 --- a/mqtt/loop/init.lua +++ b/mqtt/loop/init.lua @@ -1,6 +1,28 @@ ---- auto detect the connector to use. --- This is based on a.o. libraries already loaded, so 'require' this --- module as late as possible (after the other modules) +--- Auto detect the IO loop to use. +-- Interacting with the supported IO loops (ioloop, copas, and nginx) requires +-- specific implementations to get it right. +-- This module is will auto-detect the environment and return the proper +-- module from; +-- +-- * `mqtt.loop.ioloop` +-- +-- * `mqtt.loop.copas` +-- +-- * `mqtt.loop.nginx` +-- +-- Since the selection is based on a.o. packages loaded, make sure that in case +-- of using the `copas` scheduler, you require it before the `mqtt` modules. +-- +-- @usage +-- --local copas = require "copas" -- only if you use Copas +-- local mqtt = require "mqtt" +-- local add_client = require("mqtt.loop").add -- returns a loop-specific function +-- +-- local client = mqtt.create { ... options ... } +-- add_client(client) -- works for ioloop, copas, and nginx +-- +-- @module mqtt.loop + local loops = setmetatable({ copas = "mqtt.loop.copas", nginx = "mqtt.loop.nginx", diff --git a/mqtt/loop/ioloop.lua b/mqtt/loop/ioloop.lua index 4b83038..d585651 100644 --- a/mqtt/loop/ioloop.lua +++ b/mqtt/loop/ioloop.lua @@ -1,7 +1,17 @@ +--- IOloop specific client handling module. +-- Typically this module is not used directly, but through `mqtt.loop` when +-- auto-detecting the environment. +-- @module mqtt.loop.ioloop + local _M = {} local mqtt = require "mqtt" +--- Add MQTT client to the integrated ioloop. +-- The client will automatically be removed after it exits. It will set up a +-- function to call `Client:check_keep_alive` in the ioloop. +-- @param client mqtt-client to add to the ioloop +-- @return `true` on success or `false` and error message on failure function _M.add(client) local default_loop = mqtt.get_ioloop() return default_loop:add(client) diff --git a/mqtt/loop/nginx.lua b/mqtt/loop/nginx.lua index bcea387..3a8a3d9 100644 --- a/mqtt/loop/nginx.lua +++ b/mqtt/loop/nginx.lua @@ -1,3 +1,7 @@ +--- Nginx specific client handling module. +-- Typically this module is not used directly, but through `mqtt.loop` when +-- auto-detecting the environment. +-- @module mqtt.loop.nginx local client_registry = {} @@ -5,9 +9,10 @@ local _M = {} --- Add MQTT client to the Nginx environment. --- The client will automatically be removed after it exits. --- @tparam cl client to add --- @return true on success or false and error message on failure +-- The client will automatically be removed after it exits. It will set up a +-- thread to call `Client:check_keep_alive`. +-- @param client mqtt-client to add to the Nginx environment +-- @return `true` on success or `false` and error message on failure function _M.add(client) if client_registry[client] then ngx.log(ngx.WARN, "MQTT client '%s' was already added to Nginx", client.opts.id) @@ -46,7 +51,10 @@ function _M.add(client) if not ok then ngx.log(ngx.CRIT, "Failed to start timer-context for device '", client.id,"': ", err) + return false, "timer failed: " .. err end + + return true end From ca71e1bd5d5ea9b2316ad4cb4cd62eff0564f90f Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Tue, 9 Nov 2021 15:47:17 +0100 Subject: [PATCH 35/65] chore(docs) reduce tab-size in docs for readability --- docs/config.ld | 1 + docs/ldoc.css | 1 + mqtt/init.lua | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/config.ld b/docs/config.ld index 8d97a61..050ed24 100644 --- a/docs/config.ld +++ b/docs/config.ld @@ -40,3 +40,4 @@ topics = { format = "markdown" plain = true +style = true diff --git a/docs/ldoc.css b/docs/ldoc.css index 52c4ad2..fac66f4 100644 --- a/docs/ldoc.css +++ b/docs/ldoc.css @@ -120,6 +120,7 @@ pre { margin: 10px 0 10px 0; overflow: auto; font-family: "Andale Mono", monospace; + tab-size: 2; } pre.example { diff --git a/mqtt/init.lua b/mqtt/init.lua index c97333e..fd2605a 100644 --- a/mqtt/init.lua +++ b/mqtt/init.lua @@ -189,7 +189,7 @@ function mqtt.topic_match(topic, opts) return false, "expected topic to be a string" end if type(opts) ~= "table" then - return false, "expected optionss to be a table" + return false, "expected options to be a table" end local pattern = opts.pattern if not pattern then From 1c04d085a9c6c2d0ee7f2dd5f1b62f4f93312734 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Fri, 19 Nov 2021 17:39:30 +0100 Subject: [PATCH 36/65] fix(reconnect) set a number default (30 secs) if true --- mqtt/client.lua | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/mqtt/client.lua b/mqtt/client.lua index e81a245..129f792 100644 --- a/mqtt/client.lua +++ b/mqtt/client.lua @@ -162,6 +162,11 @@ function Client:__init(opts) -- default connector a.connector = a.connector or require("mqtt.connector") + -- default reconnect interval + if a.reconnect == true then + a.reconnect = 30 + end + -- validate connector content assert(type(a.connector) == "table", "expecting connector to be a table") assert(type(a.connector.validate) == "function", "expecting connector.validate to be a function") @@ -421,7 +426,7 @@ end -- @tparam string opts.topic topic to unsubscribe -- @tparam[opt] table opts.properties properties for unsubscribe operation -- @tparam[opt] table opts.user_properties user properties for unsubscribe operation --- @tparam[opt] function opts.callback callback function to be called when the unsubscre is acknowledged by the broker +-- @tparam[opt] function opts.callback callback function to be called when the unsubscribe is acknowledged by the broker -- @return packet id on success or false and error message on failure function Client:unsubscribe(opts) -- fetch and validate opts From c8583cf146965365fda7cc1b5eb6ba652ae76920 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Fri, 19 Nov 2021 17:40:49 +0100 Subject: [PATCH 37/65] chore(test) rename test file --- tests/spec/{topics.lua => 09-topics.lua} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/spec/{topics.lua => 09-topics.lua} (100%) diff --git a/tests/spec/topics.lua b/tests/spec/09-topics.lua similarity index 100% rename from tests/spec/topics.lua rename to tests/spec/09-topics.lua From aa8f5c0eef0d8979e34e1cf5ec375ec8d040abc4 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Fri, 19 Nov 2021 17:48:04 +0100 Subject: [PATCH 38/65] fix(timeouts) copas and nginx should have indefinite timeouts on reading --- mqtt/connector/copas.lua | 11 ++++++++++- mqtt/connector/nginx.lua | 12 ++++++++++-- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/mqtt/connector/copas.lua b/mqtt/connector/copas.lua index 83453af..1a8d4c8 100644 --- a/mqtt/connector/copas.lua +++ b/mqtt/connector/copas.lua @@ -47,7 +47,7 @@ end function connector:connect() self:validate() local sock = copas.wrap(socket.tcp(), self.secure_params) - sock:settimeout(self.timeout) + sock:settimeouts(self.timeout, self.timeout, -1) -- no timout on reading local ok, err = sock:connect(self.host, self.port) if not ok then @@ -57,6 +57,12 @@ function connector:connect() return true end +-- the packet was fully read, we can clear the bufer. +function connector:buffer_clear() + -- since the packet is complete, we wait now indefinitely for the next one + self.sock:settimeouts(nil, nil, -1) -- no timeout on reading +end + -- Shutdown network connection function connector:shutdown() self.sock:close() @@ -80,6 +86,9 @@ function connector:receive(size) local sock = self.sock local data, err = sock:receive(size) if data then + -- bytes received, so change from idefinite timeout to regular until + -- packet is complete (see buffer_clear method) + self.sock:settimeouts(nil, nil, self.timeout) return data end diff --git a/mqtt/connector/nginx.lua b/mqtt/connector/nginx.lua index 00bdadb..d34130b 100644 --- a/mqtt/connector/nginx.lua +++ b/mqtt/connector/nginx.lua @@ -32,7 +32,7 @@ ngxsocket.super = super -- load required stuff local ngx_socket_tcp = ngx.socket.tcp - +local long_timeout = 7*24*60*60*1000 -- one week -- validate connection options function ngxsocket:validate() @@ -49,7 +49,7 @@ end function ngxsocket:connect() local sock = ngx_socket_tcp() -- set read-timeout to 'nil' to not timeout at all - sock:settimeouts(self.timeout * 1000, self.timeout * 1000, 24*60*60*1000) -- millisecs + sock:settimeouts(self.timeout * 1000, self.timeout * 1000, long_timeout) -- no timeout on reading local ok, err = sock:connect(self.host, self.port) if not ok then return false, "socket:connect failed: "..err @@ -71,11 +71,19 @@ function ngxsocket:send(data) return self.sock:send(data) end +function ngxsocket:buffer_clear() + -- since the packet is complete, we wait now indefinitely for the next one + self.sock:settimeouts(self.timeout * 1000, self.timeout * 1000, long_timeout) -- no timeout on reading +end + -- Receive given amount of data from network connection function ngxsocket:receive(size) local sock = self.sock local data, err = sock:receive(size) if data then + -- bytes received, so change from idefinite timeout to regular until + -- packet is complete (see buffer_clear method) + self.sock:settimeouts(self.timeout * 1000, self.timeout * 1000, self.timeout * 1000) return data end From 1300b0d4ab83ced3af1e2827d385ac01851b68f1 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Fri, 19 Nov 2021 18:44:50 +0100 Subject: [PATCH 39/65] fix(loop) make packet handling async for copas/nginx the thread reading should not execute the tasks, it has to return to reading the socket again asap, to keep the device responsive. --- mqtt/loop/copas.lua | 10 ++++++++++ mqtt/loop/nginx.lua | 11 +++++++++++ 2 files changed, 21 insertions(+) diff --git a/mqtt/loop/copas.lua b/mqtt/loop/copas.lua index 3491528..d42216d 100644 --- a/mqtt/loop/copas.lua +++ b/mqtt/loop/copas.lua @@ -23,6 +23,16 @@ function _M.add(cl) end client_registry[cl] = true + do -- make mqtt device async for incoming packets + local handle_received_packet = cl.handle_received_packet + + -- replace packet handler; create a new thread for each packet received + cl.handle_received_packet = function(mqttdevice, packet) + copas.addthread(handle_received_packet, mqttdevice, packet) + return true + end + end + -- add keep-alive timer local timer = copas.addthread(function() while client_registry[cl] do diff --git a/mqtt/loop/nginx.lua b/mqtt/loop/nginx.lua index 3a8a3d9..d3d944d 100644 --- a/mqtt/loop/nginx.lua +++ b/mqtt/loop/nginx.lua @@ -19,6 +19,17 @@ function _M.add(client) return false, "MQTT client was already added to Nginx" end + do -- make mqtt device async for incoming packets + local handle_received_packet = client.handle_received_packet + + -- replace packet handler; create a new thread for each packet received + client.handle_received_packet = function(mqttdevice, packet) + ngx.thread.spawn(handle_received_packet, mqttdevice, packet) + return true + end + end + + local ok, err = ngx.timer.at(0, function() -- spawn a thread to listen on the socket local coro = ngx.thread.spawn(function() From 043badc8f56ff405ca7272f1fa7acf7941f29645 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Sun, 9 Jan 2022 14:21:59 +0100 Subject: [PATCH 40/65] fix(send) add a lock when sedning for Copas connector when sending, the sending thread may yield if the socket isn't ready for writing yet. To prevent another thread from coming in and writing, the send operation is now wrapped in a lock. Added a note to the Nginx one that it needs a similar construct. --- mqtt/connector/copas.lua | 17 +++++++++++++++-- mqtt/connector/nginx.lua | 2 ++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/mqtt/connector/copas.lua b/mqtt/connector/copas.lua index 1a8d4c8..2c787d8 100644 --- a/mqtt/connector/copas.lua +++ b/mqtt/connector/copas.lua @@ -29,6 +29,7 @@ connector.super = super local socket = require("socket") local copas = require("copas") +local new_lock = require("copas.lock").new local validate_luasec = require("mqtt.connector.base.luasec") @@ -54,6 +55,7 @@ function connector:connect() return false, "copas.connect failed: "..err end self.sock = sock + self.send_lock = new_lock(30) -- 30 second timeout return true end @@ -66,18 +68,29 @@ end -- Shutdown network connection function connector:shutdown() self.sock:close() + self.send_lock:destroy() end -- Send data to network connection function connector:send(data) + -- cache locally in case lock/sock gets replaced while we were sending + local sock = self.sock + local lock = self.send_lock + + local ok, err = lock:get() + if not ok then + return nil, "failed acquiring send_lock: "..tostring(err) + end + local i = 1 - local err while i < #data do - i, err = self.sock:send(data, i) + i, err = sock:send(data, i) if not i then + lock:release() return false, err end end + lock:release() return true end diff --git a/mqtt/connector/nginx.lua b/mqtt/connector/nginx.lua index d34130b..97c30a9 100644 --- a/mqtt/connector/nginx.lua +++ b/mqtt/connector/nginx.lua @@ -47,6 +47,8 @@ end -- Store opened socket to conn table -- Returns true on success, or false and error text on failure function ngxsocket:connect() + -- TODO: add a lock for sending to prevent multiple threads from writing to + -- the same socket simultaneously (see the Copas connector) local sock = ngx_socket_tcp() -- set read-timeout to 'nil' to not timeout at all sock:settimeouts(self.timeout * 1000, self.timeout * 1000, long_timeout) -- no timeout on reading From 5bc4a9238dc3007be145a324756b230ed0829076 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Sat, 30 Jul 2022 07:30:40 +0200 Subject: [PATCH 41/65] fix(deps) bump to Copas 4 --- mqtt/connector/base/non_buffered_base.lua | 2 +- mqtt/connector/copas.lua | 2 ++ mqtt/loop/copas.lua | 13 +++++++------ 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/mqtt/connector/base/non_buffered_base.lua b/mqtt/connector/base/non_buffered_base.lua index eb9e0f8..617813c 100644 --- a/mqtt/connector/base/non_buffered_base.lua +++ b/mqtt/connector/base/non_buffered_base.lua @@ -26,7 +26,7 @@ non_buffered.signal_closed = {} -- remote closed the connection --- Validate connection options. function non_buffered:shutdown() -- luacheck: ignore - error("method 'validate' on connector wasn't implemented") + error("method 'validate' on connector wasn't implemented") --TODO: comments and text doesn't match name end --- Clears consumed bytes. diff --git a/mqtt/connector/copas.lua b/mqtt/connector/copas.lua index 2c787d8..e7114a0 100644 --- a/mqtt/connector/copas.lua +++ b/mqtt/connector/copas.lua @@ -48,6 +48,8 @@ end function connector:connect() self:validate() local sock = copas.wrap(socket.tcp(), self.secure_params) + copas.setsocketname("mqtt@"..self.host..":"..self.port, sock) + sock:settimeouts(self.timeout, self.timeout, -1) -- no timout on reading local ok, err = sock:connect(self.host, self.port) diff --git a/mqtt/loop/copas.lua b/mqtt/loop/copas.lua index d42216d..32fc84e 100644 --- a/mqtt/loop/copas.lua +++ b/mqtt/loop/copas.lua @@ -25,23 +25,24 @@ function _M.add(cl) do -- make mqtt device async for incoming packets local handle_received_packet = cl.handle_received_packet - + local count = 0 -- replace packet handler; create a new thread for each packet received cl.handle_received_packet = function(mqttdevice, packet) - copas.addthread(handle_received_packet, mqttdevice, packet) + count = count + 1 + copas.addnamedthread(handle_received_packet, cl.opts.id..":receive_"..count, mqttdevice, packet) return true end end -- add keep-alive timer - local timer = copas.addthread(function() + local timer = copas.addnamedthread(function() while client_registry[cl] do copas.sleep(cl:check_keep_alive()) end - end) + end, cl.opts.id .. ":keep_alive") -- add client to connect and listen - copas.addthread(function() + copas.addnamedthread(function() while client_registry[cl] do local timeout = cl:step() if not timeout then @@ -54,7 +55,7 @@ function _M.add(cl) end end end - end) + end, cl.opts.id .. ":listener") return true end From e9d8b392b8480cb60c6faecc3ae60f98d81248b9 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Sun, 7 Aug 2022 15:14:40 +0200 Subject: [PATCH 42/65] fix(docs) minor documentation updates --- docs_topics/05-connectors.md | 2 +- mqtt/connector/base/non_buffered_base.lua | 2 +- mqtt/connector/copas.lua | 2 +- mqtt/loop/copas.lua | 2 ++ 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs_topics/05-connectors.md b/docs_topics/05-connectors.md index 9023962..1f14b68 100644 --- a/docs_topics/05-connectors.md +++ b/docs_topics/05-connectors.md @@ -15,7 +15,7 @@ to build on, which to pick depends on the environment. The main question is what event/io loop mechanism does your implementation have? -* a single main (co)routione that runs, and doesn't yield when doing network IO. In this case +* a single main (co)routine that runs, and doesn't yield when doing network IO. In this case you should use the `buffered_base` and read on sockets with a `0` timeout. Check the `mqtt.connector.luasocket` implementation for an example (this is what `ioloop` uses). diff --git a/mqtt/connector/base/non_buffered_base.lua b/mqtt/connector/base/non_buffered_base.lua index 617813c..ded109e 100644 --- a/mqtt/connector/base/non_buffered_base.lua +++ b/mqtt/connector/base/non_buffered_base.lua @@ -26,7 +26,7 @@ non_buffered.signal_closed = {} -- remote closed the connection --- Validate connection options. function non_buffered:shutdown() -- luacheck: ignore - error("method 'validate' on connector wasn't implemented") --TODO: comments and text doesn't match name + error("method 'shutdown' on connector wasn't implemented") end --- Clears consumed bytes. diff --git a/mqtt/connector/copas.lua b/mqtt/connector/copas.lua index e7114a0..14da185 100644 --- a/mqtt/connector/copas.lua +++ b/mqtt/connector/copas.lua @@ -17,7 +17,7 @@ -- upon receiving a packet, to call an event handler. The handler must return -- swiftly, since while the handler runs the socket will not be reading. -- Any task that might take longer than a few milliseconds should be off --- loaded to another thread. +-- loaded to another thread (the Copas-loop will take care of this). -- -- NOTE: you will need to install copas like this: `luarocks install copas`. -- @module mqtt.connector.copas diff --git a/mqtt/loop/copas.lua b/mqtt/loop/copas.lua index 32fc84e..823ceba 100644 --- a/mqtt/loop/copas.lua +++ b/mqtt/loop/copas.lua @@ -12,6 +12,8 @@ local _M = {} --- Add MQTT client to the Copas scheduler. +-- Each received packet will be handled by a new thread, such that the thread +-- listening on the socket can return immediately. -- The client will automatically be removed after it exits. It will set up a -- thread to call `Client:check_keep_alive`. -- @param cl mqtt-client to add to the Copas scheduler From 196f7af1cf201c58ea95c4b3006bf01f74fafb93 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Fri, 14 Oct 2022 16:39:14 +0200 Subject: [PATCH 43/65] fix(copas) prevent accidental disabling of keep-alive if arithmetic accidentally returns a number < 0 then Copas will put the thread to sleep-until-woken, effectively disabling keepalives. --- mqtt/loop/copas.lua | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mqtt/loop/copas.lua b/mqtt/loop/copas.lua index 823ceba..a227d55 100644 --- a/mqtt/loop/copas.lua +++ b/mqtt/loop/copas.lua @@ -39,7 +39,10 @@ function _M.add(cl) -- add keep-alive timer local timer = copas.addnamedthread(function() while client_registry[cl] do - copas.sleep(cl:check_keep_alive()) + local next_check = cl:check_keep_alive() + if next_check > 0 then + copas.sleep(next_check) + end end end, cl.opts.id .. ":keep_alive") From 8bafeb64e2915cbdf0d75e30d3b13f4b4cb4361f Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Wed, 26 Oct 2022 03:16:48 +0200 Subject: [PATCH 44/65] fix(copas) switch to pause instead of sleep --- examples/copas-example.lua | 2 +- mqtt/client.lua | 2 +- mqtt/loop/copas.lua | 4 ++-- tests/spec/08-copas_spec.lua | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/copas-example.lua b/examples/copas-example.lua index 9e3368b..51a24fa 100644 --- a/examples/copas-example.lua +++ b/examples/copas-example.lua @@ -29,7 +29,7 @@ local ping = mqtt.client{ -- code below does both, sleeping, and writing (implicit in 'publish') copas.addthread(function() for i = 1, num_pings do - copas.sleep(delay) + copas.pause(delay) print("ping", i) assert(self:publish{ topic = "luamqtt/copas-ping/"..suffix, payload = "ping"..i, qos = 1 }) end diff --git a/mqtt/client.lua b/mqtt/client.lua index 129f792..e8d3ed8 100644 --- a/mqtt/client.lua +++ b/mqtt/client.lua @@ -898,7 +898,7 @@ end -- if not my_client then -- return -- exiting, client was destroyed -- end --- copas.sleep(my_client:check_keep_alive()) +-- copas.pause(my_client:check_keep_alive()) -- end -- end) function Client:check_keep_alive() diff --git a/mqtt/loop/copas.lua b/mqtt/loop/copas.lua index a227d55..026ccd9 100644 --- a/mqtt/loop/copas.lua +++ b/mqtt/loop/copas.lua @@ -41,7 +41,7 @@ function _M.add(cl) while client_registry[cl] do local next_check = cl:check_keep_alive() if next_check > 0 then - copas.sleep(next_check) + copas.pause(next_check) end end end, cl.opts.id .. ":keep_alive") @@ -56,7 +56,7 @@ function _M.add(cl) copas.wakeup(timer) else if timeout > 0 then - copas.sleep(timeout) + copas.pause(timeout) end end end diff --git a/tests/spec/08-copas_spec.lua b/tests/spec/08-copas_spec.lua index 11ccb6f..14d2324 100644 --- a/tests/spec/08-copas_spec.lua +++ b/tests/spec/08-copas_spec.lua @@ -58,7 +58,7 @@ describe("copas connector", function() return end if timeout > 0 then - copas.sleep(timeout) + copas.pause(timeout) end end end) From 7b8cb786a6659dcefbf092d40e5cbdfc93a7efb1 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Sat, 14 Jan 2023 23:26:25 +0100 Subject: [PATCH 45/65] chore(docs) fix some typos --- mqtt/client.lua | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/mqtt/client.lua b/mqtt/client.lua index e8d3ed8..d17f8d2 100644 --- a/mqtt/client.lua +++ b/mqtt/client.lua @@ -248,23 +248,23 @@ end -- @tparam function events.connect `function(connack_packet, client_obj)`
-- After a connect attempt, after receiving the CONNACK packet from the broker. -- check `connack_packet.rc == 0` for a succesful connect. --- @tparam functon events.error `function(errmsg, client_obj [, packet])`
+-- @tparam function events.error `function(errmsg, client_obj [, packet])`
-- on errors, optional `packet` is only provided if the -- received `CONNACK.rc ~= 0` when connecting. --- @tparam functon events.close `function(connection_obj, client_obj)`
+-- @tparam function events.close `function(connection_obj, client_obj)`
-- upon closing the connection. `connection_obj.close_reason` -- (string) will hold the close reason. --- @tparam functon events.shutdown `function(client_obj)`
+-- @tparam function events.shutdown `function(client_obj)`
-- upon shutting down the client (diconnecting an no more reconnects). --- @tparam functon events.subscribe `function(suback_packet, client_obj)`
+-- @tparam function events.subscribe `function(suback_packet, client_obj)`
-- upon a succesful subscription, after receiving the SUBACK packet from the broker --- @tparam functon events.unsubscribe `function(unsuback_packet, client_obj)`
+-- @tparam function events.unsubscribe `function(unsuback_packet, client_obj)`
-- upon a succesful unsubscription, after receiving the UNSUBACK packet from the broker --- @tparam functon events.message `function(publish_packet, client_obj)`
+-- @tparam function events.message `function(publish_packet, client_obj)`
-- upon receiving a PUBLISH packet from the broker --- @tparam functon events.acknowledge `function(ack_packet, client_obj)`
+-- @tparam function events.acknowledge `function(ack_packet, client_obj)`
-- upon receiving a PUBACK or PUBREC packet from the broker --- @tparam functon events.auth `function(auth_packet, client_obj)`
+-- @tparam function events.auth `function(auth_packet, client_obj)`
-- upon receiving an AUTH packet -- @usage -- client:on { From bbd5226bc3c237ee155156344c30998d80f9e4df Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Sat, 14 Jan 2023 23:27:13 +0100 Subject: [PATCH 46/65] fix(patterns) escape lua magic characters --- mqtt/init.lua | 20 ++++++++++++++++++-- tests/spec/09-topics.lua | 10 ++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/mqtt/init.lua b/mqtt/init.lua index fd2605a..22df3e1 100644 --- a/mqtt/init.lua +++ b/mqtt/init.lua @@ -148,8 +148,24 @@ function mqtt.compile_topic_pattern(t) if t == "#" then t = "(.+)" -- matches anything at least 1 character long else - t = t:gsub("#","(.-)") -- match anything, can be empty - t = t:gsub("%+","([^/]-)") -- match anything between '/', can be empty + -- first replace valid mqtt '+' and '#' with placeholders + local hash = string.char(1) + t = t:gsub("/#$", "/" .. hash) + local plus = string.char(2) + t = t:gsub("^%+$", plus) + t = t:gsub("^%+/", plus .. "/") + local c = 1 + while c ~= 0 do -- must loop, since adjacent patterns can overlap + t, c = t:gsub("/%+/", "/" .. plus .. "/") + end + t = t:gsub("/%+$", "/" .. plus) + + -- now escape any special Lua pattern characters + t = t:gsub("[%\\%(%)%.%%%+%-%*%?%[%^%$]", function(cap) return "%"..cap end) + + -- finally replace placeholders with captures + t = t:gsub(hash,"(.-)") -- match anything, can be empty + t = t:gsub(plus,"([^/]-)") -- match anything between '/', can be empty end return "^"..t.."$" end diff --git a/tests/spec/09-topics.lua b/tests/spec/09-topics.lua index 36070ce..b1932c1 100644 --- a/tests/spec/09-topics.lua +++ b/tests/spec/09-topics.lua @@ -144,6 +144,16 @@ describe("topics", function() describe("pattern compiler & matcher", function() + it("escapes Lua pattern magic characters", function() + local t = mqtt.compile_topic_pattern("+/dash-dash/+/+/back\\slash/+") + assert.equal("^([^/]-)/dash%-dash/([^/]-)/([^/]-)/back%\\slash/([^/]-)$", t) + local h, m, d, w = ("hello/dash-dash/my/dear/back\\slash/world"):match(t) + assert.equal("hello", h) + assert.equal("my", m) + assert.equal("dear", d) + assert.equal("world", w) + end) + it("basic parsing works", function() local opts = { topic = "+/+", From e452151538843b8945a2adad9d80826b63bb2837 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Wed, 25 Jan 2023 10:20:46 +0100 Subject: [PATCH 47/65] chore(docs) small doc updates/fixes --- mqtt/connector/copas.lua | 3 +-- mqtt/loop/init.lua | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/mqtt/connector/copas.lua b/mqtt/connector/copas.lua index 14da185..9926480 100644 --- a/mqtt/connector/copas.lua +++ b/mqtt/connector/copas.lua @@ -10,8 +10,7 @@ -- * the `client` option `ssl_module` is not supported by the Copas connector, -- It will always use the module named `ssl`. -- --- * multiple threads cannot send simultaneously (simple scenarios will just --- work) +-- * multiple threads can send simultaneously (sending is wrapped in a lock) -- -- * since the client creates a long lived connection for reading, it returns -- upon receiving a packet, to call an event handler. The handler must return diff --git a/mqtt/loop/init.lua b/mqtt/loop/init.lua index f7a4c3d..d499c2e 100644 --- a/mqtt/loop/init.lua +++ b/mqtt/loop/init.lua @@ -1,7 +1,7 @@ --- Auto detect the IO loop to use. -- Interacting with the supported IO loops (ioloop, copas, and nginx) requires -- specific implementations to get it right. --- This module is will auto-detect the environment and return the proper +-- This module will auto-detect the environment and return the proper -- module from; -- -- * `mqtt.loop.ioloop` From b44634a7ec7d25ca99ccf1775b133a1a467877ec Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Fri, 24 Nov 2023 17:02:43 +0100 Subject: [PATCH 48/65] feat(clean): option to only connect clean on first connect --- mqtt/client.lua | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/mqtt/client.lua b/mqtt/client.lua index d17f8d2..0b5e1f4 100644 --- a/mqtt/client.lua +++ b/mqtt/client.lua @@ -55,7 +55,7 @@ Client.__index = Client --
`[mqtt[s]://][username[:password]@]hostname[:port]` --
Any option specifically added to the options -- table will take precedence over the option specified in this uri. --- @tparam boolean opts.clean clean session start flag +-- @tparam boolean|string opts.clean clean session start flag, use "first" to start clean only on first connect -- @tparam[opt] string opts.protocol either `"mqtt"` or `"mqtts"` -- @tparam[opt] string opts.username username for authorization on MQTT broker -- @tparam[opt] string opts.password password for authorization on MQTT broker; not acceptable in absence of username @@ -64,7 +64,7 @@ Client.__index = Client -- @tparam[opt=4] number opts.version MQTT protocol version to use, either `4` (for MQTT v3.1.1) or `5` (for MQTT v5.0). -- Also you may use special values `mqtt.v311` or `mqtt.v50` for this field. -- @tparam[opt] string opts.id MQTT client ID, will be generated by luamqtt library if absent --- @tparam[opt=false] boolean,table opts.secure use secure network connection, provided by the lua module set in `opts.ssl_module`. +-- @tparam[opt=false] boolean|table opts.secure use secure network connection, provided by the lua module set in `opts.ssl_module`. -- Set to true to select default parameters, check individual `mqtt.connectors` for supported options. -- @tparam[opt] table opts.will will message table with required fields `{ topic="...", payload="..." }` -- and optional fields `{ qos=0...2, retain=true/false }` @@ -83,7 +83,7 @@ Client.__index = Client -- -- local my_client = Client.create { -- uri = "mqtts://broker.host.com", --- clean = true, +-- clean = "first", -- version = mqtt.v50, -- } function Client:__init(opts) @@ -104,7 +104,7 @@ function Client:__init(opts) assert(value_type == "string", "expecting uri to be a string") a.uri = value elseif key == "clean" then - assert(value_type == "boolean", "expecting clean to be a boolean") + assert(value_type == "boolean" or value == "first", "expecting clean to be a boolean, or 'first'") a.clean = value elseif key == "version" then assert(value_type == "number", "expecting version to be a number") @@ -152,7 +152,7 @@ function Client:__init(opts) -- check required arguments assert(a.uri, 'expecting uri="..." to create MQTT client') - assert(a.clean ~= nil, "expecting clean=true or clean=false to create MQTT client") + assert(a.clean ~= nil, "expecting clean=true, clean=false, or clean='first' to create MQTT client") if not a.id then -- generate random client id @@ -858,7 +858,7 @@ function Client:send_connect() local connect = self._make_packet{ type = packet_type.CONNECT, id = opts.id, - clean = opts.clean, + clean = not not opts.clean, -- force to boolean, in case "first" username = opts.username, password = opts.password, will = opts.will, @@ -1058,6 +1058,10 @@ function Client:handle_received_packet(packet) log:info("client '%s' connected successfully to '%s:%s'", self.opts.id, conn.host, conn.port) -- fire connect event + if self.opts.clean == "first" then + self.opts.clean = false -- reset clean flag to false, so next connection resumes previous session + log:debug("client '%s'; switching clean flag to false (was 'first')", self.opts.id) + end self:handle("connect", packet, self) self.first_connect = false else From be50bd127ec83dfc355938b0a8ee43fb1665447e Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Sat, 25 Nov 2023 15:21:21 +0100 Subject: [PATCH 49/65] change(topic): parsing topics distinguish no-match from error --- mqtt/init.lua | 20 ++++++++++---------- tests/spec/09-topics.lua | 14 +++++++------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/mqtt/init.lua b/mqtt/init.lua index 22df3e1..2e874f8 100644 --- a/mqtt/init.lua +++ b/mqtt/init.lua @@ -179,27 +179,27 @@ end -- stored in this field for future use (cache). -- @tparam array opts.keys array of field names. The order must be the same as the -- order of the wildcards in `topic` --- @return[1] `fields` table: the array part will have the values of the wildcards, in +-- @treturn[1] table `fields`: the array part will have the values of the wildcards, in -- the order they appeared. The hash part, will have the field names provided -- in `opts.keys`, with the values of the corresponding wildcard. If a `#` -- wildcard was used, that one will be the last in the table. --- @return[1] `varargs` table: will only be returned if the wildcard topic contained the --- `#` wildcard. The returned table is an array, with all segments that were --- matched by the `#` wildcard. --- @return[2] false+err on error, eg. topic didn't match or pattern was invalid. +-- @treturn[1] `varargs`: The returned table is an array, with all segments that were +-- matched by the `#` wildcard (empty if there was no `#` wildcard). +-- @treturn[2] boolean `false` if there was no match +-- @return[3] `false`+err on error, eg. pattern was invalid. -- @usage -- local opts = { -- topic = "homes/+/+/#", -- keys = { "homeid", "roomid", "varargs"}, -- } --- local fields, varargs = topic_match("homes/myhome/living/mainlights/brightness", opts) +-- local fields, varargst = topic_match("homes/myhome/living/mainlights/brightness", opts) -- -- print(fields[1], fields.homeid) -- "myhome myhome" -- print(fields[2], fields.roomid) -- "living living" -- print(fields[3], fields.varargs) -- "mainlights/brightness mainlights/brightness" -- --- print(varargs[1]) -- "mainlights" --- print(varargs[2]) -- "brightness" +-- print(varargst[1]) -- "mainlights" +-- print(varargst[2]) -- "brightness" function mqtt.topic_match(topic, opts) if type(topic) ~= "string" then return false, "expected topic to be a string" @@ -223,7 +223,7 @@ function mqtt.topic_match(topic, opts) end local values = { topic:match(pattern) } if values[1] == nil then - return false, "topic does not match wildcard pattern" + return false end local keys = opts.keys if keys ~= nil then @@ -240,7 +240,7 @@ function mqtt.topic_match(topic, opts) end if not pattern:find("%(%.[%-%+]%)%$$") then -- pattern for "#" as last char -- we're done - return values + return values, {} end -- we have a '#' wildcard local vararg = values[#values] diff --git a/tests/spec/09-topics.lua b/tests/spec/09-topics.lua index b1932c1..c8bca35 100644 --- a/tests/spec/09-topics.lua +++ b/tests/spec/09-topics.lua @@ -160,8 +160,8 @@ describe("topics", function() pattern = nil, keys = { "hello", "world"} } - local res, err = mqtt.topic_match("hello/world", opts) - assert.is_nil(err) + local res, varargs = mqtt.topic_match("hello/world", opts) + assert.is_same({}, varargs) assert.same(res, { "hello", "world", hello = "hello", @@ -201,7 +201,7 @@ describe("topics", function() } local ok, err = mqtt.topic_match("hello/world", opts) assert.is_false(ok) - assert.is_string(err) + assert.is_nil(err) end) it("pattern '+' works", function() @@ -211,8 +211,8 @@ describe("topics", function() keys = { "hello" } } -- matches topic - local res, err = mqtt.topic_match("hello", opts) - assert.is_nil(err) + local res, varargs = mqtt.topic_match("hello", opts) + assert.are.same({}, varargs) assert.same(res, { "hello", hello = "hello", @@ -225,8 +225,8 @@ describe("topics", function() pattern = nil, keys = { "hello", "there", "world"} } - local res, err = mqtt.topic_match("//", opts) - assert.is_nil(err) + local res, varargs = mqtt.topic_match("//", opts) + assert.are.same({}, varargs) assert.same(res, { "", "", "", hello = "", From b19fb525e8a72156c93484c0aed1511c22d73824 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Sat, 2 Dec 2023 10:00:31 +0100 Subject: [PATCH 50/65] chore(ci): fix linter warnings --- README.md | 2 ++ docs_topics/02-dependencies.md | 10 +++++----- docs_topics/05-connectors.md | 3 --- examples/openresty/README.md | 2 +- mqtt/protocol5.lua | 4 ++-- 5 files changed, 10 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index deddeae..be45310 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,7 @@ And of course, any contribution are welcome! To run tests in this git repo you need [**busted**](https://luarocks.org/modules/olivine-labs/busted) as well as some dependencies: Prepare: + ```sh luarocks install busted luarocks install luacov @@ -54,6 +55,7 @@ luarocks install lualogging ``` Running the tests: + ```sh busted ``` diff --git a/docs_topics/02-dependencies.md b/docs_topics/02-dependencies.md index aede65e..f52befd 100644 --- a/docs_topics/02-dependencies.md +++ b/docs_topics/02-dependencies.md @@ -5,27 +5,27 @@ The dependencies differ slightly based on the environment you use, and the requi * [**luasocket**](https://luarocks.org/modules/luasocket/luasocket) to establish TCP connections to the MQTT broker. This is a listed dependency in the luamqtt rockspec, so it will automatically be installed if you use LuaRocks to install luamqtt. To install it manually: - luarocks install luasocket + luarocks install luasocket * [**copas**](https://github.com/keplerproject/copas) module for asynchoneous IO. Copas is an advanced co-routine scheduler with far more features than the included `ioloop`. For anything more than a few devices, or for devices which require network IO beyond mqtt alone, Copas is the better alternative. Copas is also pure-Lua, but has parallel network IO (as opposed to sequential network IO in `ioloop`), and has features like; threads, timers, locks, semaphores, and non-blocking clients for http(s), (s)ftp, and smtp. - luarocks install copas + luarocks install copas * [**luasec**](https://github.com/brunoos/luasec) module for SSL/TLS based connections. This is optional and may be skipped if you don't need secure network connections (e.g. broker is located in your local network). It's not listed in package dependencies, please install it manually like this: - luarocks install luasec + luarocks install luasec * [**LuaBitOp**](http://bitop.luajit.org/) library to perform bitwise operations, which is required only on Lua 5.1. It's not listed in package dependencies, please install it manually like this: - luarocks install luabitop + luarocks install luabitop * [**LuaLogging**](https://github.com/lunarmodules/lualogging/) to enable logging by the MQTT client. This is optional but highly recommended for long running clients. This is a great debugging aid when developing your clients. Also when using OpenResty as your runtime, you'll definitely want to use this, see [openresty.lua](https://xhaskx.github.io/luamqtt/examples/openresty.lua.html) for an example. It's not listed in package dependencies, please install it manually like this: - luarocks install lualogging + luarocks install lualogging diff --git a/docs_topics/05-connectors.md b/docs_topics/05-connectors.md index 1f14b68..8575278 100644 --- a/docs_topics/05-connectors.md +++ b/docs_topics/05-connectors.md @@ -25,6 +25,3 @@ The main question is what event/io loop mechanism does your implementation have? The main thing to look for when checking out the existing implementations is the network timeout settings, and the returned `signals`. - - - diff --git a/examples/openresty/README.md b/examples/openresty/README.md index 6479473..8fcebd3 100644 --- a/examples/openresty/README.md +++ b/examples/openresty/README.md @@ -12,7 +12,7 @@ context. In the timer we'll spawn a thread that will do the listening, and the timer itself will go in an endless loop to do the keepalives. -**Caveats** +# Caveats * Due to the socket limitation we cannot Publish anything from another context. If you run into "bad request" errors on socket operations, you diff --git a/mqtt/protocol5.lua b/mqtt/protocol5.lua index 17842c6..9055d74 100644 --- a/mqtt/protocol5.lua +++ b/mqtt/protocol5.lua @@ -1324,7 +1324,7 @@ local function parse_packet_unsuback(ptype, flags, input) end -- Parse PINGREQ packet, DOC: 3.12 PINGREQ – PING request -local function parse_packet_pingreq(ptype, flags, input_) +local function parse_packet_pingreq(ptype, flags, _) -- DOC: 3.12.1 PINGREQ Fixed Header if flags ~= 0 then -- Reserved return false, packet_type[ptype]..": unexpected flags value: "..flags @@ -1333,7 +1333,7 @@ local function parse_packet_pingreq(ptype, flags, input_) end -- Parse PINGRESP packet, DOC: 3.13 PINGRESP – PING response -local function parse_packet_pingresp(ptype, flags, input_) +local function parse_packet_pingresp(ptype, flags, _) -- DOC: 3.13.1 PINGRESP Fixed Header if flags ~= 0 then -- Reserved return false, packet_type[ptype]..": unexpected flags value: "..flags From b16be62e5a3d00c5c3551cbc797cb9dc524650c1 Mon Sep 17 00:00:00 2001 From: Alexander Kiranov Date: Fri, 22 Dec 2023 14:21:26 +0200 Subject: [PATCH 51/65] markdownlint: improved config --- .markdownlint.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.markdownlint.yaml b/.markdownlint.yaml index d95827a..1cc0dda 100644 --- a/.markdownlint.yaml +++ b/.markdownlint.yaml @@ -1,3 +1,4 @@ default: true single-title: false line-length: false +code-block-style: false From 23a070bdb7fa24f49f3a2f42ab958cf51386f178 Mon Sep 17 00:00:00 2001 From: Alexander Kiranov Date: Fri, 22 Dec 2023 14:22:30 +0200 Subject: [PATCH 52/65] lua version fix md file --- docs_topics/03-lua_versions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs_topics/03-lua_versions.md b/docs_topics/03-lua_versions.md index 3674f37..52c376c 100644 --- a/docs_topics/03-lua_versions.md +++ b/docs_topics/03-lua_versions.md @@ -2,7 +2,7 @@ It's tested to work on Debian 9 GNU/Linux with Lua versions: -* Lua 5.1 ... Lua 5.3 (**i.e. any modern Lua version**) +* Lua 5.1 ... Lua 5.4 (**i.e. any modern Lua version**) * LuaJIT 2.0.0 ... LuaJIT 2.1.0 beta3 * It may also work on other Lua versions without any guarantees From 82c621342c3625f55661697fd57f535cfb9aa166 Mon Sep 17 00:00:00 2001 From: Alexander Kiranov Date: Fri, 22 Dec 2023 14:23:08 +0200 Subject: [PATCH 53/65] README.md: added short installation header --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index be45310..66f31c9 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,14 @@ This library is written in **pure-lua** to provide maximum portability. * Full MQTT v5.0 client-side support * Support for Copas, OpenResty/Nginx, and an included lightweight ioloop. +# Installation + +From LuaRocks: + + luarocks install luamqtt + +[More details](./docs_topics/01-installation.md) + # Documentation See [https://xhaskx.github.io/luamqtt/](https://xhaskx.github.io/luamqtt/) From aa207ac4d8289850bd65ea37f2821bc0a4e5b2b3 Mon Sep 17 00:00:00 2001 From: Alexander Kiranov Date: Fri, 22 Dec 2023 14:23:33 +0200 Subject: [PATCH 54/65] README.md: busted tests dependency fix --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 66f31c9..27d9f14 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,7 @@ luarocks install luasocket luarocks install luasec luarocks install copas luarocks install lualogging +luarocks install ansicolors ``` Running the tests: From a84c7e40259009d837625155ca79ca667f28a8af Mon Sep 17 00:00:00 2001 From: Alexander Kiranov Date: Fri, 22 Dec 2023 14:33:32 +0200 Subject: [PATCH 55/65] markdown link fix --- docs_topics/01-installation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs_topics/01-installation.md b/docs_topics/01-installation.md index 94f0cd9..eead3b5 100644 --- a/docs_topics/01-installation.md +++ b/docs_topics/01-installation.md @@ -12,4 +12,4 @@ in the rockspec. To install from source clone the repo and make sure the `./mqtt/` folder is in your Lua search path. -Check the [dependencies](./02-dependencies.md.html) on how (and when) to install those. +Check the [dependencies](./02-dependencies.md) on how (and when) to install those. From c6520644a50925325059f54b5624717c0e35dd6e Mon Sep 17 00:00:00 2001 From: Alexander Kiranov Date: Fri, 22 Dec 2023 14:36:50 +0200 Subject: [PATCH 56/65] markdown fix --- docs_topics/02-dependencies.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs_topics/02-dependencies.md b/docs_topics/02-dependencies.md index f52befd..6aff5af 100644 --- a/docs_topics/02-dependencies.md +++ b/docs_topics/02-dependencies.md @@ -5,6 +5,7 @@ The dependencies differ slightly based on the environment you use, and the requi * [**luasocket**](https://luarocks.org/modules/luasocket/luasocket) to establish TCP connections to the MQTT broker. This is a listed dependency in the luamqtt rockspec, so it will automatically be installed if you use LuaRocks to install luamqtt. To install it manually: + luarocks install luasocket * [**copas**](https://github.com/keplerproject/copas) module for asynchoneous IO. Copas is an advanced co-routine @@ -12,15 +13,18 @@ The dependencies differ slightly based on the environment you use, and the requi require network IO beyond mqtt alone, Copas is the better alternative. Copas is also pure-Lua, but has parallel network IO (as opposed to sequential network IO in `ioloop`), and has features like; threads, timers, locks, semaphores, and non-blocking clients for http(s), (s)ftp, and smtp. + luarocks install copas * [**luasec**](https://github.com/brunoos/luasec) module for SSL/TLS based connections. This is optional and may be skipped if you don't need secure network connections (e.g. broker is located in your local network). It's not listed in package dependencies, please install it manually like this: + luarocks install luasec * [**LuaBitOp**](http://bitop.luajit.org/) library to perform bitwise operations, which is required only on Lua 5.1. It's not listed in package dependencies, please install it manually like this: + luarocks install luabitop * [**LuaLogging**](https://github.com/lunarmodules/lualogging/) to enable logging by the MQTT client. This is optional @@ -28,4 +32,5 @@ The dependencies differ slightly based on the environment you use, and the requi using OpenResty as your runtime, you'll definitely want to use this, see [openresty.lua](https://xhaskx.github.io/luamqtt/examples/openresty.lua.html) for an example. It's not listed in package dependencies, please install it manually like this: + luarocks install lualogging From 46fb48a74153cd3e56b6ea11ca9450ce8756cebc Mon Sep 17 00:00:00 2001 From: Alexander Kiranov Date: Fri, 22 Dec 2023 14:58:27 +0200 Subject: [PATCH 57/65] markdown improve --- README.md | 2 ++ docs_topics/03-lua_versions.md | 2 ++ docs_topics/README.md | 11 +++++++++++ 3 files changed, 15 insertions(+) create mode 100644 docs_topics/README.md diff --git a/README.md b/README.md index 27d9f14..91d563f 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,8 @@ From LuaRocks: See [https://xhaskx.github.io/luamqtt/](https://xhaskx.github.io/luamqtt/) +[More details](./docs_topics/README.md) + # Forum See [flespi forum thread](https://forum.flespi.com/d/97-luamqtt-mqtt-client-written-in-pure-lua) diff --git a/docs_topics/03-lua_versions.md b/docs_topics/03-lua_versions.md index 52c376c..b71ad81 100644 --- a/docs_topics/03-lua_versions.md +++ b/docs_topics/03-lua_versions.md @@ -6,4 +6,6 @@ It's tested to work on Debian 9 GNU/Linux with Lua versions: * LuaJIT 2.0.0 ... LuaJIT 2.1.0 beta3 * It may also work on other Lua versions without any guarantees +So basically it should work on any modern Linux-based OS with Lua interpreter available. + Also has run under **Windows** and it was ok, but installing luarocks-modules may be a non-trivial task on this OS. diff --git a/docs_topics/README.md b/docs_topics/README.md new file mode 100644 index 0000000..221fa39 --- /dev/null +++ b/docs_topics/README.md @@ -0,0 +1,11 @@ +# Short documentation topics + +[Installation](./01-installation.md) + +[Dependencies](./02-dependencies.md) + +[Lua versions](./03-lua_versions.md) + +[MQTT versions](./04-mqtt_versions.md) + +[Connectors](./05-connectors.md) From fe730bad9748bfe34e533b76dff7898c83d8f2a0 Mon Sep 17 00:00:00 2001 From: Alexander Kiranov Date: Fri, 22 Dec 2023 15:03:08 +0200 Subject: [PATCH 58/65] markdown improved a bit --- docs_topics/02-dependencies.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs_topics/02-dependencies.md b/docs_topics/02-dependencies.md index 6aff5af..9dd6843 100644 --- a/docs_topics/02-dependencies.md +++ b/docs_topics/02-dependencies.md @@ -8,7 +8,7 @@ The dependencies differ slightly based on the environment you use, and the requi luarocks install luasocket -* [**copas**](https://github.com/keplerproject/copas) module for asynchoneous IO. Copas is an advanced co-routine +* [**copas**](https://github.com/keplerproject/copas) _[optional dependency]_ module for asynchoneous IO. Copas is an advanced co-routine scheduler with far more features than the included `ioloop`. For anything more than a few devices, or for devices which require network IO beyond mqtt alone, Copas is the better alternative. Copas is also pure-Lua, but has parallel network IO (as opposed to sequential network IO in `ioloop`), and has features like; threads, timers, locks, semaphores, and @@ -16,18 +16,18 @@ The dependencies differ slightly based on the environment you use, and the requi luarocks install copas -* [**luasec**](https://github.com/brunoos/luasec) module for SSL/TLS based connections. This is optional and may be +* [**luasec**](https://github.com/brunoos/luasec) _[optional dependency]_ module for SSL/TLS based connections. This is optional and may be skipped if you don't need secure network connections (e.g. broker is located in your local network). It's not listed in package dependencies, please install it manually like this: luarocks install luasec -* [**LuaBitOp**](http://bitop.luajit.org/) library to perform bitwise operations, which is required only on +* [**LuaBitOp**](http://bitop.luajit.org/) _[optional dependency]_ library to perform bitwise operations, which is required only on Lua 5.1. It's not listed in package dependencies, please install it manually like this: luarocks install luabitop -* [**LuaLogging**](https://github.com/lunarmodules/lualogging/) to enable logging by the MQTT client. This is optional +* [**LuaLogging**](https://github.com/lunarmodules/lualogging/) _[optional dependency]_ to enable logging by the MQTT client. This is optional but highly recommended for long running clients. This is a great debugging aid when developing your clients. Also when using OpenResty as your runtime, you'll definitely want to use this, see [openresty.lua](https://xhaskx.github.io/luamqtt/examples/openresty.lua.html) for an example. From 8312df321a444fa5305f8e6cdd2b453da59b7dd1 Mon Sep 17 00:00:00 2001 From: Alexander Kiranov Date: Fri, 22 Dec 2023 15:06:18 +0200 Subject: [PATCH 59/65] markdown improved a bit + openresty.lua path fix --- docs_topics/02-dependencies.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs_topics/02-dependencies.md b/docs_topics/02-dependencies.md index 9dd6843..118ba1e 100644 --- a/docs_topics/02-dependencies.md +++ b/docs_topics/02-dependencies.md @@ -8,7 +8,7 @@ The dependencies differ slightly based on the environment you use, and the requi luarocks install luasocket -* [**copas**](https://github.com/keplerproject/copas) _[optional dependency]_ module for asynchoneous IO. Copas is an advanced co-routine +* _[optional dependency]_ [**copas**](https://github.com/keplerproject/copas) module for asynchoneous IO. Copas is an advanced co-routine scheduler with far more features than the included `ioloop`. For anything more than a few devices, or for devices which require network IO beyond mqtt alone, Copas is the better alternative. Copas is also pure-Lua, but has parallel network IO (as opposed to sequential network IO in `ioloop`), and has features like; threads, timers, locks, semaphores, and @@ -16,21 +16,21 @@ The dependencies differ slightly based on the environment you use, and the requi luarocks install copas -* [**luasec**](https://github.com/brunoos/luasec) _[optional dependency]_ module for SSL/TLS based connections. This is optional and may be +* _[optional dependency]_ [**luasec**](https://github.com/brunoos/luasec) module for SSL/TLS based connections. This is optional and may be skipped if you don't need secure network connections (e.g. broker is located in your local network). It's not listed in package dependencies, please install it manually like this: luarocks install luasec -* [**LuaBitOp**](http://bitop.luajit.org/) _[optional dependency]_ library to perform bitwise operations, which is required only on +* _[optional dependency]_ [**LuaBitOp**](http://bitop.luajit.org/) library to perform bitwise operations, which is required only on Lua 5.1. It's not listed in package dependencies, please install it manually like this: luarocks install luabitop -* [**LuaLogging**](https://github.com/lunarmodules/lualogging/) _[optional dependency]_ to enable logging by the MQTT client. This is optional +* _[optional dependency]_ [**LuaLogging**](https://github.com/lunarmodules/lualogging/) to enable logging by the MQTT client. This is optional but highly recommended for long running clients. This is a great debugging aid when developing your clients. Also when using OpenResty as your runtime, you'll definitely want to use this, see - [openresty.lua](https://xhaskx.github.io/luamqtt/examples/openresty.lua.html) for an example. + [openresty.lua](../examples/openresty/app/openresty.lua) for an example. It's not listed in package dependencies, please install it manually like this: luarocks install lualogging From 1669a15c076deff64e15855e060537c7825a812d Mon Sep 17 00:00:00 2001 From: Alexander Kiranov Date: Fri, 22 Dec 2023 15:09:24 +0200 Subject: [PATCH 60/65] markdown fix --- examples/openresty/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/openresty/README.md b/examples/openresty/README.md index 8fcebd3..c8d3a03 100644 --- a/examples/openresty/README.md +++ b/examples/openresty/README.md @@ -25,5 +25,5 @@ timer itself will go in an endless loop to do the keepalives. * [conf/nginx.conf](conf/nginx.conf): configuration for the nginx daemon to run lua scripts * [app/openresty.lua](app/openresty.lua): example lua script maintaining connection -* [mqtt/loop/nginx.lua](mqtt/loop/nginx.lua): how to add a client in an Nginx environment +* [mqtt/loop/nginx.lua](../../mqtt/loop/nginx.lua): how to add a client in an Nginx environment * `start.sh`, `stop.sh`, `quit.sh`, `restart.sh`: optional scripts to manage the OpenResty instance From f8411de3c9b6d1130d7d0907999ef03bad67cf8d Mon Sep 17 00:00:00 2001 From: Alexander Kiranov Date: Fri, 22 Dec 2023 16:14:35 +0200 Subject: [PATCH 61/65] markdown fix --- docs_topics/03-lua_versions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs_topics/03-lua_versions.md b/docs_topics/03-lua_versions.md index b71ad81..9496cd9 100644 --- a/docs_topics/03-lua_versions.md +++ b/docs_topics/03-lua_versions.md @@ -6,6 +6,6 @@ It's tested to work on Debian 9 GNU/Linux with Lua versions: * LuaJIT 2.0.0 ... LuaJIT 2.1.0 beta3 * It may also work on other Lua versions without any guarantees -So basically it should work on any modern Linux-based OS with Lua interpreter available. +So basically it should work on any modern OS with Lua interpreter available. Also has run under **Windows** and it was ok, but installing luarocks-modules may be a non-trivial task on this OS. From de92f3c001dc24fdecf3ba0dda665c98fab2211f Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Sat, 20 Jul 2024 12:17:16 +0200 Subject: [PATCH 62/65] fix(buffered): properly calculate the remainder to be read --- mqtt/connector/base/buffered_base.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mqtt/connector/base/buffered_base.lua b/mqtt/connector/base/buffered_base.lua index 034dc6f..eda9113 100644 --- a/mqtt/connector/base/buffered_base.lua +++ b/mqtt/connector/base/buffered_base.lua @@ -46,7 +46,7 @@ function buffered:receive(size) while size > (#buf - idx) do -- buffer is lacking bytes, read more... - local data, err = self:plain_receive(#buf - idx + size) + local data, err = self:plain_receive(size - (#buf - idx)) if not data then if err == self.signal_idle then -- read timedout, retry entire packet later, reset buffer From d71226977afdac0eec6fdeadde9d582d04a392a3 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Sat, 20 Jul 2024 12:28:25 +0200 Subject: [PATCH 63/65] fix(luasocket): return the partial results read --- mqtt/connector/base/buffered_base.lua | 3 +++ mqtt/connector/luasocket.lua | 6 ++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/mqtt/connector/base/buffered_base.lua b/mqtt/connector/base/buffered_base.lua index eda9113..48ac0ce 100644 --- a/mqtt/connector/base/buffered_base.lua +++ b/mqtt/connector/base/buffered_base.lua @@ -74,6 +74,9 @@ end -- is no data to read. If there is no data, then it MUST return -- `nil, self.signal_idle` to indicate it no data was there and we need to retry later. -- +-- If there is partial data, it should return that data (less than the requested +-- number of bytes), with no error/signal. +-- -- If the receive errors, because of a closed connection it should return -- `nil, self.signal_closed` to indicate this. Any other errors can be returned -- as a regular `nil, err`. diff --git a/mqtt/connector/luasocket.lua b/mqtt/connector/luasocket.lua index 6a25431..b5bf004 100644 --- a/mqtt/connector/luasocket.lua +++ b/mqtt/connector/luasocket.lua @@ -118,8 +118,10 @@ function luasocket:plain_receive(size) sock:settimeout(0) - local data, err = sock:receive(size) - if data then + local data, err, partial = sock:receive(size) + + data = data or partial or "" + if #data > 0 then return data end From 1e2a11824106fad9b5052323dd83035a822923e3 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Sat, 20 Jul 2024 17:04:14 +0200 Subject: [PATCH 64/65] docs(mqtt): fix some doc comments --- mqtt/connector/nginx.lua | 2 +- mqtt/init.lua | 34 ++++++++++++++++++++-------------- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/mqtt/connector/nginx.lua b/mqtt/connector/nginx.lua index 97c30a9..ca7c954 100644 --- a/mqtt/connector/nginx.lua +++ b/mqtt/connector/nginx.lua @@ -19,7 +19,7 @@ -- loaded to another thread. -- -- * Nginx timers should be short lived because memory is only released after --- the context is destroyed. In this case we're using the fro prolonged periods +-- the context is destroyed. In this case we're using them for prolonged periods -- of time, so be aware of this and implement client restarts if required. -- -- thanks to @irimiab: https://github.com/xHasKx/luamqtt/issues/13 diff --git a/mqtt/init.lua b/mqtt/init.lua index 2e874f8..f41a75b 100644 --- a/mqtt/init.lua +++ b/mqtt/init.lua @@ -1,5 +1,11 @@ --- MQTT module -- @module mqtt +-- @usage +-- local client = mqtt.client { +-- uri = "mqtts://aladdin:soopersecret@mqttbroker.com", +-- clean = true, +-- version = mqtt.v50, -- specify constant for MQTT version +-- } --[[ MQTT protocol DOC: http://docs.oasis-open.org/mqtt/mqtt/v3.1.1/errata01/os/mqtt-v3.1.1-errata01-os-complete.html @@ -18,13 +24,6 @@ CONVENTIONS: -- @tfield number v50 MQTT v5.0 protocol version constant -- @tfield string _VERSION luamqtt library version string -- @table mqtt --- @see mqtt.const --- @usage --- local client = mqtt.client { --- uri = "mqtts://aladdin:soopersecret@mqttbroker.com", --- clean = true, --- version = mqtt.v50, -- specify constant for MQTT version --- } local mqtt = {} -- copy all values from const module @@ -47,6 +46,7 @@ local ioloop_get = ioloop.get --- Create new MQTT client instance -- @param ... Same as for `Client.create`(...) +-- @treturn Client new client instance -- @see Client:__init function mqtt.client(...) return client_create(...) @@ -76,8 +76,10 @@ end --- Validates a topic with wildcards. --- @param t (string) wildcard topic to validate --- @return topic, or false+error +-- @tparam string t wildcard-topic to validate +-- @treturn[1] string the input topic if valid +-- @treturn[2] boolean false if invalid +-- @treturn[2] string error description -- @usage local t = assert(mqtt.validate_subscribe_topic("base/+/thermostat/#")) function mqtt.validate_subscribe_topic(t) if type(t) ~= "string" then @@ -114,8 +116,10 @@ function mqtt.validate_subscribe_topic(t) end --- Validates a topic without wildcards. --- @param t (string) topic to validate --- @return topic, or false+error +-- @tparam string t topic to validate +-- @treturn[1] string the input topic if valid +-- @treturn[2] boolean false if invalid +-- @treturn[2] string error description -- @usage local t = assert(mqtt.validate_publish_topic("base/living/thermostat/setpoint")) function mqtt.validate_publish_topic(t) if type(t) ~= "string" then @@ -133,8 +137,10 @@ end --- Returns a Lua pattern from topic. -- Takes a wildcarded-topic and returns a Lua pattern that can be used -- to validate if a received topic matches the wildcard-topic --- @param t (string) the wildcard topic --- @return Lua-pattern (string) or false+err +-- @tparam string t the wildcard topic +-- @treturn[1] string Lua-pattern that matches the topic and returns the captures +-- @treturn[2] boolean false if the topic was invalid +-- @treturn[2] string error description -- @usage -- local patt = compile_topic_pattern("homes/+/+/#") -- @@ -171,7 +177,7 @@ function mqtt.compile_topic_pattern(t) end --- Parses wildcards in a topic into a table. --- @tparam topic string incoming topic string +-- @tparam string topic incoming topic string -- @tparam table opts parsing options table -- @tparam string opts.topic the wild-carded topic to match against (optional if `opts.pattern` is given) -- @tparam string opts.pattern the compiled pattern for the wild-carded topic (optional if `opts.topic` From d977890d967d6b042c306f34417aacc39b773876 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Sun, 13 Oct 2024 17:25:34 +0200 Subject: [PATCH 65/65] chore(log): namespace all logs with '[LuaMQTT]' --- mqtt/client.lua | 70 ++++++++++++++++++++++---------------------- mqtt/init.lua | 2 +- mqtt/ioloop.lua | 28 +++++++++--------- mqtt/loop/copas.lua | 4 +-- mqtt/loop/detect.lua | 6 ++-- 5 files changed, 55 insertions(+), 55 deletions(-) diff --git a/mqtt/client.lua b/mqtt/client.lua index 0b5e1f4..08828b1 100644 --- a/mqtt/client.lua +++ b/mqtt/client.lua @@ -240,7 +240,7 @@ function Client:__init(opts) self:on(self.opts.on) end - log:info("MQTT client '%s' created", a.id) + log:info("[LuaMQTT] client '%s' created", a.id) end --- Add functions as handlers of given events. @@ -393,13 +393,13 @@ function Client:subscribe(opts) local packet_id = pargs.packet_id local subscribe = self._make_packet(pargs) - log:info("subscribing client '%s' to topic '%s' (packet: %s)", self.opts.id, opts.topic, packet_id or "n.a.") + log:info("[LuaMQTT] subscribing client '%s' to topic '%s' (packet: %s)", self.opts.id, opts.topic, packet_id or "n.a.") -- send SUBSCRIBE packet local ok, err = self:_send_packet(subscribe) if not ok then err = "failed to send SUBSCRIBE: "..err - log:error("client '%s': %s", self.opts.id, err) + log:error("[LuaMQTT] client '%s': %s", self.opts.id, err) self:handle("error", err, self) self:close_connection("error") return false, err @@ -453,13 +453,13 @@ function Client:unsubscribe(opts) local packet_id = pargs.packet_id local unsubscribe = self._make_packet(pargs) - log:info("unsubscribing client '%s' from topic '%s' (packet: %s)", self.opts.id, opts.topic, packet_id or "n.a.") + log:info("[LuaMQTT] unsubscribing client '%s' from topic '%s' (packet: %s)", self.opts.id, opts.topic, packet_id or "n.a.") -- send UNSUBSCRIBE packet local ok, err = self:_send_packet(unsubscribe) if not ok then err = "failed to send UNSUBSCRIBE: "..err - log:error("client '%s' %s", self.opts.id, err) + log:error("[LuaMQTT] client '%s' %s", self.opts.id, err) self:handle("error", err, self) self:close_connection("error") return false, err @@ -519,13 +519,13 @@ function Client:publish(opts) local packet_id = opts.packet_id local publish = self._make_packet(opts) - log:debug("client '%s' publishing to topic '%s' (packet: %s)", self.opts.id, opts.topic, packet_id or "n.a.") + log:debug("[LuaMQTT] client '%s' publishing to topic '%s' (packet: %s)", self.opts.id, opts.topic, packet_id or "n.a.") -- send PUBLISH packet local ok, err = self:_send_packet(publish) if not ok then err = "failed to send PUBLISH: "..err - log:error("client '%s' %s", self.opts.id, err) + log:error("[LuaMQTT] client '%s' %s", self.opts.id, err) self:handle("error", err, self) self:close_connection("error") return false, err @@ -580,7 +580,7 @@ function Client:acknowledge(msg, rc, properties, user_properties) return true end - log:debug("client '%s' acknowledging packet %s", self.opts.id, packet_id or "n.a.") + log:debug("[LuaMQTT] client '%s' acknowledging packet %s", self.opts.id, packet_id or "n.a.") if msg.qos == 1 then -- PUBACK should be sent @@ -598,7 +598,7 @@ function Client:acknowledge(msg, rc, properties, user_properties) local ok, err = self:_send_packet(puback) if not ok then err = "failed to send PUBACK: "..err - log:error("client '%s' %s", self.opts.id, err) + log:error("[LuaMQTT] client '%s' %s", self.opts.id, err) self:handle("error", err, self) self:close_connection("error") return false, err @@ -619,7 +619,7 @@ function Client:acknowledge(msg, rc, properties, user_properties) local ok, err = self:_send_packet(pubrec) if not ok then err = "failed to send PUBREC: "..err - log:error("client '%s' %s", self.opts.id, err) + log:error("[LuaMQTT] client '%s' %s", self.opts.id, err) self:handle("error", err, self) self:close_connection("error") return false, err @@ -658,13 +658,13 @@ function Client:disconnect(rc, properties, user_properties) user_properties = user_properties, } - log:info("client '%s' disconnecting (rc = %d)", self.opts.id, rc or 0) + log:info("[LuaMQTT] client '%s' disconnecting (rc = %d)", self.opts.id, rc or 0) -- send DISCONNECT packet local ok, err = self:_send_packet(disconnect) if not ok then err = "failed to send DISCONNECT: "..err - log:error("client '%s' %s", self.opts.id, err) + log:error("[LuaMQTT] client '%s' %s", self.opts.id, err) self:handle("error", err, self) self:close_connection("error") return false, err @@ -683,7 +683,7 @@ end -- @param ... see `Client:disconnect` -- @return `true` function Client:shutdown(...) - log:debug("client '%s' shutting down", self.opts.id) + log:debug("[LuaMQTT] client '%s' shutting down", self.opts.id) self.first_connect = false self.opts.reconnect = false self:disconnect(...) @@ -716,13 +716,13 @@ function Client:auth(rc, properties, user_properties) user_properties = user_properties, } - log:info("client '%s' authenticating") + log:info("[LuaMQTT] client '%s' authenticating") -- send AUTH packet local ok, err = self:_send_packet(auth) if not ok then err = "failed to send AUTH: "..err - log:error("client '%s' %s", self.opts.id, err) + log:error("[LuaMQTT] client '%s' %s", self.opts.id, err) self:handle("error", err, self) self:close_connection("error") return false, err @@ -742,7 +742,7 @@ function Client:close_connection(reason) reason = reason or "unspecified" - log:info("client '%s' closing connection (reason: %s)", self.opts.id, reason) + log:info("[LuaMQTT] client '%s' closing connection (reason: %s)", self.opts.id, reason) conn:shutdown() self.connection = nil @@ -788,13 +788,13 @@ function Client:send_pingreq() type = packet_type.PINGREQ, } - log:debug("client '%s' sending PINGREQ", self.opts.id) + log:debug("[LuaMQTT] client '%s' sending PINGREQ", self.opts.id) -- send PINGREQ packet local ok, err = self:_send_packet(pingreq) if not ok then err = "failed to send PINGREQ: "..err - log:error("client '%s' %s", self.opts.id, err) + log:error("[LuaMQTT] client '%s' %s", self.opts.id, err) self:handle("error", err, self) self:close_connection("error") return false, err @@ -824,12 +824,12 @@ function Client:open_connection() }, connector) Client._parse_connection_opts(opts, conn) - log:info("client '%s' connecting to broker '%s' (using: %s)", self.opts.id, opts.uri, conn.type or "unknown") + log:info("[LuaMQTT] client '%s' connecting to broker '%s' (using: %s)", self.opts.id, opts.uri, conn.type or "unknown") -- perform connect local ok, err = conn:connect() if not ok then - log:error("client '%s' %s", self.opts.id, err) + log:error("[LuaMQTT] client '%s' %s", self.opts.id, err) err = "failed to open network connection: "..err self:handle("error", err, self) return false, err @@ -867,13 +867,13 @@ function Client:send_connect() user_properties = opts.user_properties, } - log:info("client '%s' sending CONNECT (user '%s')", self.opts.id, opts.username or "not specified") + log:info("[LuaMQTT] client '%s' sending CONNECT (user '%s')", self.opts.id, opts.username or "not specified") -- send CONNECT packet local ok, err = self:_send_packet(connect) if not ok then err = "failed to send CONNECT: "..err - log:error("client '%s' %s", self.opts.id, err) + log:error("[LuaMQTT] client '%s' %s", self.opts.id, err) self:handle("error", err, self) self:close_connection("error") return false, err @@ -915,7 +915,7 @@ function Client:check_keep_alive() if t_timeout and t_timeout <= t_now then -- we timed-out, close and exit local err = str_format("failed to receive PINGRESP within %d seconds", interval) - log:error("client '%s' %s", self.opts.id, err) + log:error("[LuaMQTT] client '%s' %s", self.opts.id, err) self:handle("error", err, self) self:close_connection("error") return interval, err @@ -948,13 +948,13 @@ function Client:acknowledge_pubrel(packet_id) -- create PUBREL packet local pubrel = self._make_packet{type=packet_type.PUBREL, packet_id=packet_id, rc=0} - log:debug("client '%s' sending PUBREL (packet: %s)", self.opts.id, packet_id or "n.a.") + log:debug("[LuaMQTT] client '%s' sending PUBREL (packet: %s)", self.opts.id, packet_id or "n.a.") -- send PUBREL packet local ok, err = self:_send_packet(pubrel) if not ok then err = "failed to send PUBREL: "..err - log:error("client '%s' %s", self.opts.id, err) + log:error("[LuaMQTT] client '%s' %s", self.opts.id, err) self:handle("error", err, self) self:close_connection("error") return false, err @@ -974,13 +974,13 @@ function Client:acknowledge_pubcomp(packet_id) -- create PUBCOMP packet local pubcomp = self._make_packet{type=packet_type.PUBCOMP, packet_id=packet_id, rc=0} - log:debug("client '%s' sending PUBCOMP (packet: %s)", self.opts.id, packet_id or "n.a.") + log:debug("[LuaMQTT] client '%s' sending PUBCOMP (packet: %s)", self.opts.id, packet_id or "n.a.") -- send PUBCOMP packet local ok, err = self:_send_packet(pubcomp) if not ok then err = "failed to send PUBCOMP: "..err - log:error("client '%s' %s", self.opts.id, err) + log:error("[LuaMQTT] client '%s' %s", self.opts.id, err) self:handle("error", err, self) self:close_connection("error") return false, err @@ -1028,13 +1028,13 @@ function Client:handle_received_packet(packet) local conn = self.connection local err - log:debug("client '%s' received '%s' (packet: %s)", self.opts.id, packet_type[packet.type], packet.packet_id or "n.a.") + log:debug("[LuaMQTT] client '%s' received '%s' (packet: %s)", self.opts.id, packet_type[packet.type], packet.packet_id or "n.a.") if not conn.connack then -- expecting only CONNACK packet here if packet.type ~= packet_type.CONNACK then err = "expecting CONNACK but received "..packet.type - log:error("client '%s' %s", self.opts.id, err) + log:error("[LuaMQTT] client '%s' %s", self.opts.id, err) self:handle("error", err, self) self:close_connection("error") self.first_connect = false @@ -1047,7 +1047,7 @@ function Client:handle_received_packet(packet) -- check CONNACK rc if packet.rc ~= 0 then err = str_format("CONNECT failed with CONNACK [rc=%d]: %s", packet.rc, packet:reason_string()) - log:error("client '%s' %s", self.opts.id, err) + log:error("[LuaMQTT] client '%s' %s", self.opts.id, err) self:handle("error", err, self, packet) self:handle("connect", packet, self) self:close_connection("connection failed") @@ -1055,12 +1055,12 @@ function Client:handle_received_packet(packet) return false, err end - log:info("client '%s' connected successfully to '%s:%s'", self.opts.id, conn.host, conn.port) + log:info("[LuaMQTT] client '%s' connected successfully to '%s:%s'", self.opts.id, conn.host, conn.port) -- fire connect event if self.opts.clean == "first" then self.opts.clean = false -- reset clean flag to false, so next connection resumes previous session - log:debug("client '%s'; switching clean flag to false (was 'first')", self.opts.id) + log:debug("[LuaMQTT] client '%s'; switching clean flag to false (was 'first')", self.opts.id) end self:handle("connect", packet, self) self.first_connect = false @@ -1108,7 +1108,7 @@ function Client:handle_received_packet(packet) elseif ptype == packet_type.AUTH then self:handle("auth", packet, self) else - log:warn("client '%s' don't know how to handle %s", self.opts.id, ptype) + log:warn("[LuaMQTT] client '%s' don't know how to handle %s", self.opts.id, ptype) end end return true @@ -1121,7 +1121,7 @@ do if not self.first_connect and not reconnect then -- this would be a re-connect, but we're not supposed to auto-reconnect - log:debug("client '%s' was disconnected and not set to auto-reconnect", self.opts.id) + log:debug("[LuaMQTT] client '%s' was disconnected and not set to auto-reconnect", self.opts.id) return false, "network connection is not opened" end @@ -1182,7 +1182,7 @@ do return reconnect and 0, err else err = "failed to receive next packet: "..tostring(err) - log:error("client '%s' %s", self.opts.id, err) + log:error("[LuaMQTT] client '%s' %s", self.opts.id, err) self:handle("error", err, self) self:close_connection("error") return reconnect and 0, err diff --git a/mqtt/init.lua b/mqtt/init.lua index f41a75b..dc295b6 100644 --- a/mqtt/init.lua +++ b/mqtt/init.lua @@ -65,7 +65,7 @@ mqtt.get_ioloop = ioloop_get -- @usage -- mqtt.run_ioloop(client1, client2, func1) function mqtt.run_ioloop(...) - log:info("starting default ioloop instance") + log:info("[LuaMQTT] starting default ioloop instance") local loop = ioloop_get() for i = 1, select("#", ...) do local cl = select(i, ...) diff --git a/mqtt/ioloop.lua b/mqtt/ioloop.lua index 94b895f..63cfb99 100644 --- a/mqtt/ioloop.lua +++ b/mqtt/ioloop.lua @@ -47,7 +47,7 @@ Ioloop.__index = Ioloop -- @tparam[opt=luasocket.sleep] function opts.sleep_function custom sleep function to call after each iteration -- @treturn Ioloop ioloop instance function Ioloop:__init(opts) - log:debug("initializing ioloop instance '%s'", tostring(self)) + log:debug("[LuaMQTT] initializing ioloop instance '%s'", tostring(self)) opts = opts or {} opts.sleep_min = opts.sleep_min or 0 opts.sleep_step = opts.sleep_step or 0.002 @@ -88,10 +88,10 @@ function Ioloop:add(client) local clients = self.clients if clients[client] then if type(client) == "table" then - log:warn("MQTT client '%s' was already added to ioloop '%s'", client.opts.id, tostring(self)) + log:warn("[LuaMQTT] client '%s' was already added to ioloop '%s'", client.opts.id, tostring(self)) return false, "MQTT client was already added to this ioloop" else - log:warn("MQTT loop function '%s' was already added to this ioloop '%s'", tostring(client), tostring(self)) + log:warn("[LuaMQTT] loop function '%s' was already added to this ioloop '%s'", tostring(client), tostring(self)) return false, "MQTT loop function was already added to this ioloop" end end @@ -100,7 +100,7 @@ function Ioloop:add(client) self.timeouts[client] = self.opts.sleep_min if type(client) == "table" then - log:info("adding client '%s' to ioloop '%s'", client.opts.id, tostring(self)) + log:info("[LuaMQTT] adding client '%s' to ioloop '%s'", client.opts.id, tostring(self)) -- create and add function for PINGREQ local function f() if not clients[client] then @@ -112,7 +112,7 @@ function Ioloop:add(client) -- add it to start doing keepalive checks self:add(f) else - log:info("adding function '%s' to ioloop '%s'", tostring(client), tostring(self)) + log:info("[LuaMQTT] adding function '%s' to ioloop '%s'", tostring(client), tostring(self)) end return true @@ -125,10 +125,10 @@ function Ioloop:remove(client) local clients = self.clients if not clients[client] then if type(client) == "table" then - log:warn("MQTT client not found '%s' in ioloop '%s'", client.opts.id, tostring(self)) + log:warn("[LuaMQTT] client not found '%s' in ioloop '%s'", client.opts.id, tostring(self)) return false, "MQTT client not found" else - log:warn("MQTT loop function not found '%s' in ioloop '%s'", tostring(client), tostring(self)) + log:warn("[LuaMQTT] loop function not found '%s' in ioloop '%s'", tostring(client), tostring(self)) return false, "MQTT loop function not found" end end @@ -144,9 +144,9 @@ function Ioloop:remove(client) end if type(client) == "table" then - log:info("removed client '%s' from ioloop '%s'", client.opts.id, tostring(self)) + log:info("[LuaMQTT] removed client '%s' from ioloop '%s'", client.opts.id, tostring(self)) else - log:info("removed loop function '%s' from ioloop '%s'", tostring(client), tostring(self)) + log:info("[LuaMQTT] removed loop function '%s' from ioloop '%s'", tostring(client), tostring(self)) end return true @@ -175,12 +175,12 @@ function Ioloop:iteration() -- an error from a client was returned if not client.opts.reconnect then -- error and not reconnecting, remove the client - log:info("client '%s' returned '%s', no re-connect set, removing client", client.opts.id, err) + log:info("[LuaMQTT] client '%s' returned '%s', no re-connect set, removing client", client.opts.id, err) self:remove(client) t = opts.sleep_max else -- error, but will reconnect - log:error("client '%s' failed with '%s', will try re-connecting", client.opts.id, err) + log:error("[LuaMQTT] client '%s' failed with '%s', will try re-connecting", client.opts.id, err) t = opts.sleep_min -- try asap end else @@ -200,7 +200,7 @@ end -- While there is at least one client/function in the ioloop it will continue -- iterating. After all clients/functions are gone, it will return. function Ioloop:run_until_clients() - log:info("ioloop started with %d clients/functions", #self.clients) + log:info("[LuaMQTT] ioloop started with %d clients/functions", #self.clients) self.running = true while next(self.clients) do @@ -208,7 +208,7 @@ function Ioloop:run_until_clients() end self.running = false - log:info("ioloop finished with %d clients/functions", #self.clients) + log:info("[LuaMQTT] ioloop finished with %d clients/functions", #self.clients) end --- Exported functions @@ -238,7 +238,7 @@ function _M.get(autocreate, opts) autocreate = true end if autocreate and not ioloop_instance then - log:info("auto-creating default ioloop instance") + log:info("[LuaMQTT] auto-creating default ioloop instance") ioloop_instance = _M.create(opts) end return ioloop_instance diff --git a/mqtt/loop/copas.lua b/mqtt/loop/copas.lua index 026ccd9..94b4d13 100644 --- a/mqtt/loop/copas.lua +++ b/mqtt/loop/copas.lua @@ -20,7 +20,7 @@ local _M = {} -- @return `true` on success or `false` and error message on failure function _M.add(cl) if client_registry[cl] then - log:warn("MQTT client '%s' was already added to Copas", cl.opts.id) + log:warn("[LuaMQTT] client '%s' was already added to Copas", cl.opts.id) return false, "MQTT client was already added to Copas" end client_registry[cl] = true @@ -52,7 +52,7 @@ function _M.add(cl) local timeout = cl:step() if not timeout then client_registry[cl] = nil -- exiting - log:debug("MQTT client '%s' exited, removed from Copas", cl.opts.id) + log:debug("[LuaMQTT] client '%s' exited, removed from Copas", cl.opts.id) copas.wakeup(timer) else if timeout > 0 then diff --git a/mqtt/loop/detect.lua b/mqtt/loop/detect.lua index 4fc15b6..eaa8cae 100644 --- a/mqtt/loop/detect.lua +++ b/mqtt/loop/detect.lua @@ -7,19 +7,19 @@ return function() if loop then return loop end if type(ngx) == "table" then -- there is a global 'ngx' table, so we're running OpenResty - log:info("LuaMQTT auto-detected Nginx as the runtime environment") + log:info("[LuaMQTT] auto-detected Nginx as the runtime environment") loop = "nginx" return loop elseif package.loaded.copas then -- 'copas' was already loaded - log:info("LuaMQTT auto-detected Copas as the io-loop in use") + log:info("[LuaMQTT] auto-detected Copas as the io-loop in use") loop = "copas" return loop elseif pcall(require, "socket") and tostring(require("socket")._VERSION):find("LuaSocket") then -- LuaSocket is available - log:info("LuaMQTT auto-detected LuaSocket as the socket library to use with mqtt-ioloop") + log:info("[LuaMQTT] auto-detected LuaSocket as the socket library to use with mqtt-ioloop") loop = "ioloop" return loop