From d9fb729f010508f24d65eeb7cd33d9788ed0d6ea Mon Sep 17 00:00:00 2001 From: AlirezaPourchali Date: Fri, 15 Nov 2024 00:14:47 +0330 Subject: [PATCH] userguide/runmodes: fix typo Documentation: #7383 --- doc/userguide/performance/runmodes.rst | 2 +- etc/schema.json | 36 ++++ examples/plugins/c-json-filetype/Makefile.in | 2 +- src/Makefile.am | 12 ++ src/app-layer-ftp.c | 10 + src/app-layer-ftp.h | 1 + src/app-layer-htp-range.c | 24 ++- src/app-layer-htp-range.h | 6 +- src/app-layer.c | 7 + src/detect-engine-register.c | 6 +- src/detect-engine-register.h | 1 + src/flow-callbacks.c | 129 +++++++++++ src/flow-callbacks.h | 121 +++++++++++ src/flow-hash.c | 8 +- src/flow-manager.c | 3 +- src/flow-util.c | 5 +- src/flow-util.h | 2 +- src/flow.c | 3 + src/output-eve-stream.c | 2 +- src/output-eve.c | 39 ++++ src/output-eve.h | 43 ++++ src/output-json-alert.c | 6 +- src/output-json-anomaly.c | 21 +- src/output-json-arp.c | 2 +- src/output-json-dcerpc.c | 2 +- src/output-json-dhcp.c | 2 +- src/output-json-dnp3.c | 4 +- src/output-json-dns.c | 8 +- src/output-json-drop.c | 6 +- src/output-json-file.c | 8 +- src/output-json-flow.c | 2 +- src/output-json-frame.c | 10 +- src/output-json-http.c | 2 +- src/output-json-ike.c | 2 +- src/output-json-metadata.c | 2 +- src/output-json-mqtt.c | 2 +- src/output-json-netflow.c | 4 +- src/output-json-nfs.c | 2 +- src/output-json-pgsql.c | 2 +- src/output-json-smb.c | 2 +- src/output-json-smtp.c | 2 +- src/output-json-tls.c | 2 +- src/output-json.c | 5 +- src/output-json.h | 3 +- src/output.c | 2 +- src/runmode-unittests.c | 1 + src/runmode-unix-socket.c | 68 ++---- src/suricata.c | 2 + src/tests/fuzz/fuzz_siginit.c | 1 + src/thread-callbacks.c | 55 +++++ src/thread-callbacks.h | 54 +++++ src/thread-storage.c | 212 +++++++++++++++++++ src/thread-storage.h | 45 ++++ src/threads.c | 2 + src/threadvars.h | 2 + src/tm-threads.c | 8 +- src/util-storage.c | 2 + src/util-storage.h | 1 + src/util-thash.c | 22 +- src/util-thash.h | 7 +- 60 files changed, 923 insertions(+), 124 deletions(-) create mode 100644 src/flow-callbacks.c create mode 100644 src/flow-callbacks.h create mode 100644 src/thread-callbacks.c create mode 100644 src/thread-callbacks.h create mode 100644 src/thread-storage.c create mode 100644 src/thread-storage.h diff --git a/doc/userguide/performance/runmodes.rst b/doc/userguide/performance/runmodes.rst index 4afc5d56ccd2..050a98e7273e 100644 --- a/doc/userguide/performance/runmodes.rst +++ b/doc/userguide/performance/runmodes.rst @@ -12,7 +12,7 @@ a queue. Packets will be processed by one thread at a time, but there can be multiple packets being processed at a time by the engine (see :ref:`suricata-yaml-max-pending-packets`). A thread can have one or more thread-modules. If they have more modules, they can only be -active one a a time. The way threads, modules and queues are arranged +active one at a time. The way threads, modules and queues are arranged together is called the "Runmode". Different runmodes diff --git a/etc/schema.json b/etc/schema.json index cf03a2db30b6..08959e3c7a1d 100644 --- a/etc/schema.json +++ b/etc/schema.json @@ -6389,6 +6389,42 @@ "additionalProperties": false }, "http": { + "type": "object", + "properties": { + "memcap": { + "type": "integer" + }, + "memuse": { + "type": "integer" + }, + "byterange": { + "type": "object", + "properties": { + "memcap": { + "type": "integer" + }, + "memuse": { + "type": "integer" + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "host": { + "type": "object", + "properties": { + "memcap": { + "type": "integer" + }, + "memuse": { + "type": "integer" + } + }, + "additionalProperties": false + }, + "ippair": { "type": "object", "properties": { "memcap": { diff --git a/examples/plugins/c-json-filetype/Makefile.in b/examples/plugins/c-json-filetype/Makefile.in index 0d4ec381ee4a..4d85c8ee2d59 100644 --- a/examples/plugins/c-json-filetype/Makefile.in +++ b/examples/plugins/c-json-filetype/Makefile.in @@ -5,7 +5,7 @@ # But as this is an example in the Suricata source tree we'll look for # includes in the source tree. -CPPFLAGS += -I@top_srcdir@/src -DHAVE_CONFIG_H +CPPFLAGS += -I@top_srcdir@/src -I@top_srcdir@/rust/gen -I@top_srcdir@/rust/dist -DHAVE_CONFIG_H # Currently the Suricata logging system requires this to be even for # plugins. diff --git a/src/Makefile.am b/src/Makefile.am index 7d05751ec254..b0f841cfd0c2 100755 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -317,6 +317,7 @@ noinst_HEADERS = \ feature.h \ flow-bit.h \ flow-bypass.h \ + flow-callbacks.h \ flow.h \ flow-hash.h \ flow-manager.h \ @@ -437,6 +438,8 @@ noinst_HEADERS = \ suricata-common.h \ suricata.h \ suricata-plugin.h \ + thread-callbacks.h \ + thread-storage.h \ threads-debug.h \ threads.h \ threads-profile.h \ @@ -875,6 +878,7 @@ libsuricata_c_a_SOURCES = \ feature.c \ flow-bit.c \ flow-bypass.c \ + flow-callbacks.c \ flow.c \ flow-hash.c \ flow-manager.c \ @@ -988,6 +992,8 @@ libsuricata_c_a_SOURCES = \ stream-tcp-sack.c \ stream-tcp-util.c \ suricata.c \ + thread-callbacks.c \ + thread-storage.c \ threads.c \ tm-modules.c \ tmqh-flow.c \ @@ -1168,6 +1174,12 @@ install-headers: for header in $(noinst_HEADERS); do \ $(INSTALL_DATA) $$header "$(DESTDIR)${includedir}/suricata"; \ done + if test -e ../rust/dist/rust-bindings.h; then \ + $(INSTALL_DATA) ../rust/dist/rust-bindings.h "$(DESTDIR)${includedir}/suricata"; \ + fi + if test -e ../rust/gen/rust-bindings.h; then \ + $(INSTALL_DATA) ../rust/gen/rust-bindings.h "$(DESTDIR)${includedir}/suricata"; \ + fi # Until we can remove autoconf.h from our headers, we need to to # provide this for library/plugin users. diff --git a/src/app-layer-ftp.c b/src/app-layer-ftp.c index a1a99d4bd701..15238b9f65ef 100644 --- a/src/app-layer-ftp.c +++ b/src/app-layer-ftp.c @@ -174,6 +174,16 @@ uint64_t FTPMemcapGlobalCounter(void) return tmpval; } +int FTPSetMemcap(uint64_t size) +{ + if ((uint64_t)SC_ATOMIC_GET(ftp_memcap) < size) { + SC_ATOMIC_SET(ftp_memcap, size); + return 1; + } + + return 0; +} + /** * \brief Check if alloc'ing "size" would mean we're over memcap * diff --git a/src/app-layer-ftp.h b/src/app-layer-ftp.h index 5be11d81f81c..e69415d8cf13 100644 --- a/src/app-layer-ftp.h +++ b/src/app-layer-ftp.h @@ -185,6 +185,7 @@ typedef struct FtpDataState_ { void RegisterFTPParsers(void); void FTPParserRegisterTests(void); void FTPParserCleanup(void); +int FTPSetMemcap(uint64_t size); uint64_t FTPMemuseGlobalCounter(void); uint64_t FTPMemcapGlobalCounter(void); diff --git a/src/app-layer-htp-range.c b/src/app-layer-htp-range.c index b1f2b62423fc..9e8a4e1e641f 100644 --- a/src/app-layer-htp-range.c +++ b/src/app-layer-htp-range.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2021 Open Information Security Foundation +/* Copyright (C) 2024 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -42,6 +42,28 @@ static void HttpRangeBlockDerefContainer(HttpRangeContainerBlock *b); #define CONTAINER_URLRANGE_HASH_SIZE 256 +int HTPByteRangeSetMemcap(uint64_t size) +{ + if (size == 0 || (uint64_t)SC_ATOMIC_GET(ContainerUrlRangeList.ht->memuse) < size) { + SC_ATOMIC_SET(ContainerUrlRangeList.ht->config.memcap, size); + return 1; + } + + return 0; +} + +uint64_t HTPByteRangeMemcapGlobalCounter(void) +{ + uint64_t tmpval = SC_ATOMIC_GET(ContainerUrlRangeList.ht->config.memcap); + return tmpval; +} + +uint64_t HTPByteRangeMemuseGlobalCounter(void) +{ + uint64_t tmpval = SC_ATOMIC_GET(ContainerUrlRangeList.ht->memuse); + return tmpval; +} + int HttpRangeContainerBufferCompare(HttpRangeContainerBuffer *a, HttpRangeContainerBuffer *b) { // lexical order : start, buflen, offset diff --git a/src/app-layer-htp-range.h b/src/app-layer-htp-range.h index fb0dc5b76875..8fb561020b0a 100644 --- a/src/app-layer-htp-range.h +++ b/src/app-layer-htp-range.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2021 Open Information Security Foundation +/* Copyright (C) 2024 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -111,4 +111,8 @@ HttpRangeContainerBlock *HttpRangeContainerOpenFile(const unsigned char *key, ui void HttpRangeFreeBlock(HttpRangeContainerBlock *b); +uint64_t HTPByteRangeMemcapGlobalCounter(void); +uint64_t HTPByteRangeMemuseGlobalCounter(void); +int HTPByteRangeSetMemcap(uint64_t); + #endif /* SURICATA_APP_LAYER_HTP_RANGE_H */ diff --git a/src/app-layer.c b/src/app-layer.c index 94f99f44f83e..9654c7d82e64 100644 --- a/src/app-layer.c +++ b/src/app-layer.c @@ -31,6 +31,7 @@ #include "app-layer-protos.h" #include "app-layer-expectation.h" #include "app-layer-ftp.h" +#include "app-layer-htp-range.h" #include "app-layer-detect-proto.h" #include "app-layer-frames.h" #include "stream-tcp-reassemble.h" @@ -1113,6 +1114,12 @@ void AppLayerRegisterGlobalCounters(void) StatsRegisterGlobalCounter("ftp.memuse", FTPMemuseGlobalCounter); StatsRegisterGlobalCounter("ftp.memcap", FTPMemcapGlobalCounter); StatsRegisterGlobalCounter("app_layer.expectations", ExpectationGetCounter); + StatsRegisterGlobalCounter("http.byterange.memuse", HTPByteRangeMemuseGlobalCounter); + StatsRegisterGlobalCounter("http.byterange.memcap", HTPByteRangeMemcapGlobalCounter); + StatsRegisterGlobalCounter("ippair.memuse", IPPairGetMemuse); + StatsRegisterGlobalCounter("ippair.memcap", IPPairGetMemuse); + StatsRegisterGlobalCounter("host.memuse", HostGetMemuse); + StatsRegisterGlobalCounter("host.memcap", HostGetMemcap); } static bool IsAppLayerErrorExceptionPolicyStatsValid(enum ExceptionPolicy policy) diff --git a/src/detect-engine-register.c b/src/detect-engine-register.c index 37fbc98d8597..903cc158cf5f 100644 --- a/src/detect-engine-register.c +++ b/src/detect-engine-register.c @@ -439,7 +439,7 @@ void SigTableCleanup(void) } } -void SigTableSetup(void) +void SigTableInit(void) { if (sigmatch_table == NULL) { DETECT_TBLSIZE = DETECT_TBLSIZE_STATIC + DETECT_TBLSIZE_STEP; @@ -447,10 +447,12 @@ void SigTableSetup(void) if (sigmatch_table == NULL) { DETECT_TBLSIZE = 0; FatalError("Could not allocate sigmatch_table"); - return; } } +} +void SigTableSetup(void) +{ DetectSidRegister(); DetectPriorityRegister(); DetectPrefilterRegister(); diff --git a/src/detect-engine-register.h b/src/detect-engine-register.h index c9134c77b83a..db4cd957af9d 100644 --- a/src/detect-engine-register.h +++ b/src/detect-engine-register.h @@ -338,6 +338,7 @@ extern int DETECT_TBLSIZE_IDX; #define DETECT_TBLSIZE_STEP 256 int SigTableList(const char *keyword); void SigTableCleanup(void); +void SigTableInit(void); void SigTableSetup(void); void SigTableRegisterTests(void); diff --git a/src/flow-callbacks.c b/src/flow-callbacks.c new file mode 100644 index 000000000000..30e703c3efb3 --- /dev/null +++ b/src/flow-callbacks.c @@ -0,0 +1,129 @@ +/* Copyright (C) 2024 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +#include "flow-callbacks.h" + +typedef struct FlowInitCallback_ { + SCFlowInitCallbackFn Callback; + void *user; + struct FlowInitCallback_ *next; +} FlowInitCallback; + +static FlowInitCallback *init_callbacks = NULL; + +typedef struct FlowUpdateCallback_ { + SCFlowUpdateCallbackFn Callback; + void *user; + struct FlowUpdateCallback_ *next; +} FlowUpdateCallback; + +static FlowUpdateCallback *update_callbacks = NULL; + +typedef struct FlowFinishCallback_ { + SCFlowFinishCallbackFn Callback; + void *user; + struct FlowFinishCallback_ *next; +} FlowFinishCallback; + +static FlowFinishCallback *finish_callbacks = NULL; + +bool SCFlowRegisterInitCallback(SCFlowInitCallbackFn fn, void *user) +{ + FlowInitCallback *cb = SCCalloc(1, sizeof(*cb)); + if (cb == NULL) { + return false; + } + cb->Callback = fn; + cb->user = user; + if (init_callbacks == NULL) { + init_callbacks = cb; + } else { + FlowInitCallback *current = init_callbacks; + while (current->next != NULL) { + current = current->next; + } + current->next = cb; + } + return true; +} + +void SCFlowRunInitCallbacks(ThreadVars *tv, Flow *f, const Packet *p) +{ + FlowInitCallback *cb = init_callbacks; + while (cb != NULL) { + cb->Callback(tv, f, p, cb->user); + cb = cb->next; + } +} + +bool SCFlowRegisterUpdateCallback(SCFlowUpdateCallbackFn fn, void *user) +{ + FlowUpdateCallback *cb = SCCalloc(1, sizeof(*cb)); + if (cb == NULL) { + return false; + } + cb->Callback = fn; + cb->user = user; + if (update_callbacks == NULL) { + update_callbacks = cb; + } else { + FlowUpdateCallback *current = update_callbacks; + while (current->next != NULL) { + current = current->next; + } + current->next = cb; + } + return true; +} + +void SCFlowRunUpdateCallbacks(ThreadVars *tv, Flow *f, Packet *p) +{ + FlowUpdateCallback *cb = update_callbacks; + while (cb != NULL) { + cb->Callback(tv, f, p, cb->user); + cb = cb->next; + } +} + +bool SCFlowRegisterFinishCallback(SCFlowFinishCallbackFn fn, void *user) +{ + FlowFinishCallback *cb = SCCalloc(1, sizeof(*cb)); + if (cb == NULL) { + return false; + } + cb->Callback = fn; + cb->user = user; + if (finish_callbacks == NULL) { + finish_callbacks = cb; + } else { + FlowFinishCallback *current = finish_callbacks; + while (current->next != NULL) { + current = current->next; + } + current->next = cb; + } + return true; +} + +void SCFlowRunFinishCallbacks(ThreadVars *tv, Flow *f) +{ + FlowFinishCallback *cb = finish_callbacks; + while (cb != NULL) { + cb->Callback(tv, f, cb->user); + cb = cb->next; + } +} diff --git a/src/flow-callbacks.h b/src/flow-callbacks.h new file mode 100644 index 000000000000..4c694807753f --- /dev/null +++ b/src/flow-callbacks.h @@ -0,0 +1,121 @@ +/* Copyright (C) 2024 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +#ifndef SURICATA_FLOW_CALLBACKS_H +#define SURICATA_FLOW_CALLBACKS_H + +#include "suricata-common.h" +#include "flow.h" + +/** \brief Function type for flow initialization callbacks. + * + * Once registered with SCFlowRegisterInitCallback, this function will + * be called every time a flow is initialized, or in other words, + * every time Suricata picks up a flow. + * + * \param tv The ThreadVars data structure for the thread creating the + * flow. + * \param f The newly initialized flow. + * \param p The packet related to creating the new flow. + * \param user The user data provided during callback registration. + */ +typedef void (*SCFlowInitCallbackFn)(ThreadVars *tv, Flow *f, const Packet *p, void *user); + +/** \brief Register a flow init callback. + * + * Register a user provided function to be called every time a flow is + * initialized for use. + * + * \param fn Pointer to function to be called + * \param user Additional user data to be passed to callback + * + * \returns true if callback was registered, otherwise false if the + * callback could not be registered due to memory allocation error. + */ +bool SCFlowRegisterInitCallback(SCFlowInitCallbackFn fn, void *user); + +/** \internal + * + * Run all registered flow init callbacks. + */ +void SCFlowRunInitCallbacks(ThreadVars *tv, Flow *f, const Packet *p); + +/** \brief Function type for flow update callbacks. + * + * Once registered with SCFlowRegisterUpdateCallback, this function + * will be called every time a flow is updated by a packet (basically + * everytime a packet is seen on a flow). + * + * \param tv The ThreadVars data structure for the thread updating the + * flow. + * \param f The flow being updated. + * \param p The packet responsible for the flow update. + * \param user The user data provided during callback registration. + */ +typedef void (*SCFlowUpdateCallbackFn)(ThreadVars *tv, Flow *f, Packet *p, void *user); + +/** \brief Register a flow update callback. + * + * Register a user provided function to be called everytime a flow is + * updated. + * + * \param fn Pointer to function to be called + * \param user Additional user data to be passed to callback + * + * \returns true if callback was registered, otherwise false if the + * callback could not be registered due to memory allocation error. + */ +bool SCFlowRegisterUpdateCallback(SCFlowUpdateCallbackFn fn, void *user); + +/** \internal + * + * Run all registered flow update callbacks. + */ +void SCFlowRunUpdateCallbacks(ThreadVars *tv, Flow *f, Packet *p); + +/** \brief Function type for flow finish callbacks. + * + * Once registered with SCFlowRegisterFinshCallback, this function + * will be called when Suricata is done with a flow. + * + * \param tv The ThreadVars data structure for the thread finishing + * the flow. + * \param f The flow being finshed. + * \param user The user data provided during callback registration. + */ +typedef void (*SCFlowFinishCallbackFn)(ThreadVars *tv, Flow *f, void *user); + +/** \brief Register a flow init callback. + * + * Register a user provided function to be called every time a flow is + * finished. + * + * \param fn Pointer to function to be called + * \param user Additional user data to be passed to callback + * + * \returns true if callback was registered, otherwise false if the + * callback could not be registered due to memory allocation error. + */ +bool SCFlowRegisterFinishCallback(SCFlowFinishCallbackFn fn, void *user); + +/** \internal + * + * Run all registered flow init callbacks. + */ +void SCFlowRunFinishCallbacks(ThreadVars *tv, Flow *f); + +#endif /* SURICATA_FLOW_CALLBACKS_H */ diff --git a/src/flow-hash.c b/src/flow-hash.c index ddab01cd5b69..fcd957c72e27 100644 --- a/src/flow-hash.c +++ b/src/flow-hash.c @@ -38,6 +38,7 @@ #include "flow-storage.h" #include "flow-timeout.h" #include "flow-spare-pool.h" +#include "flow-callbacks.h" #include "app-layer-parser.h" #include "util-time.h" @@ -781,7 +782,7 @@ static Flow *TcpReuseReplace(ThreadVars *tv, FlowLookupStruct *fls, FlowBucket * fb->head = f; /* initialize and return */ - FlowInit(f, p); + FlowInit(tv, f, p); f->flow_hash = hash; f->fb = fb; FlowUpdateState(f, FLOW_STATE_NEW); @@ -886,7 +887,7 @@ Flow *FlowGetFlowFromHash(ThreadVars *tv, FlowLookupStruct *fls, Packet *p, Flow fb->head = f; /* got one, now lock, initialize and return */ - FlowInit(f, p); + FlowInit(tv, f, p); f->flow_hash = hash; f->fb = fb; FlowUpdateState(f, FLOW_STATE_NEW); @@ -951,7 +952,7 @@ Flow *FlowGetFlowFromHash(ThreadVars *tv, FlowLookupStruct *fls, Packet *p, Flow fb->head = f; /* initialize and return */ - FlowInit(f, p); + FlowInit(tv, f, p); f->flow_hash = hash; f->fb = fb; FlowUpdateState(f, FLOW_STATE_NEW); @@ -1242,6 +1243,7 @@ static Flow *FlowGetUsedFlow(ThreadVars *tv, DecodeThreadVars *dtv, const SCTime } #endif + SCFlowRunFinishCallbacks(tv, f); FlowClearMemory(f, f->protomap); /* leave locked */ diff --git a/src/flow-manager.c b/src/flow-manager.c index 05b791ee612e..9da986b22df6 100644 --- a/src/flow-manager.c +++ b/src/flow-manager.c @@ -39,6 +39,7 @@ #include "flow-manager.h" #include "flow-storage.h" #include "flow-spare-pool.h" +#include "flow-callbacks.h" #include "stream-tcp.h" #include "stream-tcp-cache.h" @@ -1059,7 +1060,7 @@ static void Recycler(ThreadVars *tv, FlowRecyclerThreadData *ftd, Flow *f) StatsDecr(tv, ftd->counter_tcp_active_sessions); } StatsDecr(tv, ftd->counter_flow_active); - + SCFlowRunFinishCallbacks(tv, f); FlowClearMemory(f, f->protomap); FLOWLOCK_UNLOCK(f); } diff --git a/src/flow-util.c b/src/flow-util.c index 7e11da41f527..31e22b9341ac 100644 --- a/src/flow-util.c +++ b/src/flow-util.c @@ -29,6 +29,7 @@ #include "flow.h" #include "flow-private.h" #include "flow-util.h" +#include "flow-callbacks.h" #include "flow-var.h" #include "app-layer.h" @@ -142,7 +143,7 @@ static inline void FlowSetICMPv6CounterPart(Flow *f) /* initialize the flow from the first packet * we see from it. */ -void FlowInit(Flow *f, const Packet *p) +void FlowInit(ThreadVars *tv, Flow *f, const Packet *p) { SCEnter(); SCLogDebug("flow %p", f); @@ -203,6 +204,8 @@ void FlowInit(Flow *f, const Packet *p) FlowSetStorageById(f, MacSetGetFlowStorageID(), ms); } + SCFlowRunInitCallbacks(tv, f, p); + SCReturn; } diff --git a/src/flow-util.h b/src/flow-util.h index 2d813bd9ee4d..368c955d876a 100644 --- a/src/flow-util.h +++ b/src/flow-util.h @@ -140,7 +140,7 @@ Flow *FlowAlloc(void); void FlowFree(Flow *); uint8_t FlowGetProtoMapping(uint8_t); -void FlowInit(Flow *, const Packet *); +void FlowInit(ThreadVars *, Flow *, const Packet *); uint8_t FlowGetReverseProtoMapping(uint8_t rproto); /* flow end counter logic */ diff --git a/src/flow.c b/src/flow.c index 7bfa80ea0a9b..aea79d23bf08 100644 --- a/src/flow.c +++ b/src/flow.c @@ -44,6 +44,7 @@ #include "flow-storage.h" #include "flow-bypass.h" #include "flow-spare-pool.h" +#include "flow-callbacks.h" #include "stream-tcp-private.h" @@ -503,6 +504,8 @@ void FlowHandlePacketUpdate(Flow *f, Packet *p, ThreadVars *tv, DecodeThreadVars SCLogDebug("setting FLOW_NOPAYLOAD_INSPECTION flag on flow %p", f); DecodeSetNoPayloadInspectionFlag(p); } + + SCFlowRunUpdateCallbacks(tv, f, p); } /** \brief Entry point for packet flow handling diff --git a/src/output-eve-stream.c b/src/output-eve-stream.c index fcdf0c2e5c0b..4b44d86835e7 100644 --- a/src/output-eve-stream.c +++ b/src/output-eve-stream.c @@ -425,7 +425,7 @@ static int EveStreamLogger(ThreadVars *tv, void *thread_data, const Packet *p) /* Close stream. */ jb_close(js); - OutputJsonBuilderBuffer(js, td->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, js, td->ctx); jb_free(js); return TM_ECODE_OK; diff --git a/src/output-eve.c b/src/output-eve.c index d0d775cba7f7..2c67f3b6c38a 100644 --- a/src/output-eve.c +++ b/src/output-eve.c @@ -15,11 +15,50 @@ * 02110-1301, USA. */ +#include "suricata-common.h" #include "output-eve.h" #include "util-debug.h" +#include "rust.h" + +typedef struct EveUserCallback_ { + SCEveUserCallbackFn Callback; + void *user; + struct EveUserCallback_ *next; +} EveUserCallback; + +static EveUserCallback *eve_user_callbacks = NULL; static TAILQ_HEAD(, SCEveFileType_) output_types = TAILQ_HEAD_INITIALIZER(output_types); +bool SCEveRegisterCallback(SCEveUserCallbackFn fn, void *user) +{ + EveUserCallback *cb = SCCalloc(1, sizeof(*cb)); + if (cb == NULL) { + return false; + } + cb->Callback = fn; + cb->user = user; + if (eve_user_callbacks == NULL) { + eve_user_callbacks = cb; + } else { + EveUserCallback *current = eve_user_callbacks; + while (current->next != NULL) { + current = current->next; + } + current->next = cb; + } + return true; +} + +void SCEveRunCallbacks(ThreadVars *tv, const Packet *p, Flow *f, JsonBuilder *jb) +{ + EveUserCallback *cb = eve_user_callbacks; + while (cb != NULL) { + cb->Callback(tv, p, f, jb, cb->user); + cb = cb->next; + } +} + static bool IsBuiltinTypeName(const char *name) { const char *builtin[] = { diff --git a/src/output-eve.h b/src/output-eve.h index 7046c7b98005..7e55ce28f8e2 100644 --- a/src/output-eve.h +++ b/src/output-eve.h @@ -31,6 +31,7 @@ #define SURICATA_OUTPUT_EVE_H #include "suricata-common.h" +#include "rust.h" #include "conf.h" typedef uint32_t ThreadId; @@ -173,4 +174,46 @@ bool SCRegisterEveFileType(SCEveFileType *); SCEveFileType *SCEveFindFileType(const char *name); +/** \brief Function type for EVE callbacks. + * + * The function type for callbacks registered with + * SCEveRegisterCallback. This function will be called with the + * JsonBuilder just prior to the top-level object being closed. New + * fields maybe added, however there is no way to alter existing + * objects already added to the JsonBuilder. + * + * \param tv The ThreadVars for the thread performing the logging. + * \param p Packet if available. + * \param f Flow if available. + * \param user User data provided during callback registration. + */ +typedef void (*SCEveUserCallbackFn)( + ThreadVars *tv, const Packet *p, Flow *f, JsonBuilder *jb, void *user); + +/** \brief Register a callback for adding extra information to EVE logs. + * + * Allow users to register a callback for each EVE log. The callback + * is called just before the root object on the JsonBuilder is to be + * closed. + * + * New objects and fields can be append, but exist entries cannot be modified. + * + * Packet and Flow will be provided if available, but will other be + * NULL. + * + * Limitations: At this time the callbacks will only be called for EVE + * loggers that use JsonBuilder, notably this means it won't be called + * for stats records at this time. + * + * \returns true if callback is registered, false is not due to memory + * allocation error. + */ +bool SCEveRegisterCallback(SCEveUserCallbackFn fn, void *user); + +/** \internal + * + * Run EVE callbacks. + */ +void SCEveRunCallbacks(ThreadVars *tv, const Packet *p, Flow *f, JsonBuilder *jb); + #endif diff --git a/src/output-json-alert.c b/src/output-json-alert.c index 7822cc798045..91a55828a7a1 100644 --- a/src/output-json-alert.c +++ b/src/output-json-alert.c @@ -757,7 +757,7 @@ static int AlertJson(ThreadVars *tv, JsonAlertLogThread *aft, const Packet *p) EveAddVerdict(jb, p); } - OutputJsonBuilderBuffer(jb, aft->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, jb, aft->ctx); jb_free(jb); } @@ -767,7 +767,7 @@ static int AlertJson(ThreadVars *tv, JsonAlertLogThread *aft, const Packet *p) CreateEveHeader(p, LOG_DIR_PACKET, "packet", NULL, json_output_ctx->eve_ctx); if (unlikely(packetjs != NULL)) { EvePacket(p, packetjs, 0); - OutputJsonBuilderBuffer(packetjs, aft->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, packetjs, aft->ctx); jb_free(packetjs); } } @@ -801,7 +801,7 @@ static int AlertJsonDecoderEvent(ThreadVars *tv, JsonAlertLogThread *aft, const AlertJsonHeader(p, pa, jb, json_output_ctx->flags, NULL, NULL); - OutputJsonBuilderBuffer(jb, aft->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, jb, aft->ctx); jb_free(jb); } diff --git a/src/output-json-anomaly.c b/src/output-json-anomaly.c index 241cb974a758..00f82fa3685e 100644 --- a/src/output-json-anomaly.c +++ b/src/output-json-anomaly.c @@ -143,16 +143,16 @@ static int AnomalyDecodeEventJson(ThreadVars *tv, JsonAnomalyLogThread *aft, EvePacket(p, js, GET_PKT_LEN(p) < 32 ? GET_PKT_LEN(p) : 32); } - OutputJsonBuilderBuffer(js, aft->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, js, aft->ctx); jb_free(js); } return TM_ECODE_OK; } -static int AnomalyAppLayerDecoderEventJson(JsonAnomalyLogThread *aft, - const Packet *p, AppLayerDecoderEvents *decoder_events, - bool is_pktlayer, const char *layer, uint64_t tx_id) +static int AnomalyAppLayerDecoderEventJson(ThreadVars *tv, JsonAnomalyLogThread *aft, + const Packet *p, AppLayerDecoderEvents *decoder_events, bool is_pktlayer, const char *layer, + uint64_t tx_id) { const char *alprotoname = AppLayerGetProtoName(p->flow->alproto); @@ -201,7 +201,7 @@ static int AnomalyAppLayerDecoderEventJson(JsonAnomalyLogThread *aft, /* anomaly */ jb_close(js); - OutputJsonBuilderBuffer(js, aft->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, js, aft->ctx); jb_free(js); /* Current implementation assumes a single owner for this value */ @@ -223,8 +223,7 @@ static int JsonAnomalyTxLogger(ThreadVars *tv, void *thread_data, const Packet * decoder_events = AppLayerParserGetEventsByTx(f->proto, f->alproto, tx); if (decoder_events && decoder_events->event_last_logged < decoder_events->cnt) { SCLogDebug("state %p, tx: %p, tx_id: %"PRIu64, state, tx, tx_id); - AnomalyAppLayerDecoderEventJson(aft, p, decoder_events, false, - "proto_parser", tx_id); + AnomalyAppLayerDecoderEventJson(tv, aft, p, decoder_events, false, "proto_parser", tx_id); } return TM_ECODE_OK; } @@ -255,8 +254,8 @@ static int AnomalyJson(ThreadVars *tv, JsonAnomalyLogThread *aft, const Packet * if (aft->json_output_ctx->flags & LOG_JSON_APPLAYER_TYPE) { /* app layer proto detect events */ if (rc == TM_ECODE_OK && AnomalyHasPacketAppLayerEvents(p)) { - rc = AnomalyAppLayerDecoderEventJson(aft, p, p->app_layer_events, - true, "proto_detect", TX_ID_UNUSED); + rc = AnomalyAppLayerDecoderEventJson( + tv, aft, p, p->app_layer_events, true, "proto_detect", TX_ID_UNUSED); } /* parser state events */ @@ -264,8 +263,8 @@ static int AnomalyJson(ThreadVars *tv, JsonAnomalyLogThread *aft, const Packet * SCLogDebug("Checking for anomaly events; alproto %d", p->flow->alproto); AppLayerDecoderEvents *parser_events = AppLayerParserGetDecoderEvents(p->flow->alparser); if (parser_events && (parser_events->event_last_logged < parser_events->cnt)) { - rc = AnomalyAppLayerDecoderEventJson(aft, p, parser_events, - false, "parser", TX_ID_UNUSED); + rc = AnomalyAppLayerDecoderEventJson( + tv, aft, p, parser_events, false, "parser", TX_ID_UNUSED); } } } diff --git a/src/output-json-arp.c b/src/output-json-arp.c index 0490c6b54d1e..87a80d8cdb0b 100644 --- a/src/output-json-arp.c +++ b/src/output-json-arp.c @@ -90,7 +90,7 @@ static int JsonArpLogger(ThreadVars *tv, void *thread_data, const Packet *p) JSONFormatAndAddMACAddr(jb, "dest_mac", arph->dest_mac, false); jb_set_string(jb, "dest_ip", dstip); jb_close(jb); /* arp */ - OutputJsonBuilderBuffer(jb, thread); + OutputJsonBuilderBuffer(tv, p, p->flow, jb, thread); jb_free(jb); return TM_ECODE_OK; diff --git a/src/output-json-dcerpc.c b/src/output-json-dcerpc.c index 17e0199ed727..3b3bff90feac 100644 --- a/src/output-json-dcerpc.c +++ b/src/output-json-dcerpc.c @@ -47,7 +47,7 @@ static int JsonDCERPCLogger(ThreadVars *tv, void *thread_data, jb_close(jb); MemBufferReset(thread->buffer); - OutputJsonBuilderBuffer(jb, thread); + OutputJsonBuilderBuffer(tv, p, p->flow, jb, thread); jb_free(jb); return TM_ECODE_OK; diff --git a/src/output-json-dhcp.c b/src/output-json-dhcp.c index 9c7d9dff9230..a4a4a29990e8 100644 --- a/src/output-json-dhcp.c +++ b/src/output-json-dhcp.c @@ -72,7 +72,7 @@ static int JsonDHCPLogger(ThreadVars *tv, void *thread_data, rs_dhcp_logger_log(ctx->rs_logger, tx, js); - OutputJsonBuilderBuffer(js, thread->thread); + OutputJsonBuilderBuffer(tv, p, p->flow, js, thread->thread); jb_free(js); return TM_ECODE_OK; diff --git a/src/output-json-dnp3.c b/src/output-json-dnp3.c index 53cecd78a1aa..ea557ff206a8 100644 --- a/src/output-json-dnp3.c +++ b/src/output-json-dnp3.c @@ -246,7 +246,7 @@ static int JsonDNP3LoggerToServer(ThreadVars *tv, void *thread_data, jb_open_object(js, "dnp3"); JsonDNP3LogRequest(js, tx); jb_close(js); - OutputJsonBuilderBuffer(js, thread->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, js, thread->ctx); jb_free(js); SCReturnInt(TM_ECODE_OK); @@ -267,7 +267,7 @@ static int JsonDNP3LoggerToClient(ThreadVars *tv, void *thread_data, jb_open_object(js, "dnp3"); JsonDNP3LogResponse(js, tx); jb_close(js); - OutputJsonBuilderBuffer(js, thread->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, js, thread->ctx); jb_free(js); SCReturnInt(TM_ECODE_OK); diff --git a/src/output-json-dns.c b/src/output-json-dns.c index 3954da2336dc..cb60a4509a32 100644 --- a/src/output-json-dns.c +++ b/src/output-json-dns.c @@ -331,7 +331,7 @@ static int JsonDoh2Logger(ThreadVars *tv, void *thread_data, const Packet *p, Fl } out: if (r || r2) { - OutputJsonBuilderBuffer(jb, td->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, jb, td->ctx); } jb_free(jb); return TM_ECODE_OK; @@ -363,7 +363,7 @@ static int JsonDnsLoggerToServer(ThreadVars *tv, void *thread_data, } jb_close(jb); - OutputJsonBuilderBuffer(jb, td->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, jb, td->ctx); jb_free(jb); } @@ -392,7 +392,7 @@ static int JsonDnsLoggerToClient(ThreadVars *tv, void *thread_data, jb_set_int(jb, "version", 2); SCDnsLogJsonAnswer(txptr, td->dnslog_ctx->flags, jb); jb_close(jb); - OutputJsonBuilderBuffer(jb, td->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, jb, td->ctx); jb_free(jb); } @@ -432,7 +432,7 @@ static int JsonDnsLogger(ThreadVars *tv, void *thread_data, const Packet *p, Flo } if (SCDnsLogJson(txptr, td->dnslog_ctx->flags, jb)) { - OutputJsonBuilderBuffer(jb, td->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, jb, td->ctx); } jb_free(jb); } diff --git a/src/output-json-drop.c b/src/output-json-drop.c index b82c632daf65..1ac27a209d2a 100644 --- a/src/output-json-drop.c +++ b/src/output-json-drop.c @@ -85,7 +85,7 @@ static int g_droplog_flows_start = 1; * * \return return TM_ECODE_OK on success */ -static int DropLogJSON (JsonDropLogThread *aft, const Packet *p) +static int DropLogJSON(ThreadVars *tv, JsonDropLogThread *aft, const Packet *p) { JsonDropOutputCtx *drop_ctx = aft->drop_ctx; @@ -191,7 +191,7 @@ static int DropLogJSON (JsonDropLogThread *aft, const Packet *p) } } - OutputJsonBuilderBuffer(js, aft->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, js, aft->ctx); jb_free(js); return TM_ECODE_OK; @@ -326,7 +326,7 @@ static OutputInitResult JsonDropLogInitCtxSub(ConfNode *conf, OutputCtx *parent_ static int JsonDropLogger(ThreadVars *tv, void *thread_data, const Packet *p) { JsonDropLogThread *td = thread_data; - int r = DropLogJSON(td, p); + int r = DropLogJSON(tv, td, p); if (r < 0) return -1; diff --git a/src/output-json-file.c b/src/output-json-file.c index 509ae488bbee..e1f33893806e 100644 --- a/src/output-json-file.c +++ b/src/output-json-file.c @@ -213,8 +213,8 @@ JsonBuilder *JsonBuildFileInfoRecord(const Packet *p, const File *ff, void *tx, * \internal * \brief Write meta data on a single line json record */ -static void FileWriteJsonRecord(JsonFileLogThread *aft, const Packet *p, const File *ff, void *tx, - const uint64_t tx_id, uint8_t dir, OutputJsonCtx *eve_ctx) +static void FileWriteJsonRecord(ThreadVars *tv, JsonFileLogThread *aft, const Packet *p, + const File *ff, void *tx, const uint64_t tx_id, uint8_t dir, OutputJsonCtx *eve_ctx) { HttpXFFCfg *xff_cfg = aft->filelog_ctx->xff_cfg != NULL ? aft->filelog_ctx->xff_cfg : aft->filelog_ctx->parent_xff_cfg; @@ -223,7 +223,7 @@ static void FileWriteJsonRecord(JsonFileLogThread *aft, const Packet *p, const F return; } - OutputJsonBuilderBuffer(js, aft->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, js, aft->ctx); jb_free(js); } @@ -237,7 +237,7 @@ static int JsonFileLogger(ThreadVars *tv, void *thread_data, const Packet *p, co SCLogDebug("ff %p", ff); - FileWriteJsonRecord(aft, p, ff, tx, tx_id, dir, aft->filelog_ctx->eve_ctx); + FileWriteJsonRecord(tv, aft, p, ff, tx, tx_id, dir, aft->filelog_ctx->eve_ctx); return 0; } diff --git a/src/output-json-flow.c b/src/output-json-flow.c index f7826734f0cb..051d530fb1ef 100644 --- a/src/output-json-flow.c +++ b/src/output-json-flow.c @@ -340,7 +340,7 @@ static int JsonFlowLogger(ThreadVars *tv, void *thread_data, Flow *f) EveFlowLogJSON(thread, jb, f); - OutputJsonBuilderBuffer(jb, thread); + OutputJsonBuilderBuffer(tv, NULL, f, jb, thread); jb_free(jb); SCReturnInt(TM_ECODE_OK); diff --git a/src/output-json-frame.c b/src/output-json-frame.c index 09ec4aaab110..90224240f43f 100644 --- a/src/output-json-frame.c +++ b/src/output-json-frame.c @@ -287,8 +287,8 @@ void FrameJsonLogOneFrame(const uint8_t ipproto, const Frame *frame, Flow *f, jb_close(jb); } -static int FrameJsonUdp( - JsonFrameLogThread *aft, const Packet *p, Flow *f, FramesContainer *frames_container) +static int FrameJsonUdp(ThreadVars *tv, JsonFrameLogThread *aft, const Packet *p, Flow *f, + FramesContainer *frames_container) { FrameJsonOutputCtx *json_output_ctx = aft->json_output_ctx; @@ -315,7 +315,7 @@ static int FrameJsonUdp( jb_set_string(jb, "app_proto", AppProtoToString(f->alproto)); FrameJsonLogOneFrame(IPPROTO_UDP, frame, p->flow, NULL, p, jb, aft->payload_buffer); - OutputJsonBuilderBuffer(jb, aft->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, jb, aft->ctx); jb_free(jb); frame->flags |= FRAME_FLAG_LOGGED; } @@ -333,7 +333,7 @@ static int FrameJson(ThreadVars *tv, JsonFrameLogThread *aft, const Packet *p) return TM_ECODE_OK; if (p->proto == IPPROTO_UDP) { - return FrameJsonUdp(aft, p, p->flow, frames_container); + return FrameJsonUdp(tv, aft, p, p->flow, frames_container); } BUG_ON(p->proto != IPPROTO_TCP); @@ -387,7 +387,7 @@ static int FrameJson(ThreadVars *tv, JsonFrameLogThread *aft, const Packet *p) jb_set_string(jb, "app_proto", AppProtoToString(p->flow->alproto)); FrameJsonLogOneFrame(IPPROTO_TCP, frame, p->flow, stream, p, jb, aft->payload_buffer); - OutputJsonBuilderBuffer(jb, aft->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, jb, aft->ctx); jb_free(jb); frame->flags |= FRAME_FLAG_LOGGED; } else if (frame != NULL) { diff --git a/src/output-json-http.c b/src/output-json-http.c index 0c5b875ee9ad..b45be9a45b6b 100644 --- a/src/output-json-http.c +++ b/src/output-json-http.c @@ -493,7 +493,7 @@ static int JsonHttpLogger(ThreadVars *tv, void *thread_data, const Packet *p, Fl } } - OutputJsonBuilderBuffer(js, jhl->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, js, jhl->ctx); jb_free(js); SCReturnInt(TM_ECODE_OK); diff --git a/src/output-json-ike.c b/src/output-json-ike.c index 470026fde13b..a13ef0e1d944 100644 --- a/src/output-json-ike.c +++ b/src/output-json-ike.c @@ -90,7 +90,7 @@ static int JsonIKELogger(ThreadVars *tv, void *thread_data, const Packet *p, Flo goto error; } - OutputJsonBuilderBuffer(jb, thread->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, jb, thread->ctx); jb_free(jb); return TM_ECODE_OK; diff --git a/src/output-json-metadata.c b/src/output-json-metadata.c index f97547c551b4..2602e4b9b3ef 100644 --- a/src/output-json-metadata.c +++ b/src/output-json-metadata.c @@ -74,7 +74,7 @@ static int MetadataJson(ThreadVars *tv, OutputJsonThreadCtx *aft, const Packet * if (!aft->ctx->cfg.include_metadata) { EveAddMetadata(p, p->flow, js); } - OutputJsonBuilderBuffer(js, aft); + OutputJsonBuilderBuffer(tv, p, p->flow, js, aft); jb_free(js); return TM_ECODE_OK; diff --git a/src/output-json-mqtt.c b/src/output-json-mqtt.c index 66cf67a0334b..c912ddcc1835 100644 --- a/src/output-json-mqtt.c +++ b/src/output-json-mqtt.c @@ -85,7 +85,7 @@ static int JsonMQTTLogger(ThreadVars *tv, void *thread_data, if (!rs_mqtt_logger_log(tx, thread->mqttlog_ctx->flags, thread->mqttlog_ctx->max_log_len, js)) goto error; - OutputJsonBuilderBuffer(js, thread->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, js, thread->ctx); jb_free(js); return TM_ECODE_OK; diff --git a/src/output-json-netflow.c b/src/output-json-netflow.c index 2e359bb909c5..e448ecd33bc4 100644 --- a/src/output-json-netflow.c +++ b/src/output-json-netflow.c @@ -276,7 +276,7 @@ static int JsonNetFlowLogger(ThreadVars *tv, void *thread_data, Flow *f) return TM_ECODE_OK; NetFlowLogEveToServer(jb, f); EveAddCommonOptions(&jhl->ctx->cfg, NULL, f, jb, LOG_DIR_FLOW_TOSERVER); - OutputJsonBuilderBuffer(jb, jhl); + OutputJsonBuilderBuffer(tv, NULL, f, jb, jhl); jb_free(jb); /* only log a response record if we actually have seen response packets */ @@ -286,7 +286,7 @@ static int JsonNetFlowLogger(ThreadVars *tv, void *thread_data, Flow *f) return TM_ECODE_OK; NetFlowLogEveToClient(jb, f); EveAddCommonOptions(&jhl->ctx->cfg, NULL, f, jb, LOG_DIR_FLOW_TOCLIENT); - OutputJsonBuilderBuffer(jb, jhl); + OutputJsonBuilderBuffer(tv, NULL, f, jb, jhl); jb_free(jb); } SCReturnInt(TM_ECODE_OK); diff --git a/src/output-json-nfs.c b/src/output-json-nfs.c index 72274a6b7865..0b08c0e5105d 100644 --- a/src/output-json-nfs.c +++ b/src/output-json-nfs.c @@ -94,7 +94,7 @@ static int JsonNFSLogger(ThreadVars *tv, void *thread_data, jb_close(jb); MemBufferReset(thread->buffer); - OutputJsonBuilderBuffer(jb, thread); + OutputJsonBuilderBuffer(tv, p, p->flow, jb, thread); jb_free(jb); return TM_ECODE_OK; } diff --git a/src/output-json-pgsql.c b/src/output-json-pgsql.c index 71bcd10f071d..9cba28d25d4e 100644 --- a/src/output-json-pgsql.c +++ b/src/output-json-pgsql.c @@ -80,7 +80,7 @@ static int JsonPgsqlLogger(ThreadVars *tv, void *thread_data, const Packet *p, F goto error; } - OutputJsonBuilderBuffer(jb, thread->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, jb, thread->ctx); jb_free(jb); return TM_ECODE_OK; diff --git a/src/output-json-smb.c b/src/output-json-smb.c index 279ee772e8f0..4be1fce93e72 100644 --- a/src/output-json-smb.c +++ b/src/output-json-smb.c @@ -59,7 +59,7 @@ static int JsonSMBLogger(ThreadVars *tv, void *thread_data, } jb_close(jb); - OutputJsonBuilderBuffer(jb, thread); + OutputJsonBuilderBuffer(tv, p, p->flow, jb, thread); jb_free(jb); return TM_ECODE_OK; diff --git a/src/output-json-smtp.c b/src/output-json-smtp.c index bddbc4a9fcc4..592645cb3c09 100644 --- a/src/output-json-smtp.c +++ b/src/output-json-smtp.c @@ -85,7 +85,7 @@ static int JsonSmtpLogger(ThreadVars *tv, void *thread_data, const Packet *p, Fl jb_close(jb); EveEmailLogJson(jhl, jb, p, f, state, tx, tx_id); - OutputJsonBuilderBuffer(jb, jhl->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, jb, jhl->ctx); jb_free(jb); diff --git a/src/output-json-tls.c b/src/output-json-tls.c index aa24b3380a0d..c4ba0e249e62 100644 --- a/src/output-json-tls.c +++ b/src/output-json-tls.c @@ -501,7 +501,7 @@ static int JsonTlsLogger(ThreadVars *tv, void *thread_data, const Packet *p, /* Close the tls object. */ jb_close(js); - OutputJsonBuilderBuffer(js, aft->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, js, aft->ctx); jb_free(js); return 0; diff --git a/src/output-json.c b/src/output-json.c index 1f411cc110b8..18376fd428a5 100644 --- a/src/output-json.c +++ b/src/output-json.c @@ -955,7 +955,8 @@ int OutputJSONBuffer(json_t *js, LogFileCtx *file_ctx, MemBuffer **buffer) return 0; } -int OutputJsonBuilderBuffer(JsonBuilder *js, OutputJsonThreadCtx *ctx) +int OutputJsonBuilderBuffer( + ThreadVars *tv, const Packet *p, Flow *f, JsonBuilder *js, OutputJsonThreadCtx *ctx) { LogFileCtx *file_ctx = ctx->file_ctx; MemBuffer **buffer = &ctx->buffer; @@ -967,6 +968,8 @@ int OutputJsonBuilderBuffer(JsonBuilder *js, OutputJsonThreadCtx *ctx) jb_set_string(js, "pcap_filename", PcapFileGetFilename()); } + SCEveRunCallbacks(tv, p, f, js); + jb_close(js); MemBufferReset(*buffer); diff --git a/src/output-json.h b/src/output-json.h index 761064f7e10a..89597e616a0f 100644 --- a/src/output-json.h +++ b/src/output-json.h @@ -103,7 +103,8 @@ JsonBuilder *CreateEveHeader(const Packet *p, enum OutputJsonLogDirection dir, JsonBuilder *CreateEveHeaderWithTxId(const Packet *p, enum OutputJsonLogDirection dir, const char *event_type, JsonAddrInfo *addr, uint64_t tx_id, OutputJsonCtx *eve_ctx); int OutputJSONBuffer(json_t *js, LogFileCtx *file_ctx, MemBuffer **buffer); -int OutputJsonBuilderBuffer(JsonBuilder *js, OutputJsonThreadCtx *ctx); +int OutputJsonBuilderBuffer( + ThreadVars *tv, const Packet *p, Flow *f, JsonBuilder *js, OutputJsonThreadCtx *ctx); OutputInitResult OutputJsonInitCtx(ConfNode *); OutputInitResult OutputJsonLogInitSub(ConfNode *conf, OutputCtx *parent_ctx); diff --git a/src/output.c b/src/output.c index 002f33b5abc6..b99897509c0f 100644 --- a/src/output.c +++ b/src/output.c @@ -927,7 +927,7 @@ static int JsonGenericLogger(ThreadVars *tv, void *thread_data, const Packet *p, goto error; } - OutputJsonBuilderBuffer(js, thread); + OutputJsonBuilderBuffer(tv, p, p->flow, js, thread); jb_free(js); return TM_ECODE_OK; diff --git a/src/runmode-unittests.c b/src/runmode-unittests.c index e116e86d5be5..35780ab101a0 100644 --- a/src/runmode-unittests.c +++ b/src/runmode-unittests.c @@ -240,6 +240,7 @@ void RunUnittests(int list_unittests, const char *regex_arg) AppLayerSetup(); /* hardcoded initialization code */ + SigTableInit(); SigTableSetup(); /* load the rule keywords */ TmqhSetup(); diff --git a/src/runmode-unix-socket.c b/src/runmode-unix-socket.c index e0b314a1cd2a..3c390e99a6db 100644 --- a/src/runmode-unix-socket.c +++ b/src/runmode-unix-socket.c @@ -44,7 +44,9 @@ #include "defrag-hash.h" #include "ippair.h" #include "app-layer.h" +#include "app-layer-ftp.h" #include "app-layer-htp-mem.h" +#include "app-layer-htp-range.h" #include "host-bit.h" #include "util-misc.h" @@ -86,50 +88,23 @@ const char *RunModeUnixSocketGetDefaultMode(void) return "autofp"; } -#define MEMCAPS_MAX 7 -static MemcapCommand memcaps[MEMCAPS_MAX] = { +static MemcapCommand memcaps[] = { { - "stream", - StreamTcpSetMemcap, - StreamTcpGetMemcap, - StreamTcpMemuseCounter, - }, - { - "stream-reassembly", - StreamTcpReassembleSetMemcap, - StreamTcpReassembleGetMemcap, - StreamTcpReassembleMemuseGlobalCounter - }, - { - "flow", - FlowSetMemcap, - FlowGetMemcap, - FlowGetMemuse - }, - { - "applayer-proto-http", - HTPSetMemcap, - HTPGetMemcap, - HTPMemuseGlobalCounter - }, - { - "defrag", - DefragTrackerSetMemcap, - DefragTrackerGetMemcap, - DefragTrackerGetMemuse - }, - { - "ippair", - IPPairSetMemcap, - IPPairGetMemcap, - IPPairGetMemuse - }, - { - "host", - HostSetMemcap, - HostGetMemcap, - HostGetMemuse + "stream", + StreamTcpSetMemcap, + StreamTcpGetMemcap, + StreamTcpMemuseCounter, }, + { "stream-reassembly", StreamTcpReassembleSetMemcap, StreamTcpReassembleGetMemcap, + StreamTcpReassembleMemuseGlobalCounter }, + { "flow", FlowSetMemcap, FlowGetMemcap, FlowGetMemuse }, + { "applayer-proto-http", HTPSetMemcap, HTPGetMemcap, HTPMemuseGlobalCounter }, + { "applayer-proto-http-byterange", HTPByteRangeSetMemcap, HTPByteRangeMemcapGlobalCounter, + HTPByteRangeMemuseGlobalCounter }, + { "defrag", DefragTrackerSetMemcap, DefragTrackerGetMemcap, DefragTrackerGetMemuse }, + { "ippair", IPPairSetMemcap, IPPairGetMemcap, IPPairGetMemuse }, + { "host", HostSetMemcap, HostGetMemcap, HostGetMemuse }, + { "ftp", FTPSetMemcap, FTPMemcapGlobalCounter, FTPMemuseGlobalCounter }, }; float MemcapsGetPressure(void) @@ -1523,7 +1498,6 @@ TmEcode UnixSocketSetMemcap(json_t *cmd, json_t* answer, void *data) char *memcap = NULL; char *value_str = NULL; uint64_t value; - int i; json_t *jarg = json_object_get(cmd, "config"); if (!json_is_string(jarg)) { @@ -1549,7 +1523,7 @@ TmEcode UnixSocketSetMemcap(json_t *cmd, json_t* answer, void *data) return TM_ECODE_FAILED; } - for (i = 0; i < MEMCAPS_MAX; i++) { + for (size_t i = 0; i < ARRAY_SIZE(memcaps); i++) { if (strcmp(memcaps[i].name, memcap) == 0 && memcaps[i].SetFunc) { int updated = memcaps[i].SetFunc(value); char message[150]; @@ -1592,7 +1566,6 @@ TmEcode UnixSocketSetMemcap(json_t *cmd, json_t* answer, void *data) TmEcode UnixSocketShowMemcap(json_t *cmd, json_t *answer, void *data) { char *memcap = NULL; - int i; json_t *jarg = json_object_get(cmd, "config"); if (!json_is_string(jarg)) { @@ -1601,7 +1574,7 @@ TmEcode UnixSocketShowMemcap(json_t *cmd, json_t *answer, void *data) } memcap = (char *)json_string_value(jarg); - for (i = 0; i < MEMCAPS_MAX; i++) { + for (size_t i = 0; i < ARRAY_SIZE(memcaps); i++) { if (strcmp(memcaps[i].name, memcap) == 0 && memcaps[i].GetFunc) { char str[50]; uint64_t val = memcaps[i].GetFunc(); @@ -1632,7 +1605,6 @@ TmEcode UnixSocketShowMemcap(json_t *cmd, json_t *answer, void *data) TmEcode UnixSocketShowAllMemcap(json_t *cmd, json_t *answer, void *data) { json_t *jmemcaps = json_array(); - int i; if (jmemcaps == NULL) { json_object_set_new(answer, "message", @@ -1640,7 +1612,7 @@ TmEcode UnixSocketShowAllMemcap(json_t *cmd, json_t *answer, void *data) return TM_ECODE_FAILED; } - for (i = 0; i < MEMCAPS_MAX; i++) { + for (size_t i = 0; i < ARRAY_SIZE(memcaps); i++) { json_t *jobj = json_object(); if (jobj == NULL) { json_decref(jmemcaps); diff --git a/src/suricata.c b/src/suricata.c index 6bdd6edb90f6..ee9dfc0b5b69 100644 --- a/src/suricata.c +++ b/src/suricata.c @@ -2670,6 +2670,8 @@ int PostConfLoadedSetup(SCInstance *suri) MacSetRegisterFlowStorage(); + SigTableInit(); + #ifdef HAVE_PLUGINS SCPluginsLoad(suri->capture_plugin_name, suri->capture_plugin_args); #endif diff --git a/src/tests/fuzz/fuzz_siginit.c b/src/tests/fuzz/fuzz_siginit.c index a98148cfac8d..a50e1fd67ebf 100644 --- a/src/tests/fuzz/fuzz_siginit.c +++ b/src/tests/fuzz/fuzz_siginit.c @@ -27,6 +27,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) MpmTableSetup(); SpmTableSetup(); EngineModeSetIDS(); + SigTableInit(); SigTableSetup(); } if (cnt++ == 1024) { diff --git a/src/thread-callbacks.c b/src/thread-callbacks.c new file mode 100644 index 000000000000..ede35d7107ce --- /dev/null +++ b/src/thread-callbacks.c @@ -0,0 +1,55 @@ +/* Copyright (C) 2024 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +#include "thread-callbacks.h" + +typedef struct ThreadInitCallback_ { + SCThreadInitCallbackFn Callback; + void *user; + struct ThreadInitCallback_ *next; +} ThreadInitCallback; + +static ThreadInitCallback *init_callbacks = NULL; + +bool SCThreadRegisterInitCallback(SCThreadInitCallbackFn fn, void *user) +{ + ThreadInitCallback *cb = SCCalloc(1, sizeof(*cb)); + if (cb == NULL) { + return false; + } + cb->Callback = fn; + cb->user = user; + if (init_callbacks == NULL) { + init_callbacks = cb; + } else { + ThreadInitCallback *current = init_callbacks; + while (current->next != NULL) { + current = current->next; + } + current->next = cb; + } + return true; +} + +void SCThreadRunInitCallbacks(ThreadVars *tv) +{ + ThreadInitCallback *cb = init_callbacks; + while (cb != NULL) { + cb->Callback(tv, cb->user); + cb = cb->next; + } +} diff --git a/src/thread-callbacks.h b/src/thread-callbacks.h new file mode 100644 index 000000000000..5bcd638bf86b --- /dev/null +++ b/src/thread-callbacks.h @@ -0,0 +1,54 @@ +/* Copyright (C) 2024 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +#ifndef SURICATA_THREAD_CALLBACKS_H +#define SURICATA_THREAD_CALLBACKS_H + +#include "suricata-common.h" +#include "threadvars.h" + +/** \brief Function type for thread intialization callbacks. + * + * Once registered by SCThreadRegisterInitCallback, this function will + * be called for every thread being initialized during Suricata + * startup. + * + * \param tv The ThreadVars struct that has just been initialized. + * \param user The user data provided when registering the callback. + */ +typedef void (*SCThreadInitCallbackFn)(ThreadVars *tv, void *user); + +/** \brief Register a thread init callback. + * + * Register a user provided function to be called every time a thread is + * initialized for use. + * + * \param fn Pointer to function to be called + * \param user Additional user data to be passed to callback + * + * \returns true if callback was registered, otherwise false if the + * callback could not be registered due to memory allocation error. + */ +bool SCThreadRegisterInitCallback(SCThreadInitCallbackFn fn, void *user); + +/** \internal + * + * Run all registered flow init callbacks. + */ +void SCThreadRunInitCallbacks(ThreadVars *tv); + +#endif /* SURICATA_THREAD_CALLBACKS_H */ diff --git a/src/thread-storage.c b/src/thread-storage.c new file mode 100644 index 000000000000..977f4fde9752 --- /dev/null +++ b/src/thread-storage.c @@ -0,0 +1,212 @@ +/* Copyright (C) 2024 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +#include "suricata-common.h" +#include "thread-storage.h" +#include "util-storage.h" +#include "util-unittest.h" + +const StorageEnum storage_type = STORAGE_THREAD; + +unsigned int ThreadStorageSize(void) +{ + return StorageGetSize(storage_type); +} + +void *ThreadGetStorageById(const ThreadVars *tv, ThreadStorageId id) +{ + return StorageGetById(tv->storage, storage_type, id.id); +} + +int ThreadSetStorageById(ThreadVars *tv, ThreadStorageId id, void *ptr) +{ + return StorageSetById(tv->storage, storage_type, id.id, ptr); +} + +void *ThreadAllocStorageById(ThreadVars *tv, ThreadStorageId id) +{ + return StorageAllocByIdPrealloc(tv->storage, storage_type, id.id); +} + +void ThreadFreeStorageById(ThreadVars *tv, ThreadStorageId id) +{ + StorageFreeById(tv->storage, storage_type, id.id); +} + +void ThreadFreeStorage(ThreadVars *tv) +{ + if (ThreadStorageSize() > 0) + StorageFreeAll(tv->storage, storage_type); +} + +ThreadStorageId ThreadStorageRegister(const char *name, const unsigned int size, + void *(*Alloc)(unsigned int), void (*Free)(void *)) +{ + int id = StorageRegister(storage_type, name, size, Alloc, Free); + ThreadStorageId tsi = { .id = id }; + return tsi; +} + +#ifdef UNITTESTS + +static void *StorageTestAlloc(unsigned int size) +{ + return SCCalloc(1, size); +} + +static void StorageTestFree(void *x) +{ + SCFree(x); +} + +static int ThreadStorageTest01(void) +{ + StorageInit(); + + ThreadStorageId id1 = ThreadStorageRegister("test", 8, StorageTestAlloc, StorageTestFree); + FAIL_IF(id1.id < 0); + + ThreadStorageId id2 = ThreadStorageRegister("variable", 24, StorageTestAlloc, StorageTestFree); + FAIL_IF(id2.id < 0); + + ThreadStorageId id3 = + ThreadStorageRegister("store", sizeof(void *), StorageTestAlloc, StorageTestFree); + FAIL_IF(id3.id < 0); + + FAIL_IF(StorageFinalize() < 0); + + ThreadVars *tv = SCCalloc(1, sizeof(ThreadVars) + ThreadStorageSize()); + FAIL_IF_NULL(tv); + + void *ptr = ThreadGetStorageById(tv, id1); + FAIL_IF_NOT_NULL(ptr); + + ptr = ThreadGetStorageById(tv, id2); + FAIL_IF_NOT_NULL(ptr); + + ptr = ThreadGetStorageById(tv, id3); + FAIL_IF_NOT_NULL(ptr); + + void *ptr1a = ThreadAllocStorageById(tv, id1); + FAIL_IF_NULL(ptr1a); + + void *ptr2a = ThreadAllocStorageById(tv, id2); + FAIL_IF_NULL(ptr2a); + + void *ptr3a = ThreadAllocStorageById(tv, id3); + FAIL_IF_NULL(ptr3a); + + void *ptr1b = ThreadGetStorageById(tv, id1); + FAIL_IF(ptr1a != ptr1b); + + void *ptr2b = ThreadGetStorageById(tv, id2); + FAIL_IF(ptr2a != ptr2b); + + void *ptr3b = ThreadGetStorageById(tv, id3); + FAIL_IF(ptr3a != ptr3b); + + ThreadFreeStorage(tv); + StorageCleanup(); + SCFree(tv); + PASS; +} + +static int ThreadStorageTest02(void) +{ + StorageInit(); + + ThreadStorageId id1 = ThreadStorageRegister("test", sizeof(void *), NULL, StorageTestFree); + FAIL_IF(id1.id < 0); + + FAIL_IF(StorageFinalize() < 0); + + ThreadVars *tv = SCCalloc(1, sizeof(ThreadVars) + ThreadStorageSize()); + FAIL_IF_NULL(tv); + + void *ptr = ThreadGetStorageById(tv, id1); + FAIL_IF_NOT_NULL(ptr); + + void *ptr1a = SCMalloc(128); + FAIL_IF_NULL(ptr1a); + + ThreadSetStorageById(tv, id1, ptr1a); + + void *ptr1b = ThreadGetStorageById(tv, id1); + FAIL_IF(ptr1a != ptr1b); + + ThreadFreeStorage(tv); + StorageCleanup(); + PASS; +} + +static int ThreadStorageTest03(void) +{ + StorageInit(); + + ThreadStorageId id1 = ThreadStorageRegister("test1", sizeof(void *), NULL, StorageTestFree); + FAIL_IF(id1.id < 0); + + ThreadStorageId id2 = ThreadStorageRegister("test2", sizeof(void *), NULL, StorageTestFree); + FAIL_IF(id2.id < 0); + + ThreadStorageId id3 = ThreadStorageRegister("test3", 32, StorageTestAlloc, StorageTestFree); + FAIL_IF(id3.id < 0); + + FAIL_IF(StorageFinalize() < 0); + + ThreadVars *tv = SCCalloc(1, sizeof(ThreadVars) + ThreadStorageSize()); + FAIL_IF_NULL(tv); + + void *ptr = ThreadGetStorageById(tv, id1); + FAIL_IF_NOT_NULL(ptr); + + void *ptr1a = SCMalloc(128); + FAIL_IF_NULL(ptr1a); + + ThreadSetStorageById(tv, id1, ptr1a); + + void *ptr2a = SCMalloc(256); + FAIL_IF_NULL(ptr2a); + + ThreadSetStorageById(tv, id2, ptr2a); + + void *ptr3a = ThreadAllocStorageById(tv, id3); + FAIL_IF_NULL(ptr3a); + + void *ptr1b = ThreadGetStorageById(tv, id1); + FAIL_IF(ptr1a != ptr1b); + + void *ptr2b = ThreadGetStorageById(tv, id2); + FAIL_IF(ptr2a != ptr2b); + + void *ptr3b = ThreadGetStorageById(tv, id3); + FAIL_IF(ptr3a != ptr3b); + + ThreadFreeStorage(tv); + StorageCleanup(); + PASS; +} +#endif + +void RegisterThreadStorageTests(void) +{ +#ifdef UNITTESTS + UtRegisterTest("ThreadStorageTest01", ThreadStorageTest01); + UtRegisterTest("ThreadStorageTest02", ThreadStorageTest02); + UtRegisterTest("ThreadStorageTest03", ThreadStorageTest03); +#endif +} diff --git a/src/thread-storage.h b/src/thread-storage.h new file mode 100644 index 000000000000..5dd22570b0fa --- /dev/null +++ b/src/thread-storage.h @@ -0,0 +1,45 @@ +/* Copyright (C) 2024 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * Thread wrapper around storage API. + */ + +#ifndef SURICATA_THREAD_STORAGE_H +#define SURICATA_THREAD_STORAGE_H + +#include "threadvars.h" + +typedef struct ThreadStorageId { + int id; +} ThreadStorageId; + +unsigned int ThreadStorageSize(void); + +void *ThreadGetStorageById(const ThreadVars *tv, ThreadStorageId id); +int ThreadSetStorageById(ThreadVars *tv, ThreadStorageId id, void *ptr); +void *ThreadAllocStorageById(ThreadVars *tv, ThreadStorageId id); + +void ThreadFreeStorageById(ThreadVars *tv, ThreadStorageId id); +void ThreadFreeStorage(ThreadVars *tv); + +void RegisterThreadStorageTests(void); + +ThreadStorageId ThreadStorageRegister(const char *name, const unsigned int size, + void *(*Alloc)(unsigned int), void (*Free)(void *)); + +#endif /* SURICATA_THREAD_STORAGE_H */ diff --git a/src/threads.c b/src/threads.c index 1708a8f5cd37..919e6422e32f 100644 --- a/src/threads.c +++ b/src/threads.c @@ -25,6 +25,7 @@ */ #include "suricata-common.h" +#include "thread-storage.h" #include "util-unittest.h" #include "util-debug.h" #include "threads.h" @@ -149,5 +150,6 @@ void ThreadMacrosRegisterTests(void) UtRegisterTest("ThreadMacrosTest03RWLocks", ThreadMacrosTest03RWLocks); UtRegisterTest("ThreadMacrosTest04RWLocks", ThreadMacrosTest04RWLocks); // UtRegisterTest("ThreadMacrosTest05RWLocks", ThreadMacrosTest05RWLocks); + RegisterThreadStorageTests(); #endif /* UNIT TESTS */ } diff --git a/src/threadvars.h b/src/threadvars.h index cebcdb4e3ac1..6f339e9839d5 100644 --- a/src/threadvars.h +++ b/src/threadvars.h @@ -28,6 +28,7 @@ #include "counters.h" #include "packet-queue.h" #include "util-atomic.h" +#include "util-storage.h" struct TmSlot_; @@ -135,6 +136,7 @@ typedef struct ThreadVars_ { struct FlowQueue_ *flow_queue; bool break_loop; + Storage storage[]; } ThreadVars; /** Thread setup flags: */ diff --git a/src/tm-threads.c b/src/tm-threads.c index b0d0f8686ba0..07f9a9390df0 100644 --- a/src/tm-threads.c +++ b/src/tm-threads.c @@ -29,7 +29,9 @@ #include "suricata.h" #include "stream.h" #include "runmodes.h" +#include "thread-callbacks.h" #include "threadvars.h" +#include "thread-storage.h" #include "tm-queues.h" #include "tm-queuehandlers.h" #include "tm-threads.h" @@ -919,7 +921,7 @@ ThreadVars *TmThreadCreate(const char *name, const char *inq_name, const char *i SCLogDebug("creating thread \"%s\"...", name); /* XXX create separate function for this: allocate a thread container */ - tv = SCCalloc(1, sizeof(ThreadVars)); + tv = SCCalloc(1, sizeof(ThreadVars) + ThreadStorageSize()); if (unlikely(tv == NULL)) goto error; @@ -1011,6 +1013,8 @@ ThreadVars *TmThreadCreate(const char *name, const char *inq_name, const char *i if (mucond != 0) TmThreadInitMC(tv); + SCThreadRunInitCallbacks(tv); + return tv; error: @@ -1577,6 +1581,8 @@ static void TmThreadFree(ThreadVars *tv) SCLogDebug("Freeing thread '%s'.", tv->name); + ThreadFreeStorage(tv); + if (tv->flow_queue) { BUG_ON(tv->flow_queue->qlen != 0); SCFree(tv->flow_queue); diff --git a/src/util-storage.c b/src/util-storage.c index 02f69a568cd2..bae251432315 100644 --- a/src/util-storage.c +++ b/src/util-storage.c @@ -59,6 +59,8 @@ static const char *StoragePrintType(StorageEnum type) return "ippair"; case STORAGE_DEVICE: return "livedevice"; + case STORAGE_THREAD: + return "thread"; case STORAGE_MAX: return "max"; } diff --git a/src/util-storage.h b/src/util-storage.h index 11d64bdbecbd..fce1f964eb14 100644 --- a/src/util-storage.h +++ b/src/util-storage.h @@ -31,6 +31,7 @@ typedef enum StorageEnum_ { STORAGE_FLOW, STORAGE_IPPAIR, STORAGE_DEVICE, + STORAGE_THREAD, STORAGE_MAX, } StorageEnum; diff --git a/src/util-thash.c b/src/util-thash.c index 3787454a37f4..be6e930bcc5f 100644 --- a/src/util-thash.c +++ b/src/util-thash.c @@ -228,12 +228,15 @@ static int THashInitConfig(THashTableContext *ctx, const char *cnf_prefix) GET_VAR(cnf_prefix, "memcap"); if ((ConfGet(varname, &conf_val)) == 1) { - if (ParseSizeStringU64(conf_val, &ctx->config.memcap) < 0) { + uint64_t memcap; + if (ParseSizeStringU64(conf_val, &memcap) < 0) { SCLogError("Error parsing %s " "from conf file - %s. Killing engine", varname, conf_val); return -1; } + SC_ATOMIC_INIT(ctx->config.memcap); + SC_ATOMIC_SET(ctx->config.memcap, memcap); } GET_VAR(cnf_prefix, "hash-size"); if ((ConfGet(varname, &conf_val)) == 1) @@ -261,7 +264,7 @@ static int THashInitConfig(THashTableContext *ctx, const char *cnf_prefix) "Memcap: %" PRIu64 ", Hash table size %" PRIu64 ". Calculate " "total hash size by multiplying \"hash-size\" with %" PRIuMAX ", " "which is the hash bucket size.", - ctx->config.memcap, hash_size, (uintmax_t)sizeof(THashHashRow)); + SC_ATOMIC_GET(ctx->config.memcap), hash_size, (uintmax_t)sizeof(THashHashRow)); return -1; } ctx->array = SCMallocAligned(ctx->config.hash_size * sizeof(THashHashRow), CLS); @@ -283,7 +286,7 @@ static int THashInitConfig(THashTableContext *ctx, const char *cnf_prefix) SCLogError("preallocating data failed: " "max thash memcap reached. Memcap %" PRIu64 ", " "Memuse %" PRIu64 ".", - ctx->config.memcap, + SC_ATOMIC_GET(ctx->config.memcap), ((uint64_t)SC_ATOMIC_GET(ctx->memuse) + THASH_DATA_SIZE(ctx))); return -1; } @@ -323,12 +326,12 @@ THashTableContext *THashInit(const char *cnf_prefix, size_t data_size, unless defined by the rule keyword */ #ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION // limit memcap size to default when fuzzing - ctx->config.memcap = THASH_DEFAULT_MEMCAP; + SC_ATOMIC_SET(ctx->config.memcap, THASH_DEFAULT_MEMCAP); #else if (memcap > 0) { - ctx->config.memcap = memcap; + SC_ATOMIC_SET(ctx->config.memcap, memcap); } else { - ctx->config.memcap = reset_memcap ? UINT64_MAX : THASH_DEFAULT_MEMCAP; + SC_ATOMIC_SET(ctx->config.memcap, reset_memcap ? UINT64_MAX : THASH_DEFAULT_MEMCAP); } #endif ctx->config.prealloc = THASH_DEFAULT_PREALLOC; @@ -349,8 +352,9 @@ THashTableContext *THashInit(const char *cnf_prefix, size_t data_size, * */ void THashConsolidateMemcap(THashTableContext *ctx) { - ctx->config.memcap = MAX(SC_ATOMIC_GET(ctx->memuse), ctx->config.memcap); - SCLogDebug("memcap after load set to: %" PRIu64, ctx->config.memcap); + SC_ATOMIC_SET( + ctx->config.memcap, MAX(SC_ATOMIC_GET(ctx->memuse), SC_ATOMIC_GET(ctx->config.memcap))); + SCLogDebug("memcap after load set to: %" PRIu64, SC_ATOMIC_GET(ctx->config.memcap)); } /** \brief shutdown the flow engine @@ -598,7 +602,7 @@ static THashData *THashDataGetNew(THashTableContext *ctx, void *data) SC_ATOMIC_SET(ctx->memcap_reached, true); } SCLogError("Adding data will exceed memcap: %" PRIu64 ", current memuse: %" PRIu64, - (ctx)->config.memcap, SC_ATOMIC_GET(ctx->memuse)); + SC_ATOMIC_GET((ctx)->config.memcap), SC_ATOMIC_GET(ctx->memuse)); } } } diff --git a/src/util-thash.h b/src/util-thash.h index 803a5f477c3c..5d4a61f10b12 100644 --- a/src/util-thash.h +++ b/src/util-thash.h @@ -122,7 +122,7 @@ typedef int (*THashOutputFunc)(void *output_ctx, const uint8_t *data, const uint typedef int (*THashFormatFunc)(const void *in_data, char *output, size_t output_size); typedef struct THashDataConfig_ { - uint64_t memcap; + SC_ATOMIC_DECLARE(uint64_t, memcap); uint32_t hash_rand; uint32_t hash_size; uint32_t prealloc; @@ -161,8 +161,9 @@ typedef struct THashTableContext_ { * \retval 1 it fits * \retval 0 no fit */ -#define THASH_CHECK_MEMCAP(ctx, size) \ - ((((uint64_t)SC_ATOMIC_GET((ctx)->memuse) + (uint64_t)(size)) <= (ctx)->config.memcap)) +#define THASH_CHECK_MEMCAP(ctx, size) \ + ((((uint64_t)SC_ATOMIC_GET((ctx)->memuse) + (uint64_t)(size)) <= \ + SC_ATOMIC_GET((ctx)->config.memcap))) #define THashIncrUsecnt(h) \ (void)SC_ATOMIC_ADD((h)->use_cnt, 1)