From 3ddc993a70915c2f7ed5c34e174b2d2a18043d90 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Mon, 9 Dec 2024 11:45:22 +0000 Subject: [PATCH] custom_calyptia: switch to YAML for Fleet config Signed-off-by: Patrick Stephens --- .github/workflows/unit-tests.yaml | 4 +- .gitignore | 1 + CMakeLists.txt | 6 +- dockerfiles/Dockerfile | 2 +- .../fluent-bit/calyptia/calyptia_constants.h | 4 +- .../include/cprofiles/cprof_encode_msgpack.h | 20 +- plugins/custom_calyptia/calyptia.c | 8 +- plugins/custom_calyptia/calyptia.h | 5 +- plugins/in_calyptia_fleet/in_calyptia_fleet.c | 189 +++++++++--------- plugins/in_calyptia_fleet/in_calyptia_fleet.h | 80 ++++++++ tests/runtime/CMakeLists.txt | 4 +- tests/runtime/custom_calyptia_input_test.c | 5 +- tests/runtime/in_calyptia_fleet_test.c | 121 +++++++++++ tests/runtime_shell/CMakeLists.txt | 1 + .../conf/custom_calyptia_fleet.conf | 17 ++ tests/runtime_shell/custom_calyptia.sh | 65 ++++++ 16 files changed, 424 insertions(+), 108 deletions(-) create mode 100644 plugins/in_calyptia_fleet/in_calyptia_fleet.h create mode 100644 tests/runtime/in_calyptia_fleet_test.c create mode 100644 tests/runtime_shell/conf/custom_calyptia_fleet.conf create mode 100755 tests/runtime_shell/custom_calyptia.sh diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index 24d93fc61ba..167a75f578f 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -78,6 +78,7 @@ jobs: CC: ${{ matrix.compiler }} CXX: ${{ matrix.compiler }} FLB_OPT: ${{ matrix.flb_option }} + CALYPTIA_FLEET_TOKEN: ${{ secrets.CALYPTIA_FLEET_TOKEN }} run-macos-unit-tests: # We chain this after Linux one as there are costs and restrictions associated @@ -111,6 +112,7 @@ jobs: CC: gcc CXX: g++ FLB_OPT: ${{ matrix.flb_option }} + CALYPTIA_FLEET_TOKEN: ${{ secrets.CALYPTIA_FLEET_TOKEN }} run-aarch64-unit-tests: # Ensure for OSS Fluent Bit repo we enable usage of Actuated runners for ARM builds, for forks it should keep existing ubuntu-latest usage. @@ -167,6 +169,7 @@ jobs: env: CC: ${{ matrix.config.compiler }} CXX: ${{ matrix.config.compiler }} + CALYPTIA_FLEET_TOKEN: ${{ secrets.CALYPTIA_FLEET_TOKEN }} run-qemu-ubuntu-unit-tests: # We chain this after Linux one as there are CPU time costs for QEMU emulation @@ -217,7 +220,6 @@ jobs: make -j $nparallel ctest -j $nparallel --build-run-dir . --output-on-failure - # Required check looks at this so do not remove run-all-unit-tests: if: always() diff --git a/.gitignore b/.gitignore index ba37a7bb55f..1543d19fe19 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ init/fluent-bit.service lib/chunkio/include/chunkio/cio_version.h lib/monkey/monkey.service lib/monkey/include/monkey/mk_core/mk_core_info.h +lib/cprofiles/include/cprofiles/cprof_info.h packaging/.env packaging/packages diff --git a/CMakeLists.txt b/CMakeLists.txt index 41904bffd0f..228948d3fd1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,9 +6,9 @@ cmake_policy(SET CMP0069 NEW) set(CMAKE_POLICY_DEFAULT_CMP0069 NEW) # Fluent Bit Version -set(FLB_VERSION_MAJOR 4) -set(FLB_VERSION_MINOR 0) -set(FLB_VERSION_PATCH 0) +set(FLB_VERSION_MAJOR 3) +set(FLB_VERSION_MINOR 2) +set(FLB_VERSION_PATCH 5) set(FLB_VERSION_STR "${FLB_VERSION_MAJOR}.${FLB_VERSION_MINOR}.${FLB_VERSION_PATCH}") set(CMAKE_POSITION_INDEPENDENT_CODE ON) diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile index 7cdcfe6bc88..c6ad9060347 100644 --- a/dockerfiles/Dockerfile +++ b/dockerfiles/Dockerfile @@ -13,7 +13,7 @@ # docker buildx build --platform "linux/amd64,linux/arm64,linux/arm/v7,linux/s390x" -f ./dockerfiles/Dockerfile.multiarch --build-arg FLB_TARBALL=https://github.com/fluent/fluent-bit/archive/v1.8.11.tar.gz ./dockerfiles/ # Set this to the current release version: it gets done so as part of the release. -ARG RELEASE_VERSION=3.2.4 +ARG RELEASE_VERSION=3.2.5 # For multi-arch builds - assumption is running on an AMD64 host FROM multiarch/qemu-user-static:x86_64-arm AS qemu-arm32 diff --git a/include/fluent-bit/calyptia/calyptia_constants.h b/include/fluent-bit/calyptia/calyptia_constants.h index f2271b87a67..dc1c2a0216d 100644 --- a/include/fluent-bit/calyptia/calyptia_constants.h +++ b/include/fluent-bit/calyptia/calyptia_constants.h @@ -36,8 +36,10 @@ #define CALYPTIA_ENDPOINT_TRACE "/v1/traces/%s" #define CALYPTIA_ENDPOINT_FLEETS "/v1/fleets" -#define CALYPTIA_ENDPOINT_FLEET_CONFIG_INI "/v1/fleets/%s/config?format=ini" +#define CALYPTIA_ENDPOINT_FLEET_CONFIG_INI "/v1/fleets/%s/config?format=ini&config_format=ini" +#define CALYPTIA_ENDPOINT_FLEET_CONFIG_YAML "/v1/fleets/%s/config?format=yaml&config_format=yaml" #define CALYPTIA_ENDPOINT_FLEET_FILES "/v1/fleets/%s/files" +#define CALYPTIA_ENDPOINT_FLEET_BY_NAME "/v1/search?project_id=%s&resource=fleet&term=%s&exact=true" /* Storage */ #define CALYPTIA_SESSION_FILE "session.CALYPTIA" diff --git a/lib/cprofiles/include/cprofiles/cprof_encode_msgpack.h b/lib/cprofiles/include/cprofiles/cprof_encode_msgpack.h index b02c86477cf..d8e166f01c5 100644 --- a/lib/cprofiles/include/cprofiles/cprof_encode_msgpack.h +++ b/lib/cprofiles/include/cprofiles/cprof_encode_msgpack.h @@ -20,13 +20,21 @@ #ifndef CPROF_ENCODE_MSGPACK_H #define CPROF_ENCODE_MSGPACK_H -#include -#include -#include +#define CPROF_SOURCE_DIR "/Users/pat/github/fluent/fluent-bit" -#define CPROF_ENCODE_MSGPACK_SUCCESS 0 -#define CPROF_ENCODE_MSGPACK_ALLOCATION_ERROR 1 -#define CPROF_ENCODE_MSGPACK_INVALID_ARGUMENT_ERROR 2 +/* General flags set by /CMakeLists.txt */ +#ifndef CPROF_HAVE_TIMESPEC_GET +#define CPROF_HAVE_TIMESPEC_GET +#endif +#ifndef CPROF_HAVE_CLOCK_GET_TIME +#define CPROF_HAVE_CLOCK_GET_TIME +#endif +#ifndef CPROF_HAVE_CFL +#define CPROF_HAVE_CFL +#endif +#ifndef CPROF_HAVE_FLUENT_OTEL_PROTO +#define CPROF_HAVE_FLUENT_OTEL_PROTO +#endif struct cprof_msgpack_encoding_context { mpack_writer_t writer; diff --git a/plugins/custom_calyptia/calyptia.c b/plugins/custom_calyptia/calyptia.c index dba68a25f10..c296b944a29 100644 --- a/plugins/custom_calyptia/calyptia.c +++ b/plugins/custom_calyptia/calyptia.c @@ -232,6 +232,7 @@ int set_fleet_input_properties(struct calyptia *ctx, struct flb_input_instance * flb_input_set_property(fleet, "host", ctx->cloud_host); flb_input_set_property(fleet, "port", ctx->cloud_port); flb_input_set_property(fleet, "config_dir", ctx->fleet_config_dir); + flb_input_set_property(fleet, "fleet_config_legacy_format", ctx->fleet_config_legacy_format == 1 ? "on" : "off"); /* Set TLS properties */ flb_input_set_property(fleet, "tls", ctx->cloud_tls == 1 ? "on" : "off"); @@ -617,7 +618,7 @@ static int cb_calyptia_init(struct flb_custom_instance *ins, flb_free(ctx); return -1; } - ctx->machine_id_auto_configured = 1; + ctx->machine_id_auto_configured = FLB_TRUE; } /* input collector */ @@ -774,6 +775,11 @@ static struct flb_config_map config_map[] = { 0, FLB_TRUE, offsetof(struct calyptia, register_retry_on_flush), "Retry agent registration on flush if failed on init." }, + { + FLB_CONFIG_MAP_BOOL, "fleet_config_legacy_format", "true", + 0, FLB_TRUE, offsetof(struct calyptia, fleet_config_legacy_format), + "If set, use legacy (TOML) format for configuration files." + }, /* EOF */ {0} }; diff --git a/plugins/custom_calyptia/calyptia.h b/plugins/custom_calyptia/calyptia.h index 5da14d80ace..0dff625c694 100644 --- a/plugins/custom_calyptia/calyptia.h +++ b/plugins/custom_calyptia/calyptia.h @@ -20,8 +20,6 @@ #ifndef FLB_CALYPTIA_H #define FLB_CALYPTIA_H -#include - struct calyptia { /* config map options */ flb_sds_t api_key; @@ -55,7 +53,8 @@ struct calyptia { flb_sds_t fleet_max_http_buffer_size; flb_sds_t fleet_interval_sec; flb_sds_t fleet_interval_nsec; - bool register_retry_on_flush; /* retry registration on flush if failed */ + int register_retry_on_flush; /* retry registration on flush if failed */ + int fleet_config_legacy_format; /* Fleet config format to use: INI (true) or YAML (false) */ }; int set_fleet_input_properties(struct calyptia *ctx, struct flb_input_instance *fleet); diff --git a/plugins/in_calyptia_fleet/in_calyptia_fleet.c b/plugins/in_calyptia_fleet/in_calyptia_fleet.c index 5a423856c32..143bf90ce31 100644 --- a/plugins/in_calyptia_fleet/in_calyptia_fleet.c +++ b/plugins/in_calyptia_fleet/in_calyptia_fleet.c @@ -42,6 +42,8 @@ #include +#include "in_calyptia_fleet.h" + /* Glob support */ #ifndef _MSC_VER #include @@ -53,70 +55,15 @@ #define PATH_MAX MAX_PATH #endif -#define CALYPTIA_HEADERS_PROJECT "X-Project-Token" -#define CALYPTIA_HEADERS_CTYPE "Content-Type" -#define CALYPTIA_HEADERS_CTYPE_JSON "application/json" - #define DEFAULT_INTERVAL_SEC "15" #define DEFAULT_INTERVAL_NSEC "0" #define DEFAULT_MAX_HTTP_BUFFER_SIZE "10485760" -struct flb_in_calyptia_fleet_config { - /* Time interval check */ - int interval_sec; - int interval_nsec; - - /* maximum http buffer size */ - int max_http_buffer_size; - - /* Grabbed from the cfg_path, used to check if configuration has - * has been updated. - */ - long config_timestamp; - - flb_sds_t api_key; - - flb_sds_t fleet_id; - - /* flag used to mark fleet_id for release when found automatically. */ - int fleet_id_found; - - flb_sds_t fleet_name; - flb_sds_t machine_id; - flb_sds_t config_dir; - flb_sds_t cloud_host; - flb_sds_t cloud_port; - - flb_sds_t fleet_url; - flb_sds_t fleet_files_url; - - struct flb_input_instance *ins; /* plugin instance */ - - /* Networking */ - struct flb_upstream *u; - - int collect_fd; -}; - -struct reload_ctx { - flb_ctx_t *flb; - flb_sds_t cfg_path; -}; - -static flb_sds_t fleet_config_filename(struct flb_in_calyptia_fleet_config *ctx, char *fname); - -#define new_fleet_config_filename(a) fleet_config_filename((a), "new") -#define cur_fleet_config_filename(a) fleet_config_filename((a), "cur") -#define old_fleet_config_filename(a) fleet_config_filename((a), "old") -#define hdr_fleet_config_filename(a) fleet_config_filename((a), "header") - +static int fleet_cur_chdir(struct flb_in_calyptia_fleet_config *ctx); static int get_calyptia_files(struct flb_in_calyptia_fleet_config *ctx, - const char *url, time_t timestamp); -static int fleet_cur_chdir(struct flb_in_calyptia_fleet_config *ctx); - #ifndef FLB_SYSTEM_WINDOWS static int is_link(const char *path) { @@ -231,12 +178,17 @@ static flb_sds_t generate_base_fleet_directory(struct flb_in_calyptia_fleet_conf } if (*fleet_dir == NULL) { - *fleet_dir = flb_sds_create_size(4096); + *fleet_dir = flb_sds_create_size(CALYPTIA_MAX_DIR_SIZE); if (*fleet_dir == NULL) { return NULL; } } + /* Ensure we have a valid value */ + if (ctx->config_dir == NULL) { + ctx->config_dir = FLEET_DEFAULT_CONFIG_DIR; + } + if (ctx->fleet_name != NULL) { return flb_sds_printf(fleet_dir, "%s" PATH_SEPARATOR "%s" PATH_SEPARATOR "%s", ctx->config_dir, ctx->machine_id, ctx->fleet_name); @@ -245,7 +197,7 @@ static flb_sds_t generate_base_fleet_directory(struct flb_in_calyptia_fleet_conf ctx->config_dir, ctx->machine_id, ctx->fleet_id); } -static flb_sds_t fleet_config_filename(struct flb_in_calyptia_fleet_config *ctx, char *fname) +flb_sds_t fleet_config_filename(struct flb_in_calyptia_fleet_config *ctx, char *fname) { flb_sds_t cfgname = NULL; flb_sds_t ret; @@ -258,7 +210,12 @@ static flb_sds_t fleet_config_filename(struct flb_in_calyptia_fleet_config *ctx, return NULL; } - ret = flb_sds_printf(&cfgname, PATH_SEPARATOR "%s.conf", fname); + if (ctx->fleet_config_legacy_format) { + ret = flb_sds_printf(&cfgname, PATH_SEPARATOR "%s.conf", fname); + } else { + ret = flb_sds_printf(&cfgname, PATH_SEPARATOR "%s.yaml", fname); + } + if (ret == NULL) { flb_sds_destroy(cfgname); return NULL; @@ -366,7 +323,7 @@ static int is_timestamped_fleet_config_path(struct flb_in_calyptia_fleet_config char *end; long val; - if (path == NULL) { + if (path == NULL || ctx == NULL) { return FLB_FALSE; } @@ -384,7 +341,12 @@ static int is_timestamped_fleet_config_path(struct flb_in_calyptia_fleet_config return FLB_FALSE; } - if (strcmp(end, ".conf") == 0) { + if (ctx->fleet_config_legacy_format) { + if (strcmp(end, ".conf") == 0) { + return FLB_TRUE; + } + } + else if (strcmp(end, ".yaml") == 0) { return FLB_TRUE; } @@ -528,7 +490,7 @@ static int test_config_is_valid(struct flb_in_calyptia_fleet_config *ctx, conf = flb_cf_create(); if (conf == NULL) { - flb_plg_debug(ctx->ins, "unable to create conf during validation test: %s", + flb_plg_debug(ctx->ins, "unable to create config during validation test: %s", cfgpath); goto config_init_error; } @@ -536,7 +498,7 @@ static int test_config_is_valid(struct flb_in_calyptia_fleet_config *ctx, conf = flb_cf_create_from_file(conf, cfgpath); if (conf == NULL) { flb_plg_debug(ctx->ins, - "unable to create conf from file during validation test: %s", + "unable to create config from file during validation test: %s", cfgpath); goto cf_create_from_file_error; } @@ -555,7 +517,7 @@ static int parse_config_name_timestamp(struct flb_in_calyptia_fleet_config *ctx, { char *ext = NULL; long timestamp; - char realname[4096] = {0}; + char realname[CALYPTIA_MAX_DIR_SIZE] = {0}; char *fname; ssize_t len; @@ -564,13 +526,17 @@ static int parse_config_name_timestamp(struct flb_in_calyptia_fleet_config *ctx, } switch (is_link(cfgpath)) { + /* Prevent undefined references due to use of readlink */ +#ifndef FLB_SYSTEM_WINDOWS case FLB_TRUE: + len = readlink(cfgpath, realname, sizeof(realname)); if (len > sizeof(realname)) { return FLB_FALSE; } break; +#endif /* FLB_SYSTEM_WINDOWS */ case FLB_FALSE: strncpy(realname, cfgpath, sizeof(realname)-1); break; @@ -957,13 +923,13 @@ static int get_calyptia_fleet_id_by_name(struct flb_in_calyptia_fleet_config *ct return -1; } - url = flb_sds_create_size(4096); + url = flb_sds_create_size(CALYPTIA_MAX_DIR_SIZE); if (url == NULL) { flb_sds_destroy(project_id); return -1; } - flb_sds_printf(&url, "/v1/search?project_id=%s&resource=fleet&term=%s&exact=true", + flb_sds_printf(&url, CALYPTIA_ENDPOINT_FLEET_BY_NAME, project_id, ctx->fleet_name); client = fleet_http_do(ctx, url); @@ -1349,7 +1315,12 @@ static int calyptia_config_delete_old(struct flb_in_calyptia_fleet_config *ctx) return -1; } - if (flb_sds_cat_safe(&glob_files, PATH_SEPARATOR "*.conf", strlen(PATH_SEPARATOR "*.conf")) != 0) { + if (ctx->fleet_config_legacy_format) { + if (flb_sds_cat_safe(&glob_files, PATH_SEPARATOR "*.conf", strlen(PATH_SEPARATOR "*.conf")) != 0) { + flb_sds_destroy(glob_files); + return -1; + } + } else if (flb_sds_cat_safe(&glob_files, PATH_SEPARATOR "*.yaml", strlen(PATH_SEPARATOR "*.yaml")) != 0) { flb_sds_destroy(glob_files); return -1; } @@ -1413,7 +1384,13 @@ static flb_sds_t calyptia_config_get_newest(struct flb_in_calyptia_fleet_config return NULL; } - if (flb_sds_cat_safe(&glob_conf_files, PATH_SEPARATOR "*.conf", strlen(PATH_SEPARATOR "*.conf")) != 0) { + if (ctx->fleet_config_legacy_format) { + if (flb_sds_cat_safe(&glob_conf_files, PATH_SEPARATOR "*.conf", strlen(PATH_SEPARATOR "*.conf")) != 0) { + flb_plg_error(ctx->ins, "unable to concatenate fleet glob"); + flb_sds_destroy(glob_conf_files); + return NULL; + } + } else if (flb_sds_cat_safe(&glob_conf_files, PATH_SEPARATOR "*.yaml", strlen(PATH_SEPARATOR "*.yaml")) != 0) { flb_plg_error(ctx->ins, "unable to concatenate fleet glob"); flb_sds_destroy(glob_conf_files); return NULL; @@ -1421,7 +1398,7 @@ static flb_sds_t calyptia_config_get_newest(struct flb_in_calyptia_fleet_config inis = read_glob(glob_conf_files); if (inis == NULL) { - flb_plg_error(ctx->ins, "unable to read fleet directory for conf files: %s", + flb_plg_error(ctx->ins, "unable to read fleet directory for config files: %s", glob_conf_files); flb_sds_destroy(glob_conf_files); return NULL; @@ -1608,7 +1585,7 @@ static int calyptia_config_rollback(struct flb_in_calyptia_fleet_config *ctx, } #endif -static void fleet_config_get_properties(flb_sds_t *buf, struct mk_list *props) +static void fleet_config_get_properties(flb_sds_t *buf, struct mk_list *props, int fleet_config_legacy_format) { struct mk_list *head; struct flb_kv *kv; @@ -1617,7 +1594,11 @@ static void fleet_config_get_properties(flb_sds_t *buf, struct mk_list *props) kv = mk_list_entry(head, struct flb_kv, _head); if (kv->key != NULL && kv->val != NULL) { - flb_sds_printf(buf, " %s ", kv->key); + if (fleet_config_legacy_format) { + flb_sds_printf(buf, " %s ", kv->key); + } else { + flb_sds_printf(buf, " %s: ", kv->key); + } flb_sds_cat_safe(buf, kv->val, strlen(kv->val)); flb_sds_cat_safe(buf, "\n", 1); } @@ -1637,7 +1618,6 @@ static flb_sds_t get_fleet_id_from_header(struct flb_in_calyptia_fleet_config *c cf_hdr = flb_cf_create_from_file(NULL, hdr_fleet_config_filename(ctx)); if (cf_hdr == NULL) { - flb_cf_destroy(cf_hdr); return NULL; } @@ -1673,9 +1653,10 @@ static flb_sds_t get_fleet_id_from_header(struct flb_in_calyptia_fleet_config *c flb_cf_destroy(cf_hdr); return fleet_id; } + + flb_cf_destroy(cf_hdr); } - flb_cf_destroy(cf_hdr); return NULL; } @@ -1687,8 +1668,11 @@ flb_sds_t fleet_config_get(struct flb_in_calyptia_fleet_config *ctx) flb_ctx_t *flb = flb_context_get(); flb_sds_t fleet_id = NULL; + if (!ctx) { + return NULL; + } - buf = flb_sds_create_size(2048); + buf = flb_sds_create_size(CALYPTIA_MAX_DIR_SIZE); if (!buf) { return NULL; @@ -1699,14 +1683,23 @@ flb_sds_t fleet_config_get(struct flb_in_calyptia_fleet_config *ctx) if (strcasecmp(c_ins->p->name, "calyptia")) { continue; } - flb_sds_printf(&buf, "[CUSTOM]\n"); - flb_sds_printf(&buf, " name %s\n", c_ins->p->name); + if (ctx->fleet_config_legacy_format) { + flb_sds_printf(&buf, "[CUSTOM]\n"); + flb_sds_printf(&buf, " name %s\n", c_ins->p->name); + } else { + flb_sds_printf(&buf, "customs:\n"); + flb_sds_printf(&buf, " - name: %s\n", c_ins->p->name); + } - fleet_config_get_properties(&buf, &c_ins->properties); + fleet_config_get_properties(&buf, &c_ins->properties, ctx->fleet_config_legacy_format); if (flb_config_prop_get("fleet_id", &c_ins->properties) == NULL) { if (ctx->fleet_id != NULL) { - flb_sds_printf(&buf, " fleet_id %s\n", ctx->fleet_id); + if (ctx->fleet_config_legacy_format) { + flb_sds_printf(&buf, " fleet_id %s\n", ctx->fleet_id); + } else { + flb_sds_printf(&buf, " fleet_id: %s\n", ctx->fleet_id); + } } else { fleet_id = get_fleet_id_from_header(ctx); @@ -1716,7 +1709,11 @@ flb_sds_t fleet_config_get(struct flb_in_calyptia_fleet_config *ctx) return NULL; } - flb_sds_printf(&buf, " fleet_id %s\n", fleet_id); + if (ctx->fleet_config_legacy_format) { + flb_sds_printf(&buf, " fleet_id %s\n", fleet_id); + } else { + flb_sds_printf(&buf, " fleet_id: %s\n", fleet_id); + } flb_sds_destroy(fleet_id); } } @@ -1765,7 +1762,7 @@ static int create_fleet_header(struct flb_in_calyptia_fleet_config *ctx) return rc; } -static int get_calyptia_fleet_config(struct flb_in_calyptia_fleet_config *ctx) +int get_calyptia_fleet_config(struct flb_in_calyptia_fleet_config *ctx) { flb_sds_t cfgname; flb_sds_t cfgnewname; @@ -1775,17 +1772,21 @@ static int get_calyptia_fleet_config(struct flb_in_calyptia_fleet_config *ctx) int ret = -1; if (ctx->fleet_url == NULL) { - ctx->fleet_url = flb_sds_create_size(4096); + ctx->fleet_url = flb_sds_create_size(CALYPTIA_MAX_DIR_SIZE); if (ctx->fleet_url == NULL) { return -1; } - flb_sds_printf(&ctx->fleet_url, CALYPTIA_ENDPOINT_FLEET_CONFIG_INI, ctx->fleet_id); + if (ctx->fleet_config_legacy_format) { + flb_sds_printf(&ctx->fleet_url, CALYPTIA_ENDPOINT_FLEET_CONFIG_INI, ctx->fleet_id); + } else { + flb_sds_printf(&ctx->fleet_url, CALYPTIA_ENDPOINT_FLEET_CONFIG_YAML, ctx->fleet_id); + } } if (ctx->fleet_files_url == NULL) { - ctx->fleet_files_url = flb_sds_create_size(4096); + ctx->fleet_files_url = flb_sds_create_size(CALYPTIA_MAX_DIR_SIZE); if (ctx->fleet_files_url == NULL) { return -1; @@ -1797,8 +1798,12 @@ static int get_calyptia_fleet_config(struct flb_in_calyptia_fleet_config *ctx) create_fleet_header(ctx); hdrname = fleet_config_filename(ctx, "header"); - header = flb_sds_create_size(32); - flb_sds_printf(&header, "@include %s\n\n", hdrname); + header = flb_sds_create_size(CALYPTIA_MAX_DIR_SIZE); + if (ctx->fleet_config_legacy_format) { + flb_sds_printf(&header, "@include %s\n\n", hdrname); + } else { + flb_sds_printf(&header, "includes: \n - %s\n", hdrname); + } flb_sds_destroy(hdrname); /* create the base file. */ @@ -1807,7 +1812,7 @@ static int get_calyptia_fleet_config(struct flb_in_calyptia_fleet_config *ctx) /* new file created! */ if (ret == 1) { - get_calyptia_files(ctx, ctx->fleet_files_url, time_last_modified); + get_calyptia_files(ctx, time_last_modified); cfgname = time_fleet_config_filename(ctx, time_last_modified); @@ -1831,6 +1836,8 @@ static int get_calyptia_fleet_config(struct flb_in_calyptia_fleet_config *ctx) return -1; } #endif + + flb_sds_destroy(cfgname); } return 0; @@ -2119,13 +2126,12 @@ static int create_fleet_files(struct flb_in_calyptia_fleet_config *ctx, } static int get_calyptia_files(struct flb_in_calyptia_fleet_config *ctx, - const char *url, time_t timestamp) { struct flb_http_client *client; int ret = -1; - if (ctx == NULL || url == NULL) { + if (ctx == NULL || ctx->fleet_files_url == NULL) { return -1; } @@ -2194,7 +2200,7 @@ static int in_calyptia_fleet_init(struct flb_input_instance *in, return -1; } - ctx->config_dir = flb_sds_create_size(4096); + ctx->config_dir = flb_sds_create_size(CALYPTIA_MAX_DIR_SIZE); if (ctx->config_dir == NULL) { flb_plg_error(in, "unable to allocate config-dir."); @@ -2355,6 +2361,11 @@ static struct flb_config_map config_map[] = { 0, FLB_TRUE, offsetof(struct flb_in_calyptia_fleet_config, interval_nsec), "Set the collector interval (nanoseconds)" }, + { + FLB_CONFIG_MAP_BOOL, "fleet_config_legacy_format", "true", + 0, FLB_TRUE, offsetof(struct flb_in_calyptia_fleet_config, fleet_config_legacy_format), + "If set, use legacy (TOML) format for configuration files." + }, /* EOF */ {0} }; diff --git a/plugins/in_calyptia_fleet/in_calyptia_fleet.h b/plugins/in_calyptia_fleet/in_calyptia_fleet.h new file mode 100644 index 00000000000..ed97c73f376 --- /dev/null +++ b/plugins/in_calyptia_fleet/in_calyptia_fleet.h @@ -0,0 +1,80 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_IN_CALYPTIA_FLEET_H +#define FLB_IN_CALYPTIA_FLEET_H + +#include +#include + +struct flb_in_calyptia_fleet_config { + /* Time interval check */ + int interval_sec; + int interval_nsec; + + /* maximum http buffer size */ + int max_http_buffer_size; + + /* Grabbed from the cfg_path, used to check if configuration has + * has been updated. + */ + long config_timestamp; + + flb_sds_t api_key; + + flb_sds_t fleet_id; + + /* flag used to mark fleet_id for release when found automatically. */ + int fleet_id_found; + + flb_sds_t fleet_name; + flb_sds_t machine_id; + flb_sds_t config_dir; + flb_sds_t cloud_host; + flb_sds_t cloud_port; + + flb_sds_t fleet_url; + flb_sds_t fleet_files_url; + + /* whether to use legacy INI/TOML or YAML format */ + int fleet_config_legacy_format; + + struct flb_input_instance *ins; /* plugin instance */ + + /* Networking */ + struct flb_upstream *u; + + int collect_fd; +}; + +struct reload_ctx { + flb_ctx_t *flb; + flb_sds_t cfg_path; +}; + +flb_sds_t fleet_config_filename(struct flb_in_calyptia_fleet_config *ctx, char *fname); + +#define new_fleet_config_filename(a) fleet_config_filename((a), "new") +#define cur_fleet_config_filename(a) fleet_config_filename((a), "cur") +#define old_fleet_config_filename(a) fleet_config_filename((a), "old") +#define hdr_fleet_config_filename(a) fleet_config_filename((a), "header") + +int get_calyptia_fleet_config(struct flb_in_calyptia_fleet_config *ctx); + +#endif /* FLB_IN_CALYPTIA_FLEET_H */ diff --git a/tests/runtime/CMakeLists.txt b/tests/runtime/CMakeLists.txt index 7c5143cbfca..49ed0e0cf5d 100644 --- a/tests/runtime/CMakeLists.txt +++ b/tests/runtime/CMakeLists.txt @@ -73,6 +73,7 @@ if (FLB_CUSTOM_CALYPTIA) "custom_calyptia_test.c" "custom_calyptia_registration_retry_test.c" "custom_calyptia_input_test.c" + "in_calyptia_fleet_test.c" ) foreach(TEST_SOURCE ${CALYPTIA_TESTS}) @@ -82,6 +83,7 @@ if (FLB_CUSTOM_CALYPTIA) add_executable(${TEST_TARGET} ${TEST_SOURCE} "../../plugins/custom_calyptia/calyptia.c" + "../../plugins/in_calyptia_fleet/in_calyptia_fleet.c" ) target_link_libraries(${TEST_TARGET} @@ -113,7 +115,7 @@ if(FLB_IN_EBPF) # Function to create eBPF handler tests function(add_ebpf_handler_test name source_file handler_file) set(TEST_TARGET "flb-rt-in_ebpf_${name}_handler") - + # Add executable add_executable(${TEST_TARGET} ${source_file} diff --git a/tests/runtime/custom_calyptia_input_test.c b/tests/runtime/custom_calyptia_input_test.c index 5cf023afa76..7b8be6b2a77 100644 --- a/tests/runtime/custom_calyptia_input_test.c +++ b/tests/runtime/custom_calyptia_input_test.c @@ -67,6 +67,7 @@ static struct test_context *init_test_context() t_ctx->ctx->fleet_max_http_buffer_size = flb_strdup("1024"); t_ctx->ctx->fleet_interval_sec = flb_strdup("60"); t_ctx->ctx->fleet_interval_nsec = flb_strdup("500000000"); + t_ctx->ctx->fleet_config_legacy_format = FLB_TRUE; t_ctx->fleet = flb_input_new(t_ctx->config, "calyptia_fleet", NULL, FLB_FALSE); if (!t_ctx->fleet) { @@ -160,13 +161,13 @@ void test_set_fleet_input_properties() TEST_MSG("max_http_buffer_size expected=%s got=%s", t_ctx->ctx->fleet_max_http_buffer_size, value); TEST_CHECK(value && strcmp(value, t_ctx->ctx->fleet_max_http_buffer_size) == 0); - // /* Check interval_sec */ + /* Check interval_sec */ value = flb_input_get_property("interval_sec", t_ctx->fleet); TEST_CHECK(value != NULL); TEST_MSG("interval_sec expected=%s got=%s", t_ctx->ctx->fleet_interval_sec, value); TEST_CHECK(value && strcmp(value, t_ctx->ctx->fleet_interval_sec) == 0); - // /* Check interval_nsec */ + /* Check interval_nsec */ value = flb_input_get_property("interval_nsec", t_ctx->fleet); TEST_CHECK(value != NULL); TEST_MSG("interval_nsec expected=%s got=%s", t_ctx->ctx->fleet_interval_nsec, value); diff --git a/tests/runtime/in_calyptia_fleet_test.c b/tests/runtime/in_calyptia_fleet_test.c new file mode 100644 index 00000000000..c9cd8cb1da2 --- /dev/null +++ b/tests/runtime/in_calyptia_fleet_test.c @@ -0,0 +1,121 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#include +#include +#include "flb_tests_runtime.h" +#include "../../plugins/in_calyptia_fleet/in_calyptia_fleet.h" + +flb_sds_t fleet_config_filename(struct flb_in_calyptia_fleet_config *ctx, char *fname); +int get_calyptia_fleet_config(struct flb_in_calyptia_fleet_config *ctx); + +/* Test context structure */ +struct test_context { + struct flb_in_calyptia_fleet_config *ctx; + struct flb_config *config; +}; + +/* Initialize test context */ +static struct test_context *init_test_context() +{ + struct test_context *t_ctx = flb_calloc(1, sizeof(struct test_context)); + if (!t_ctx) { + return NULL; + } + + t_ctx->config = flb_config_init(); + if (!t_ctx->config) { + flb_free(t_ctx); + return NULL; + } + + t_ctx->ctx = flb_calloc(1, sizeof(struct flb_in_calyptia_fleet_config)); + if (!t_ctx->ctx) { + flb_config_exit(t_ctx->config); + flb_free(t_ctx); + return NULL; + } + + /* Initialize plugin instance for logging */ + t_ctx->ctx->ins = flb_calloc(1, sizeof(struct flb_input_instance)); + if (!t_ctx->ctx->ins) { + flb_free(t_ctx->ctx); + flb_config_exit(t_ctx->config); + flb_free(t_ctx); + return NULL; + } + + /* Initialize test values in ctx */ + t_ctx->ctx->api_key = flb_strdup("test_api_key"); + t_ctx->ctx->fleet_id = flb_strdup("test_fleet_id"); + + t_ctx->ctx->fleet_name = flb_strdup("test_fleet"); + t_ctx->ctx->machine_id = flb_strdup("test_machine_id"); + + t_ctx->ctx->fleet_config_legacy_format = FLB_TRUE; + + return t_ctx; +} + +static void cleanup_test_context(struct test_context *t_ctx) +{ + if (!t_ctx) { + return; + } + + if (t_ctx->ctx) { + if (t_ctx->ctx->api_key) flb_free(t_ctx->ctx->api_key); + if (t_ctx->ctx->fleet_id) flb_free(t_ctx->ctx->fleet_id); + + if (t_ctx->ctx->fleet_name) flb_free(t_ctx->ctx->fleet_name); + if (t_ctx->ctx->machine_id) flb_free(t_ctx->ctx->machine_id); + if (t_ctx->ctx->fleet_files_url) flb_free(t_ctx->ctx->fleet_files_url); + + if (t_ctx->ctx->ins) flb_free(t_ctx->ctx->ins); + flb_free(t_ctx->ctx); + } + + if (t_ctx->config) { + /* Destroy the config which will cleanup any remaining instances */ + flb_config_exit(t_ctx->config); + } + + flb_free(t_ctx); +} + +static void test_in_fleet_format() { + struct test_context *t_ctx = init_test_context(); + TEST_CHECK(t_ctx != NULL); + + /* Ensure we create TOML files by default */ + char expectedValue[CALYPTIA_MAX_DIR_SIZE]; + int ret = sprintf(expectedValue, "%s/%s/%s/test.conf", FLEET_DEFAULT_CONFIG_DIR, t_ctx->ctx->machine_id, t_ctx->ctx->fleet_name); + TEST_CHECK(ret > 0); + + flb_sds_t value = fleet_config_filename( t_ctx->ctx, "test" ); + TEST_CHECK(value != NULL); + TEST_MSG("fleet_config_filename expected=%s got=%s", expectedValue, value); + TEST_CHECK(value && strcmp(value, expectedValue) == 0); + flb_sds_destroy(value); + value = NULL; + + /* Ensure we create YAML files if configured to do so */ + t_ctx->ctx->fleet_config_legacy_format = FLB_FALSE; + + ret = sprintf(expectedValue, "%s/%s/%s/test.yaml", FLEET_DEFAULT_CONFIG_DIR, t_ctx->ctx->machine_id, t_ctx->ctx->fleet_name); + TEST_CHECK(ret > 0); + + value = fleet_config_filename( t_ctx->ctx, "test" ); + TEST_CHECK(value != NULL); + TEST_MSG("fleet_config_filename expected=%s got=%s", expectedValue, value); + TEST_CHECK(value && strcmp(value, expectedValue) == 0); + flb_sds_destroy(value); + value = NULL; + + cleanup_test_context(t_ctx); +} + +/* Define test list */ +TEST_LIST = { + {"in_calyptia_fleet_format", test_in_fleet_format}, + {NULL, NULL} +}; \ No newline at end of file diff --git a/tests/runtime_shell/CMakeLists.txt b/tests/runtime_shell/CMakeLists.txt index 79c96de5fc9..0f13baae82c 100644 --- a/tests/runtime_shell/CMakeLists.txt +++ b/tests/runtime_shell/CMakeLists.txt @@ -4,6 +4,7 @@ configure_file( ) set(UNIT_TESTS_SH + custom_calyptia.sh in_dummy_expect.sh in_tail_expect.sh in_http_tls_expect.sh diff --git a/tests/runtime_shell/conf/custom_calyptia_fleet.conf b/tests/runtime_shell/conf/custom_calyptia_fleet.conf new file mode 100644 index 00000000000..944c9ca5bc2 --- /dev/null +++ b/tests/runtime_shell/conf/custom_calyptia_fleet.conf @@ -0,0 +1,17 @@ +[INPUT] + name random + tag random.local + + # Read interval (sec) Default: 1 + interval_sec 1 + +[CUSTOM] + name calyptia + API_Key ${CALYPTIA_FLEET_TOKEN} + Fleet_Name fluent-bit-ci-fleet + fleet_config_legacy_format ${CALYPTIA_FLEET_FORMAT} + fleet.config_dir ${CALYPTIA_FLEET_DIR} + +[OUTPUT] + name null + match * diff --git a/tests/runtime_shell/custom_calyptia.sh b/tests/runtime_shell/custom_calyptia.sh new file mode 100755 index 00000000000..2094302b236 --- /dev/null +++ b/tests/runtime_shell/custom_calyptia.sh @@ -0,0 +1,65 @@ +#!/bin/sh + +test_custom_calyptia_fleet_yaml() { + [ -z "${CALYPTIA_FLEET_TOKEN:-}" ] && startSkipping + + export CALYPTIA_FLEET_TOKEN + export CALYPTIA_FLEET_FORMAT="off" + export CALYPTIA_FLEET_DIR="${CALYPTIA_FLEET_DIR:-/tmp/fleet-test}" + rm -rf "$CALYPTIA_FLEET_DIR" + + # Dry-run to check it is valid + if ! $FLB_BIN -c "$FLB_RUNTIME_SHELL_CONF/custom_calyptia_fleet.conf" --dry-run; then + fail 'Dry run failed' + fi + + $FLB_BIN -c "$FLB_RUNTIME_SHELL_CONF/custom_calyptia_fleet.conf" & + FLB_PID=$! + + # Allow us to register and retrieve fleet config + sleep 30 + + # Check we have YAML files + if find "$CALYPTIA_FLEET_DIR" -name '*.yaml' -type f -exec false {} +; then + fail 'No YAML files found' + else + find "$CALYPTIA_FLEET_DIR" -name '*.yaml' -type f -exec cat {} \; + fi + + # Clean up + kill -15 $FLB_PID +} + +test_custom_calyptia_fleet_toml() { + [ -z "${CALYPTIA_FLEET_TOKEN:-}" ] && startSkipping + + export CALYPTIA_FLEET_TOKEN + export CALYPTIA_FLEET_FORMAT="on" + export CALYPTIA_FLEET_DIR="${CALYPTIA_FLEET_DIR:-/tmp/fleet-test}" + rm -rf "$CALYPTIA_FLEET_DIR" + + # Dry-run to check it is valid + if ! $FLB_BIN -c "$FLB_RUNTIME_SHELL_CONF/custom_calyptia_fleet.conf" --dry-run; then + fail 'Dry run failed' + fi + + $FLB_BIN -c "$FLB_RUNTIME_SHELL_CONF/custom_calyptia_fleet.conf" & + FLB_PID=$! + + # Allow us to register and retrieve fleet config + sleep 30 + + # Check we have no YAML files + if find "$CALYPTIA_FLEET_DIR" -name '*.yaml' -type f -exec false {} +; then + echo 'No YAML files found' + else + fail 'YAML files found' + fi + + # Clean up + kill -15 $FLB_PID +} + +# The following command launch the unit test +# shellcheck source=/dev/null +. "$FLB_RUNTIME_SHELL_PATH/runtime_shell.env"