diff --git a/.github/workflows/cifuzz.yml b/.github/workflows/cifuzz.yml new file mode 100644 index 000000000..dc89c0b26 --- /dev/null +++ b/.github/workflows/cifuzz.yml @@ -0,0 +1,41 @@ +name: CIFuzz +on: + pull_request: + paths: + - 'src/**' + - 'fuzzing/**' + - '.github/workflows/cifuzz.yml' + +permissions: {} +jobs: + Fuzzing: + runs-on: ubuntu-latest + permissions: + security-events: write + steps: + - name: Build Fuzzers + id: build + uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master + with: + oss-fuzz-project-name: 'unit' + language: c + - name: Run Fuzzers + uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master + with: + oss-fuzz-project-name: 'unit' + language: c + fuzz-seconds: 300 + output-sarif: true + - name: Upload Crash + uses: actions/upload-artifact@v3 + if: failure() && steps.build.outcome == 'success' + with: + name: artifacts + path: ./out/artifacts + - name: Upload Sarif + if: always() && steps.build.outcome == 'success' + uses: github/codeql-action/upload-sarif@v3 + with: + # Path to SARIF file relative to the root of the repository + sarif_file: cifuzz-sarif/results.sarif + checkout_path: cifuzz-sarif diff --git a/fuzzing/README.md b/fuzzing/README.md index b15093277..9c70c8016 100644 --- a/fuzzing/README.md +++ b/fuzzing/README.md @@ -43,11 +43,11 @@ $ mkdir -p build/fuzz_http_h1p_seed $ mkdir -p build/fuzz_http_h1p_peer_seed $ mkdir -p build/fuzz_json_seed -$ ./build/fuzz_basic build/fuzz_basic_seed src/fuzz/fuzz_basic_seed_corpus -$ ./build/fuzz_http_controller build/fuzz_http_controller_seed src/fuzz/fuzz_http_controller_seed_corpus -$ ./build/fuzz_http_h1p build/fuzz_http_h1p_seed src/fuzz/fuzz_http_h1p_seed_corpus -$ ./build/fuzz_http_h1p_peer build/fuzz_http_h1p_peer_seed src/fuzz/fuzz_http_h1p_peer_seed_corpus -$ ./build/fuzz_json build/fuzz_json_seed src/fuzz/fuzz_json_seed_corpus +$ ./build/fuzz_basic build/fuzz_basic_seed fuzzing/fuzz_basic_seed_corpus +$ ./build/fuzz_http_controller build/fuzz_http_controller_seed fuzzing/fuzz_http_seed_corpus +$ ./build/fuzz_http_h1p build/fuzz_http_h1p_seed fuzzing/fuzz_http_seed_corpus +$ ./build/fuzz_http_h1p_peer build/fuzz_http_h1p_peer_seed fuzzing/fuzz_http_seed_corpus +$ ./build/fuzz_json build/fuzz_json_seed fuzzing/fuzz_json_seed_corpus ``` Here is more information about [LibFuzzer](https://llvm.org/docs/LibFuzzer.html). diff --git a/fuzzing/build-fuzz.sh b/fuzzing/build-fuzz.sh index 04f080d94..62f7a6761 100644 --- a/fuzzing/build-fuzz.sh +++ b/fuzzing/build-fuzz.sh @@ -16,5 +16,5 @@ mkdir -p build/fuzz_http_h1p_peer_seed mkdir -p build/fuzz_json_seed echo "" -echo "Run: ./build/\${fuzzer} build/\${fuzzer}_seed src/fuzz/\${fuzzer}_seed_corpus" +echo "Run: ./build/\${fuzzer} build/\${fuzzer}_seed fuzzing/\${fuzzer}_seed_corpus" echo "" diff --git a/fuzzing/nxt_http_controller_fuzz.c b/fuzzing/nxt_http_controller_fuzz.c index b7c6c2724..eac54d7b0 100644 --- a/fuzzing/nxt_http_controller_fuzz.c +++ b/fuzzing/nxt_http_controller_fuzz.c @@ -76,6 +76,14 @@ LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) goto failed; } + r_controller->conn = nxt_mp_zget(mp, sizeof(nxt_conn_t)); + if (r_controller->conn == NULL) { + goto failed; + } + + nxt_main_log.level = NXT_LOG_ALERT; + r_controller->conn->log = nxt_main_log; + nxt_http_fields_process(rp.fields, &nxt_controller_fields_hash, r_controller); diff --git a/fuzzing/nxt_http_h1p_fuzz.c b/fuzzing/nxt_http_h1p_fuzz.c index 471e87a4f..a170463a0 100644 --- a/fuzzing/nxt_http_h1p_fuzz.c +++ b/fuzzing/nxt_http_h1p_fuzz.c @@ -75,6 +75,8 @@ LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) goto failed; } + r_h1p->mem_pool = mp; + nxt_http_fields_process(rp.fields, &nxt_h1p_fields_hash, r_h1p); failed: diff --git a/fuzzing/nxt_json_fuzz.c b/fuzzing/nxt_json_fuzz.c index 532babb1b..cfeb395da 100644 --- a/fuzzing/nxt_json_fuzz.c +++ b/fuzzing/nxt_json_fuzz.c @@ -4,7 +4,7 @@ #include #include - +#include #define KMININPUTLENGTH 2 #define KMAXINPUTLENGTH 1024 @@ -33,6 +33,8 @@ LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { nxt_mp_t *mp; nxt_str_t input; + nxt_thread_t *thr; + nxt_runtime_t *rt; nxt_conf_value_t *conf; nxt_conf_validation_t vldt; @@ -40,11 +42,21 @@ LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) return 0; } + thr = nxt_thread(); + mp = nxt_mp_create(1024, 128, 256, 32); if (mp == NULL) { return 0; } + rt = nxt_mp_zget(mp, sizeof(nxt_runtime_t)); + if (rt == NULL) { + goto failed; + } + + thr->runtime = rt; + rt->mem_pool = mp; + input.start = (u_char *)data; input.length = size; @@ -64,6 +76,11 @@ LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) vldt.conf_pool = mp; vldt.ver = NXT_VERNUM; + rt->languages = nxt_array_create(mp, 1, sizeof(nxt_app_lang_module_t)); + if (rt->languages == NULL) { + goto failed; + } + nxt_conf_validate(&vldt); nxt_mp_destroy(vldt.pool); diff --git a/src/nxt_application.c b/src/nxt_application.c index e0247bf08..629aa11c4 100644 --- a/src/nxt_application.c +++ b/src/nxt_application.c @@ -32,6 +32,7 @@ typedef struct { nxt_app_type_t type; + nxt_str_t name; nxt_str_t version; nxt_str_t file; nxt_array_t *mounts; @@ -257,12 +258,14 @@ nxt_discovery_modules(nxt_task_t *task, const char *path) module[i].type, &module[i].version, &module[i].file); size += nxt_length("{\"type\": ,"); + size += nxt_length(" \"name\": \"\","); size += nxt_length(" \"version\": \"\","); size += nxt_length(" \"file\": \"\","); size += nxt_length(" \"mounts\": []},"); size += NXT_INT_T_LEN + module[i].version.length + + module[i].name.length + module[i].file.length; mounts = module[i].mounts; @@ -294,9 +297,10 @@ nxt_discovery_modules(nxt_task_t *task, const char *path) for (i = 0; i < n; i++) { mounts = module[i].mounts; - p = nxt_sprintf(p, end, "{\"type\": %d, \"version\": \"%V\", " - "\"file\": \"%V\", \"mounts\": [", - module[i].type, &module[i].version, &module[i].file); + p = nxt_sprintf(p, end, "{\"type\": %d, \"name\": \"%V\", " + "\"version\": \"%V\", \"file\": \"%V\", \"mounts\": [", + module[i].type, &module[i].name, &module[i].version, + &module[i].file); mnt = mounts->elts; for (j = 0; j < mounts->nelts; j++) { @@ -412,6 +416,11 @@ nxt_discovery_module(nxt_task_t *task, nxt_mp_t *mp, nxt_array_t *modules, goto fail; } + nxt_str_dup(mp, &module->name, &app->type); + if (module->name.start == NULL) { + goto fail; + } + module->file.length = nxt_strlen(name); module->file.start = nxt_mp_alloc(mp, module->file.length); diff --git a/src/nxt_application.h b/src/nxt_application.h index f5d7a9df3..a3b4230a0 100644 --- a/src/nxt_application.h +++ b/src/nxt_application.h @@ -35,6 +35,7 @@ typedef nxt_int_t (*nxt_application_setup_t)(nxt_task_t *task, typedef struct { nxt_app_type_t type; + char *name; u_char *version; char *file; nxt_app_module_t *module; diff --git a/src/nxt_buf.h b/src/nxt_buf.h index f1e2879f2..a561ef4e1 100644 --- a/src/nxt_buf.h +++ b/src/nxt_buf.h @@ -13,7 +13,7 @@ * should be allocated by appropriate nxt_buf_XXX_alloc() function. * * 1) Memory-only buffers, their size is less than nxt_buf_t size, it - * is equal to offsetof(nxt_buf_t, file_pos), that is it is nxt_buf_t + * is equal to offsetof(nxt_buf_t, file), that is it is nxt_buf_t * without file and mmap part. The buffers are frequently used, so * the reduction allows to save 20-32 bytes depending on platform. * diff --git a/src/nxt_main_process.c b/src/nxt_main_process.c index c302cb02f..00318226b 100644 --- a/src/nxt_main_process.c +++ b/src/nxt_main_process.c @@ -1354,6 +1354,12 @@ static nxt_conf_map_t nxt_app_lang_module_map[] = { offsetof(nxt_app_lang_module_t, type), }, + { + nxt_string("name"), + NXT_CONF_MAP_CSTRZ, + offsetof(nxt_app_lang_module_t, name), + }, + { nxt_string("version"), NXT_CONF_MAP_CSTRZ, diff --git a/src/nxt_status.c b/src/nxt_status.c index f8002e86e..92cbf2e65 100644 --- a/src/nxt_status.c +++ b/src/nxt_status.c @@ -6,40 +6,131 @@ #include #include #include +#include nxt_conf_value_t * nxt_status_get(nxt_status_report_t *report, nxt_mp_t *mp) { - size_t i; - nxt_str_t name; - nxt_int_t ret; - nxt_status_app_t *app; - nxt_conf_value_t *status, *obj, *apps, *app_obj; - - static nxt_str_t conns_str = nxt_string("connections"); - static nxt_str_t acc_str = nxt_string("accepted"); - static nxt_str_t active_str = nxt_string("active"); - static nxt_str_t idle_str = nxt_string("idle"); - static nxt_str_t closed_str = nxt_string("closed"); - static nxt_str_t reqs_str = nxt_string("requests"); - static nxt_str_t total_str = nxt_string("total"); - static nxt_str_t apps_str = nxt_string("applications"); - static nxt_str_t procs_str = nxt_string("processes"); - static nxt_str_t run_str = nxt_string("running"); - static nxt_str_t start_str = nxt_string("starting"); - - status = nxt_conf_create_object(mp, 3); + size_t i, nr_langs; + uint16_t lang_cnts[NXT_APP_UNKNOWN] = { 1 }; + uint32_t idx = 0; + nxt_str_t name; + nxt_int_t ret; + nxt_array_t *langs; + nxt_thread_t *thr; + nxt_app_type_t type, prev_type; + nxt_status_app_t *app; + nxt_conf_value_t *status, *obj, *mods, *apps, *app_obj, *mod_obj; + nxt_app_lang_module_t *modules; + + static const nxt_str_t modules_str = nxt_string("modules"); + static const nxt_str_t version_str = nxt_string("version"); + static const nxt_str_t lib_str = nxt_string("lib"); + static const nxt_str_t conns_str = nxt_string("connections"); + static const nxt_str_t acc_str = nxt_string("accepted"); + static const nxt_str_t active_str = nxt_string("active"); + static const nxt_str_t idle_str = nxt_string("idle"); + static const nxt_str_t closed_str = nxt_string("closed"); + static const nxt_str_t reqs_str = nxt_string("requests"); + static const nxt_str_t total_str = nxt_string("total"); + static const nxt_str_t apps_str = nxt_string("applications"); + static const nxt_str_t procs_str = nxt_string("processes"); + static const nxt_str_t run_str = nxt_string("running"); + static const nxt_str_t start_str = nxt_string("starting"); + + status = nxt_conf_create_object(mp, 4); if (nxt_slow_path(status == NULL)) { return NULL; } + thr = nxt_thread(); + langs = thr->runtime->languages; + + modules = langs->elts; + /* + * We need to count the number of unique languages to correctly + * allocate the below mods object. + * + * We also need to count how many of each language. + * + * Start by skipping past NXT_APP_EXTERNAL which is always the + * first entry. + */ + for (i = 1, nr_langs = 0, prev_type = NXT_APP_UNKNOWN; i < langs->nelts; + i++) + { + type = modules[i].type; + + lang_cnts[type]++; + + if (type == prev_type) { + continue; + } + + nr_langs++; + prev_type = type; + } + + mods = nxt_conf_create_object(mp, nr_langs); + if (nxt_slow_path(mods == NULL)) { + return NULL; + } + + nxt_conf_set_member(status, &modules_str, mods, idx++); + + i = 1; + obj = mod_obj = NULL; + prev_type = NXT_APP_UNKNOWN; + for (size_t l = 0, a = 0; i < langs->nelts; i++) { + nxt_str_t item, mod_name; + + type = modules[i].type; + if (type != prev_type) { + a = 0; + + if (lang_cnts[type] == 1) { + mod_obj = nxt_conf_create_object(mp, 2); + obj = mod_obj; + } else { + mod_obj = nxt_conf_create_array(mp, lang_cnts[type]); + } + + if (nxt_slow_path(mod_obj == NULL)) { + return NULL; + } + + mod_name.start = (u_char *)modules[i].name; + mod_name.length = strlen(modules[i].name); + nxt_conf_set_member(mods, &mod_name, mod_obj, l++); + } + + if (lang_cnts[type] > 1) { + obj = nxt_conf_create_object(mp, 2); + if (nxt_slow_path(obj == NULL)) { + return NULL; + } + + nxt_conf_set_element(mod_obj, a++, obj); + } + + item.start = modules[i].version; + item.length = nxt_strlen(modules[i].version); + nxt_conf_set_member_string(obj, &version_str, &item, 0); + + item.start = (u_char *)modules[i].file; + item.length = strlen(modules[i].file); + nxt_conf_set_member_string(obj, &lib_str, &item, 1); + + prev_type = type; + } + obj = nxt_conf_create_object(mp, 4); if (nxt_slow_path(obj == NULL)) { return NULL; } - nxt_conf_set_member(status, &conns_str, obj, 0); + nxt_conf_set_member(status, &conns_str, obj, idx++); nxt_conf_set_member_integer(obj, &acc_str, report->accepted_conns, 0); nxt_conf_set_member_integer(obj, &active_str, report->accepted_conns @@ -53,7 +144,7 @@ nxt_status_get(nxt_status_report_t *report, nxt_mp_t *mp) return NULL; } - nxt_conf_set_member(status, &reqs_str, obj, 1); + nxt_conf_set_member(status, &reqs_str, obj, idx++); nxt_conf_set_member_integer(obj, &total_str, report->requests, 0); @@ -62,7 +153,7 @@ nxt_status_get(nxt_status_report_t *report, nxt_mp_t *mp) return NULL; } - nxt_conf_set_member(status, &apps_str, apps, 2); + nxt_conf_set_member(status, &apps_str, apps, idx++); for (i = 0; i < report->apps_count; i++) { app = &report->apps[i]; diff --git a/test/unit/status.py b/test/unit/status.py index 95096a96f..d8bb4e41e 100644 --- a/test/unit/status.py +++ b/test/unit/status.py @@ -6,16 +6,16 @@ class Status: control = Control() def _check_zeros(): - assert Status.control.conf_get('/status') == { - 'connections': { + status = Status.control.conf_get('/status') + + assert status['connections'] == { 'accepted': 0, 'active': 0, 'idle': 0, 'closed': 0, - }, - 'requests': {'total': 0}, - 'applications': {}, } + assert status['requests'] == {'total': 0} + assert status['applications'] == {} def init(status=None): Status._status = ( @@ -31,6 +31,9 @@ def find_diffs(d1, d2): if k in d2 } + if isinstance(d1, str): + return d1 == d2 + return d1 - d2 return find_diffs(Status.control.conf_get('/status'), Status._status) diff --git a/tools/unitctl/Cargo.lock b/tools/unitctl/Cargo.lock index 202799638..bcbe53b7a 100644 --- a/tools/unitctl/Cargo.lock +++ b/tools/unitctl/Cargo.lock @@ -1185,9 +1185,9 @@ checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "openssl" -version = "0.10.64" +version = "0.10.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" +checksum = "9529f4786b70a3e8c61e11179af17ab6188ad8d0ded78c5529441ed39d4bd9c1" dependencies = [ "bitflags 2.4.1", "cfg-if", @@ -1217,9 +1217,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.102" +version = "0.9.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c597637d56fbc83893a35eb0dd04b2b8e7a50c91e64e9493e398b5df4fb45fa2" +checksum = "7f9e8deee91df40a943c71b917e5874b951d32a802526c85721ce3b776c929d6" dependencies = [ "cc", "libc", diff --git a/tools/unitctl/README.md b/tools/unitctl/README.md index 4aa6068c5..e6fca4772 100644 --- a/tools/unitctl/README.md +++ b/tools/unitctl/README.md @@ -51,11 +51,44 @@ desired. ## Features (Current) +``` +CLI interface to the NGINX Unit Control API + +Usage: unitctl [OPTIONS] + +Commands: + instances List all running Unit processes + edit Open current Unit configuration in editor + import Import configuration from a directory + execute Sends raw JSON payload to Unit + status Get the current status of Unit + listeners List active listeners + help Print this message or the help of the given subcommand(s) + +Options: + -s, --control-socket-address + Path (unix:/var/run/unit/control.sock), tcp address with port (127.0.0.1:80), or URL + -w, --wait-timeout-seconds + Number of seconds to wait for control socket to become available + -t, --wait-max-tries + Number of times to try to access control socket when waiting [default: 3] + -h, --help + Print help + -V, --version + Print version +``` + - Consumes alternative configuration formats Like YAML and converts them +- Can convert output to multiple different formats (YAML, plain JSON, highlighted JSON) - Syntactic highlighting of JSON output - Interpretation of Unit errors with (arguably more) useful error messages ### Lists all running Unit processes and provides details about each process. +Unitctl will detect and connect to running process of Unit on the host. +It will pull information about the running Unit configuration +(including how to access its control API) from the process information of +each detected Unit process. + ``` $ unitctl instances No socket path provided - attempting to detect from running instance @@ -68,6 +101,13 @@ unitd instance [pid: 79489, version: 1.32.0]: ``` ### Start a new Unit process via docker +Unitctl can launch new containers of Unit. +These can be official Unit images or custom Unit images. +Any container that calls `unitd` in a CMD declaration will suffice. + +The new containers will then be shown in a call to +`unitctl instances` + ``` $ unitctl instances new /tmp/2 $(pwd) 'unit:wasm' Pulling and starting a container from unit:wasm @@ -77,21 +117,32 @@ Note: Container will be on host network ``` -To the subcommand `unitctl instances new` the user must provide three things: -1. **A directory such as `/tmp/2`.** - The Unit container will mount this to `/var/run` internally. - Thus, the control socket and pid file will be accessible from the host. -2. **A path to an application.** +To the subcommand `unitctl instances new` the user must provide three arguments: +1. **A means of showing the control API:** + There are two possibilities for this argument. + A filepath on which to open a unix socket, + or a TCP address. + - If a directory is specified the Unit container + will mount this to `/var/run` internally. + Thus, the control socket and pid file will be + accessible from the host. For example: `/tmp/2`. + - If a TCP endpoint is specified Unit will be configured + to offer its control API on the given port and address. + For example: `127.0.0.1:7171`. +2. **A path to an application:** In the example, `$(pwd)` is provided. The Unit container will mount this READ ONLY to `/www/`. This will allow the user to configure their Unit container to expose an application stored on the host. -3. **An image tag.** +3. **An image tag:** In the example, `unit:wasm` is used. This will be the image that unitctl will deploy. Custom repos and images can be deployed in this manner. After deployment the user will have one Unit container running on the host network. ### Lists active applications and provides means to restart them +Unitctl can list running applications by accessing the specified control API. +Unitctl can also request from the API that an application be restarted. + Listing applications: ``` $ unitctl app list @@ -120,6 +171,9 @@ $ unitctl -s '127.0.0.1:8001' -s /run/nginx-unit.control.sock app list ``` ### Lists active listeners from running Unit processes +Unitctl can query a given control API to fetch all configured +listeners. + ``` unitctl listeners No socket path provided - attempting to detect from running instance @@ -138,6 +192,9 @@ $ unitctl -s '127.0.0.1:8001' -s /run/nginx-unit.control.sock listeners ``` ### Get the current status of NGINX Unit processes +Unitctl can query the control API to provide the status of the running +Unit daemon. + ``` $ unitctl status -t yaml No socket path provided - attempting to detect from running instance @@ -159,6 +216,10 @@ $ unitctl -s '127.0.0.1:8001' -s /run/nginx-unit.control.sock status ``` ### Send arbitrary configuration payloads to Unit +Unitctl can accept custom request payloads and query given API endpoints with them. +The request payload must be passed in using the `-f` flag either as a filename or +using the `-` filename to denote the use of stdin as shown in the example below. + ``` $ echo '{ "listeners": { @@ -188,6 +249,12 @@ $ unitctl -s '127.0.0.1:8001' -s /run/nginx-unit.control.sock execute ... ``` ### Edit current configuration in your favorite editor +Unitctl can fetch the configuration from a running instance of Unit and +load it in any number of preconfigured editors on your command line. + +Unitctl will try to use whatever editor is configured with the `EDITOR` +environment variable, but will default to vim, emacs, nano, vi, or pico. + ``` $ unitctl edit [[EDITOR LOADS SHOWING CURRENT CONFIGURATION - USER EDITS AND SAVES]] @@ -200,6 +267,10 @@ $ unitctl edit *Note:* This command does not support operating on multiple instances of Unit at once. ### Import configuration, certificates, and NJS modules from directory +Unitctl will parse existing configuration, certificates, and NJS modules +stored in a directory and convert them into a payload to reconfigure a +given Unit daemon. + ``` $ unitctl import /opt/unit/config Imported /opt/unit/config/certificates/snake.pem -> /certificates/snake.pem @@ -209,12 +280,15 @@ Imported 3 files ``` ### Export configuration from a running Unit instance -``` -$ unitctl export -f config.tar -``` +Unitctl will query a control API to fetch running configuration +and NJS modules from a Unit process. Due to a technical limitation +this output will not contain currently stored certificate bundles. +The output is saved as a tarball at the filename given with the `-f` +argument. Standard out may be used with `-f -` as shown in the +following examples. -Addtionally, standard out can be used: ``` +$ unitctl export -f config.tar $ unitctl export -f - $ unitctl export -f - | tar xf - config.json $ unitctl export -f - > config.tar @@ -225,6 +299,8 @@ $ unitctl export -f - > config.tar *Note:* This command does not support operating on multiple instances of Unit at once. ### Wait for socket to become available +All commands support waiting on unix sockets for availability. + ``` $ unitctl --wait-timeout-seconds=3 --wait-max-tries=4 import /opt/unit/config` Waiting for 3s control socket to be available try 2/4...