From 93521615674da2cc744487c1ca5d224bc969275a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 31 Dec 2024 20:51:01 +0100 Subject: [PATCH 1/2] Make maximum number of threads used by web server configurable Signed-off-by: DL6ER --- src/api/docs/content/specs/config.yaml | 3 ++ src/config/config.c | 6 ++++ src/config/config.h | 1 + src/webserver/webserver.c | 47 +++++++++++++++++++++----- src/webserver/webserver.h | 5 +++ test/pihole.toml | 22 ++++++++++-- 6 files changed, 72 insertions(+), 12 deletions(-) diff --git a/src/api/docs/content/specs/config.yaml b/src/api/docs/content/specs/config.yaml index d9d7c612d..ddadde0a1 100644 --- a/src/api/docs/content/specs/config.yaml +++ b/src/api/docs/content/specs/config.yaml @@ -407,6 +407,8 @@ components: type: string port: type: string + threads: + type: integer session: type: object properties: @@ -739,6 +741,7 @@ components: domain: pi.hole acl: "+0.0.0.0/0,::/0" port: 80,[::]:80 + threads: -1 session: timeout: 300 restore: true diff --git a/src/config/config.c b/src/config/config.c index b4fc3d916..f7448a204 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -1007,6 +1007,12 @@ static void initConfig(struct config *conf) conf->webserver.port.d.s = (char*)"80,[::]:80,443s,[::]:443s"; conf->webserver.port.c = validate_stub; // Type-based checking + civetweb syntax checking + conf->webserver.threads.k = "webserver.threads"; + conf->webserver.threads.h = "Maximum number of worker threads allowed.\n The Pi-hole web server handles each incoming connection in a separate thread. Therefore, the value of this option is effectively the number of concurrent HTTP connections that can be handled. Any other connections are queued until they can be processed by a unoccupied thread.\n The default value of -1 means that the number of threads is automatically determined by the number of online CPU cores minus 1 (e.g., launching up to 8-1 = 7 threads on 8 cores). A value of -2 means the same automatism but for twice the number of cores (e.g., launching up to 16-1 = 15 threads on 8 cores). A value of 0 (or any other negative number) means that the web server is disabled. Positive values specify the number of threads explicitly. A hard-coded maximum of 64 threads is enforced for this option.\n The total number of threads you see may be lower than the configured value as threads are only created when needed due to incoming connections."; + conf->webserver.threads.t = CONF_INT; + conf->webserver.threads.d.ui = -1; + conf->webserver.threads.c = validate_stub; // Only type-based checking + conf->webserver.tls.cert.k = "webserver.tls.cert"; conf->webserver.tls.cert.h = "Path to the TLS (SSL) certificate file. All directories along the path must be readable and accessible by the user running FTL (typically 'pihole'). This option is only required when at least one of webserver.port is TLS. The file must be in PEM format, and it must have both, private key and certificate (the *.pem file created must contain a 'CERTIFICATE' section as well as a 'RSA PRIVATE KEY' section).\n The *.pem file can be created using\n cp server.crt server.pem\n cat server.key >> server.pem\n if you have these files instead"; conf->webserver.tls.cert.a = cJSON_CreateStringReference(""); diff --git a/src/config/config.h b/src/config/config.h index 38523385b..cb679039a 100644 --- a/src/config/config.h +++ b/src/config/config.h @@ -242,6 +242,7 @@ struct config { struct conf_item domain; struct conf_item acl; struct conf_item port; + struct conf_item threads; struct { struct conf_item timeout; struct conf_item restore; diff --git a/src/webserver/webserver.c b/src/webserver/webserver.c index e44a8cc89..15641e730 100644 --- a/src/webserver/webserver.c +++ b/src/webserver/webserver.c @@ -351,6 +351,44 @@ void http_init(void) return; } + char num_threads[3] = { 0 }; + // Calculate number of threads for the web server + // any positive number = number of threads (limited to at most MAX_WEBTHREADS) + // 0 = web server not available + // -1 = the number of online processors (at least 1, no more than 16) + // -2 = two times the number of online processors (at least 1, no more + // than 16) + // any other negative number = web server not available + // For the automatic options, we use the number of available (= online) + // cores which may be less than the total number of cores in the system, + // e.g., if a virtualization environment is used and fewer cores are + // assigned to the VM than are available on the host. + sprintf(num_threads, "%d", get_nprocs() > 8 ? 16 : 2*get_nprocs()); + + if(config.webserver.threads.v.i > 0) + { + const unsigned int threads = LIMIT_MIN_MAX(config.webserver.threads.v.i, 1, MAX_WEBTHREADS); + snprintf(num_threads, sizeof(num_threads), "%u", threads); + } + else if(config.webserver.threads.v.i == -1) + { + const int nprocs = get_nprocs(); + const unsigned int threads = LIMIT_MIN_MAX(nprocs - 1, 1, 16); + snprintf(num_threads, sizeof(num_threads), "%u", threads); + } + else if(config.webserver.threads.v.i == -2) + { + const int nprocs = 2 * get_nprocs(); + const unsigned int threads = LIMIT_MIN_MAX(nprocs - 1, 1, 16); + snprintf(num_threads, sizeof(num_threads), "%u", threads); + } + else + { + log_info("Web server not available as webserver.threads is set to %d. API will not be available!", + config.webserver.threads.v.i); + return; + } + /* Initialize the library */ log_web("Initializing HTTP server on ports \"%s\"", config.webserver.port.v.s); unsigned int features = MG_FEATURES_FILES | @@ -397,15 +435,6 @@ void http_init(void) // A referrer will be sent for same-site origins, but cross-origin requests will // send no referrer information. // The latter four headers are set as expected by https://securityheaders.io - char num_threads[3] = { 0 }; - // Use 16 threads if more than 8 cores are available, otherwise use - // 2*cores. This is to prevent overloading the system with too many - // threads. - // We use the number of available (= online) cores which may be less - // than the total number of cores in the system, e.g., if a - // virtualization environment is used and fewer cores are assigned to - // the VM than are available on the host. - sprintf(num_threads, "%d", get_nprocs() > 8 ? 16 : 2*get_nprocs()); const char *options[] = { "document_root", config.webserver.paths.webroot.v.s, "error_pages", error_pages, diff --git a/src/webserver/webserver.h b/src/webserver/webserver.h index 4fff052a6..4a671f293 100644 --- a/src/webserver/webserver.h +++ b/src/webserver/webserver.h @@ -12,6 +12,11 @@ #include +// Hard-coded maximum number of allowed web server threads +#define MAX_WEBTHREADS 64 +// Macro to limiting a numeric value to a certain minimum and maximum +#define LIMIT_MIN_MAX(a, b, c) ((a) < (b) ? (b) : (a) > (c) ? (c) : (a)) + void http_init(void); void http_terminate(void); diff --git a/test/pihole.toml b/test/pihole.toml index 2b1be8b73..87c4d8453 100644 --- a/test/pihole.toml +++ b/test/pihole.toml @@ -1,7 +1,7 @@ -# Pi-hole configuration file (v5.25.2-1921-gd3948088-dirty) +# Pi-hole configuration file (v5.25.2-2439-gdc1c4c17) # Encoding: UTF-8 # This file is managed by pihole-FTL -# Last updated on 2024-06-15 09:10:13 UTC +# Last updated on 2025-01-01 11:30:22 UTC [dns] # Array of upstream DNS servers used by Pi-hole @@ -659,6 +659,22 @@ # comma-separated list of <[ip_address:]port> port = "80,[::]:80,443s,[::]:443s" + # Maximum number of worker threads allowed. + # The Pi-hole web server handles each incoming connection in a separate thread. + # Therefore, the value of this option is effectively the number of concurrent HTTP + # connections that can be handled. Any other connections are queued until they can be + # processed by a unoccupied thread. + # The default value of -1 means that the number of threads is automatically determined + # by the number of online CPU cores minus 1 (e.g., launching up to 8-1 = 7 threads on + # 8 cores). A value of -2 means the same automatism but for twice the number of cores + # (e.g., launching up to 16-1 = 15 threads on 8 cores). A value of 0 (or any other + # negative number) means that the web server is disabled. Positive values specify the + # number of threads explicitly. A hard-coded maximum of 64 threads is enforced for + # this option. + # The total number of threads you see may be lower than the configured value as + # threads are only created when needed due to incoming connections. + threads = -1 + [webserver.session] # Session timeout in seconds. If a session is inactive for more than this time, it will # be terminated. Sessions are continuously refreshed by the web interface, preventing @@ -1124,7 +1140,7 @@ all = true ### CHANGED, default = false # Configuration statistics: -# 150 total entries out of which 94 entries are default +# 151 total entries out of which 95 entries are default # --> 56 entries are modified # 2 entries are forced through environment: # - misc.nice From 4132963313fc27f9bfa8c743bca498db4f4bd8ce Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 1 Jan 2025 18:44:43 +0100 Subject: [PATCH 2/2] Simplify webserver.threads options Signed-off-by: DL6ER --- src/api/docs/content/specs/config.yaml | 2 +- src/config/config.c | 6 +++--- src/webserver/webserver.c | 26 +++++--------------------- test/pihole.toml | 15 ++++++--------- 4 files changed, 15 insertions(+), 34 deletions(-) diff --git a/src/api/docs/content/specs/config.yaml b/src/api/docs/content/specs/config.yaml index ddadde0a1..8ddf1e150 100644 --- a/src/api/docs/content/specs/config.yaml +++ b/src/api/docs/content/specs/config.yaml @@ -741,7 +741,7 @@ components: domain: pi.hole acl: "+0.0.0.0/0,::/0" port: 80,[::]:80 - threads: -1 + threads: 0 session: timeout: 300 restore: true diff --git a/src/config/config.c b/src/config/config.c index f7448a204..cdacd61d2 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -1008,9 +1008,9 @@ static void initConfig(struct config *conf) conf->webserver.port.c = validate_stub; // Type-based checking + civetweb syntax checking conf->webserver.threads.k = "webserver.threads"; - conf->webserver.threads.h = "Maximum number of worker threads allowed.\n The Pi-hole web server handles each incoming connection in a separate thread. Therefore, the value of this option is effectively the number of concurrent HTTP connections that can be handled. Any other connections are queued until they can be processed by a unoccupied thread.\n The default value of -1 means that the number of threads is automatically determined by the number of online CPU cores minus 1 (e.g., launching up to 8-1 = 7 threads on 8 cores). A value of -2 means the same automatism but for twice the number of cores (e.g., launching up to 16-1 = 15 threads on 8 cores). A value of 0 (or any other negative number) means that the web server is disabled. Positive values specify the number of threads explicitly. A hard-coded maximum of 64 threads is enforced for this option.\n The total number of threads you see may be lower than the configured value as threads are only created when needed due to incoming connections."; - conf->webserver.threads.t = CONF_INT; - conf->webserver.threads.d.ui = -1; + conf->webserver.threads.h = "Maximum number of worker threads allowed.\n The Pi-hole web server handles each incoming connection in a separate thread. Therefore, the value of this option is effectively the number of concurrent HTTP connections that can be handled. Any other connections are queued until they can be processed by a unoccupied thread.\n The default value of 0 means that the number of threads is automatically determined by the number of online CPU cores minus 1 (e.g., launching up to 8-1 = 7 threads on 8 cores). Any other value specifies the number of threads explicitly. A hard-coded maximum of 64 threads is enforced for this option.\n The total number of threads you see may be lower than the configured value as threads are only created when needed due to incoming connections."; + conf->webserver.threads.t = CONF_UINT; + conf->webserver.threads.d.ui = 0; conf->webserver.threads.c = validate_stub; // Only type-based checking conf->webserver.tls.cert.k = "webserver.tls.cert"; diff --git a/src/webserver/webserver.c b/src/webserver/webserver.c index 15641e730..c3fa16a5b 100644 --- a/src/webserver/webserver.c +++ b/src/webserver/webserver.c @@ -354,40 +354,24 @@ void http_init(void) char num_threads[3] = { 0 }; // Calculate number of threads for the web server // any positive number = number of threads (limited to at most MAX_WEBTHREADS) - // 0 = web server not available - // -1 = the number of online processors (at least 1, no more than 16) - // -2 = two times the number of online processors (at least 1, no more - // than 16) - // any other negative number = web server not available - // For the automatic options, we use the number of available (= online) + // 0 = the number of online processors (at least 1, no more than 16) + // For the automatic option, we use the number of available (= online) // cores which may be less than the total number of cores in the system, // e.g., if a virtualization environment is used and fewer cores are // assigned to the VM than are available on the host. sprintf(num_threads, "%d", get_nprocs() > 8 ? 16 : 2*get_nprocs()); - if(config.webserver.threads.v.i > 0) + if(config.webserver.threads.v.ui > 0) { - const unsigned int threads = LIMIT_MIN_MAX(config.webserver.threads.v.i, 1, MAX_WEBTHREADS); + const unsigned int threads = LIMIT_MIN_MAX(config.webserver.threads.v.ui, 1, MAX_WEBTHREADS); snprintf(num_threads, sizeof(num_threads), "%u", threads); } - else if(config.webserver.threads.v.i == -1) + else // Automatic thread calculation { const int nprocs = get_nprocs(); const unsigned int threads = LIMIT_MIN_MAX(nprocs - 1, 1, 16); snprintf(num_threads, sizeof(num_threads), "%u", threads); } - else if(config.webserver.threads.v.i == -2) - { - const int nprocs = 2 * get_nprocs(); - const unsigned int threads = LIMIT_MIN_MAX(nprocs - 1, 1, 16); - snprintf(num_threads, sizeof(num_threads), "%u", threads); - } - else - { - log_info("Web server not available as webserver.threads is set to %d. API will not be available!", - config.webserver.threads.v.i); - return; - } /* Initialize the library */ log_web("Initializing HTTP server on ports \"%s\"", config.webserver.port.v.s); diff --git a/test/pihole.toml b/test/pihole.toml index 87c4d8453..644e58074 100644 --- a/test/pihole.toml +++ b/test/pihole.toml @@ -1,7 +1,7 @@ -# Pi-hole configuration file (v5.25.2-2439-gdc1c4c17) +# Pi-hole configuration file (v5.25.2-2439-g93521615-dirty) # Encoding: UTF-8 # This file is managed by pihole-FTL -# Last updated on 2025-01-01 11:30:22 UTC +# Last updated on 2025-01-01 12:33:13 UTC [dns] # Array of upstream DNS servers used by Pi-hole @@ -664,16 +664,13 @@ # Therefore, the value of this option is effectively the number of concurrent HTTP # connections that can be handled. Any other connections are queued until they can be # processed by a unoccupied thread. - # The default value of -1 means that the number of threads is automatically determined + # The default value of 0 means that the number of threads is automatically determined # by the number of online CPU cores minus 1 (e.g., launching up to 8-1 = 7 threads on - # 8 cores). A value of -2 means the same automatism but for twice the number of cores - # (e.g., launching up to 16-1 = 15 threads on 8 cores). A value of 0 (or any other - # negative number) means that the web server is disabled. Positive values specify the - # number of threads explicitly. A hard-coded maximum of 64 threads is enforced for - # this option. + # 8 cores). Any other value specifies the number of threads explicitly. A hard-coded + # maximum of 64 threads is enforced for this option. # The total number of threads you see may be lower than the configured value as # threads are only created when needed due to incoming connections. - threads = -1 + threads = 0 [webserver.session] # Session timeout in seconds. If a session is inactive for more than this time, it will