Merge branch 'development-v6' into fix/delete_dhcp_lease

Signed-off-by: DL6ER <[email protected]>

.devcontainer/devcontainer.json

@@ -1,15 +1,20 @@
 {
   "name": "FTL x86_64 Build Env",
-  "image": "ghcr.io/pi-hole/ftl-build:v2.3-alpine",
+  "image": "ghcr.io/pi-hole/ftl-build:v2.4.1",
   "runArgs": [ "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined" ],
-  "extensions": [
-    "jetmartin.bats",
-    "ms-vscode.cpptools",
-    "ms-vscode.cmake-tools",
-    "eamodio.gitlens"
-  ],
+  "customizations": {
+    "vscode": {
+      "extensions": [
+        "jetmartin.bats",
+        "ms-vscode.cpptools",
+        "ms-vscode.cmake-tools",
+        "eamodio.gitlens"
+      ]
+    }
+  },
   "mounts": [
-    "type=bind,source=/home/${localEnv:USER}/.ssh,target=/root/.ssh,readonly"
+    "type=bind,source=/home/${localEnv:USER}/.ssh,target=/root/.ssh,readonly",
+    "type=bind,source=/var/www/html,target=/var/www/html,readonly"
   ]
 
 }

.github/.codespellignore

@@ -5,3 +5,4 @@ nd
 doubleclick
 requestor
 requestors
+punycode

.github/Dockerfile

@@ -1,5 +1,4 @@
-ARG BUILDER="alpine"
-FROM ghcr.io/pi-hole/ftl-build:v2.3-${BUILDER} AS builder
+FROM ghcr.io/pi-hole/ftl-build:v2.4.1 AS builder
 
 WORKDIR /app
 

.github/workflows/build.yml

@@ -119,9 +119,9 @@ jobs:
       -
         name: Store binary artifacts for later deployoment
         if: github.event_name != 'pull_request'
-        uses: actions/upload-artifact@v3.1.3
+        uses: actions/upload-artifact@v4.0.0
         with:
-          name: tmp-storage
+          name: ${{ matrix.bin_name }}-binary
           path: '${{ matrix.bin_name }}*'
       -
         name: Extract documentation files from container
@@ -131,9 +131,9 @@ jobs:
       -
         name: Upload documentation artifacts for deployoment
         if: github.event_name != 'pull_request' && matrix.platform == 'linux/amd64'
-        uses: actions/upload-artifact@v3.1.3
+        uses: actions/upload-artifact@v4.0.0
         with:
-          name: tmp-storage
+          name: pihole-api-docs
           path: 'api-docs.tar.gz'
 
   deploy:
@@ -146,15 +146,17 @@ jobs:
         uses: actions/[email protected]
       -
         name: Get Binaries and documentation built in previous jobs
-        uses: actions/download-artifact@v3.0.2
+        uses: actions/download-artifact@v4.1.0
         id: download
         with:
-          name: tmp-storage
-          path: ftl-builds/
+          path: ftl_builds/
+          pattern: pihole-*
+          merge-multiple: true
       -
         name: Display structure of downloaded files
         run: ls -R
         working-directory: ${{steps.download.outputs.download-path}}
+
       -
         name: Install SSH Key
         uses: benoitchantre/[email protected]
@@ -163,22 +165,22 @@ jobs:
           known-hosts: ${{ secrets.KNOWN_HOSTS }}
       -
         name: Untar documentation files
-        working-directory: ${{steps.download.outputs.download-path}}
+        working-directory: ftl_builds/
         run: |
           mkdir docs/
           tar xzvf api-docs.tar.gz -C docs/
       -
         name: Display structure of files ready for upload
         run: ls -R
-        working-directory: ${{steps.download.outputs.download-path}}
+        working-directory: ftl_builds/
       -
         name: Transfer Builds to Pi-hole server for pihole checkout
         if: github.actor != 'dependabot[bot]'
         env:
           USER: ${{ secrets.SSH_USER }}
           HOST: ${{ secrets.SSH_HOST }}
           TARGET_DIR: ${{ needs.smoke-tests.outputs.OUTPUT_DIR }}
-          SOURCE_DIR: ${{ steps.download.outputs.download-path }}
+          SOURCE_DIR: ftl_builds/
         run: |
           bash ./deploy.sh
       -
@@ -187,4 +189,4 @@ jobs:
         uses: softprops/action-gh-release@v1
         with:
           files: |
-            ${{ steps.download.outputs.download-path }}/*
+            ftl_builds/*

.github/workflows/codespell.yml

@@ -1,5 +1,8 @@
 name: Codespell
 on:
+  push:
+    branches:
+      - '**'
   pull_request:
     types: [opened, synchronize, reopened, ready_for_review]
 

.github/workflows/openapi-validator.yml

@@ -17,7 +17,7 @@ jobs:
       - name: Set Node.js version
         uses: actions/setup-node@v4
         with:
-          node-version: "18"
+          node-version: "20"
 
       - name: Install npm dependencies
         run: npm ci

.github/workflows/stale.yml

@@ -17,7 +17,7 @@ jobs:
       issues: write
 
     steps:
-    - uses: actions/stale@v8.0.0
+    - uses: actions/stale@v9.0.0
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         days-before-stale: 30

.github/workflows/stale_pr.yml

@@ -17,7 +17,7 @@ jobs:
       pull-requests: write
 
     steps:
-      - uses: actions/stale@v8.0.0
+      - uses: actions/stale@v9.0.0
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           # Do not automatically mark PR/issue as stale

.gitignore

@@ -7,6 +7,7 @@ version~
 
 # CMake files generated during compilation
 /cmake/
+/cmake_ci/
 /cmake-build-debug/
 /cmake-build-release/
 
@@ -18,6 +19,15 @@ version~
 /.vscode/
 /build/
 
+# __pycache__ files (API tests)
+__pycache__/
+
+# When patch fails to apply a patch segment to the original file, it saves the
+# temporary original file copy out durably as *.orig, dumps the rejected segment
+# to *.rej, and continues trying to apply patch segments.
+*.orig
+*.rej
+
 # MAC->Vendor database files
 tools/manuf.data
 tools/macvendor.db

patch/civetweb.sh

@@ -7,7 +7,7 @@ patch -p1 < patch/civetweb/0001-Always-Kepler-syntax-for-Lua-server-pages.patch
 patch -p1 < patch/civetweb/0001-Add-FTL-URI-rewriting-changes-to-CivetWeb.patch
 patch -p1 < patch/civetweb/0001-Add-mbedTLS-debug-logging-hook.patch
 patch -p1 < patch/civetweb/0001-Register-CSRF-token-in-conn-request_info.patch
-patch -p1 < patch/civetweb/0001-Do-not-try-to-guess-server-hostname-in-Civetweb-when.patch
 patch -p1 < patch/civetweb/0001-Log-debug-messages-to-webserver.log-when-debug.webse.patch
+patch -p1 < patch/civetweb/0001-Allow-extended-ASCII-characters-in-URIs.patch
 
 echo "ALL PATCHES APPLIED OKAY"

patch/civetweb/0001-Allow-extended-ASCII-characters-in-URIs.patch

@@ -0,0 +1,35 @@
+From ebb27741b10ed2eac51ac356708800ae96cdd17a Mon Sep 17 00:00:00 2001
+From: DL6ER <[email protected]>
+Date: Tue, 31 Oct 2023 08:35:31 +0100
+Subject: [PATCH] Allow extended ASCII characters in URIs
+
+Signed-off-by: DL6ER <[email protected]>
+---
+ src/webserver/civetweb/civetweb.c | 4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/src/webserver/civetweb/civetweb.c b/src/webserver/civetweb/civetweb.c
+index 9b0c6308..5320c4d4 100644
+--- a/src/webserver/civetweb/civetweb.c
++++ b/src/webserver/civetweb/civetweb.c
+@@ -10734,7 +10734,7 @@ skip_to_end_of_word_and_terminate(char **ppw, int eol)
+ {
+ 	/* Forward until a space is found - use isgraph here */
+ 	/* See http://www.cplusplus.com/reference/cctype/ */
+-	while (isgraph((unsigned char)**ppw)) {
++	while ((unsigned char)**ppw > 127 || isgraph((unsigned char)**ppw)) {
+ 		(*ppw)++;
+ 	}
+
+@@ -18473,7 +18473,7 @@ get_uri_type(const char *uri)
+ 	 * and % encoded symbols.
+ 	 */
+ 	for (i = 0; uri[i] != 0; i++) {
+-		if (uri[i] < 33) {
++		if ((unsigned char)uri[i] < 33) {
+ 			/* control characters and spaces are invalid */
+ 			return 0;
+ 		}
+-- 
+2.34.1
+

patch/sqlite3/0001-print-FTL-version-in-interactive-shell.patch

@@ -25,6 +25,6 @@ index 6280ebf6..a5e82f70 100644
        char *zHistory;
        int nHistory;
 +      print_FTL_version();
-       printf(
-         "SQLite version %s %.19s\n" /*extra-version-info*/
-         "Enter \".help\" for usage hints.\n",
+ #if CIO_WIN_WC_XLATE
+ # define SHELL_CIO_CHAR_SET (stdout_is_console? " (UTF-16 console I/O)" : "")
+ #else

src/CMakeLists.txt

@@ -24,14 +24,20 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR})
 # SQLITE_DEFAULT_MEMSTATUS=0: This setting causes the sqlite3_status() interfaces that track memory usage to be disabled. This helps the sqlite3_malloc() routines run much faster, and since SQLite uses sqlite3_malloc() internally, this helps to make the entire library faster.
 # SQLITE_OMIT_DEPRECATED: Omitting deprecated interfaces and features will not help SQLite to run any faster. It will reduce the library footprint, however. And it is the right thing to do.
 # SQLITE_OMIT_PROGRESS_CALLBACK: The progress handler callback counter must be checked in the inner loop of the bytecode engine. By omitting this interface, a single conditional is removed from the inner loop of the bytecode engine, helping SQL statements to run slightly faster.
+# SQLITE_OMIT_SHARED_CACHE: This option builds SQLite without support for shared cache mode. The sqlite3_enable_shared_cache() is omitted along with a fair amount of logic within the B-Tree subsystem associated with shared cache management. This compile-time option is recommended most applications as it results in improved performance and reduced library footprint.
 # SQLITE_DEFAULT_FOREIGN_KEYS=1: This macro determines whether enforcement of foreign key constraints is enabled or disabled by default for new database connections.
 # SQLITE_DQS=0: This setting disables the double-quoted string literal misfeature.
 # SQLITE_ENABLE_DBPAGE_VTAB: Enables the SQLITE_DBPAGE virtual table. Warning: writing to the SQLITE_DBPAGE virtual table can very easily cause unrecoverably database corruption.
 # SQLITE_TEMP_STORE=2: Store temporary tables in memory for reduced IO and higher performance (can be overwritten by the user at runtime).
-# SQLITE_USE_URI=1: The advantage of using a URI filename is that query parameters on the URI can be used to control details of the newly created database connection.
 # HAVE_READLINE: Enable readline support to allow easy editing, history and auto-completion
 # SQLITE_DEFAULT_CACHE_SIZE=-16384: Allow up to 16 MiB of cache to be used by SQLite3 (default is 2000 kiB)
-set(SQLITE_DEFINES "-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_DEFAULT_FOREIGN_KEYS=1 -DSQLITE_DQS=0 -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_TEMP_STORE=2 -DSQLITE_USE_URI=1 -DHAVE_READLINE -DSQLITE_DEFAULT_CACHE_SIZE=16384")
+# SQLITE_DEFAULT_SYNCHRONOUS=1: Use normal synchronous mode (default is 2)
+# SQLITE_LIKE_DOESNT_MATCH_BLOBS: This option causes the LIKE operator to only match BLOB values against BLOB values and TEXT values against TEXT values. This compile-time option makes SQLite run more efficiently when processing queries that use the LIKE operator.
+# HAVE_MALLOC_USABLE_SIZE: This option causes SQLite to try to use the malloc_usable_size() function to obtain the actual size of memory allocations from the underlying malloc() system interface. Applications are encouraged to use HAVE_MALLOC_USABLE_SIZE whenever possible.
+# HAVE_FDATASYNC: This option causes SQLite to try to use the fdatasync() system call to sync the database file to disk when committing a transaction. Syncing using fdatasync() is faster than syncing using fsync() as fdatasync() does not wait for the file metadata to be written to disk.
+# SQLITE_DEFAULT_WORKER_THREADS=4: This option sets the default number of worker threads to use when doing parallel sorting and indexing. The default is 0 which means to use a single thread. The default for SQLITE_MAX_WORKER_THREADS is 8.
+# SQLITE_MAX_PREPARE_RETRY=200: This option sets the maximum number of automatic re-preparation attempts that can occur after encountering a schema change. This can be caused by running ANALYZE which is done periodically by FTL.
+set(SQLITE_DEFINES "-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_OMIT_SHARED_CACHE -DSQLITE_DEFAULT_FOREIGN_KEYS=1 -DSQLITE_DQS=0 -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_TEMP_STORE=2 -DHAVE_READLINE -DSQLITE_DEFAULT_CACHE_SIZE=16384 -DSQLITE_DEFAULT_SYNCHRONOUS=1 -DSQLITE_LIKE_DOESNT_MATCH_BLOBS -DHAVE_MALLOC_USABLE_SIZE -DHAVE_FDATASYNC -DSQLITE_DEFAULT_WORKER_THREADS=4 -DSQLITE_MAX_PREPARE_RETRY=200")
 
 # Code hardening and debugging improvements
 # -fstack-protector-strong: The program will be resistant to having its stack overflowed
@@ -154,6 +160,11 @@ set(EXTRAWARN "${EXTRAWARN_GCC6} \
                ${EXTRAWARN_GCC8} \
                ${EXTRAWARN_GCC12} \
                ${EXTRAWARN_GCC13}")
+
+# Remove extra spaces from EXTRAWARN
+string(REGEX REPLACE " +" " " EXTRAWARN "${EXTRAWARN}")
+
+# Separate EXTRAWARN into a list of arguments
 separate_arguments(EXTRAWARN)
 
 # -Wxor-used-as-pow
@@ -219,8 +230,6 @@ set(sources
         regex_r.h
         resolve.c
         resolve.h
-        setupVars.c
-        setupVars.h
         shmem.c
         shmem.h
         signals.c
@@ -283,9 +292,12 @@ find_package(Threads REQUIRED)
 find_library(LIBHOGWEED NAMES libhogweed${CMAKE_STATIC_LIBRARY_SUFFIX} hogweed HINTS /usr/local/lib64)
 find_library(LIBGMP NAMES libgmp${CMAKE_STATIC_LIBRARY_SUFFIX} gmp)
 find_library(LIBNETTLE NAMES libnettle${CMAKE_STATIC_LIBRARY_SUFFIX} nettle HINTS /usr/local/lib64)
-find_library(LIBIDN NAMES libidn${CMAKE_STATIC_LIBRARY_SUFFIX} idn)
 
-target_link_libraries(pihole-FTL rt Threads::Threads ${LIBHOGWEED} ${LIBGMP} ${LIBNETTLE} ${LIBIDN})
+# for IDN2 we need the idn2 library which in turn depends on the unistring library
+find_library(LIBIDN2 NAMES libidn2${CMAKE_STATIC_LIBRARY_SUFFIX} idn)
+find_library(LIBUNISTRING NAMES libunistring${CMAKE_STATIC_LIBRARY_SUFFIX} unistring)
+
+target_link_libraries(pihole-FTL rt Threads::Threads ${LIBHOGWEED} ${LIBGMP} ${LIBNETTLE} ${LIBIDN2} ${LIBUNISTRING})
 
 if(LUA_DL STREQUAL "true")
     find_library(LIBDL dl)

src/FTL.h

@@ -50,14 +50,15 @@
 // Number of elements in an array
 #define ArraySize(X) (sizeof(X)/sizeof(X[0]))
 
+// Constant socket buffer length
 #define SOCKETBUFFERLEN 1024
 
 // How often do we garbage collect (to ensure we only have data fitting to the MAXLOGAGE defined above)? [seconds]
 // Default: 600 (10 minute intervals)
 #define GCinterval 600
 
 // Delay applied to the garbage collecting [seconds]
-// Default: -60 (one minute before a full hour)
+// Default: -60 (one minute before the end of the interval set above)
 #define GCdelay (-60)
 
 // How many client connection do we accept at once?
@@ -133,12 +134,20 @@
 // Special exit code used to signal that FTL wants to restart
 #define RESTART_FTL_CODE 22
 
+// How often should the database be analyzed?
+// Default: 604800 (once per week)
+#define DATABASE_ANALYZE_INTERVAL 604800
+
+// How often should we update client vendor's from the MAC vendor database?
+// Default: 2592000 (once per month)
+#define DATABASE_MACVENDOR_INTERVAL 2592000
+
 // Use out own syscalls handling functions that will detect possible errors
 // and report accordingly in the log. This will make debugging FTL crash
 // caused by insufficient memory or by code bugs (not properly dealing
 // with NULL pointers) much easier.
 #undef strdup // strdup() is a macro in itself, it needs special handling
-#define free(ptr) FTLfree(ptr, __FILE__,  __FUNCTION__,  __LINE__)
+#define free(ptr) FTLfree((void**)&ptr, __FILE__,  __FUNCTION__,  __LINE__)
 #define strdup(str_in) FTLstrdup(str_in, __FILE__,  __FUNCTION__,  __LINE__)
 #define calloc(numer_of_elements, element_size) FTLcalloc(numer_of_elements, element_size, __FILE__,  __FUNCTION__,  __LINE__)
 #define realloc(ptr, new_size) FTLrealloc(ptr, new_size, __FILE__,  __FUNCTION__,  __LINE__)
@@ -161,6 +170,22 @@
 #define pthread_mutex_lock(mutex) FTLpthread_mutex_lock(mutex, __FILE__,  __FUNCTION__,  __LINE__)
 #define fopen(pathname, mode) FTLfopen(pathname, mode, __FILE__,  __FUNCTION__,  __LINE__)
 #define ftlallocate(fd, offset, len) FTLfallocate(fd, offset, len, __FILE__,  __FUNCTION__,  __LINE__)
+#define strlen(str) FTLstrlen(str, __FILE__,  __FUNCTION__,  __LINE__)
+#define strnlen(str, maxlen) FTLstrnlen(str, maxlen, __FILE__,  __FUNCTION__,  __LINE__)
+#define strcpy(dest, src) FTLstrcpy(dest, src, __FILE__,  __FUNCTION__,  __LINE__)
+#define strncpy(dest, src, n) FTLstrncpy(dest, src, n, __FILE__,  __FUNCTION__,  __LINE__)
+#define memset(s, c, n) FTLmemset(s, c, n, __FILE__,  __FUNCTION__,  __LINE__)
+#define memcpy(dest, src, n) FTLmemcpy(dest, src, n, __FILE__,  __FUNCTION__,  __LINE__)
+#define memmove(dest, src, n) FTLmemmove(dest, src, n, __FILE__,  __FUNCTION__,  __LINE__)
+#define strstr(haystack, needle) FTLstrstr(haystack, needle, __FILE__,  __FUNCTION__,  __LINE__)
+#define strcmp(s1, s2) FTLstrcmp(s1, s2, __FILE__,  __FUNCTION__,  __LINE__)
+#define strncmp(s1, s2, n) FTLstrncmp(s1, s2, n, __FILE__,  __FUNCTION__,  __LINE__)
+#define strcasecmp(s1, s2) FTLstrcasecmp(s1, s2, __FILE__,  __FUNCTION__,  __LINE__)
+#define strncasecmp(s1, s2, n) FTLstrncasecmp(s1, s2, n, __FILE__,  __FUNCTION__,  __LINE__)
+#define strcat(dest, src) FTLstrcat(dest, src, __FILE__,  __FUNCTION__,  __LINE__)
+#define strncat(dest, src, n) FTLstrncat(dest, src, n, __FILE__,  __FUNCTION__,  __LINE__)
+#define memcmp(s1, s2, n) FTLmemcmp(s1, s2, n, __FILE__,  __FUNCTION__,  __LINE__)
+#define memmem(haystack, haystacklen, needle, needlelen) FTLmemmem(haystack, haystacklen, needle, needlelen, __FILE__,  __FUNCTION__,  __LINE__)
 #include "syscalls/syscalls.h"
 
 // Preprocessor help functions