From 2aff05c78a18ff0c6e07d13b2346510c701a6dfa Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 23 Oct 2023 18:13:12 +0200 Subject: [PATCH 001/221] Reuse existing shared clients, domains, and cache memory slots which are not referenced by any active query within the past 24 hours. Furthermore, always scan the shared strings and reuse them before allocating new memory with the same content Signed-off-by: DL6ER --- src/FTL.h | 2 +- src/database/aliasclients.c | 2 +- src/database/gravity-db.c | 2 +- src/database/query-table.c | 5 +- src/datastructure.c | 101 +++++++++++++++++++++---- src/datastructure.h | 10 ++- src/dnsmasq_interface.c | 4 - src/gc.c | 147 ++++++++++++++++++++++++++++++++++++ src/resolve.c | 2 +- src/shmem.c | 108 ++++++++++++++++++++------ src/shmem.h | 3 +- 11 files changed, 334 insertions(+), 52 deletions(-) diff --git a/src/FTL.h b/src/FTL.h index 7c4151b98..c333d5de3 100644 --- a/src/FTL.h +++ b/src/FTL.h @@ -57,7 +57,7 @@ #define GCinterval 600 // Delay applied to the garbage collecting [seconds] -// Default: -60 (one minute before a full hour) +// Default: -60 (one minute before the end of the interval set above) #define GCdelay (-60) // How many client connection do we accept at once? diff --git a/src/database/aliasclients.c b/src/database/aliasclients.c index 16278cf05..f060e217f 100644 --- a/src/database/aliasclients.c +++ b/src/database/aliasclients.c @@ -195,7 +195,7 @@ static int get_aliasclient_ID(sqlite3 *db, const clientsData *client) const clientsData *alias_client = getClient(aliasclientID, true); // Skip clients that are not alias-clients - if(!alias_client->flags.aliasclient) + if(alias_client == NULL || !alias_client->flags.aliasclient) continue; // Compare MAC address of the current client to the diff --git a/src/database/gravity-db.c b/src/database/gravity-db.c index 34cc819b5..6055ffc8e 100644 --- a/src/database/gravity-db.c +++ b/src/database/gravity-db.c @@ -320,7 +320,7 @@ static bool get_client_groupids(clientsData* client) ip, sqlite3_errstr(rc)); sqlite3_reset(table_stmt); sqlite3_finalize(table_stmt); - return NULL; + return false; } // Perform query diff --git a/src/database/query-table.c b/src/database/query-table.c index a3a47ab3a..084180b98 100644 --- a/src/database/query-table.c +++ b/src/database/query-table.c @@ -1093,6 +1093,10 @@ void DB_read_queries(void) // Update overTime data structure with the new client change_clientcount(client, 0, 0, timeidx, 1); + // Get domain pointer + domainsData *domain = getDomain(domainID, true); + domain->lastQuery = queryTimeStamp; + // Increase DNS queries counter counters->queries++; @@ -1152,7 +1156,6 @@ void DB_read_queries(void) case QUERY_SPECIAL_DOMAIN: // Blocked by special domain handling query->flags.blocked = true; // Get domain pointer - domainsData *domain = getDomain(domainID, true); domain->blockedcount++; change_clientcount(client, 0, 1, -1, 0); break; diff --git a/src/datastructure.c b/src/datastructure.c index 7998d99bc..543eeed4d 100644 --- a/src/datastructure.c +++ b/src/datastructure.c @@ -83,13 +83,13 @@ int findQueryID(const int id) return -1; } -int findUpstreamID(const char * upstreamString, const in_port_t port) +int _findUpstreamID(const char *upstreamString, const in_port_t port, int line, const char *func, const char *file) { // Go through already knows upstream servers and see if we used one of those - for(int upstreamID=0; upstreamID < counters->upstreams; upstreamID++) + for(int upstreamID = 0; upstreamID < counters->upstreams; upstreamID++) { // Get upstream pointer - upstreamsData* upstream = getUpstream(upstreamID, true); + upstreamsData* upstream = _getUpstream(upstreamID, false, line, func, file); // Check if the returned pointer is valid before trying to access it if(upstream == NULL) @@ -101,10 +101,10 @@ int findUpstreamID(const char * upstreamString, const in_port_t port) // This upstream server is not known // Store ID const int upstreamID = counters->upstreams; - log_debug(DEBUG_ANY, "New upstream server: %s:%u (%i/%i)", upstreamString, port, upstreamID, counters->upstreams_MAX); + log_debug(DEBUG_GC, "New upstream server: %s:%u (ID %i)", upstreamString, port, upstreamID); // Get upstream pointer - upstreamsData* upstream = getUpstream(upstreamID, false); + upstreamsData* upstream = _getUpstream(upstreamID, false, line, func, file); if(upstream == NULL) { log_err("Encountered serious memory error in findupstreamID()"); @@ -137,13 +137,34 @@ int findUpstreamID(const char * upstreamString, const in_port_t port) return upstreamID; } -int findDomainID(const char *domainString, const bool count) +static int get_next_domainID(void) +{ + // Compare content of domain against known domain IP addresses + for(int domainID=0; domainID < counters->domains; domainID++) + { + // Get domain pointer + domainsData* domain = getDomain(domainID, false); + + // Check if the returned pointer is valid before trying to access it + if(domain == NULL) + continue; + + // Check if the magic byte is set + if(domain->magic == 0x00) + return domainID; + } + + // If we did not return until here, then we need to allocate a new domain ID + return counters->domains; +} + +int _findDomainID(const char *domainString, const bool count, int line, const char *func, const char *file) { uint32_t domainHash = hashStr(domainString); for(int domainID = 0; domainID < counters->domains; domainID++) { // Get domain pointer - domainsData* domain = getDomain(domainID, true); + domainsData* domain = _getDomain(domainID, false, line, func, file); // Check if the returned pointer is valid before trying to access it if(domain == NULL) @@ -157,23 +178,28 @@ int findDomainID(const char *domainString, const bool count) if(strcmp(getstr(domain->domainpos), domainString) == 0) { if(count) + { domain->count++; + domain->lastQuery = double_time(); + } return domainID; } } // If we did not return until here, then this domain is not known // Store ID - const int domainID = counters->domains; + const int domainID = get_next_domainID(); // Get domain pointer - domainsData* domain = getDomain(domainID, false); + domainsData* domain = _getDomain(domainID, false, line, func, file); if(domain == NULL) { log_err("Encountered serious memory error in findDomainID()"); return -1; } + log_debug(DEBUG_GC, "New domain: %s (ID %d)", domainString, domainID); + // Set magic byte domain->magic = MAGICBYTE; // Set its counter to 1 only if this domain is to be counted @@ -185,19 +211,41 @@ int findDomainID(const char *domainString, const bool count) domain->domainpos = addstr(domainString); // Store pre-computed hash of domain for faster lookups later on domain->domainhash = hashStr(domainString); + domain->lastQuery = 0.0; // Increase counter by one counters->domains++; return domainID; } -int findClientID(const char *clientIP, const bool count, const bool aliasclient) +static int get_next_clientID(void) { // Compare content of client against known client IP addresses for(int clientID=0; clientID < counters->clients; clientID++) { // Get client pointer - clientsData* client = getClient(clientID, true); + clientsData* client = getClient(clientID, false); + + // Check if the returned pointer is valid before trying to access it + if(client == NULL) + continue; + + // Check if the magic byte is unset + if(client->magic == 0x00) + return clientID; + } + + // If we did not return until here, then we need to allocate a new client ID + return counters->clients; +} + +int _findClientID(const char *clientIP, const bool count, const bool aliasclient, int line, const char *func, const char *file) +{ + // Compare content of client against known client IP addresses + for(int clientID=0; clientID < counters->clients; clientID++) + { + // Get client pointer + clientsData* client = _getClient(clientID, true, line, func, file); // Check if the returned pointer is valid before trying to access it if(client == NULL) @@ -223,16 +271,18 @@ int findClientID(const char *clientIP, const bool count, const bool aliasclient) // If we did not return until here, then this client is definitely new // Store ID - const int clientID = counters->clients; + const int clientID = get_next_clientID(); // Get client pointer - clientsData* client = getClient(clientID, false); + clientsData* client = _getClient(clientID, false, line, func, file); if(client == NULL) { log_err("Encountered serious memory error in findClientID()"); return -1; } + log_debug(DEBUG_GC, "New client: %s (ID %d)", clientIP, clientID); + // Set magic byte client->magic = MAGICBYTE; // Set its counter to 1 @@ -319,6 +369,27 @@ void change_clientcount(clientsData *client, int total, int blocked, int overTim } } +static int get_next_cacheID(void) +{ + // Compare content of cache against known cache IP addresses + for(int cacheID=0; cacheID < counters->dns_cache_size; cacheID++) + { + // Get cache pointer + DNSCacheData* cache = getDNSCache(cacheID, false); + + // Check if the returned pointer is valid before trying to access it + if(cache == NULL) + continue; + + // Check if the magic byte is set + if(cache->magic == 0x00) + return cacheID; + } + + // If we did not return until here, then we need to allocate a new cache ID + return counters->dns_cache_size; +} + int _findCacheID(const int domainID, const int clientID, const enum query_type query_type, const bool create_new, const char *func, int line, const char *file) { @@ -344,7 +415,7 @@ int _findCacheID(const int domainID, const int clientID, const enum query_type q return -1; // Get ID of new cache entry - const int cacheID = counters->dns_cache_size; + const int cacheID = get_next_cacheID(); // Get client pointer DNSCacheData* dns_cache = _getDNSCache(cacheID, false, line, func, file); @@ -441,7 +512,7 @@ const char *getClientIPString(const queriesData* query) if(query->privacylevel < PRIVACY_HIDE_DOMAINS_CLIENTS) { // Get client pointer - const clientsData* client = getClient(query->clientID, false); + const clientsData* client = getClient(query->clientID, true); // Check if the returned pointer is valid before trying to access it if(client == NULL) diff --git a/src/datastructure.h b/src/datastructure.h index c115064e3..47adabbef 100644 --- a/src/datastructure.h +++ b/src/datastructure.h @@ -103,6 +103,7 @@ typedef struct { int blockedcount; uint32_t domainhash; size_t domainpos; + double lastQuery; } domainsData; typedef struct { @@ -119,9 +120,12 @@ typedef struct { void strtolower(char *str); uint32_t hashStr(const char *s) __attribute__((pure)); int findQueryID(const int id); -int findUpstreamID(const char * upstream, const in_port_t port); -int findDomainID(const char *domain, const bool count); -int findClientID(const char *client, const bool count, const bool aliasclient); +#define findUpstreamID(upstream, port) _findUpstreamID(upstream, port, __LINE__, __FUNCTION__, __FILE__) +int _findUpstreamID(const char *upstream, const in_port_t port, int line, const char *func, const char *file); +#define findDomainID(domain, count) _findDomainID(domain, count, __LINE__, __FUNCTION__, __FILE__) +int _findDomainID(const char *domain, const bool count, int line, const char *func, const char *file); +#define findClientID(client, count, aliasclient) _findClientID(client, count, aliasclient, __LINE__, __FUNCTION__, __FILE__) +int _findClientID(const char *client, const bool count, const bool aliasclient, int line, const char *func, const char *file); #define findCacheID(domainID, clientID, query_type, create_new) _findCacheID(domainID, clientID, query_type, create_new, __FUNCTION__, __LINE__, __FILE__) int _findCacheID(const int domainID, const int clientID, const enum query_type query_type, const bool create_new, const char *func, const int line, const char *file); bool isValidIPv4(const char *addr); diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index 0211afa86..930343a69 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -1700,10 +1700,6 @@ static void FTL_forwarded(const unsigned int flags, const char *name, const unio // Update overTime counts const int timeidx = getOverTimeID(query->timestamp); upstream->overTime[timeidx]++; - // Update lastQuery timestamp - upstream->lastQuery = time(NULL); - // Count forwarded query - upstream->count++; } // Proceed only if diff --git a/src/gc.c b/src/gc.c index 322a92ab0..9c147684c 100644 --- a/src/gc.c +++ b/src/gc.c @@ -39,6 +39,150 @@ bool doGC = false; +// Recycle old clients and domains in our internal data structure +// This has the side-effect of recycling intermediate domains +// seen during CNAME inspection, too, as they are never referenced +// by any query (only head and tail of the CNAME chain are) +static void recycle(void) +{ + bool *client_used = calloc(counters->clients, sizeof(bool)); + bool *domain_used = calloc(counters->domains, sizeof(bool)); + bool *upstreams_used = calloc(counters->upstreams, sizeof(bool)); + if(client_used == NULL || domain_used == NULL || upstreams_used == NULL) + { + log_err("Cannot allocate memory for recycling"); + return; + } + + // Find list of client and domain IDs no active query is referencing anymore + // and recycle them + for(int queryID = 0; queryID < counters->queries; queryID++) + { + queriesData* query = getQuery(queryID, true); + if(query == NULL) + continue; + + // Mark client and domain as used + client_used[query->clientID] = true; + domain_used[query->domainID] = true; + + // Mark upstream as used (if any) + if(query->upstreamID > -1) + upstreams_used[query->upstreamID] = true; + + // Mark CNAME domain as used (if any) + if(query->CNAME_domainID >= 0) + domain_used[query->CNAME_domainID] = true; + } + + // Recycle clients + unsigned int clients_recycled = 0; + for(int clientID = 0; clientID < counters->clients; clientID++) + { + if(client_used[clientID]) + continue; + + clientsData* client = getClient(clientID, true); + if(client == NULL) + continue; + + log_debug(DEBUG_GC, "Recycling client %s (ID %d, lastQuery at %.3f)", + getstr(client->ippos), clientID, client->lastQuery); + + // Wipe client's memory + memset(client, 0, sizeof(clientsData)); + + clients_recycled++; + } + + // Recycle domains + unsigned int domains_recycled = 0; + for(int domainID = 0; domainID < counters->domains; domainID++) + { + if(domain_used[domainID]) + continue; + + domainsData* domain = getDomain(domainID, true); + if(domain == NULL) + continue; + + log_debug(DEBUG_GC, "Recycling domain %s (ID %d, lastQuery at %.3f)", + getstr(domain->domainpos), domainID, domain->lastQuery); + + // Wipe domain's memory + memset(domain, 0, sizeof(domainsData)); + + domains_recycled++; + } + + // Recycle cache records + unsigned int cache_recycled = 0; + for(int cacheID = 0; cacheID < counters->dns_cache_size; cacheID++) + { + DNSCacheData *cache = getDNSCache(cacheID, true); + if(cache == NULL) + continue; + + // Skip cache entries that are still in use + if(cache->magic != 0x00) + continue; + + log_debug(DEBUG_GC, "Recycling cache entry with ID %d", cacheID); + + // Wipe cache entry's memory + memset(cache, 0, sizeof(DNSCacheData)); + + cache_recycled++; + } + + // Free memory + free(client_used); + free(domain_used); + free(upstreams_used); + + // Scan number of recycled clients and domains if in debug mode + if(config.debug.gc.v.b) + { + unsigned int free_domains = 0, free_clients = 0, free_cache = 0; + for(int clientID = 0; clientID < counters->clients; clientID++) + { + // Do not check magic to avoid skipping recycled clients + clientsData *client = getClient(clientID, false); + if(client == NULL) + continue; + if(client->magic == 0x00) + free_clients++; + } + for(int domainID = 0; domainID < counters->domains; domainID++) + { + // Do not check magic to avoid skipping recycled domains + domainsData *domain = getDomain(domainID, false); + if(domain == NULL) + continue; + if(domain->magic == 0x00) + free_domains++; + } + for(int cacheID = 0; cacheID < counters->dns_cache_size; cacheID++) + { + // Do not check magic to avoid skipping recycled cache entries + DNSCacheData *cache = getDNSCache(cacheID, false); + if(cache == NULL) + continue; + if(cache->magic == 0x00) + free_cache++; + } + + log_debug(DEBUG_GC, "Recycler result: %u clients, %u domains and %u cache records are free", + free_clients, free_domains, free_cache); + + log_debug(DEBUG_GC, "Recycled additional %u/%d (max %d) clients, %u/%d (max %d) domains, and %u/%d (max %d) cache records (scanned %d queries)", + clients_recycled, counters->clients, counters->clients_MAX, + domains_recycled, counters->domains, counters->domains_MAX, + cache_recycled, counters->dns_cache_size, counters->dns_cache_MAX, + counters->queries); + } +} + // Subtract rate-limitation count from individual client counters // As long as client->rate_limit is still larger than the allowed // maximum count, the rate-limitation will just continue @@ -271,6 +415,9 @@ void runGC(const time_t now, time_t *lastGCrun, const bool flush) memset(tail, 0, (counters->queries_MAX - counters->queries)*sizeof(queriesData)); } + // Recycle old clients and domains + recycle(); + // Determine if overTime memory needs to get moved moveOverTimeMemory(mintime); diff --git a/src/resolve.c b/src/resolve.c index 6eead4a37..0f0c47de7 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -568,7 +568,7 @@ static void resolveUpstreams(const bool onlynew) upstreamsData* upstream = getUpstream(upstreamID, true); if(upstream == NULL) { - log_warn("Unable to get upstream pointer with ID %i in resolveUpstreams(), skipping...", upstreamID); + // This is not a fatal error, as the upstream may have been recycled skipped++; unlock_shm(); continue; diff --git a/src/shmem.c b/src/shmem.c index d30c5676a..415305f69 100644 --- a/src/shmem.c +++ b/src/shmem.c @@ -273,7 +273,7 @@ bool strcmp_escaped(const char *a, const char *b) } -size_t addstr(const char *input) +size_t _addstr(const char *input, const char *func, const int line, const char *file) { if(input == NULL) { @@ -308,11 +308,26 @@ size_t addstr(const char *input) char *str = str_escape(input, &N); if(N > 0) - log_info("INFO: FTL replaced %u invalid characters with ~ in the query \"%s\"", N, str); + log_info("FTL replaced %u invalid characters with ~ in the query \"%s\"", N, str); + + // Search buffer for existence of exact same string + char *str_pos = memmem(shm_strings.ptr, shmSettings->next_str_pos, str, len); + if(str_pos != NULL) + { + log_debug(DEBUG_SHMEM, "Reusing existing string \"%s\" at position %zd in %s() (%s:%i)", + str, str_pos - (char*)shm_strings.ptr, func, short_path(file), line); + + // If the string already exists, we can free the memory allocated + // for the escaped string + free(str); + + // Return position of existing string + return (str_pos - (char*)shm_strings.ptr); + } // Debugging output - log_debug(DEBUG_SHMEM, "Adding \"%s\" (len %zu) to buffer. next_str_pos is %u", - str, len, shmSettings->next_str_pos); + log_debug(DEBUG_SHMEM, "Adding \"%s\" (len %zu) to buffer in %s() (%s:%i), next_str_pos is %u", + str, len, func, short_path(file), line, shmSettings->next_str_pos); // Copy the C string pointed by str into the shared string buffer strncpy(&((char*)shm_strings.ptr)[shmSettings->next_str_pos], str, len); @@ -1121,11 +1136,20 @@ queriesData* _getQuery(int queryID, bool checkMagic, int line, const char *func, return NULL; } - if(check_range(queryID, counters->queries_MAX, "query", func, line, file) && - check_magic(queryID, checkMagic, queries[queryID].magic, "query", func, line, file)) - return &queries[queryID]; - else + // Check allowed range + if(!check_range(queryID, counters->queries_MAX, "query", func, line, file)) return NULL; + + // May have been recycled, do not return recycled queries if we are checking + // the magic byte + if(checkMagic && queries[queryID].magic == 0x00) + return NULL; + + // Check magic byte + if(check_magic(queryID, checkMagic, queries[queryID].magic, "query", func, line, file)) + return &queries[queryID]; + + return NULL; } clientsData* _getClient(int clientID, bool checkMagic, int line, const char *func, const char *file) @@ -1146,11 +1170,20 @@ clientsData* _getClient(int clientID, bool checkMagic, int line, const char *fun return NULL; } - if(check_range(clientID, counters->clients_MAX, "client", func, line, file) && - check_magic(clientID, checkMagic, clients[clientID].magic, "client", func, line, file)) - return &clients[clientID]; - else + // Check allowed range + if(!check_range(clientID, counters->clients_MAX, "client", func, line, file)) + return NULL; + + // May have been recycled, do not return recycled clients if we are checking + // the magic byte + if(checkMagic && clients[clientID].magic == 0x00) return NULL; + + // Check magic byte + if(check_magic(clientID, checkMagic, clients[clientID].magic, "client", func, line, file)) + return &clients[clientID]; + + return NULL; } domainsData* _getDomain(int domainID, bool checkMagic, int line, const char *func, const char *file) @@ -1171,11 +1204,20 @@ domainsData* _getDomain(int domainID, bool checkMagic, int line, const char *fun return NULL; } - if(check_range(domainID, counters->domains_MAX, "domain", func, line, file) && - check_magic(domainID, checkMagic, domains[domainID].magic, "domain", func, line, file)) - return &domains[domainID]; - else + // Check allowed range + if(!check_range(domainID, counters->domains_MAX, "domain", func, line, file)) + return NULL; + + // May have been recycled, do not return recycled domains if we are checking + // the magic byte + if(checkMagic && domains[domainID].magic == 0x00) return NULL; + + // Check magic byte + if(check_magic(domainID, checkMagic, domains[domainID].magic, "domain", func, line, file)) + return &domains[domainID]; + + return NULL; } upstreamsData* _getUpstream(int upstreamID, bool checkMagic, int line, const char *func, const char *file) @@ -1196,11 +1238,20 @@ upstreamsData* _getUpstream(int upstreamID, bool checkMagic, int line, const cha return NULL; } - if(check_range(upstreamID, counters->upstreams_MAX, "upstream", func, line, file) && - check_magic(upstreamID, checkMagic, upstreams[upstreamID].magic, "upstream", func, line, file)) - return &upstreams[upstreamID]; - else + // Check allowed range + if(!check_range(upstreamID, counters->upstreams_MAX, "upstream", func, line, file)) + return NULL; + + // May have been recycled, do not return recycled upstreams if we are checking + // the magic byte + if(checkMagic && upstreams[upstreamID].magic == 0x00) return NULL; + + // Check magic byte + if(check_magic(upstreamID, checkMagic, upstreams[upstreamID].magic, "upstream", func, line, file)) + return &upstreams[upstreamID]; + + return NULL; } DNSCacheData* _getDNSCache(int cacheID, bool checkMagic, int line, const char *func, const char *file) @@ -1221,9 +1272,18 @@ DNSCacheData* _getDNSCache(int cacheID, bool checkMagic, int line, const char *f return NULL; } - if(check_range(cacheID, counters->dns_cache_MAX, "dns_cache", func, line, file) && - check_magic(cacheID, checkMagic, dns_cache[cacheID].magic, "dns_cache", func, line, file)) - return &dns_cache[cacheID]; - else + // Check allowed range + if(!check_range(cacheID, counters->dns_cache_MAX, "dns_cache", func, line, file)) return NULL; + + // May have been recycled, do not return recycled upstreams if we are checking + // the magic byte + if(checkMagic && dns_cache[cacheID].magic == 0x00) + return NULL; + + // Check magic byte + if(check_magic(cacheID, checkMagic, dns_cache[cacheID].magic, "dns_cache", func, line, file)) + return &dns_cache[cacheID]; + + return NULL; } diff --git a/src/shmem.h b/src/shmem.h index 5b2a9a5d5..f92de6545 100644 --- a/src/shmem.h +++ b/src/shmem.h @@ -111,7 +111,8 @@ void _unlock_shm(const char* func, const int line, const char* file); bool init_shmem(void); void destroy_shmem(void); -size_t addstr(const char *str); +#define addstr(str) _addstr(str, __FUNCTION__, __LINE__, __FILE__) +size_t _addstr(const char *str, const char *func, const int line, const char *file); #define getstr(pos) _getstr(pos, __FUNCTION__, __LINE__, __FILE__) const char *_getstr(const size_t pos, const char *func, const int line, const char *file); From 1dc4449a3488e718b5acdee86c971820f9a81733 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 23 Oct 2023 21:47:36 +0200 Subject: [PATCH 002/221] Improve recycler reporting Signed-off-by: DL6ER --- src/gc.c | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/gc.c b/src/gc.c index 9c147684c..bd5bd9d18 100644 --- a/src/gc.c +++ b/src/gc.c @@ -172,14 +172,13 @@ static void recycle(void) free_cache++; } - log_debug(DEBUG_GC, "Recycler result: %u clients, %u domains and %u cache records are free", - free_clients, free_domains, free_cache); - - log_debug(DEBUG_GC, "Recycled additional %u/%d (max %d) clients, %u/%d (max %d) domains, and %u/%d (max %d) cache records (scanned %d queries)", - clients_recycled, counters->clients, counters->clients_MAX, - domains_recycled, counters->domains, counters->domains_MAX, - cache_recycled, counters->dns_cache_size, counters->dns_cache_MAX, - counters->queries); + log_debug(DEBUG_GC, "Recycler summary: %u/%d (max %d) clients, %u/%d (max %d) domains and %u/%d (max %d) cache records are free", + free_clients, counters->clients, counters->clients_MAX, + free_domains, counters->domains, counters->domains_MAX, + free_cache, counters->dns_cache_size, counters->dns_cache_MAX); + + log_debug(DEBUG_GC, "Recycled additional %u clients, %u domains, and %u cache records (scanned %d queries)", + clients_recycled, domains_recycled, cache_recycled, counters->queries); } } From c4fd7dd04fdf4409b261c1abd3bc45656377aca2 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 26 Oct 2023 18:15:06 +0200 Subject: [PATCH 003/221] Do not issue a warning if encountering a recycled client during periodic name resolution Signed-off-by: DL6ER --- src/resolve.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/resolve.c b/src/resolve.c index 0f0c47de7..28733cc74 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -442,9 +442,9 @@ static void resolveClients(const bool onlynew, const bool force_refreshing) clientsData* client = getClient(clientID, true); if(client == NULL) { - log_warn("Unable to get client pointer (1) with ID %i in resolveClients(), skipping...", clientID); - skipped++; + // Client has been recycled, skip it unlock_shm(); + skipped++; continue; } @@ -452,6 +452,7 @@ static void resolveClients(const bool onlynew, const bool force_refreshing) if(client->flags.aliasclient) { unlock_shm(); + skipped++; continue; } @@ -467,6 +468,7 @@ static void resolveClients(const bool onlynew, const bool force_refreshing) getstr(ippos), getstr(oldnamepos), (int)(now - client->lastQuery)); unlock_shm(); + skipped++; continue; } From 0b4abb2bd33aa4da4a178f3c83beb647922ec54e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 26 Oct 2023 19:13:57 +0200 Subject: [PATCH 004/221] Optimize status and reply handling in the code. Add status object to /api/stats/summary Signed-off-by: DL6ER --- src/api/docs/content/specs/stats.yaml | 76 +++++++++++++++++++++++++++ src/api/stats.c | 4 ++ src/database/query-table.c | 37 ++++++------- src/datastructure.c | 62 +++++++++++++--------- src/datastructure.h | 5 +- src/dnsmasq_interface.c | 11 ++-- src/gc.c | 11 ++-- src/overTime.c | 9 ---- 8 files changed, 145 insertions(+), 70 deletions(-) diff --git a/src/api/docs/content/specs/stats.yaml b/src/api/docs/content/specs/stats.yaml index 60fa3f69d..b2dfba01a 100644 --- a/src/api/docs/content/specs/stats.yaml +++ b/src/api/docs/content/specs/stats.yaml @@ -406,6 +406,82 @@ components: type: integer description: Queries of remaining types example: 845 + status: + type: object + description: Number of individual queries (by status) + properties: + UNKNOWN: + type: integer + description: Type UNKNOWN queries + example: 3 + GRAVITY: + type: integer + description: Type GRAVITY queries + example: 72 + FORWARDED: + type: integer + description: Type FORWARDED queries + example: 533 + CACHE: + type: integer + description: Type CACHE queries + example: 32 + REGEX: + type: integer + description: Type REGEX queries + example: 84 + DENYLIST: + type: integer + description: Type DENYLIST queries + example: 31 + EXTERNAL_BLOCKED_IP: + type: integer + description: Type EXTERNAL_BLOCKED_IP queries + example: 0 + EXTERNAL_BLOCKED_NULL: + type: integer + description: Type EXTERNAL_BLOCKED_NULL queries + example: 0 + EXTERNAL_BLOCKED_NXRA: + type: integer + description: Type EXTERNAL_BLOCKED_NXRA queries + example: 0 + GRAVITY_CNAME: + type: integer + description: Type GRAVITY_CNAME queries + example: 0 + REGEX_CNAME: + type: integer + description: Type REGEX_CNAME queries + example: 0 + DENYLIST_CNAME: + type: integer + description: Type DENYLIST_CNAME queries + example: 0 + RETRIED: + type: integer + description: Type RETRIED queries + example: 0 + RETRIED_DNSSEC: + type: integer + description: Type RETRIED_DNSSEC queries + example: 0 + IN_PROGRESS: + type: integer + description: Type IN_PROGRESS queries + example: 0 + DBBUSY: + type: integer + description: Type DBBUSY queries + example: 0 + SPECIAL_DOMAIN: + type: integer + description: Type SPECIAL_DOMAIN queries + example: 0 + CACHE_STALE: + type: integer + description: Type CACHE_STALE queries + example: 0 replies: type: object description: Number of individual replies diff --git a/src/api/stats.c b/src/api/stats.c index 0c06b1f78..a9874750f 100644 --- a/src/api/stats.c +++ b/src/api/stats.c @@ -100,6 +100,10 @@ int api_stats_summary(struct ftl_conn *api) return ret; JSON_ADD_ITEM_TO_OBJECT(queries, "types", types); + cJSON *statuses = JSON_NEW_OBJECT(); + for(enum query_status status = 0; status < QUERY_STATUS_MAX; status++) + JSON_ADD_NUMBER_TO_OBJECT(statuses, get_query_status_str(status), counters->status[status]); + JSON_ADD_ITEM_TO_OBJECT(queries, "status", statuses); cJSON *replies = JSON_NEW_OBJECT(); for(enum reply_type reply = 0; reply type = TYPE_OTHER; query->qtype = type - 100; } + counters->querytype[query->type]++; // Status is set below query->domainID = domainID; @@ -1084,13 +1085,9 @@ void DB_read_queries(void) clientsData *client = getClient(clientID, true); client->lastQuery = queryTimeStamp; - // Handle type counters - if(type >= TYPE_A && type < TYPE_MAX) - counters->querytype[type]++; - // Update overTime data overTime[timeidx].total++; - // Update overTime data structure with the new client + // Update client's overTime data structure change_clientcount(client, 0, 0, timeidx, 1); // Increase DNS queries counter @@ -1126,12 +1123,8 @@ void DB_read_queries(void) cache->domainlist_id = sqlite3_column_int(stmt, 7); } - // Increment status counters, we first have to add one to the count of - // unknown queries because query_set_status() will subtract from there - // when setting a different status - if(status != QUERY_UNKNOWN) - counters->status[QUERY_UNKNOWN]++; - query_set_status(query, status); + // Increment status counters + query_set_status_init(query, status); // Do further processing based on the query status we read from the database switch(status) diff --git a/src/datastructure.c b/src/datastructure.c index 7998d99bc..78c9add46 100644 --- a/src/datastructure.c +++ b/src/datastructure.c @@ -926,19 +926,27 @@ static const char* __attribute__ ((const)) query_status_str(const enum query_sta return NULL; } -void _query_set_status(queriesData *query, const enum query_status new_status, const char *func, const int line, const char *file) +void _query_set_status(queriesData *query, const enum query_status new_status, const bool init, + const char *func, const int line, const char *file) { // Debug logging if(config.debug.status.v.b) { - const char *oldstr = query->status < QUERY_STATUS_MAX ? query_status_str(query->status) : "INVALID"; - if(query->status == new_status) + if(init) { + const char *newstr = new_status < QUERY_STATUS_MAX ? query_status_str(new_status) : "INVALID"; + log_debug(DEBUG_STATUS, "Query %i: status initialized: %s (%d) in %s() (%s:%i)", + query->id, newstr, new_status, func, short_path(file), line); + } + else if(query->status == new_status) + { + const char *oldstr = query->status < QUERY_STATUS_MAX ? query_status_str(query->status) : "INVALID"; log_debug(DEBUG_STATUS, "Query %i: status unchanged: %s (%d) in %s() (%s:%i)", query->id, oldstr, query->status, func, short_path(file), line); } else { + const char *oldstr = query->status < QUERY_STATUS_MAX ? query_status_str(query->status) : "INVALID"; const char *newstr = new_status < QUERY_STATUS_MAX ? query_status_str(new_status) : "INVALID"; log_debug(DEBUG_STATUS, "Query %i: status changed: %s (%d) -> %s (%d) in %s() (%s:%i)", query->id, oldstr, query->status, newstr, new_status, func, short_path(file), line); @@ -949,30 +957,36 @@ void _query_set_status(queriesData *query, const enum query_status new_status, c if(new_status >= QUERY_STATUS_MAX) return; - // Update counters - if(query->status != new_status) + const enum query_status old_status = query->status; + if(old_status == new_status && !init) { - counters->status[query->status]--; - counters->status[new_status]++; - - const int timeidx = getOverTimeID(query->timestamp); - if(is_blocked(query->status)) - overTime[timeidx].blocked--; - if(is_blocked(new_status)) - overTime[timeidx].blocked++; - - if(query->status == QUERY_CACHE) - overTime[timeidx].cached--; - if(new_status == QUERY_CACHE) - overTime[timeidx].cached++; - - if(query->status == QUERY_FORWARDED) - overTime[timeidx].forwarded--; - if(new_status == QUERY_FORWARDED) - overTime[timeidx].forwarded++; + // Nothing to do + return; } - // Update status + // else: update global counters, ... + if(!init) + counters->status[old_status]--; + counters->status[new_status]++; + + // ... update overTime counters, ... + const int timeidx = getOverTimeID(query->timestamp); + if(is_blocked(old_status) && !init) + overTime[timeidx].blocked--; + if(is_blocked(new_status)) + overTime[timeidx].blocked++; + + if(old_status == QUERY_CACHE && !init) + overTime[timeidx].cached--; + if(new_status == QUERY_CACHE) + overTime[timeidx].cached++; + + if(old_status == QUERY_FORWARDED && !init) + overTime[timeidx].forwarded--; + if(new_status == QUERY_FORWARDED) + overTime[timeidx].forwarded++; + + // ... and set new status query->status = new_status; } diff --git a/src/datastructure.h b/src/datastructure.h index c115064e3..66da308fe 100644 --- a/src/datastructure.h +++ b/src/datastructure.h @@ -134,8 +134,9 @@ const char *get_cached_statuslist(void) __attribute__ ((pure)); int get_blocked_count(void) __attribute__ ((pure)); int get_forwarded_count(void) __attribute__ ((pure)); int get_cached_count(void) __attribute__ ((pure)); -#define query_set_status(query, new_status) _query_set_status(query, new_status, __FUNCTION__, __LINE__, __FILE__) -void _query_set_status(queriesData *query, const enum query_status new_status, const char *func, const int line, const char *file); +#define query_set_status(query, new_status) _query_set_status(query, new_status, false, __FUNCTION__, __LINE__, __FILE__) +#define query_set_status_init(query, new_status) _query_set_status(query, new_status, true, __FUNCTION__, __LINE__, __FILE__) +void _query_set_status(queriesData *query, const enum query_status new_status, const bool init, const char *func, const int line, const char *file); void FTL_reload_all_domainlists(void); void FTL_reset_per_client_domain_data(void); diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index 0211afa86..5987a9910 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -746,12 +746,12 @@ bool _FTL_new_query(const unsigned int flags, const char *name, query->magic = MAGICBYTE; query->timestamp = querytimestamp; query->type = querytype; + counters->querytype[querytype]++; query->qtype = qtype; query->id = id; // Has to be set before calling query_set_status() // This query is unknown as long as no reply has been found and analyzed - counters->status[QUERY_UNKNOWN]++; - query_set_status(query, QUERY_UNKNOWN); + query_set_status_init(query, QUERY_UNKNOWN); query->domainID = domainID; query->clientID = clientID; // Initialize database field, will be set when the query is stored in the long-term DB @@ -797,9 +797,6 @@ bool _FTL_new_query(const unsigned int flags, const char *name, client->lastQuery = querytimestamp; client->numQueriesARP++; - // Update counters - counters->querytype[querytype]++; - // Process interface information of client (if available) // Skip interface name length 1 to skip "-". No real interface should // have a name with a length of 1... @@ -2528,7 +2525,6 @@ static void FTL_upstream_error(const union all_addr *addr, const unsigned int fl if(query->reply == REPLY_OTHER) log_debug(DEBUG_QUERIES, " Unknown rcode = %i", addr->log.rcode); - if(addr->log.ede != EDE_UNSET) log_debug(DEBUG_QUERIES, " EDE: %s (1/%d)", edestr(addr->log.ede), addr->log.ede); @@ -3364,7 +3360,10 @@ void FTL_multiple_replies(const int id, int *firstID) log_debug(DEBUG_QUERIES, "**** sending reply %d also to %d", *firstID, queryID); // Copy relevant information over + counters->reply[duplicated_query->reply]--; duplicated_query->reply = source_query->reply; + counters->reply[duplicated_query->reply]++; + duplicated_query->dnssec = source_query->dnssec; duplicated_query->flags.complete = true; duplicated_query->CNAME_domainID = source_query->CNAME_domainID; diff --git a/src/gc.c b/src/gc.c index 322a92ab0..dbb73ffe7 100644 --- a/src/gc.c +++ b/src/gc.c @@ -189,7 +189,6 @@ void runGC(const time_t now, time_t *lastGCrun, const bool flush) case QUERY_RETRIED: // (fall through) case QUERY_RETRIED_DNSSEC: // Forwarded to an upstream DNS server - // Adjusting counters is done below in moveOverTimeMemory() break; case QUERY_CACHE: case QUERY_CACHE_STALE: @@ -206,7 +205,6 @@ void runGC(const time_t now, time_t *lastGCrun, const bool flush) case QUERY_DENYLIST_CNAME: // Exactly denied domain in CNAME chain (fall through) case QUERY_DBBUSY: // Blocked because gravity database was busy case QUERY_SPECIAL_DOMAIN: // Blocked by special domain handling - //counters->blocked--; overTime[timeidx].blocked--; if(domain != NULL) domain->blockedcount--; @@ -220,16 +218,15 @@ void runGC(const time_t now, time_t *lastGCrun, const bool flush) break; } - // Update reply countersthread_running[GC] = false; + // Update reply counters counters->reply[query->reply]--; // Update type counters - if(query->type >= TYPE_A && query->type < TYPE_MAX) - counters->querytype[query->type]--; + counters->querytype[query->type]--; // Subtract UNKNOWN from the counters before - // setting the status if different. This ensure - // we are not counting them at all. + // setting the status if different. + // Minus one here and plus one below = net zero if(query->status != QUERY_UNKNOWN) counters->status[QUERY_UNKNOWN]--; diff --git a/src/overTime.c b/src/overTime.c index d29f6ed93..5148b908f 100644 --- a/src/overTime.c +++ b/src/overTime.c @@ -190,15 +190,6 @@ void moveOverTimeMemory(const time_t mintime) &overTime[moveOverTime], remainingSlots*sizeof(*overTime)); - // Correct time indices of queries. This is necessary because we just moved the slot this index points to - for(int queryID = 0; queryID < counters->queries; queryID++) - { - // Get query pointer - queriesData* query = getQuery(queryID, true); - if(query == NULL) - continue; - } - // Move client-specific overTime memory for(int clientID = 0; clientID < counters->clients; clientID++) { From f1c59db0f684e8f282c0348a086cf5e6a282be0d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 3 Nov 2023 09:46:36 +0100 Subject: [PATCH 005/221] Declare pihole.toml as UTF-8 document. we add a compile-time switch to use ASCII with UTF-8 escaping instead in case this is a necessity for anyone (I don't really expect this in the third millenial but we also know people are still using Windows XP on the web...) Signed-off-by: DL6ER --- src/config/config.h | 5 +++ src/config/toml_helper.c | 52 +++++++++++++++++++++++++------ src/config/toml_writer.c | 12 ++++--- test/api/libs/responseVerifyer.py | 2 +- 4 files changed, 56 insertions(+), 15 deletions(-) diff --git a/src/config/config.h b/src/config/config.h index c230627ee..975676eaf 100644 --- a/src/config/config.h +++ b/src/config/config.h @@ -33,6 +33,11 @@ // This static string represents an unchanged password #define PASSWORD_VALUE "********" +// Remove the following line to disable the use of UTF-8 in the config file +// As consequence, the config file will be written in ASCII and all non-ASCII +// characters will be replaced by their UTF-8 escape sequences (UCS-2) +#define TOML_UTF8 + union conf_value { bool b; // boolean value int i; // integer value diff --git a/src/config/toml_helper.c b/src/config/toml_helper.c index 7f40b683f..30a595b92 100644 --- a/src/config/toml_helper.c +++ b/src/config/toml_helper.c @@ -123,17 +123,51 @@ static void printTOMLstring(FILE *fp, const char *s, const bool toml) continue; } - // Escape special characters + // Escape special characters with simple escape sequences switch (ch) { - case 0x08: fprintf(fp, "\\b"); continue; - case 0x09: fprintf(fp, "\\t"); continue; - case 0x0a: fprintf(fp, "\\n"); continue; - case 0x0c: fprintf(fp, "\\f"); continue; - case 0x0d: fprintf(fp, "\\r"); continue; - case '"': fprintf(fp, "\\\""); continue; - case '\\': fprintf(fp, "\\\\"); continue; - default: fprintf(fp, "\\0x%02x", ch & 0xff); continue; + case '\b': fputs("\\b", fp); continue; + case '\t': fputs("\\t", fp); continue; + case '\n': fputs("\\n", fp); continue; + case '\f': fputs("\\f", fp); continue; + case '\r': fputs("\\r", fp); continue; + case '"': fputs("\\\"", fp); continue; + case '\\': fputs("\\\\", fp); continue; + } + +#ifndef TOML_UTF8 + // The Universal Coded Character Set (UCS, Unicode) is a + // standard set of characters defined by the international + // standard ISO/IEC 10646, Information technology — Universal + // Coded Character Set (UCS) (plus amendments to that standard), + // which is the basis of many character encodings, improving as + // characters from previously unrepresented typing systems are + // added. + // The following code converts a UTF-8 character to UCS and + // prints it as \UXXXXXXXX + int64_t ucs; + int bytes = toml_utf8_to_ucs(s, len, &ucs); + if(bytes > 0) + { + // Print 4-byte UCS as \UXXXXXXXX + fprintf(fp, "\\U%08X", (uint32_t)ucs); + // Advance string pointer + s += bytes - 1; + // Decrease remaining string length + len -= bytes - 1; + continue; } +#else + // Escape all other control characters as short 2-byte + // UCS sequences + if(iscntrl(ch)) + { + fprintf(fp, "\\u%04X", ch); + continue; + } + + // Print remaining characters as is + putc(ch, fp); +#endif } if(toml) fprintf(fp, "\""); } diff --git a/src/config/toml_writer.c b/src/config/toml_writer.c index 5cf96822f..92f57a0e2 100644 --- a/src/config/toml_writer.c +++ b/src/config/toml_writer.c @@ -41,15 +41,17 @@ bool writeFTLtoml(const bool verbose) log_info("Writing config file"); // Write header - fputs("# This file is managed by pihole-FTL\n#\n", fp); - fputs("# Do not edit the file while FTL is\n", fp); - fputs("# running or your changes may be overwritten\n#\n", fp); + fprintf(fp, "# Pi-hole configuration file (%s)\n", get_FTL_version()); +#ifdef TOML_UTF8 + fputs("# Encoding: UTF-8\n", fp); +#else + fputs("# Encoding: ASCII + UCS\n", fp); +#endif + fputs("# This file is managed by pihole-FTL\n", fp); char timestring[TIMESTR_SIZE] = ""; get_timestr(timestring, time(NULL), false, false); fputs("# Last updated on ", fp); fputs(timestring, fp); - fputs("\n# by FTL ", fp); - fputs(get_FTL_version(), fp); fputs("\n\n", fp); // Iterate over configuration and store it into the file diff --git a/test/api/libs/responseVerifyer.py b/test/api/libs/responseVerifyer.py index 77323beb3..128e80695 100644 --- a/test/api/libs/responseVerifyer.py +++ b/test/api/libs/responseVerifyer.py @@ -159,7 +159,7 @@ def verify_endpoint(self, endpoint: str): if expected_file not in zipfile_obj.namelist(): self.errors.append("File " + expected_file + " is missing in received archive.") pihole_toml = zipfile_obj.read("etc/pihole/pihole.toml") - if not pihole_toml.startswith(b"# This file is managed by pihole-FTL"): + if not pihole_toml.startswith(b"# Pi-hole configuration file (v"): self.errors.append("Received ZIP file's pihole.toml starts with wrong header") except Exception as err: self.errors.append("Error during ZIP analysis: " + str(err)) From fe8798e1b1258491baafd3120226dfef0611adcb Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 3 Nov 2023 12:22:26 +0100 Subject: [PATCH 006/221] Add tests for international custom DNS records Signed-off-by: DL6ER --- src/daemon.c | 15 +++++++++++++++ src/daemon.h | 1 + src/dnsmasq/dnsmasq.c | 5 +---- src/main.c | 3 +++ test/pihole.toml | 5 ++++- test/test_suite.bats | 22 +++++++++++++++++++++- 6 files changed, 45 insertions(+), 6 deletions(-) diff --git a/src/daemon.c b/src/daemon.c index 9cc65a813..60b659265 100644 --- a/src/daemon.c +++ b/src/daemon.c @@ -35,6 +35,8 @@ #include "webserver/webserver.h" // free_api() #include "api/api.h" +// setlocale() +#include pthread_t threads[THREADS_MAX] = { 0 }; bool resolver_ready = false; @@ -444,3 +446,16 @@ bool ipv6_enabled(void) // IPv6-capable interface return true; } + +void init_locale(void) +{ + // Set locale to system default, needed for libidn to work properly + // Without this, libidn will not be able to convert UTF-8 to ASCII + // (error message "Character encoding conversion error") + setlocale(LC_ALL, ""); + + // Set locale for numeric values to C to ensure that we always use + // the dot as decimal separator (even if the system locale uses a + // comma, e.g., in German) + setlocale(LC_NUMERIC, "C"); +} \ No newline at end of file diff --git a/src/daemon.h b/src/daemon.h index a2212b103..fb96c63e1 100644 --- a/src/daemon.h +++ b/src/daemon.h @@ -24,6 +24,7 @@ void set_nice(void); void calc_cpu_usage(void); float get_cpu_percentage(void) __attribute__((pure)); bool ipv6_enabled(void); +void init_locale(void); #include #include diff --git a/src/dnsmasq/dnsmasq.c b/src/dnsmasq/dnsmasq.c index 067f54fc6..ef4ab3e91 100644 --- a/src/dnsmasq/dnsmasq.c +++ b/src/dnsmasq/dnsmasq.c @@ -81,10 +81,7 @@ int main_dnsmasq (int argc, char **argv) #endif #if defined(HAVE_IDN) || defined(HAVE_LIBIDN2) || defined(LOCALEDIR) - setlocale(LC_ALL, ""); - /*** Pi-hole modification ***/ - setlocale(LC_NUMERIC, "C"); - /****************************/ + /*** Pi-hole modification: Locale is already initialized in main.c ***/ #endif #ifdef LOCALEDIR bindtextdomain("dnsmasq", LOCALEDIR); diff --git a/src/main.c b/src/main.c index 4f80e63b4..22e57fe5f 100644 --- a/src/main.c +++ b/src/main.c @@ -45,6 +45,9 @@ jmp_buf exit_jmp; int main (int argc, char *argv[]) { + // Initialize locale (needed for libidn) + init_locale(); + // Get user pihole-FTL is running as // We store this in a global variable // such that the log routine can access diff --git a/test/pihole.toml b/test/pihole.toml index 2fc68e648..565ba7a6b 100644 --- a/test/pihole.toml +++ b/test/pihole.toml @@ -100,7 +100,10 @@ # # Possible values are: # Array of custom DNS records each one in HOSTS form: "IP HOSTNAME" - hosts = [] + hosts = [ + "1.1.1.1 abc-custom.com def-custom.de", + "2.2.2.2 äste.com steä.com" + ] ### CHANGED, default = [] # If set, A and AAAA queries for plain names, without dots or domain parts, are never # forwarded to upstream nameservers diff --git a/test/test_suite.bats b/test/test_suite.bats index 9355df664..08245f644 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -1253,6 +1253,26 @@ [[ "${lines[0]}" == "192.168.1.7" ]] } +@test "Custom DNS records: Multiple domains per line are accepted" { + run bash -c "dig A abc-custom.com +short @127.0.0.1" + printf "%s\n" "${lines[@]}" + [[ "${lines[0]}" == "1.1.1.1" ]] + run bash -c "dig A def-custom.de +short @127.0.0.1" + printf "%s\n" "${lines[@]}" + [[ "${lines[0]}" == "1.1.1.1" ]] +} + +@test "Custom DNS records: International domains are converted to IDNA form" { + # äste.com ---> xn--ste-pla.com + run bash -c "dig A xn--ste-pla.com +short @127.0.0.1" + printf "%s\n" "${lines[@]}" + [[ "${lines[0]}" == "2.2.2.2" ]] + # steä.com -> xn--ste-sla.com + run bash -c "dig A xn--ste-sla.com +short @127.0.0.1" + printf "%s\n" "${lines[@]}" + [[ "${lines[0]}" == "2.2.2.2" ]] +} + @test "Environmental variable is favored over config file" { # The config file has -10 but we set FTLCONF_misc_nice="-11" run bash -c 'grep -B1 "nice = -11" /etc/pihole/pihole.toml' @@ -1410,7 +1430,7 @@ [[ "${lines[0]}" == "PI.HOLE" ]] run bash -c './pihole-FTL --config dns.hosts' printf "%s\n" "${lines[@]}" - [[ "${lines[0]}" == "[]" ]] + [[ "${lines[0]}" == "[ 1.1.1.1 abc-custom.com def-custom.de, 2.2.2.2 äste.com steä.com ]" ]] run bash -c './pihole-FTL --config webserver.port' printf "%s\n" "${lines[@]}" [[ "${lines[0]}" == "80,[::]:80,443s" ]] From a96c283c0cc19ccce2504cff11e79bbfe2c603f3 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 3 Nov 2023 19:25:21 +0100 Subject: [PATCH 007/221] Add authentication via query string Signed-off-by: DL6ER --- src/api/auth.c | 16 ++++++++++++++++ test/api/libs/FTLAPI.py | 13 ++++++++++--- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/src/api/auth.c b/src/api/auth.c index d48d171be..2f95b337a 100644 --- a/src/api/auth.c +++ b/src/api/auth.c @@ -140,6 +140,7 @@ int check_client_auth(struct ftl_conn *api, const bool is_api) } } + // If not, does the client provide a session ID via COOKIE? bool cookie_auth = false; if(!sid_avail) { @@ -151,7 +152,22 @@ int check_client_auth(struct ftl_conn *api, const bool is_api) // Mark SID as available sid_avail = true; } + } + // If not, does the client provide a session ID via URI? + if(!sid_avail && api->request->query_string && GET_VAR("sid", sid, api->request->query_string) > 0) + { + // "+" may have been replaced by " ", undo this here + for(unsigned int i = 0; i < SID_SIZE; i++) + if(sid[i] == ' ') + sid[i] = '+'; + + // Zero terminate SID string + sid[SID_SIZE-1] = '\0'; + // Mention source of SID + sid_source = "URI"; + // Mark SID as available + sid_avail = true; } if(!sid_avail) diff --git a/test/api/libs/FTLAPI.py b/test/api/libs/FTLAPI.py index a71bcec23..c8d19e8b9 100644 --- a/test/api/libs/FTLAPI.py +++ b/test/api/libs/FTLAPI.py @@ -15,6 +15,7 @@ from typing import List import json from hashlib import sha256 +import urllib.parse url = "http://pi.hole/api/auth" @@ -23,6 +24,7 @@ class AuthenticationMethods(Enum): HEADER = 1 BODY = 2 COOKIE = 3 + QUERY_STR = 4 # Class to query the FTL API class FTLAPI(): @@ -103,13 +105,18 @@ def get_jsondata_headers_cookies(self, authenticate: AuthenticationMethods): def GET(self, uri: str, params: List[str] = [], expected_mimetype: str = "application/json", authenticate: AuthenticationMethods = AuthenticationMethods.BODY): self.errors = [] try: + # Get json_data, headers and cookies + json_data, headers, cookies = self.get_jsondata_headers_cookies(authenticate) + + # Add session ID to the request if authenticating via query string + if self.auth_method == AuthenticationMethods.QUERY_STR.name: + encoded_sid = urllib.parse.quote(self.session['sid'], safe='') + params.append("sid=" + encoded_sid) + # Add parameters to the URI (if any) if len(params) > 0: uri = uri + "?" + "&".join(params) - # Get json_data, headers and cookies - json_data, headers, cookies = self.get_jsondata_headers_cookies(authenticate) - if self.verbose: print("GET " + self.api_url + uri + " with json_data: " + json.dumps(json_data)) From ff3b8db24f3d30d7d6a204dcf40549a96da21617 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 4 Nov 2023 07:27:08 +0100 Subject: [PATCH 008/221] Use webserver.domain as "account" in the TOTP QR code Signed-off-by: DL6ER --- src/api/2fa.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/2fa.c b/src/api/2fa.c index 1b608c3fa..5d6148d94 100644 --- a/src/api/2fa.c +++ b/src/api/2fa.c @@ -282,7 +282,7 @@ int generateTOTP(struct ftl_conn *api) // Create JSON object cJSON *tjson = cJSON_CreateObject(); JSON_REF_STR_IN_OBJECT(tjson, "type", "totp"); - JSON_REF_STR_IN_OBJECT(tjson, "account", "pi.hole"); + JSON_REF_STR_IN_OBJECT(tjson, "account", config.webserver.domain.v.s); JSON_REF_STR_IN_OBJECT(tjson, "issuer", "Pi-hole%20API"); JSON_REF_STR_IN_OBJECT(tjson, "algorithm", "SHA1"); JSON_ADD_NUMBER_TO_OBJECT(tjson, "digits", RFC6238_DIGITS); From 39f350af5d51e42a3c858edfaddef52c58cd09e0 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 4 Nov 2023 08:14:03 +0100 Subject: [PATCH 009/221] Only add dhcp.leaseTime if it actually set. If not given, the default lease time is one hour for IPv4 and one day for IPv6 (dnsmasq defaults, see their man page). Signed-off-by: DL6ER --- src/config/dnsmasq_config.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/config/dnsmasq_config.c b/src/config/dnsmasq_config.c index e727e3a7a..20a4a91a0 100644 --- a/src/config/dnsmasq_config.c +++ b/src/config/dnsmasq_config.c @@ -419,11 +419,13 @@ bool __attribute__((const)) write_dnsmasq_config(struct config *conf, bool test_ fputs("# DHCP server setting\n", pihole_conf); fputs("dhcp-authoritative\n", pihole_conf); fputs("dhcp-leasefile="DHCPLEASESFILE"\n", pihole_conf); - fprintf(pihole_conf, "dhcp-range=%s,%s,%s\n", + fprintf(pihole_conf, "dhcp-range=%s,%s", conf->dhcp.start.v.s, - conf->dhcp.end.v.s, - conf->dhcp.leaseTime.v.s); - fprintf(pihole_conf, "dhcp-option=option:router,%s\n", + conf->dhcp.end.v.s); + // Lease time is optional, only add it if it is set + if(strlen(conf->dhcp.leaseTime.v.s) > 0) + fprintf(pihole_conf, ",%s", conf->dhcp.leaseTime.v.s); + fprintf(pihole_conf, "\ndhcp-option=option:router,%s\n", conf->dhcp.router.v.s); if(conf->dhcp.rapidCommit.v.b) From 6f2c6fe8011ab0e9286fe6011efcaa2d6aec5794 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 4 Nov 2023 08:20:09 +0100 Subject: [PATCH 010/221] Add dhcp.netmask (type IPv4 address) and change the type of dhcp.{start,end,router} from string to IPv4 address to allow detection of invalid settings (e.g., "192.168.1.3000") early on Signed-off-by: DL6ER --- src/api/docs/content/specs/config.yaml | 3 +++ src/config/config.c | 19 +++++++++++++------ src/config/config.h | 3 ++- src/config/dnsmasq_config.c | 15 ++++++++++++--- test/pihole.toml | 11 +++++++++++ 5 files changed, 41 insertions(+), 10 deletions(-) diff --git a/src/api/docs/content/specs/config.yaml b/src/api/docs/content/specs/config.yaml index db3154d14..8158be03b 100644 --- a/src/api/docs/content/specs/config.yaml +++ b/src/api/docs/content/specs/config.yaml @@ -299,6 +299,8 @@ components: type: string router: type: string + netmask: + type: string domain: type: string leaseTime: @@ -627,6 +629,7 @@ components: end: "192.168.0.250" router: "192.168.0.1" domain: "lan" + netmask: "0.0.0.0" leaseTime: "24h" ipv6: true rapidCommit: true diff --git a/src/config/config.c b/src/config/config.c index 72c364070..184b74f36 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -687,23 +687,23 @@ void initConfig(struct config *conf) conf->dhcp.start.k = "dhcp.start"; conf->dhcp.start.h = "Start address of the DHCP address pool"; conf->dhcp.start.a = cJSON_CreateStringReference(", e.g., \"192.168.0.10\""); - conf->dhcp.start.t = CONF_STRING; + conf->dhcp.start.t = CONF_STRUCT_IN_ADDR; conf->dhcp.start.f = FLAG_RESTART_FTL; - conf->dhcp.start.d.s = (char*)""; + memset(&conf->dhcp.start.d.in_addr, 0, sizeof(struct in_addr)); conf->dhcp.end.k = "dhcp.end"; conf->dhcp.end.h = "End address of the DHCP address pool"; conf->dhcp.end.a = cJSON_CreateStringReference(", e.g., \"192.168.0.250\""); - conf->dhcp.end.t = CONF_STRING; + conf->dhcp.end.t = CONF_STRUCT_IN_ADDR; conf->dhcp.end.f = FLAG_RESTART_FTL; - conf->dhcp.end.d.s = (char*)""; + memset(&conf->dhcp.end.d.in_addr, 0, sizeof(struct in_addr)); conf->dhcp.router.k = "dhcp.router"; conf->dhcp.router.h = "Address of the gateway to be used (typically the address of your router in a home installation)"; conf->dhcp.router.a = cJSON_CreateStringReference(", e.g., \"192.168.0.1\""); - conf->dhcp.router.t = CONF_STRING; + conf->dhcp.router.t = CONF_STRUCT_IN_ADDR; conf->dhcp.router.f = FLAG_RESTART_FTL; - conf->dhcp.router.d.s = (char*)""; + memset(&conf->dhcp.router.d.in_addr, 0, sizeof(struct in_addr)); conf->dhcp.domain.k = "dhcp.domain"; conf->dhcp.domain.h = "The DNS domain used by your Pi-hole"; @@ -712,6 +712,13 @@ void initConfig(struct config *conf) conf->dhcp.domain.f = FLAG_RESTART_FTL | FLAG_ADVANCED_SETTING; conf->dhcp.domain.d.s = (char*)"lan"; + conf->dhcp.netmask.k = "dhcp.netmask"; + conf->dhcp.netmask.h = "The netmask used by your Pi-hole. For directly connected networks (i.e., networks on which the machine running Pi-hole has an interface) the netmask is optional and may be set to \"0.0.0.0\": it will then be determined from the interface configuration itself. For networks which receive DHCP service via a relay agent, we cannot determine the netmask itself, so it should explicitly be specified, otherwise Pi-hole guesses based on the class (A, B or C) of the network address."; + conf->dhcp.netmask.a = cJSON_CreateStringReference(", e.g., \"255.255.255.0\" or \"0.0.0.0\" for auto-discovery"); + conf->dhcp.netmask.t = CONF_STRUCT_IN_ADDR; + conf->dhcp.netmask.f = FLAG_RESTART_FTL | FLAG_ADVANCED_SETTING; + memset(&conf->dhcp.netmask.d.in_addr, 0, sizeof(struct in_addr)); + conf->dhcp.leaseTime.k = "dhcp.leaseTime"; conf->dhcp.leaseTime.h = "If the lease time is given, then leases will be given for that length of time. If not given, the default lease time is one hour for IPv4 and one day for IPv6."; conf->dhcp.leaseTime.a = cJSON_CreateStringReference("The lease time can be in seconds, or minutes (e.g., \"45m\") or hours (e.g., \"1h\") or days (like \"2d\") or even weeks (\"1w\"). You may also use \"infinite\" as string but be aware of the drawbacks"); diff --git a/src/config/config.h b/src/config/config.h index c230627ee..d75ef0e9d 100644 --- a/src/config/config.h +++ b/src/config/config.h @@ -88,7 +88,7 @@ enum conf_type { #define FLAG_PSEUDO_ITEM (1 << 2) #define FLAG_INVALIDATE_SESSIONS (1 << 3) #define FLAG_WRITE_ONLY (1 << 4) -#define FLAG_ENV_VAR (1 << 5) +#define FLAG_ENV_VAR (1 << 5) struct conf_item { const char *k; // item Key @@ -178,6 +178,7 @@ struct config { struct conf_item end; struct conf_item router; struct conf_item domain; + struct conf_item netmask; struct conf_item leaseTime; struct conf_item ipv6; struct conf_item rapidCommit; diff --git a/src/config/dnsmasq_config.c b/src/config/dnsmasq_config.c index 20a4a91a0..210665583 100644 --- a/src/config/dnsmasq_config.c +++ b/src/config/dnsmasq_config.c @@ -419,9 +419,18 @@ bool __attribute__((const)) write_dnsmasq_config(struct config *conf, bool test_ fputs("# DHCP server setting\n", pihole_conf); fputs("dhcp-authoritative\n", pihole_conf); fputs("dhcp-leasefile="DHCPLEASESFILE"\n", pihole_conf); - fprintf(pihole_conf, "dhcp-range=%s,%s", - conf->dhcp.start.v.s, - conf->dhcp.end.v.s); + char start[INET_ADDRSTRLEN] = { 0 }, end[INET_ADDRSTRLEN] = { 0 }; + inet_ntop(AF_INET, &conf->dhcp.start.v.in_addr, start, INET_ADDRSTRLEN); + inet_ntop(AF_INET, &conf->dhcp.end.v.in_addr, end, INET_ADDRSTRLEN); + fprintf(pihole_conf, "dhcp-range=%s,%s", start, end); + // Net mask is optional, only add if it is not 0.0.0.0 + const struct in_addr inaddr_empty = {0}; + if(memcmp(&conf->dhcp.netmask.v.in_addr, &inaddr_empty, sizeof(inaddr_empty)) != 0) + { + char netmask[INET_ADDRSTRLEN] = { 0 }; + inet_ntop(AF_INET, &conf->dhcp.netmask.v.in_addr, netmask, INET_ADDRSTRLEN); + fprintf(pihole_conf, ",%s", conf->dhcp.netmask.v.s); + } // Lease time is optional, only add it if it is set if(strlen(conf->dhcp.leaseTime.v.s) > 0) fprintf(pihole_conf, ",%s", conf->dhcp.leaseTime.v.s); diff --git a/test/pihole.toml b/test/pihole.toml index 2fc68e648..60525e433 100644 --- a/test/pihole.toml +++ b/test/pihole.toml @@ -376,6 +376,17 @@ # domain = "lan" + # The netmask used by your Pi-hole. For directly connected networks (i.e., networks on + # which the machine running Pi-hole has an interface) the netmask is optional and may + # be set to "0.0.0.0": it will then be determined from the interface configuration + # itself. For networks which receive DHCP service via a relay agent, we cannot + # determine the netmask itself, so it should explicitly be specified, otherwise + # Pi-hole guesses based on the class (A, B or C) of the network address. + # + # Possible values are: + # , e.g., "255.255.255.0" or "0.0.0.0" for auto-discovery + netmask = "0.0.0.0" + # If the lease time is given, then leases will be given for that length of time. If not # given, the default lease time is one hour for IPv4 and one day for IPv6. # From 75cd372d0e4c00ad2a3df986ca6d02bc0577092f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 4 Nov 2023 12:12:02 +0100 Subject: [PATCH 011/221] Add string format verification in API checker Signed-off-by: DL6ER --- src/api/docs/content/specs/config.yaml | 8 ++++ src/config/dnsmasq_config.c | 10 +++-- test/api/libs/responseVerifyer.py | 54 ++++++++++++++++++++------ 3 files changed, 57 insertions(+), 15 deletions(-) diff --git a/src/api/docs/content/specs/config.yaml b/src/api/docs/content/specs/config.yaml index 8158be03b..59f24bf25 100644 --- a/src/api/docs/content/specs/config.yaml +++ b/src/api/docs/content/specs/config.yaml @@ -268,8 +268,10 @@ components: type: boolean IPv4: type: string + x-format: ipv4 IPv6: type: string + x-format: ipv6 blocking: type: object properties: @@ -279,8 +281,10 @@ components: type: boolean IPv4: type: string + x-format: ipv4 IPv6: type: string + x-format: ipv6 rateLimit: type: object properties: @@ -295,12 +299,16 @@ components: type: boolean start: type: string + x-format: ipv4 end: type: string + x-format: ipv4 router: type: string + x-format: ipv4 netmask: type: string + x-format: ipv4 domain: type: string leaseTime: diff --git a/src/config/dnsmasq_config.c b/src/config/dnsmasq_config.c index 210665583..3a3894af1 100644 --- a/src/config/dnsmasq_config.c +++ b/src/config/dnsmasq_config.c @@ -419,9 +419,12 @@ bool __attribute__((const)) write_dnsmasq_config(struct config *conf, bool test_ fputs("# DHCP server setting\n", pihole_conf); fputs("dhcp-authoritative\n", pihole_conf); fputs("dhcp-leasefile="DHCPLEASESFILE"\n", pihole_conf); - char start[INET_ADDRSTRLEN] = { 0 }, end[INET_ADDRSTRLEN] = { 0 }; + char start[INET_ADDRSTRLEN] = { 0 }, + end[INET_ADDRSTRLEN] = { 0 }, + router[INET_ADDRSTRLEN] = { 0 }; inet_ntop(AF_INET, &conf->dhcp.start.v.in_addr, start, INET_ADDRSTRLEN); inet_ntop(AF_INET, &conf->dhcp.end.v.in_addr, end, INET_ADDRSTRLEN); + inet_ntop(AF_INET, &conf->dhcp.router.v.in_addr, router, INET_ADDRSTRLEN); fprintf(pihole_conf, "dhcp-range=%s,%s", start, end); // Net mask is optional, only add if it is not 0.0.0.0 const struct in_addr inaddr_empty = {0}; @@ -429,13 +432,12 @@ bool __attribute__((const)) write_dnsmasq_config(struct config *conf, bool test_ { char netmask[INET_ADDRSTRLEN] = { 0 }; inet_ntop(AF_INET, &conf->dhcp.netmask.v.in_addr, netmask, INET_ADDRSTRLEN); - fprintf(pihole_conf, ",%s", conf->dhcp.netmask.v.s); + fprintf(pihole_conf, ",%s", netmask); } // Lease time is optional, only add it if it is set if(strlen(conf->dhcp.leaseTime.v.s) > 0) fprintf(pihole_conf, ",%s", conf->dhcp.leaseTime.v.s); - fprintf(pihole_conf, "\ndhcp-option=option:router,%s\n", - conf->dhcp.router.v.s); + fprintf(pihole_conf, "\ndhcp-option=option:router,%s\n", router); if(conf->dhcp.rapidCommit.v.b) fputs("dhcp-rapid-commit\n", pihole_conf); diff --git a/test/api/libs/responseVerifyer.py b/test/api/libs/responseVerifyer.py index bc692fd04..c09d154a4 100644 --- a/test/api/libs/responseVerifyer.py +++ b/test/api/libs/responseVerifyer.py @@ -10,7 +10,7 @@ # Please see LICENSE file for your rights under this license. import io -import pprint +import ipaddress import random import zipfile from libs.openAPI import openApi @@ -221,8 +221,32 @@ def verify_teleporter_zip(self, teleporter_archive: bytes): return self.errors + # Check if a string is a valid IPv4 address + def valid_ipv4(self, addr: str) -> bool: + octets = addr.split(".") # type: list[str] + if len(octets) != 4: + return False + for octet in octets: + if not octet.isdigit(): + return False + if int(octet) < 0 or int(octet) > 255: + return False + return True + + + # Check if a string is a valid IPv6 address + def valid_ipv6(self, addr: str) -> bool: + # Split the address into parts + parts = addr.split(":") # type: list[str] + # Check if the address is a valid IPv6 address + if len(parts) != 8: + return False + + # Verify a single property's type - def verify_type(self, prop_type: any, yaml_type: str, yaml_nullable: bool): + def verify_type(self, prop: any, yaml_type: str, yaml_nullable: bool, yaml_format: str = None): + # Get the type of the property + prop_type = type(prop) # None is an acceptable reply when this is specified in the API specs if prop_type is type(None) and yaml_nullable: return True @@ -230,6 +254,14 @@ def verify_type(self, prop_type: any, yaml_type: str, yaml_nullable: bool): if yaml_type not in self.YAML_TYPES: self.errors.append("Property type \"" + yaml_type + "\" is not valid in OpenAPI specs") return False + if yaml_format is not None: + # Check if the format is correct + if yaml_format == "ipv4" and not type(ipaddress.ip_address(prop)) is ipaddress.IPv4Address: + self.errors.append("Property \"" + str(prop) + "\" is not a valid IPv4 address") + return False + elif yaml_format == "ipv6" and not type(ipaddress.ip_address(prop)) is ipaddress.IPv6Address: + self.errors.append("Property \"" + str(prop) + "\" is not a valid IPv6 address") + return False return prop_type in self.YAML_TYPES[yaml_type] @@ -306,17 +338,19 @@ def verify_property(self, YAMLprops: dict, YAMLexamples: dict, FTLprops: dict, p # if not defined as string, integer, etc.) yaml_nullable = 'nullable' in YAMLprop and YAMLprop['nullable'] == True + # Get format of this property (if defined) + yaml_format = YAMLprop['format'] if 'format' in YAMLprop else YAMLprop['x-format'] if 'x-format' in YAMLprop else None + # Add this property to the YAML response self.YAMLresponse[flat_path] = [] # Check type of YAML example (if defined) if 'example' in YAMLprop: - example_type = type(YAMLprop['example']) # Check if the type of the example matches the # type we defined in the API specs self.YAMLresponse[flat_path].append(YAMLprop['example']) - if not self.verify_type(example_type, yaml_type, yaml_nullable): - self.errors.append(f"API example ({str(example_type)}) does not match defined type ({yaml_type}) in {flat_path} (nullable: " + ("True" if yaml_nullable else "False") + ")") + if not self.verify_type(YAMLprop['example'], yaml_type, yaml_nullable, yaml_format): + self.errors.append(f"API example ({str(type(YAMLprop['example']))}) does not match defined type ({yaml_type}) in {flat_path} (nullable: " + ("True" if yaml_nullable else "False") + ")") return False # Check type of externally defined YAML examples (next to schema) @@ -340,16 +374,14 @@ def verify_property(self, YAMLprops: dict, YAMLexamples: dict, FTLprops: dict, p if skip_this: continue # Check if the type of the example matches the type we defined in the API specs - example_type = type(example) self.YAMLresponse[flat_path].append(example) - if not self.verify_type(example_type, yaml_type, yaml_nullable): - self.errors.append(f"API example ({str(example_type)}) does not match defined type ({yaml_type}) in {flat_path} (nullable: " + ("True" if yaml_nullable else "False") + ")") + if not self.verify_type(example, yaml_type, yaml_nullable, yaml_format): + self.errors.append(f"API example ({str(type(example))}) does not match defined type ({yaml_type}) in {flat_path} (nullable: " + ("True" if yaml_nullable else "False") + ")") return False # Compare type of FTL's reply against what we defined in the API specs - ftl_type = type(FTLprop) - if not self.verify_type(ftl_type, yaml_type, yaml_nullable): - self.errors.append(f"FTL's reply ({str(ftl_type)}) does not match defined type ({yaml_type}) in {flat_path}") + if not self.verify_type(FTLprop, yaml_type, yaml_nullable, yaml_format): + self.errors.append(f"FTL's reply ({str(type(FTLprop))}) does not match defined type ({yaml_type}) in {flat_path}") return False return all_okay From b96779905727e7bfccfbeefe569a36fe18b7bfff Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 4 Nov 2023 12:23:02 +0100 Subject: [PATCH 012/221] Reject invalid config items (error 400) instead of merely logging a warning to the log Signed-off-by: DL6ER --- src/api/config.c | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/api/config.c b/src/api/config.c index f04604bcb..453052dac 100644 --- a/src/api/config.c +++ b/src/api/config.c @@ -693,8 +693,23 @@ static int api_config_patch(struct ftl_conn *api) const char *response = getJSONvalue(new_item, elem, &newconf); if(response != NULL) { - log_err("/api/config: %s invalid: %s", new_item->k, response); - continue; + char *hint = calloc(strlen(new_item->k) + strlen(response) + 3, sizeof(char)); + if(hint == NULL) + { + free_config(&newconf); + return send_json_error(api, 500, + "internal_error", + "Failed to allocate memory for hint", + NULL); + } + strcpy(hint, new_item->k); + strcat(hint, ": "); + strcat(hint, response); + free_config(&newconf); + return send_json_error_free(api, 400, + "bad_request", + "Config item is invalid", + hint, true); } // Get pointer to memory location of this conf_item (global) From c54f0a7871fd73dcb6133a47a2275821eb243c55 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 4 Nov 2023 13:50:07 +0100 Subject: [PATCH 013/221] Add tests for international local CNAME records Signed-off-by: DL6ER --- test/pihole.toml | 4 +++- test/test_suite.bats | 9 +++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/test/pihole.toml b/test/pihole.toml index 565ba7a6b..b7cfd88fb 100644 --- a/test/pihole.toml +++ b/test/pihole.toml @@ -178,7 +178,9 @@ # Possible values are: # Array of static leases each on in one of the following forms: # ",[,]" - cnameRecords = [] + cnameRecords = [ + "brücke.com,äste.com,2", + ] # Port used by the DNS server port = 53 diff --git a/test/test_suite.bats b/test/test_suite.bats index 08245f644..f794ee06e 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -1273,6 +1273,15 @@ [[ "${lines[0]}" == "2.2.2.2" ]] } +@test "Local CNAME records: International domains are converted to IDNA form" { + # brücke.com ---> xn--brcke-lva.com + run bash -c "dig A xn--brcke-lva.com +short @127.0.0.1" + printf "%s\n" "${lines[@]}" + # xn--ste-pla.com ---> äste.com + [[ "${lines[0]}" == "xn--ste-pla.com." ]] + [[ "${lines[1]}" == "2.2.2.2" ]] +} + @test "Environmental variable is favored over config file" { # The config file has -10 but we set FTLCONF_misc_nice="-11" run bash -c 'grep -B1 "nice = -11" /etc/pihole/pihole.toml' From 7e094ecb3b7ee7c5e52d12b97578802140dd8eac Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 4 Nov 2023 16:36:34 +0100 Subject: [PATCH 014/221] Changing dns.hosts does not need a full FTL restart but only a cache flush and re-read Signed-off-by: DL6ER --- src/api/config.c | 28 ++++++++++++++++++++++++++++ src/config/config.c | 2 +- src/config/config.h | 2 +- 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/src/api/config.c b/src/api/config.c index f04604bcb..167763c49 100644 --- a/src/api/config.c +++ b/src/api/config.c @@ -26,6 +26,8 @@ #include "shmem.h" // hash_password() #include "config/password.h" +// main_pid() +#include "signals.h" static struct { const char *name; @@ -659,6 +661,7 @@ static int api_config_patch(struct ftl_conn *api) // Read all known config items bool config_changed = false; bool dnsmasq_changed = false; + bool rewrite_hosts = false; struct config newconf; duplicate_config(&newconf, &config); for(unsigned int i = 0; i < CONFIG_ELEMENTS; i++) @@ -730,6 +733,10 @@ static int api_config_patch(struct ftl_conn *api) if(conf_item->f & FLAG_RESTART_FTL) dnsmasq_changed = true; + // Check if this item requires rewriting the HOSTS file + if(conf_item == &config.dns.hosts) + rewrite_hosts = true; + // If the privacy level was decreased, we need to restart if(new_item == &newconf.misc.privacylevel && new_item->v.privacy_level < conf_item->v.privacy_level) @@ -768,6 +775,14 @@ static int api_config_patch(struct ftl_conn *api) // Store changed configuration to disk writeFTLtoml(true); + + // Rewrite HOSTS file if required + if(rewrite_hosts) + { + write_custom_list(); + // Reload HOSTS file + kill(main_pid(), SIGHUP); + } } else { @@ -816,6 +831,7 @@ static int api_config_put_delete(struct ftl_conn *api) // Read all known config items bool dnsmasq_changed = false; + bool rewrite_hosts = false; bool found = false; struct config newconf; duplicate_config(&newconf, &config); @@ -907,6 +923,10 @@ static int api_config_put_delete(struct ftl_conn *api) if(new_item->f & FLAG_RESTART_FTL) dnsmasq_changed = true; + // Check if this item requires rewriting the HOSTS file + if(new_item == &newconf.dns.hosts) + rewrite_hosts = true; + break; } @@ -954,6 +974,14 @@ static int api_config_put_delete(struct ftl_conn *api) // Store changed configuration to disk writeFTLtoml(true); + // Rewrite HOSTS file if required + if(rewrite_hosts) + { + write_custom_list(); + // Reload HOSTS file + kill(main_pid(), SIGHUP); + } + // Send empty reply with matching HTTP status code // 201 - Created or 204 - No content cJSON *json = JSON_NEW_OBJECT(); diff --git a/src/config/config.c b/src/config/config.c index 72c364070..553705db3 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -462,7 +462,7 @@ void initConfig(struct config *conf) conf->dns.hosts.h = "Array of custom DNS records\n Example: hosts = [ \"127.0.0.1 mylocal\", \"192.168.0.1 therouter\" ]"; conf->dns.hosts.a = cJSON_CreateStringReference("Array of custom DNS records each one in HOSTS form: \"IP HOSTNAME\""); conf->dns.hosts.t = CONF_JSON_STRING_ARRAY; - conf->dns.hosts.f = FLAG_ADVANCED_SETTING | FLAG_RESTART_FTL; + conf->dns.hosts.f = FLAG_ADVANCED_SETTING; conf->dns.hosts.d.json = cJSON_CreateArray(); conf->dns.domainNeeded.k = "dns.domainNeeded"; diff --git a/src/config/config.h b/src/config/config.h index c230627ee..72ee5aeab 100644 --- a/src/config/config.h +++ b/src/config/config.h @@ -88,7 +88,7 @@ enum conf_type { #define FLAG_PSEUDO_ITEM (1 << 2) #define FLAG_INVALIDATE_SESSIONS (1 << 3) #define FLAG_WRITE_ONLY (1 << 4) -#define FLAG_ENV_VAR (1 << 5) +#define FLAG_ENV_VAR (1 << 5) struct conf_item { const char *k; // item Key From add7ceadda65acab651e749dcab0a29da6dda38f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 4 Nov 2023 17:00:59 +0100 Subject: [PATCH 015/221] Make number of maximum concurrent API sessions adjustable Signed-off-by: DL6ER --- src/api/auth.c | 22 +++++++++++++++------- src/api/auth.h | 3 --- src/api/docs/content/specs/config.yaml | 3 +++ src/config/config.c | 6 ++++++ src/config/config.h | 1 + src/database/session-table.c | 4 ++-- test/pihole.toml | 6 ++++++ 7 files changed, 33 insertions(+), 12 deletions(-) diff --git a/src/api/auth.c b/src/api/auth.c index d48d171be..f733ba6a0 100644 --- a/src/api/auth.c +++ b/src/api/auth.c @@ -26,7 +26,7 @@ // database session functions #include "database/session-table.h" -static struct session auth_data[API_MAX_CLIENTS] = {{false, false, {false, false}, 0, 0, {0}, {0}, {0}, {0}}}; +static struct session *auth_data = NULL; static void add_request_info(struct ftl_conn *api, const char *csrf) { @@ -43,6 +43,12 @@ static void add_request_info(struct ftl_conn *api, const char *csrf) void init_api(void) { // Restore sessions from database + auth_data = calloc(config.webserver.api.max_sessions.v.u16, sizeof(struct session)); + if(auth_data == NULL) + { + log_crit("Could not allocate memory for API sessions, check config value of webserver.api.max_sessions"); + exit(EXIT_FAILURE); + } restore_db_sessions(auth_data); } @@ -50,6 +56,8 @@ void free_api(void) { // Store sessions in database backup_db_sessions(auth_data); + free(auth_data); + auth_data = NULL; } // Is this client connecting from localhost? @@ -187,7 +195,7 @@ int check_client_auth(struct ftl_conn *api, const bool is_api) } } - for(unsigned int i = 0; i < API_MAX_CLIENTS; i++) + for(unsigned int i = 0; i < config.webserver.api.max_sessions.v.u16; i++) { if(auth_data[i].used && auth_data[i].valid_until >= now && @@ -253,7 +261,7 @@ static int get_all_sessions(struct ftl_conn *api, cJSON *json) { const time_t now = time(NULL); cJSON *sessions = JSON_NEW_ARRAY(); - for(int i = 0; i < API_MAX_CLIENTS; i++) + for(int i = 0; i < config.webserver.api.max_sessions.v.u16; i++) { if(!auth_data[i].used) continue; @@ -316,7 +324,7 @@ static int get_session_object(struct ftl_conn *api, cJSON *json, const int user_ static void delete_session(const int user_id) { // Skip if nothing to be done here - if(user_id < 0 || user_id >= API_MAX_CLIENTS) + if(user_id < 0 || user_id >= config.webserver.api.max_sessions.v.u16) return; // Zero out this session (also sets valid to false == 0) @@ -326,7 +334,7 @@ static void delete_session(const int user_id) void delete_all_sessions(void) { // Zero out all sessions without looping - memset(auth_data, 0, sizeof(auth_data)); + memset(auth_data, 0, config.webserver.api.max_sessions.v.u16*sizeof(*auth_data)); } static int send_api_auth_status(struct ftl_conn *api, const int user_id, const time_t now) @@ -527,7 +535,7 @@ int api_auth(struct ftl_conn *api) } // Find unused authentication slot - for(unsigned int i = 0; i < API_MAX_CLIENTS; i++) + for(unsigned int i = 0; i < config.webserver.api.max_sessions.v.u16; i++) { // Expired slow, mark as unused if(auth_data[i].used && @@ -621,7 +629,7 @@ int api_auth_session_delete(struct ftl_conn *api) return send_json_error(api, 400, "bad_request", "Missing or invalid session ID", NULL); // Check if session ID is valid - if(uid <= API_AUTH_UNAUTHORIZED || uid >= API_MAX_CLIENTS) + if(uid <= API_AUTH_UNAUTHORIZED || uid >= config.webserver.api.max_sessions.v.u16) return send_json_error(api, 400, "bad_request", "Session ID out of bounds", NULL); // Check if session is used diff --git a/src/api/auth.h b/src/api/auth.h index 8cfd5d131..53663026d 100644 --- a/src/api/auth.h +++ b/src/api/auth.h @@ -11,9 +11,6 @@ #ifndef AUTH_H #define AUTH_H -// How many authenticated API clients are allowed simultaneously? [.] -#define API_MAX_CLIENTS 16 - // crypto library #include #include diff --git a/src/api/docs/content/specs/config.yaml b/src/api/docs/content/specs/config.yaml index db3154d14..f57a4d93f 100644 --- a/src/api/docs/content/specs/config.yaml +++ b/src/api/docs/content/specs/config.yaml @@ -386,6 +386,8 @@ components: type: boolean searchAPIauth: type: boolean + max_sessions: + type: integer prettyJSON: type: boolean password: @@ -666,6 +668,7 @@ components: api: localAPIauth: false searchAPIauth: false + max_sessions: 16 prettyJSON: false password: "********" pwhash: '' diff --git a/src/config/config.c b/src/config/config.c index 72c364070..574b1d9b6 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -903,6 +903,12 @@ void initConfig(struct config *conf) conf->webserver.api.localAPIauth.t = CONF_BOOL; conf->webserver.api.localAPIauth.d.b = true; + conf->webserver.api.max_sessions.k = "webserver.api.max_sessions"; + conf->webserver.api.max_sessions.h = "Number of concurrent sessions allowed for the API. If the number of sessions exceeds this value, no new sessions will be allowed until the number of sessions drops due to session expiration or logout. Note that the number of concurrent sessions is irrelevant if authentication is disabled as no sessions are used in this case."; + conf->webserver.api.max_sessions.t = CONF_UINT16; + conf->webserver.api.max_sessions.d.u16 = 16; + conf->webserver.api.max_sessions.f = FLAG_ADVANCED_SETTING | FLAG_RESTART_FTL; + conf->webserver.api.prettyJSON.k = "webserver.api.prettyJSON"; conf->webserver.api.prettyJSON.h = "Should FTL prettify the API output (add extra spaces, newlines and indentation)?"; conf->webserver.api.prettyJSON.t = CONF_BOOL; diff --git a/src/config/config.h b/src/config/config.h index c230627ee..b71456813 100644 --- a/src/config/config.h +++ b/src/config/config.h @@ -226,6 +226,7 @@ struct config { struct { struct conf_item localAPIauth; struct conf_item searchAPIauth; + struct conf_item max_sessions; struct conf_item prettyJSON; struct conf_item pwhash; struct conf_item password; // This is a pseudo-item diff --git a/src/database/session-table.c b/src/database/session-table.c index f99bd67c0..8c1847785 100644 --- a/src/database/session-table.c +++ b/src/database/session-table.c @@ -89,7 +89,7 @@ bool backup_db_sessions(struct session *sessions) } unsigned int api_sessions = 0; - for(unsigned int i = 0; i < API_MAX_CLIENTS; i++) + for(unsigned int i = 0; i < config.webserver.api.max_sessions.v.u16; i++) { // Get session struct session *sess = &sessions[i]; @@ -237,7 +237,7 @@ bool restore_db_sessions(struct session *sessions) // Iterate over all still valid sessions unsigned int i = 0; - while(sqlite3_step(stmt) == SQLITE_ROW && i++ < API_MAX_CLIENTS) + while(sqlite3_step(stmt) == SQLITE_ROW && i++ < config.webserver.api.max_sessions.v.u16) { // Allocate memory for new session struct session *sess = &sessions[i]; diff --git a/test/pihole.toml b/test/pihole.toml index 2fc68e648..38a0c8241 100644 --- a/test/pihole.toml +++ b/test/pihole.toml @@ -604,6 +604,12 @@ # sense of the option means only 127.0.0.1 and [::1] searchAPIauth = false + # Number of concurrent sessions allowed for the API. If the number of sessions exceeds + # this value, no new sessions will be allowed until the number of sessions drops due + # to session expiration or logout. Note that the number of concurrent sessions is + # irrelevant if authentication is disabled as no sessions are used in this case. + max_sessions = 16 + # Should FTL prettify the API output (add extra spaces, newlines and indentation)? prettyJSON = false From 11127f0f139abace6ea072ab6cdf6648a1ce21bd Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 4 Nov 2023 17:11:37 +0100 Subject: [PATCH 016/221] Fix OpenAPI checker not being able to discover properties that are returned by FTL but not documented in the OpenAPI specs Signed-off-by: DL6ER --- test/api/libs/responseVerifyer.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/api/libs/responseVerifyer.py b/test/api/libs/responseVerifyer.py index bc692fd04..710f9bf9b 100644 --- a/test/api/libs/responseVerifyer.py +++ b/test/api/libs/responseVerifyer.py @@ -152,8 +152,8 @@ def verify_endpoint(self, endpoint: str): # Check for properties in FTL that are not in the API specs for property in FTLflat.keys(): - if property not in YAMLflat.keys() and len([p.startswith(property + ".") for p in YAMLflat.keys()]) == 0: - self.errors.append("Property '" + property + "' missing in the API specs (have " + ",".join(YAMLflat.keys()) + ")") + if property not in YAMLflat.keys(): + self.errors.append("Property '" + property + "' missing in the API specs") elif expected_mimetype == "application/zip": file_like_object = io.BytesIO(FTLresponse) @@ -296,6 +296,9 @@ def verify_property(self, YAMLprops: dict, YAMLexamples: dict, FTLprops: dict, p for j in FTLprop[i]: if not self.verify_property(YAMLprop['items']['properties'], YAMLexamples, FTLprop[i], props + [i, str(j)]): all_okay = False + + # Add this property to the YAML response + self.YAMLresponse[flat_path] = [] else: # Check this property From ca97616ed16a631e19f4da67d9bda5e75e4cf48e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 5 Nov 2023 10:17:32 +0100 Subject: [PATCH 017/221] Add config option misc.etc_dnsmasq_d to allow setting whether files in /etc/dnsmasq.d are loaded as additional config files for Pi-hole. This resolves a long standing issue with Pi-hole not being compatible with software that installs custom files with conflicting lines in this directory (e.g., lxc) as well as improve the coexistence of Pi-hole with an already running dnsmasq on the host (by default, we do not share any config files in the future) Signed-off-by: DL6ER --- src/api/docs/content/specs/config.yaml | 3 +++ src/config/config.c | 6 ++++++ src/config/config.h | 1 + src/config/dnsmasq_config.c | 11 ++++++----- test/pihole.toml | 3 +++ 5 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/api/docs/content/specs/config.yaml b/src/api/docs/content/specs/config.yaml index db3154d14..9165bc4a6 100644 --- a/src/api/docs/content/specs/config.yaml +++ b/src/api/docs/content/specs/config.yaml @@ -450,6 +450,8 @@ components: type: integer addr2line: type: boolean + etc_dnsmasq_d: + type: boolean privacylevel: type: integer dnsmasq_lines: @@ -694,6 +696,7 @@ components: delay_startup: 10 addr2line: true privacylevel: 0 + etc_dnsmasq_d: false dnsmasq_lines: [ ] check: load: true diff --git a/src/config/config.c b/src/config/config.c index 72c364070..fb7a3af88 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -1074,6 +1074,12 @@ void initConfig(struct config *conf) conf->misc.addr2line.f = FLAG_ADVANCED_SETTING; conf->misc.addr2line.d.b = true; + conf->misc.etc_dnsmasq_d.k = "misc.etc_dnsmasq_d"; + conf->misc.etc_dnsmasq_d.h = "Should FTL load additional dnsmasq configuration files from /etc/dnsmasq.d/?"; + conf->misc.etc_dnsmasq_d.t = CONF_BOOL; + conf->misc.etc_dnsmasq_d.f = FLAG_ADVANCED_SETTING; + conf->misc.etc_dnsmasq_d.d.b = false; + conf->misc.dnsmasq_lines.k = "misc.dnsmasq_lines"; conf->misc.dnsmasq_lines.h = "Additional lines to inject into the generated dnsmasq configuration.\n Warning: This is an advanced setting and should only be used with care. Incorrectly formatted or duplicated lines as well as lines conflicting with the automatic configuration of Pi-hole can break the embedded dnsmasq and will stop DNS resolution from working.\n Use this option with extra care."; conf->misc.dnsmasq_lines.a = cJSON_CreateStringReference("array of valid dnsmasq config line options"); diff --git a/src/config/config.h b/src/config/config.h index c230627ee..71ef13830 100644 --- a/src/config/config.h +++ b/src/config/config.h @@ -261,6 +261,7 @@ struct config { struct conf_item delay_startup; struct conf_item nice; struct conf_item addr2line; + struct conf_item etc_dnsmasq_d; struct conf_item dnsmasq_lines; struct { struct conf_item load; diff --git a/src/config/dnsmasq_config.c b/src/config/dnsmasq_config.c index e727e3a7a..4b6080056 100644 --- a/src/config/dnsmasq_config.c +++ b/src/config/dnsmasq_config.c @@ -207,11 +207,12 @@ static void write_config_header(FILE *fp, const char *description) CONFIG_CENTER(fp, HEADER_WIDTH, "%s", "ANY CHANGES MADE TO THIS FILE WILL BE LOST WHEN THE CONFIGURATION CHANGES"); CONFIG_CENTER(fp, HEADER_WIDTH, "%s", ""); CONFIG_CENTER(fp, HEADER_WIDTH, "%s", "IF YOU WISH TO CHANGE ANY OF THESE VALUES, CHANGE THEM IN"); - CONFIG_CENTER(fp, HEADER_WIDTH, "%s", "etc/pihole/pihole.toml"); + CONFIG_CENTER(fp, HEADER_WIDTH, "%s", "/etc/pihole/pihole.toml"); CONFIG_CENTER(fp, HEADER_WIDTH, "%s", "and restart pihole-FTL"); CONFIG_CENTER(fp, HEADER_WIDTH, "%s", ""); CONFIG_CENTER(fp, HEADER_WIDTH, "%s", "ANY OTHER CHANGES SHOULD BE MADE IN A SEPARATE CONFIG FILE"); CONFIG_CENTER(fp, HEADER_WIDTH, "%s", "WITHIN /etc/dnsmasq.d/yourname.conf"); + CONFIG_CENTER(fp, HEADER_WIDTH, "%s", "(make sure misc.etc_dnsmasq_d is set to true in /etc/pihole/pihole.toml)"); CONFIG_CENTER(fp, HEADER_WIDTH, "%s", ""); CONFIG_CENTER(fp, HEADER_WIDTH, "Last updated: %s", timestring); CONFIG_CENTER(fp, HEADER_WIDTH, "by FTL version %s", get_FTL_version()); @@ -508,11 +509,11 @@ bool __attribute__((const)) write_dnsmasq_config(struct config *conf, bool test_ fputs("server=/bind/\n", pihole_conf); fputs("server=/onion/\n", pihole_conf); - if(directory_exists("/etc/dnsmasq.d")) + if(directory_exists("/etc/dnsmasq.d") && conf->misc.etc_dnsmasq_d.v.b) { - // Load possible additional user scripts from /etc/dnsmasq.d if - // the directory exists (it may not, e.g., in a container) - fputs("# Load possible additional user scripts\n", pihole_conf); + // Load additional user scripts from /etc/dnsmasq.d if the + // directory exists (it may not, e.g., in a container) + fputs("# Load additional user scripts\n", pihole_conf); fputs("conf-dir=/etc/dnsmasq.d\n", pihole_conf); fputs("\n", pihole_conf); } diff --git a/test/pihole.toml b/test/pihole.toml index 2fc68e648..ae962db29 100644 --- a/test/pihole.toml +++ b/test/pihole.toml @@ -775,6 +775,9 @@ # malfunctioning addr2line can prevent from generating any backtrace at all. addr2line = true + # Should FTL load additional dnsmasq configuration files from /etc/dnsmasq.d/? + etc_dnsmasq_d = true ### CHANGED, default = false + # Additional lines to inject into the generated dnsmasq configuration. Warning: This is # an advanced setting and should only be used with care. Incorrectly formatted or # duplicated lines as well as lines conflicting with the automatic configuration of From 9345841b1076ed7cf8cc394429fcb22f8454fe9b Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 5 Nov 2023 20:58:19 +0200 Subject: [PATCH 018/221] Fix error message shown when gravityDB_addToTable() fails Signed-off-by: DL6ER --- src/database/gravity-db.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/database/gravity-db.c b/src/database/gravity-db.c index db4e5e1e4..ee0f6137a 100644 --- a/src/database/gravity-db.c +++ b/src/database/gravity-db.c @@ -1672,7 +1672,7 @@ bool gravityDB_addToTable(const enum gravity_list_type listtype, tablerow *row, { *message = sqlite3_errmsg(gravity_db); log_err("gravityDB_addToTable(%d, %s) - SQL error prepare (%i): %s", - row->type_int, row->domain, rc, *message); + row->type_int, row->item, rc, *message); return false; } @@ -1694,7 +1694,7 @@ bool gravityDB_addToTable(const enum gravity_list_type listtype, tablerow *row, { *message = sqlite3_errmsg(gravity_db); log_err("gravityDB_addToTable(%d, %s): Failed to bind name (error %d) - %s", - row->type_int, row->name, rc, *message); + row->type_int, row->item, rc, *message); sqlite3_reset(stmt); sqlite3_finalize(stmt); return false; @@ -1706,7 +1706,7 @@ bool gravityDB_addToTable(const enum gravity_list_type listtype, tablerow *row, { *message = sqlite3_errmsg(gravity_db); log_err("gravityDB_addToTable(%d, %s): Failed to bind type (error %d) - %s", - row->type_int, row->domain, rc, *message); + row->type_int, row->item, rc, *message); sqlite3_reset(stmt); sqlite3_finalize(stmt); return false; @@ -1727,7 +1727,7 @@ bool gravityDB_addToTable(const enum gravity_list_type listtype, tablerow *row, // Error, one is not meaningful without the other *message = "Field type missing from request"; log_err("gravityDB_addToTable(%d, %s): type missing", - row->type_int, row->domain); + row->type_int, row->item); sqlite3_reset(stmt); sqlite3_finalize(stmt); return false; @@ -1737,7 +1737,7 @@ bool gravityDB_addToTable(const enum gravity_list_type listtype, tablerow *row, // Error, one is not meaningful without the other *message = "Field oldkind missing from request"; log_err("gravityDB_addToTable(%d, %s): Oldkind missing", - row->type_int, row->domain); + row->type_int, row->item); sqlite3_reset(stmt); sqlite3_finalize(stmt); return false; @@ -1760,7 +1760,7 @@ bool gravityDB_addToTable(const enum gravity_list_type listtype, tablerow *row, { *message = "Cannot interpret type/kind"; log_err("gravityDB_addToTable(%d, %s): Failed to identify type=\"%s\", kind=\"%s\"", - row->type_int, row->domain, row->type, row->kind); + row->type_int, row->item, row->type, row->kind); sqlite3_reset(stmt); sqlite3_finalize(stmt); return false; @@ -1772,7 +1772,7 @@ bool gravityDB_addToTable(const enum gravity_list_type listtype, tablerow *row, { *message = sqlite3_errmsg(gravity_db); log_err("gravityDB_addToTable(%d, %s): Failed to bind oldtype (error %d) - %s", - row->type_int, row->domain, rc, *message); + row->type_int, row->item, rc, *message); sqlite3_reset(stmt); sqlite3_finalize(stmt); return false; @@ -1785,7 +1785,7 @@ bool gravityDB_addToTable(const enum gravity_list_type listtype, tablerow *row, { *message = sqlite3_errmsg(gravity_db); log_err("gravityDB_addToTable(%d, %s): Failed to bind enabled (error %d) - %s", - row->type_int, row->domain, rc, *message); + row->type_int, row->item, rc, *message); sqlite3_reset(stmt); sqlite3_finalize(stmt); return false; @@ -1797,7 +1797,7 @@ bool gravityDB_addToTable(const enum gravity_list_type listtype, tablerow *row, { *message = sqlite3_errmsg(gravity_db); log_err("gravityDB_addToTable(%d, %s): Failed to bind comment (error %d) - %s", - row->type_int, row->domain, rc, *message); + row->type_int, row->item, rc, *message); sqlite3_reset(stmt); sqlite3_finalize(stmt); return false; From c450d488a4987e7bf51d586dc9c9cad101a12a99 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 5 Nov 2023 20:59:55 +0200 Subject: [PATCH 019/221] Fix domain modification. The UNIQUE key in the domainlist table is the combination of domain+type, not domain alone so you can add the same domain, e.g. once as allowed and once as denied entries and assign them to different clients using appropriate groups Signed-off-by: DL6ER --- src/database/gravity-db.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/database/gravity-db.c b/src/database/gravity-db.c index ee0f6137a..e1ab5d3f2 100644 --- a/src/database/gravity-db.c +++ b/src/database/gravity-db.c @@ -1664,7 +1664,7 @@ bool gravityDB_addToTable(const enum gravity_list_type listtype, tablerow *row, "ON CONFLICT(ip) DO UPDATE SET comment = :comment;"; else // domainlist querystr = "INSERT INTO domainlist (domain,type,enabled,comment) VALUES (:item,:type,:enabled,:comment) "\ - "ON CONFLICT(domain) DO UPDATE SET type = :type, enabled = :enabled, comment = :comment;"; + "ON CONFLICT(domain,type) DO UPDATE SET type = :type, enabled = :enabled, comment = :comment;"; } int rc = sqlite3_prepare_v2(gravity_db, querystr, -1, &stmt, NULL); From 1902fadd18261b24b85c59f93ddb977f83a40ed2 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 6 Nov 2023 08:55:09 +0100 Subject: [PATCH 020/221] Do not reduce the number of sent timeslots based on the real activity but instead always send everything (even if there are many zeros). This brings https://github.com/pi-hole/FTL/pull/1345 to FTL v6.0 Signed-off-by: DL6ER --- src/api/history.c | 46 ++++------------------------------------------ 1 file changed, 4 insertions(+), 42 deletions(-) diff --git a/src/api/history.c b/src/api/history.c index 202429443..2b7f772f8 100644 --- a/src/api/history.c +++ b/src/api/history.c @@ -25,51 +25,11 @@ int api_history(struct ftl_conn *api) { - unsigned int from = 0, until = OVERTIME_SLOTS; - const time_t now = time(NULL); - bool found = false; - lock_shm(); - time_t mintime = overTime[0].timestamp; - - // Start with the first non-empty overTime slot - for(unsigned int slot = 0; slot < OVERTIME_SLOTS; slot++) - { - if((overTime[slot].total > 0 || overTime[slot].blocked > 0) && - overTime[slot].timestamp >= mintime) - { - from = slot; - found = true; - break; - } - } - // End with last non-empty overTime slot or the last slot that is not - // older than the maximum history to be sent - for(unsigned int slot = 0; slot < OVERTIME_SLOTS; slot++) - { - if(overTime[slot].timestamp >= now || - overTime[slot].timestamp - now > (time_t)config.webserver.api.maxHistory.v.ui) - { - until = slot; - break; - } - } - - // If there is no data to be sent, we send back an empty array - // and thereby return early - if(!found) - { - cJSON *json = JSON_NEW_ARRAY(); - cJSON *item = JSON_NEW_OBJECT(); - JSON_ADD_ITEM_TO_ARRAY(json, item); - JSON_SEND_OBJECT_UNLOCK(json); - } - - // Minimum structure is - // {"history":[]} + // Loop over all overTime slots and add them to the array cJSON *history = JSON_NEW_ARRAY(); - for(unsigned int slot = from; slot < until; slot++) + for(unsigned int slot = 0; slot < OVERTIME_SLOTS; slot++) { cJSON *item = JSON_NEW_OBJECT(); JSON_ADD_NUMBER_TO_OBJECT(item, "timestamp", overTime[slot].timestamp); @@ -79,6 +39,8 @@ int api_history(struct ftl_conn *api) JSON_ADD_ITEM_TO_ARRAY(history, item); } + // Minimum structure is + // {"history":[]} cJSON *json = JSON_NEW_OBJECT(); JSON_ADD_ITEM_TO_OBJECT(json, "history", history); JSON_SEND_OBJECT_UNLOCK(json); From 28fe8dd4997dd50379e687c008059a9f39544ab6 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 6 Nov 2023 10:37:32 +0100 Subject: [PATCH 021/221] Return 429 Too Many Requests with useful hint when number of available API seats is exceeded, also log currently configured number of API seats in the issues warning Signed-off-by: DL6ER --- src/api/auth.c | 14 ++++++++++---- src/api/docs/content/specs/auth.yaml | 19 +++++++++++++++++++ 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/src/api/auth.c b/src/api/auth.c index f733ba6a0..2448df9cb 100644 --- a/src/api/auth.c +++ b/src/api/auth.c @@ -593,16 +593,22 @@ int api_auth(struct ftl_conn *api) } if(user_id == API_AUTH_UNAUTHORIZED) { - log_warn("No free API seats available, not authenticating client"); + log_warn("No free API seats available (webserver.api.max_sessions = %u), not authenticating client", + config.webserver.api.max_sessions.v.u16); + + return send_json_error(api, 429, + "too_many_requests", + "Too many requests", + "no free API seats available"); } } else if(result == PASSWORD_RATE_LIMITED) { // Rate limited return send_json_error(api, 429, - "too_many_requests", - "Too many requests", - "login rate limiting"); + "too_many_requests", + "Too many requests", + "login rate limiting"); } else { diff --git a/src/api/docs/content/specs/auth.yaml b/src/api/docs/content/specs/auth.yaml index 9e8dffad9..7b9734a83 100644 --- a/src/api/docs/content/specs/auth.yaml +++ b/src/api/docs/content/specs/auth.yaml @@ -96,6 +96,11 @@ components: allOf: - $ref: 'common.yaml#/components/errors/too_many_requests' - $ref: 'common.yaml#/components/schemas/took' + examples: + rate_limit: + $ref: 'auth.yaml#/components/examples/errors/rate_limit' + no_seats: + $ref: 'auth.yaml#/components/examples/errors/no_seats' delete: summary: Delete session tags: @@ -507,6 +512,20 @@ components: key: "bad_request" message: "Session ID not in use" hint: null + rate_limit: + summary: Rate limit exceeded + value: + error: + key: "too_many_requests" + message: "Too many requests" + hint: "login rate limiting" + no_seats: + summary: No free API seats available + value: + error: + key: "too_many_requests" + message: "Too many requests" + hint: "no free API seats available" parameters: id: in: path From 6e860f0c8134031853ddc84769925b9d517c139f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 6 Nov 2023 11:35:26 +0100 Subject: [PATCH 022/221] Add special reply for reused TOTP tokens Signed-off-by: DL6ER --- src/api/2fa.c | 8 +++--- src/api/api.h | 7 ++++- src/api/auth.c | 29 +++++++++++++------- src/api/docs/content/specs/auth.yaml | 40 +++++++++++++++++++++++----- 4 files changed, 63 insertions(+), 21 deletions(-) diff --git a/src/api/2fa.c b/src/api/2fa.c index 5d6148d94..6988592f2 100644 --- a/src/api/2fa.c +++ b/src/api/2fa.c @@ -198,7 +198,7 @@ static bool encode_uint8_t_array_to_base32(const uint8_t *in, const size_t in_le } static uint32_t last_code = 0; -bool verifyTOTP(const uint32_t incode) +enum totp_status verifyTOTP(const uint32_t incode) { // Decode base32 secret uint8_t decoded_secret[RFC6238_SECRET_LEN]; @@ -228,15 +228,15 @@ bool verifyTOTP(const uint32_t incode) { log_warn("2FA code has already been used (%i, %u), please wait %lu seconds", i, gencode, (unsigned long)(RFC6238_X - (now % RFC6238_X))); - return false; + return TOTP_REUSED; } log_info("2FA code verified successfully at %i", i); last_code = gencode; - return true; + return TOTP_CORRECT; } } - return false; + return TOTP_INVALID; } // Print TOTP code to stdout (for CLI use) diff --git a/src/api/api.h b/src/api/api.h index 9950ada50..efde80945 100644 --- a/src/api/api.h +++ b/src/api/api.h @@ -92,7 +92,12 @@ int api_auth_session_delete(struct ftl_conn *api); bool is_local_api_user(const char *remote_addr) __attribute__((pure)); // 2FA methods -bool verifyTOTP(const uint32_t code); +enum totp_status { + TOTP_INVALID, + TOTP_CORRECT, + TOTP_REUSED, +} __attribute__ ((packed)); +enum totp_status verifyTOTP(const uint32_t code); int generateTOTP(struct ftl_conn *api); int printTOTP(void); int generateAppPw(struct ftl_conn *api); diff --git a/src/api/auth.c b/src/api/auth.c index 2448df9cb..8fa7d6a7f 100644 --- a/src/api/auth.c +++ b/src/api/auth.c @@ -524,13 +524,22 @@ int api_auth(struct ftl_conn *api) NULL); } - if(!verifyTOTP(json_totp->valueint)) + enum totp_status totp = verifyTOTP(json_totp->valueint); + if(totp == TOTP_REUSED) + { + // 2FA token has been reused + return send_json_error(api, 401, + "unauthorized", + "Reused 2FA token", + "wait for new token"); + } + else if(totp != TOTP_CORRECT) { // 2FA token is invalid return send_json_error(api, 401, - "unauthorized", - "Invalid 2FA token", - NULL); + "unauthorized", + "Invalid 2FA token", + NULL); } } @@ -597,18 +606,18 @@ int api_auth(struct ftl_conn *api) config.webserver.api.max_sessions.v.u16); return send_json_error(api, 429, - "too_many_requests", - "Too many requests", - "no free API seats available"); + "api_seats_exceeded", + "API seats exceeded", + "increase webserver.api.max_sessions"); } } else if(result == PASSWORD_RATE_LIMITED) { // Rate limited return send_json_error(api, 429, - "too_many_requests", - "Too many requests", - "login rate limiting"); + "rate_limiting", + "Rate-limiting login attempts", + NULL); } else { diff --git a/src/api/docs/content/specs/auth.yaml b/src/api/docs/content/specs/auth.yaml index 7b9734a83..9ba516941 100644 --- a/src/api/docs/content/specs/auth.yaml +++ b/src/api/docs/content/specs/auth.yaml @@ -80,6 +80,8 @@ components: $ref: 'auth.yaml#/components/examples/errors/no_password' password_inval: $ref: 'auth.yaml#/components/examples/errors/password_inval' + totp_missing: + $ref: 'auth.yaml#/components/examples/errors/totp_missing' '401': description: Unauthorized content: @@ -88,6 +90,11 @@ components: allOf: - $ref: 'common.yaml#/components/errors/unauthorized' - $ref: 'common.yaml#/components/schemas/took' + examples: + totp_invalid: + $ref: 'auth.yaml#/components/examples/errors/totp_invalid' + totp_reused: + $ref: 'auth.yaml#/components/examples/errors/totp_reused' '429': description: Too Many Requests content: @@ -491,6 +498,13 @@ components: key: "bad_request" message: "Field password has to be of type 'string'" hint: null + totp_missing: + summary: Bad request (2FA token missing) + value: + error: + key: "bad_request" + message: "No 2FA token found in JSON payload" + hint: null missing_session_id: summary: Bad request (missing session ID) value: @@ -512,20 +526,34 @@ components: key: "bad_request" message: "Session ID not in use" hint: null + totp_invalid: + summary: 2FA token invalid + value: + error: + key: "unauthorized" + message: "Invalid 2FA token" + hint: null + totp_reused: + summary: 2FA token reused + value: + error: + key: "unauthorized" + message: "Reused 2FA token" + hint: "wait for new token" rate_limit: summary: Rate limit exceeded value: error: - key: "too_many_requests" - message: "Too many requests" - hint: "login rate limiting" + key: "rate_limiting" + message: "Rate-limiting login attempts" + hint: null no_seats: summary: No free API seats available value: error: - key: "too_many_requests" - message: "Too many requests" - hint: "no free API seats available" + key: "api_seats_exceeded" + message: "API seats exceeded" + hint: "increase webserver.api.max_sessions" parameters: id: in: path From 51fb66b80f23b977f81779943d0546050451823a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 6 Nov 2023 15:25:41 +0100 Subject: [PATCH 023/221] If reading pihole.toml failed, we recreated one from the possibly still existing setupVars.conf, etc. files. However, it arguably makes more sense to instead restore from the last known-to-be-good rotated config file in /etc/pihole/config_backups Signed-off-by: DL6ER --- src/config/config.c | 33 ++++++++++++++++++++------------- src/config/toml_helper.c | 33 +++++++++++++++++++++++++++++---- src/config/toml_helper.h | 2 +- src/config/toml_reader.c | 13 +++++++------ src/config/toml_reader.h | 3 ++- src/config/toml_writer.c | 2 +- src/files.c | 4 +--- src/files.h | 3 ++- src/zip/teleporter.c | 2 +- 9 files changed, 64 insertions(+), 31 deletions(-) diff --git a/src/config/config.c b/src/config/config.c index 72c364070..00fba277d 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -1314,22 +1314,29 @@ void readFTLconf(struct config *conf, const bool rewrite) initConfig(conf); // First try to read TOML config file - if(readFTLtoml(NULL, conf, NULL, rewrite, NULL)) + // If we cannot parse /etc/pihole.toml (due to missing or invalid syntax), + // we try to read the rotated files in /etc/pihole/config_backup starting at + // the most recent one and going back in time until we find a valid config + for(unsigned int i = 0; i < PLAIN_ROTATIONS; i++) { - // If successful, we write the config file back to disk - // to ensure that all options are present and comments - // about options deviating from the default are present - if(rewrite) + if(readFTLtoml(NULL, conf, NULL, rewrite, NULL, 0)) { - writeFTLtoml(true); - write_dnsmasq_config(conf, false, NULL); - write_custom_list(); + // If successful, we write the config file back to disk + // to ensure that all options are present and comments + // about options deviating from the default are present + if(rewrite) + { + writeFTLtoml(true); + write_dnsmasq_config(conf, false, NULL); + write_custom_list(); + } + return; } - return; } - // On error, try to read legacy (pre-v6.0) config file. If successful, - // we move the legacy config file out of our way + // If no previous config file could be read, we are likely either running + // for the first time or we are upgrading from a version prior to v6.0 + // In this case, we try to read the legacy config files const char *path = ""; if((path = readFTLlegacy(conf)) != NULL) { @@ -1506,7 +1513,7 @@ void reread_config(void) // Read TOML config file bool restart = false; - if(readFTLtoml(&config, &conf_copy, NULL, true, &restart)) + if(readFTLtoml(&config, &conf_copy, NULL, true, &restart, 0)) { // Install new configuration log_debug(DEBUG_CONFIG, "Loaded configuration is valid, installing it"); @@ -1534,7 +1541,7 @@ void reread_config(void) else { // New configuration is invalid, restore old one - log_debug(DEBUG_CONFIG, "Loaded configuration is invalid, restoring old one"); + log_debug(DEBUG_CONFIG, "Modified config file is invalid, discarding and overwriting with current configuration"); free_config(&conf_copy); } diff --git a/src/config/toml_helper.c b/src/config/toml_helper.c index 7f40b683f..dcb374615 100644 --- a/src/config/toml_helper.c +++ b/src/config/toml_helper.c @@ -20,9 +20,11 @@ #include "files.h" //set_and_check_password() #include "config/password.h" +// PATH_MAX +#include // Open the TOML file for reading or writing -FILE * __attribute((malloc)) __attribute((nonnull(1))) openFTLtoml(const char *mode) +FILE * __attribute((malloc)) __attribute((nonnull(1))) openFTLtoml(const char *mode, const unsigned int version) { FILE *fp; // Rotate config file, no rotation is done when the file is opened for @@ -30,8 +32,30 @@ FILE * __attribute((malloc)) __attribute((nonnull(1))) openFTLtoml(const char *m if(mode[0] != 'r') rotate_files(GLOBALTOMLPATH, NULL); - // No readable local file found, try global file - fp = fopen(GLOBALTOMLPATH, mode); + // This should not happen, install a safeguard anyway to unveil + // possible future coding issues early on + if(mode[0] == 'w' && version != 0) + { + log_crit("Writing to version != 0 is not supported in openFTLtoml(%s,%u)", + mode, version); + exit(EXIT_FAILURE); + } + + // Build filename based on version + char filename[PATH_MAX] = { 0 }; + if(version == 0) + { + // Use global config file + strncpy(filename, GLOBALTOMLPATH, sizeof(filename)); + } + else + { + // Use rotated config file + snprintf(filename, sizeof(filename), BACKUP_DIR"/pihole.toml.%u", version); + } + + // Try to open config file + fp = fopen(filename, mode); // Return early if opening failed if(!fp) @@ -41,7 +65,8 @@ FILE * __attribute((malloc)) __attribute((nonnull(1))) openFTLtoml(const char *m if(flock(fileno(fp), LOCK_EX) != 0) { const int _e = errno; - log_err("Cannot open FTL's config file in exclusive mode: %s", strerror(errno)); + log_err("Cannot open config file %s in exclusive mode: %s", + filename, strerror(errno)); fclose(fp); errno = _e; return NULL; diff --git a/src/config/toml_helper.h b/src/config/toml_helper.h index 4a972f447..db0d259a0 100644 --- a/src/config/toml_helper.h +++ b/src/config/toml_helper.h @@ -17,7 +17,7 @@ #include "tomlc99/toml.h" void indentTOML(FILE *fp, const unsigned int indent); -FILE *openFTLtoml(const char *mode) __attribute((malloc)) __attribute((nonnull(1))); +FILE *openFTLtoml(const char *mode, const unsigned int version) __attribute((malloc)) __attribute((nonnull(1))); void closeFTLtoml(FILE *fp); void print_comment(FILE *fp, const char *str, const char *intro, const unsigned int width, const unsigned int indent); void print_toml_allowed_values(cJSON *allowed_values, FILE *fp, const unsigned int width, const unsigned int indent); diff --git a/src/config/toml_reader.c b/src/config/toml_reader.c index 19221549e..dd993d330 100644 --- a/src/config/toml_reader.c +++ b/src/config/toml_reader.c @@ -25,18 +25,19 @@ #include "api/api.h" // Private prototypes -static toml_table_t *parseTOML(void); +static toml_table_t *parseTOML(const unsigned int version); static void reportDebugFlags(void); bool readFTLtoml(struct config *oldconf, struct config *newconf, - toml_table_t *toml, const bool verbose, bool *restart) + toml_table_t *toml, const bool verbose, bool *restart, + const unsigned int version) { // Parse lines in the config file if we did not receive a pointer to a TOML // table from an imported Teleporter file bool teleporter = (toml != NULL); if(!teleporter) { - toml = parseTOML(); + toml = parseTOML(version); if(!toml) return false; } @@ -134,11 +135,11 @@ bool readFTLtoml(struct config *oldconf, struct config *newconf, } // Parse TOML config file -static toml_table_t *parseTOML(void) +static toml_table_t *parseTOML(const unsigned int version) { // Try to open default config file. Use fallback if not found FILE *fp; - if((fp = openFTLtoml("r")) == NULL) + if((fp = openFTLtoml("r", version)) == NULL) { log_warn("No config file available (%s), using defaults", strerror(errno)); @@ -167,7 +168,7 @@ bool getLogFilePathTOML(void) { log_debug(DEBUG_CONFIG, "Reading TOML config file: log file path"); - toml_table_t *conf = parseTOML(); + toml_table_t *conf = parseTOML(0); if(!conf) return false; diff --git a/src/config/toml_reader.h b/src/config/toml_reader.h index efa8863d3..520dc09a0 100644 --- a/src/config/toml_reader.h +++ b/src/config/toml_reader.h @@ -14,7 +14,8 @@ #include "tomlc99/toml.h" bool readFTLtoml(struct config *oldconf, struct config *newconf, - toml_table_t *toml, const bool verbose, bool *restart); + toml_table_t *toml, const bool verbose, bool *restart, + const unsigned int version); bool getLogFilePathTOML(void); #endif //TOML_READER_H diff --git a/src/config/toml_writer.c b/src/config/toml_writer.c index 5cf96822f..3263c1605 100644 --- a/src/config/toml_writer.c +++ b/src/config/toml_writer.c @@ -27,7 +27,7 @@ bool writeFTLtoml(const bool verbose) // Try to open global config file FILE *fp; - if((fp = openFTLtoml("w")) == NULL) + if((fp = openFTLtoml("w", 0)) == NULL) { log_warn("Cannot write to FTL config file (%s), content not updated", strerror(errno)); // Restart watching for changes in the config file diff --git a/src/files.c b/src/files.c index 49247e4d2..79c5c1558 100644 --- a/src/files.c +++ b/src/files.c @@ -32,8 +32,6 @@ #include #include -#define BACKUP_DIR "/etc/pihole/config_backups" - // chmod_file() changes the file mode bits of a given file (relative // to the directory file descriptor) according to mode. mode is an // octal number representing the bit pattern for the new mode bits @@ -509,7 +507,7 @@ void rotate_files(const char *path, char **first_file) chown_pihole(new_path); // Compress file if we are rotating a sufficiently old file - if(i > ZIP_ROTATIONS) + if(i > PLAIN_ROTATIONS) { log_debug(DEBUG_CONFIG, "Compressing %s -> %s", new_path, new_path_compressed); diff --git a/src/files.h b/src/files.h index 007df716a..d2e33cef5 100644 --- a/src/files.h +++ b/src/files.h @@ -15,8 +15,9 @@ // setmntent() #include -#define ZIP_ROTATIONS 3 +#define PLAIN_ROTATIONS 3 #define MAX_ROTATIONS 15 +#define BACKUP_DIR "/etc/pihole/config_backups" bool chmod_file(const char *filename, const mode_t mode); bool file_exists(const char *filename); diff --git a/src/zip/teleporter.c b/src/zip/teleporter.c index 62281d897..e66161627 100644 --- a/src/zip/teleporter.c +++ b/src/zip/teleporter.c @@ -312,7 +312,7 @@ static const char *test_and_import_pihole_toml(void *ptr, size_t size, char * co // a temporary config struct (teleporter_config) struct config teleporter_config = { 0 }; duplicate_config(&teleporter_config, &config); - if(!readFTLtoml(NULL, &teleporter_config, toml, true, NULL)) + if(!readFTLtoml(NULL, &teleporter_config, toml, true, NULL, 0)) return "File etc/pihole/pihole.toml in ZIP archive contains invalid TOML configuration"; // Test dnsmasq config in the imported configuration From 5dc17a1ed33f0dd940e25d8ce7d4418ca082a6a5 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 7 Nov 2023 07:01:09 +0100 Subject: [PATCH 024/221] Move dhcp.domain -> dns.domain Signed-off-by: DL6ER --- src/api/docs/content/specs/config.yaml | 3 +++ src/config/config.c | 9 ++++++++- src/config/config.h | 1 + src/config/dnsmasq_config.c | 16 +++++++++------- src/config/toml_writer.c | 22 ++++++++++++++++++++++ src/setupVars.c | 2 +- test/pihole.toml | 26 +++++++++++++++++++++++++- 7 files changed, 69 insertions(+), 10 deletions(-) diff --git a/src/api/docs/content/specs/config.yaml b/src/api/docs/content/specs/config.yaml index db3154d14..2bbafad0d 100644 --- a/src/api/docs/content/specs/config.yaml +++ b/src/api/docs/content/specs/config.yaml @@ -206,6 +206,8 @@ components: type: boolean expandHosts: type: boolean + domain: + type: string bogusPriv: type: boolean dnssec: @@ -583,6 +585,7 @@ components: - "192.168.2.123 mymusicbox" domainNeeded: true expandHosts: true + domain: "lan" bogusPriv: true dnssec: true interface: "eth0" diff --git a/src/config/config.c b/src/config/config.c index 72c364070..8f57b205d 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -477,6 +477,13 @@ void initConfig(struct config *conf) conf->dns.expandHosts.f = FLAG_RESTART_FTL | FLAG_ADVANCED_SETTING; conf->dns.expandHosts.d.b = false; + conf->dns.domain.k = "dns.domain"; + conf->dns.domain.h = "The DNS domain used by your Pi-hole to expand hosts and for DHCP.\n\n Only if DHCP is enabled below: For DHCP, this has two effects; firstly it causes the DHCP server to return the domain to any hosts which request it, and secondly it sets the domain which it is legal for DHCP-configured hosts to claim. The intention is to constrain hostnames so that an untrusted host on the LAN cannot advertise its name via DHCP as e.g. \"google.com\" and capture traffic not meant for it. If no domain suffix is specified, then any DHCP hostname with a domain part (ie with a period) will be disallowed and logged. If a domain is specified, then hostnames with a domain part are allowed, provided the domain part matches the suffix. In addition, when a suffix is set then hostnames without a domain part have the suffix added as an optional domain part. For instance, we can set domain=mylab.com and have a machine whose DHCP hostname is \"laptop\". The IP address for that machine is available both as \"laptop\" and \"laptop.mylab.com\".\n\n You can disable setting a domain by setting this option to an empty string."; + conf->dns.domain.a = cJSON_CreateStringReference(""); + conf->dns.domain.t = CONF_STRING; + conf->dns.domain.f = FLAG_RESTART_FTL | FLAG_ADVANCED_SETTING; + conf->dns.domain.d.s = (char*)"lan"; + conf->dns.bogusPriv.k = "dns.bogusPriv"; conf->dns.bogusPriv.h = "Should all reverse lookups for private IP ranges (i.e., 192.168.x.y, etc) which are not found in /etc/hosts or the DHCP leases file be answered with \"no such domain\" rather than being forwarded upstream?"; conf->dns.bogusPriv.t = CONF_BOOL; @@ -706,7 +713,7 @@ void initConfig(struct config *conf) conf->dhcp.router.d.s = (char*)""; conf->dhcp.domain.k = "dhcp.domain"; - conf->dhcp.domain.h = "The DNS domain used by your Pi-hole"; + conf->dhcp.domain.h = "The DNS domain used by your Pi-hole (*** DEPRECATED ***)\n This setting is deprecated and will be removed in a future version. Please use dns.domain instead. Setting it to any non-default value will overwrite the value of dns.domain if it is still set to its default value."; conf->dhcp.domain.a = cJSON_CreateStringReference(""); conf->dhcp.domain.t = CONF_STRING; conf->dhcp.domain.f = FLAG_RESTART_FTL | FLAG_ADVANCED_SETTING; diff --git a/src/config/config.h b/src/config/config.h index c230627ee..b266d993a 100644 --- a/src/config/config.h +++ b/src/config/config.h @@ -125,6 +125,7 @@ struct config { struct conf_item hosts; struct conf_item domainNeeded; struct conf_item expandHosts; + struct conf_item domain; struct conf_item bogusPriv; struct conf_item dnssec; struct conf_item interface; diff --git a/src/config/dnsmasq_config.c b/src/config/dnsmasq_config.c index e727e3a7a..840e2df21 100644 --- a/src/config/dnsmasq_config.c +++ b/src/config/dnsmasq_config.c @@ -402,16 +402,18 @@ bool __attribute__((const)) write_dnsmasq_config(struct config *conf, bool test_ fputs("# Never forward A or AAAA queries for plain names, without\n",pihole_conf); fputs("# dots or domain parts, to upstream nameservers. If the name\n", pihole_conf); fputs("# is not known from /etc/hosts or DHCP a NXDOMAIN is returned\n", pihole_conf); - fprintf(pihole_conf, "local=/%s/\n", - conf->dhcp.domain.v.s); - fputs("\n", pihole_conf); + if(strlen(conf->dns.domain.v.s)) + fprintf(pihole_conf, "local=/%s/\n\n", conf->dns.domain.v.s); + else + fputs("\n", pihole_conf); } - if(strlen(conf->dhcp.domain.v.s) > 0 && strcasecmp("none", conf->dhcp.domain.v.s) != 0) + // Add domain to DNS server. It will also be used for DHCP if the DHCP + // server is enabled below + if(strlen(conf->dns.domain.v.s) > 0) { - fputs("# DNS domain for the DHCP server\n", pihole_conf); - fprintf(pihole_conf, "domain=%s\n", conf->dhcp.domain.v.s); - fputs("\n", pihole_conf); + fputs("# DNS domain for both the DNS and DHCP server\n", pihole_conf); + fprintf(pihole_conf, "domain=%s\n\n", conf->dns.domain.v.s); } if(conf->dhcp.active.v.b) diff --git a/src/config/toml_writer.c b/src/config/toml_writer.c index 5cf96822f..88252bf3d 100644 --- a/src/config/toml_writer.c +++ b/src/config/toml_writer.c @@ -20,6 +20,25 @@ // watch_config() #include "config/inotify.h" +static void migrate_config(void) +{ + // Migrating dhcp.domain -> dns.domain + if(strcmp(config.dns.domain.v.s, config.dns.domain.d.s) == 0) + { + // If the domain is the same as the default, check if the dhcp domain + // is different from the default. If so, migrate it + if(strcmp(config.dhcp.domain.v.s, config.dhcp.domain.d.s) != 0) + { + // Migrate dhcp.domain -> dns.domain + log_info("Migrating dhcp.domain = \"%s\" -> dns.domain", config.dhcp.domain.v.s); + if(config.dns.domain.t == CONF_STRING_ALLOCATED) + free(config.dns.domain.v.s); + config.dns.domain.v.s = strdup(config.dhcp.domain.v.s); + config.dns.domain.t = CONF_STRING_ALLOCATED; + } + } +} + bool writeFTLtoml(const bool verbose) { // Stop watching for changes in the config file @@ -52,6 +71,9 @@ bool writeFTLtoml(const bool verbose) fputs(get_FTL_version(), fp); fputs("\n\n", fp); + // Perform possible config migration + migrate_config(); + // Iterate over configuration and store it into the file char *last_path = (char*)""; for(unsigned int i = 0; i < CONFIG_ELEMENTS; i++) diff --git a/src/setupVars.c b/src/setupVars.c index d28640cbd..ca069d5f4 100644 --- a/src/setupVars.c +++ b/src/setupVars.c @@ -339,7 +339,7 @@ void importsetupVarsConf(void) get_conf_upstream_servers_from_setupVars(&config.dns.upstreams); // Try to get Pi-hole domain - get_conf_string_from_setupVars("PIHOLE_DOMAIN", &config.dhcp.domain); + get_conf_string_from_setupVars("PIHOLE_DOMAIN", &config.dns.domain); // Try to get bool properties (the first two are intentionally set from the same key) get_conf_bool_from_setupVars("DNS_FQDN_REQUIRED", &config.dns.domainNeeded); diff --git a/test/pihole.toml b/test/pihole.toml index 2fc68e648..7d080b35b 100644 --- a/test/pihole.toml +++ b/test/pihole.toml @@ -110,6 +110,27 @@ # same way as for DHCP-derived names expandHosts = false + # The DNS domain used by your Pi-hole to expand hosts and for DHCP. + # + # Only if DHCP is enabled below: For DHCP, this has two effects; firstly it causes the + # DHCP server to return the domain to any hosts which request it, and secondly it sets + # the domain which it is legal for DHCP-configured hosts to claim. The intention is to + # constrain hostnames so that an untrusted host on the LAN cannot advertise its name + # via DHCP as e.g. "google.com" and capture traffic not meant for it. If no domain + # suffix is specified, then any DHCP hostname with a domain part (ie with a period) + # will be disallowed and logged. If a domain is specified, then hostnames with a + # domain part are allowed, provided the domain part matches the suffix. In addition, + # when a suffix is set then hostnames without a domain part have the suffix added as + # an optional domain part. For instance, we can set domain=mylab.com and have a + # machine whose DHCP hostname is "laptop". The IP address for that machine is + # available both as "laptop" and "laptop.mylab.com". + # + # You can disable setting a domain by setting this option to an empty string. + # + # Possible values are: + # + domain = "lan" + # Should all reverse lookups for private IP ranges (i.e., 192.168.x.y, etc) which are # not found in /etc/hosts or the DHCP leases file be answered with "no such domain" # rather than being forwarded upstream? @@ -370,7 +391,10 @@ # , e.g., "192.168.0.1" router = "" - # The DNS domain used by your Pi-hole + # The DNS domain used by your Pi-hole (*** DEPRECATED ***) + # This setting is deprecated and will be removed in a future version. Please use + # dns.domain instead. Setting it to any non-default value will overwrite the value of + # dns.domain if it is still set to its default value. # # Possible values are: # From 7dec2595146671171a59db57a5d5b8df1bd17e52 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 7 Nov 2023 21:04:25 +0100 Subject: [PATCH 025/221] Suggest using sudo if insufficient permissions to edit the config file were detected Signed-off-by: DL6ER --- src/capabilities.h | 2 ++ src/config/cli.c | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/src/capabilities.h b/src/capabilities.h index 6f811301e..b5ec38cc3 100644 --- a/src/capabilities.h +++ b/src/capabilities.h @@ -10,6 +10,8 @@ #ifndef CAPABILITIES_H #define CAPABILITIES_H +#include + bool check_capability(const unsigned int cap); bool check_capabilities(void); diff --git a/src/config/cli.c b/src/config/cli.c index ddc23db98..76c6dd07c 100644 --- a/src/config/cli.c +++ b/src/config/cli.c @@ -20,6 +20,8 @@ #include "tomlc99/toml.h" // hash_password() #include "config/password.h" +// check_capability() +#include "capabilities.h" // Read a TOML value from a table depending on its type static bool readStringValue(struct conf_item *conf_item, const char *value, struct config *newconf) @@ -353,6 +355,25 @@ static bool readStringValue(struct conf_item *conf_item, const char *value, stru int set_config_from_CLI(const char *key, const char *value) { + // Check if we are either + // - root, or + // - pihole with CAP_CHOWN capability on the pihole-FTL binary + const uid_t euid = geteuid(); + const struct passwd *current_user = getpwuid(euid); + const bool is_root = euid == 0; + const bool is_pihole = current_user != NULL && strcmp(current_user->pw_name, "pihole") == 0; + const bool have_chown_cap = check_capability(CAP_CHOWN); + if(!is_root && !(is_pihole && have_chown_cap)) + { + if(is_pihole) + printf("Permission error: CAP_CHOWN is missing on the binary\n"); + else + printf("Permission error: User %s is not allowed to edit Pi-hole's config\n", current_user->pw_name); + + printf("Please run this command using sudo\n\n"); + return EXIT_FAILURE; + } + // Identify config option struct config newconf; duplicate_config(&newconf, &config); From ac534ecfb6c52cb0a92911fb78d745fd78dd75c2 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 8 Nov 2023 09:04:49 +0100 Subject: [PATCH 026/221] Also always send all history/clients data Signed-off-by: DL6ER --- src/api/history.c | 57 +++++++++++++++++------------------------------ 1 file changed, 20 insertions(+), 37 deletions(-) diff --git a/src/api/history.c b/src/api/history.c index 2b7f772f8..ada248479 100644 --- a/src/api/history.c +++ b/src/api/history.c @@ -39,35 +39,22 @@ int api_history(struct ftl_conn *api) JSON_ADD_ITEM_TO_ARRAY(history, item); } + // Unlock already here to avoid keeping the lock during JSON generation + // This is safe because we don't access any shared memory after this + // point. All numbers in the JSON are copied + unlock_shm(); + // Minimum structure is // {"history":[]} cJSON *json = JSON_NEW_OBJECT(); JSON_ADD_ITEM_TO_OBJECT(json, "history", history); - JSON_SEND_OBJECT_UNLOCK(json); + JSON_SEND_OBJECT(json); } int api_history_clients(struct ftl_conn *api) { - int sendit = false; - unsigned int from = 0, until = OVERTIME_SLOTS; - const time_t now = time(NULL); - - lock_shm(); - - // Find minimum ID to send - for(unsigned int slot = 0; slot < OVERTIME_SLOTS; slot++) - { - if((overTime[slot].total > 0 || overTime[slot].blocked > 0) && - overTime[slot].timestamp >= overTime[0].timestamp) - { - sendit = true; - from = slot; - break; - } - } - // Exit before processing any data if requested via config setting - if(config.misc.privacylevel.v.privacy_level >= PRIVACY_HIDE_DOMAINS_CLIENTS || !sendit) + if(config.misc.privacylevel.v.privacy_level >= PRIVACY_HIDE_DOMAINS_CLIENTS) { // Minimum structure is // {"history":[], "clients":[]} @@ -79,17 +66,7 @@ int api_history_clients(struct ftl_conn *api) JSON_SEND_OBJECT_UNLOCK(json); } - // End with last non-empty overTime slot or the last slot that is not - // older than the maximum history to be sent - for(unsigned int slot = 0; slot < OVERTIME_SLOTS; slot++) - { - if(overTime[slot].timestamp >= now || - overTime[slot].timestamp - now > (time_t)config.webserver.api.maxHistory.v.ui) - { - until = slot; - break; - } - } + lock_shm(); // Get clients which the user doesn't want to see // if skipclient[i] == true then this client should be hidden from @@ -116,7 +93,7 @@ int api_history_clients(struct ftl_conn *api) } } - // Also skip alias-clients + // Also skip clients included in others (in alias-clients) for(int clientID = 0; clientID < counters->clients; clientID++) { // Get client pointer @@ -127,9 +104,9 @@ int api_history_clients(struct ftl_conn *api) skipclient[clientID] = true; } - cJSON *history = JSON_NEW_ARRAY(); // Main return loop - for(unsigned int slot = from; slot < until; slot++) + cJSON *history = JSON_NEW_ARRAY(); + for(unsigned int slot = 0; slot < OVERTIME_SLOTS; slot++) { cJSON *item = JSON_NEW_OBJECT(); JSON_ADD_NUMBER_TO_OBJECT(item, "timestamp", overTime[slot].timestamp); @@ -158,8 +135,8 @@ int api_history_clients(struct ftl_conn *api) cJSON *json = JSON_NEW_OBJECT(); JSON_ADD_ITEM_TO_OBJECT(json, "history", history); - cJSON *clients = JSON_NEW_ARRAY(); // Loop over clients to generate output to be sent to the client + cJSON *clients = JSON_NEW_ARRAY(); for(int clientID = 0; clientID < counters->clients; clientID++) { if(skipclient[clientID]) @@ -178,10 +155,16 @@ int api_history_clients(struct ftl_conn *api) JSON_REF_STR_IN_OBJECT(item, "ip", client_ip); JSON_ADD_ITEM_TO_ARRAY(clients, item); } - JSON_ADD_ITEM_TO_OBJECT(json, "clients", clients); + + // Unlock already here to avoid keeping the lock during JSON generation + // This is safe because we don't access any shared memory after this + // point and all strings in the JSON are references to idempotent shared + // memory and can, thus, be accessed at any time without locking + unlock_shm(); // Free memory free(skipclient); - JSON_SEND_OBJECT_UNLOCK(json); + JSON_ADD_ITEM_TO_OBJECT(json, "clients", clients); + JSON_SEND_OBJECT(json); } From 5da282eb8dce47ee4640740dc6edc8e31075fc92 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 8 Nov 2023 09:47:00 +0100 Subject: [PATCH 027/221] Tests: Add tests that both /api/history and /api/history/clients return full 24h data Signed-off-by: DL6ER --- test/test_suite.bats | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/test_suite.bats b/test/test_suite.bats index 9355df664..b5de84d88 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -1292,6 +1292,18 @@ [[ ${lines[0]} == '"xn--bc-uia.com"' ]] } +@test "API history: Returns full 24 hours even if only a few queries are made" { + run bash -c 'curl -s 127.0.0.1/api/history | jq ".history | length"' + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == "145" ]] +} + +@test "API history/clients: Returns full 24 hours even if only a few queries are made" { + run bash -c 'curl -s 127.0.0.1/api/history/clients | jq ".history | length"' + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == "145" ]] +} + @test "API authorization (without password): No login required" { run bash -c 'curl -s 127.0.0.1/api/auth' printf "%s\n" "${lines[@]}" From 1f5b15e511a5ffda27e00c9b5adec95f44003f07 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 8 Nov 2023 11:12:37 +0100 Subject: [PATCH 028/221] Fix free() being used in the wrong page Signed-off-by: DL6ER --- src/api/search.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/api/search.c b/src/api/search.c index 3eef33040..0ed5a3bf8 100644 --- a/src/api/search.c +++ b/src/api/search.c @@ -244,9 +244,11 @@ int api_search(struct ftl_conn *api) char *allow_list = cJSON_PrintUnformatted(allow_ids); ret = search_table(api,punycode, GRAVITY_DOMAINLIST_ALLOW_REGEX, allow_list, limit, &Nregex, false, domains); free(allow_list); - free(punycode); if(ret != 200) + { + free(punycode); return ret; + } } if(cJSON_GetArraySize(deny_ids) > 0) @@ -254,9 +256,11 @@ int api_search(struct ftl_conn *api) char *deny_list = cJSON_PrintUnformatted(deny_ids); ret = search_table(api, punycode, GRAVITY_DOMAINLIST_DENY_REGEX, deny_list, limit, &Nregex, false, domains); free(deny_list); - free(punycode); if(ret != 200) + { + free(punycode); return ret; + } } cJSON *search = JSON_NEW_OBJECT(); From 1f42708f14844891132e7d2620c73de7558ee9c3 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 8 Nov 2023 13:23:02 +0100 Subject: [PATCH 029/221] Always set freed pointers to NULL Signed-off-by: DL6ER --- src/FTL.h | 2 +- src/syscalls/free.c | 13 +++++++++++-- src/syscalls/syscalls.h | 2 +- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/FTL.h b/src/FTL.h index c71d39c5e..2b11b32e9 100644 --- a/src/FTL.h +++ b/src/FTL.h @@ -138,7 +138,7 @@ // caused by insufficient memory or by code bugs (not properly dealing // with NULL pointers) much easier. #undef strdup // strdup() is a macro in itself, it needs special handling -#define free(ptr) FTLfree(ptr, __FILE__, __FUNCTION__, __LINE__) +#define free(ptr) FTLfree((void**)&ptr, __FILE__, __FUNCTION__, __LINE__) #define strdup(str_in) FTLstrdup(str_in, __FILE__, __FUNCTION__, __LINE__) #define calloc(numer_of_elements, element_size) FTLcalloc(numer_of_elements, element_size, __FILE__, __FUNCTION__, __LINE__) #define realloc(ptr, new_size) FTLrealloc(ptr, new_size, __FILE__, __FUNCTION__, __LINE__) diff --git a/src/syscalls/free.c b/src/syscalls/free.c index 260e22ea2..1091aa14c 100644 --- a/src/syscalls/free.c +++ b/src/syscalls/free.c @@ -13,17 +13,26 @@ #include "../log.h" #undef free -void FTLfree(void *ptr, const char *file, const char *func, const int line) +void FTLfree(void **ptr, const char *file, const char *func, const int line) { // The free() function frees the memory space pointed to by ptr, which // must have been returned by a previous call to malloc(), calloc(), or // realloc(). Otherwise, or if free(ptr) has already been called before, // undefined behavior occurs. If ptr is NULL, no operation is performed. if(ptr == NULL) + { + log_warn("Trying to free NULL memory location in %s() (%s:%i)", func, file, line); + return; + } + if(*ptr == NULL) { log_warn("Trying to free NULL pointer in %s() (%s:%i)", func, file, line); return; } - free(ptr); + // Actually free the memory + free(*ptr); + + // Set the pointer to NULL + *ptr = NULL; } diff --git a/src/syscalls/syscalls.h b/src/syscalls/syscalls.h index dfe1a1789..dfe510035 100644 --- a/src/syscalls/syscalls.h +++ b/src/syscalls/syscalls.h @@ -14,7 +14,7 @@ char *FTLstrdup(const char *src, const char *file, const char *func, const int line) __attribute__((malloc)); void *FTLcalloc(size_t n, size_t size, const char *file, const char *func, const int line) __attribute__((malloc)) __attribute__((alloc_size(1,2))); void *FTLrealloc(void *ptr_in, size_t size, const char *file, const char *func, const int line) __attribute__((alloc_size(2))); -void FTLfree(void *ptr, const char*file, const char *func, const int line); +void FTLfree(void **ptr, const char*file, const char *func, const int line); int FTLfallocate(const int fd, const off_t offset, const off_t len, const char *file, const char *func, const int line); From 651bb7f065420804b60a95a6bc50c56996349fb6 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 9 Nov 2023 06:29:02 +0100 Subject: [PATCH 030/221] Add X.509 certificate/key parser Signed-off-by: DL6ER --- src/args.c | 60 ++++++++++++ src/enums.h | 9 ++ src/webserver/x509.c | 228 +++++++++++++++++++++++++++++++++++++++++++ src/webserver/x509.h | 3 + 4 files changed, 300 insertions(+) diff --git a/src/args.c b/src/args.c index f80a046e6..4696b5376 100644 --- a/src/args.c +++ b/src/args.c @@ -309,6 +309,7 @@ void parse_args(int argc, char* argv[]) exit(read_teleporter_zip_from_disk(argv[2]) ? EXIT_SUCCESS : EXIT_FAILURE); } + // Generate X.509 certificate if(argc > 1 && strcmp(argv[1], "--gen-x509") == 0) { if(argc < 3 || argc > 5) @@ -327,6 +328,55 @@ void parse_args(int argc, char* argv[]) exit(generate_certificate(argv[2], rsa, domain) ? EXIT_SUCCESS : EXIT_FAILURE); } + // Parse X.509 certificate + if(argc > 1 && + (strcmp(argv[1], "--read-x509") == 0 || + strcmp(argv[1], "--read-x509-key") == 0)) + { + if(argc < 2 || argc > 4) + { + printf("Usage: %s %s [] []\n", argv[0], argv[1]); + printf("Example: %s %s /etc/pihole/tls.pem\n", argv[0], argv[1]); + printf(" with domain: %s %s /etc/pihole/tls.pem pi.hole\n", argv[0], argv[1]); + exit(EXIT_FAILURE); + } + + // Option parsing + // Should we report on the private key? + const bool private_key = strcmp(argv[1], "--read-x509-key") == 0; + // If no certificate file is given, we use the one from the config + const char *certfile = NULL; + if(argc == 2) + { + readFTLconf(&config, false); + certfile = config.webserver.tls.cert.v.s; + } + else + certfile = argv[2]; + + // If no domain is given, we only check the certificate + const char *domain = argc > 3 ? argv[3] : NULL; + + // Enable stdout printing + cli_mode = true; + log_ctrl(false, true); + + enum cert_check result = read_certificate(certfile, domain, private_key); + + if(argc < 4) + exit(result == CERT_OKAY ? EXIT_SUCCESS : EXIT_FAILURE); + else if(result == CERT_DOMAIN_MATCH) + { + printf("Certificate matches domain %s\n", argv[3]); + exit(EXIT_SUCCESS); + } + else + { + printf("Certificate does not match domain %s\n", argv[3]); + exit(EXIT_FAILURE); + } + } + // If the first argument is "gravity" (e.g., /usr/bin/pihole-FTL gravity), // we offer some specialized gravity tools if(argc > 1 && (strcmp(argv[1], "gravity") == 0 || strcmp(argv[1], "antigravity") == 0)) @@ -812,6 +862,16 @@ void parse_args(int argc, char* argv[]) printf(" an RSA (4096 bit) key will be generated instead.\n\n"); printf(" Usage: %spihole-FTL --gen-x509 %soutfile %s[rsa]%s\n\n", green, cyan, purple, normal); + printf("%sTLS X.509 certificate parser:%s\n", yellow, normal); + printf(" Parse the given X.509 certificate and optionally check if\n"); + printf(" it matches a given domain. If no domain is given, only a\n"); + printf(" human-readable output string is printed.\n\n"); + printf(" If no certificate file is given, the one from the config\n"); + printf(" is used (if applicable). If --read-x509-key is used, details\n"); + printf(" about the private key are printed as well.\n\n"); + printf(" Usage: %spihole-FTL --read-x509 %s[certfile] %s[domain]%s\n", green, cyan, purple, normal); + printf(" Usage: %spihole-FTL --read-x509-key %s[certfile] %s[domain]%s\n\n", green, cyan, purple, normal); + printf("%sGravity tools:%s\n", yellow, normal); printf(" Check domains in a given file for validity using Pi-hole's\n"); printf(" gravity filters. The expected input format is one domain\n"); diff --git a/src/enums.h b/src/enums.h index bfbcff338..c9294cdd9 100644 --- a/src/enums.h +++ b/src/enums.h @@ -311,4 +311,13 @@ enum adlist_type { ADLIST_ALLOW } __attribute__ ((packed)); +enum cert_check { + CERT_FILE_NOT_FOUND, + CERT_CANNOT_PARSE_CERT, + CERT_CANNOT_PARSE_KEY, + CERT_DOMAIN_MISMATCH, + CERT_DOMAIN_MATCH, + CERT_OKAY +} __attribute__ ((packed)); + #endif // ENUMS_H diff --git a/src/webserver/x509.c b/src/webserver/x509.c index eff3ed592..219ab4f92 100644 --- a/src/webserver/x509.c +++ b/src/webserver/x509.c @@ -282,3 +282,231 @@ bool generate_certificate(const char* certfile, bool rsa, const char *domain) return true; } + +// This function reads a X.509 certificate from a file and prints a +// human-readable representation of the certificate to stdout. If a domain is +// specified, we only check if this domain is present in the certificate. +// Otherwise, we print verbose human-readable information about the certificate +// and about the private key (if requested). +enum cert_check read_certificate(const char* certfile, const char *domain, const bool private_key) +{ + if(certfile == NULL && domain == NULL) + { + log_err("No certificate file specified\n"); + return CERT_FILE_NOT_FOUND; + } + + mbedtls_x509_crt crt; + mbedtls_pk_context key; + mbedtls_entropy_context entropy; + mbedtls_ctr_drbg_context ctr_drbg; + mbedtls_x509_crt_init(&crt); + mbedtls_pk_init(&key); + mbedtls_entropy_init(&entropy); + mbedtls_ctr_drbg_init(&ctr_drbg); + + printf("Reading certificate from %s ...\n\n", certfile); + + // Check if the file exists and is readable + if(access(certfile, R_OK) != 0) + { + log_err("Could not read certificate file: %s\n", strerror(errno)); + return CERT_FILE_NOT_FOUND; + } + + int rc = mbedtls_pk_parse_keyfile(&key, certfile, NULL, mbedtls_ctr_drbg_random, &ctr_drbg); + if (rc != 0) + { + log_err("Cannot parse key: Error code %d\n", rc); + return CERT_CANNOT_PARSE_KEY; + } + + rc = mbedtls_x509_crt_parse_file(&crt, certfile); + if (rc != 0) + { + log_err("Cannt parse certificate: Error code %d\n", rc); + return CERT_CANNOT_PARSE_CERT; + } + + // Parse mbedtls_x509_parse_subject_alt_names() + mbedtls_x509_sequence *sans = &crt.subject_alt_names; + bool found = false; + if(domain != NULL) + { + // Loop over all SANs + while(sans != NULL) + { + // Parse the SAN + mbedtls_x509_subject_alternative_name san = { 0 }; + const int ret = mbedtls_x509_parse_subject_alt_name(&sans->buf, &san); + + // Check if SAN is used (otherwise ret < 0, e.g., + // MBEDTLS_ERR_X509_FEATURE_UNAVAILABLE) and if it is a + // DNS name, skip otherwise + if(ret < 0 || san.type != MBEDTLS_X509_SAN_DNS_NAME) + goto next_san; + + // Check if the SAN matches the domain + if(strncasecmp(domain, (char*)san.san.unstructured_name.p, san.san.unstructured_name.len) == 0) + { + found = true; + break; + } +next_san: + // Go to next SAN + sans = sans->next; + } + + // Also check against the common name (CN) field + char subject[MBEDTLS_X509_MAX_DN_NAME_SIZE]; + if(mbedtls_x509_dn_gets(subject, sizeof(subject), &crt.subject) > 0 && strcasecmp(domain, subject) == 0) + found = true; + + // Free resources + mbedtls_x509_crt_free(&crt); + mbedtls_pk_free(&key); + mbedtls_entropy_free(&entropy); + mbedtls_ctr_drbg_free(&ctr_drbg); + return found ? CERT_DOMAIN_MATCH : CERT_DOMAIN_MISMATCH; + } + + // else: Print verbose information about the certificate + char certinfo[BUFFER_SIZE] = { 0 }; + mbedtls_x509_crt_info(certinfo, BUFFER_SIZE, " ", &crt); + puts("Certificate (X.509):\n"); + puts(certinfo); + + if(!private_key) + goto end; + + puts("Private key:"); + const char *keytype = mbedtls_pk_get_name(&key); + printf(" Type: %s\n", keytype); + mbedtls_pk_type_t pk_type = mbedtls_pk_get_type(&key); + if(pk_type == MBEDTLS_PK_RSA) + { + mbedtls_rsa_context *rsa = mbedtls_pk_rsa(key); + printf(" RSA modulus: %zu bit\n", 8*mbedtls_rsa_get_len(rsa)); + mbedtls_mpi E, N, P, Q, D; + mbedtls_mpi_init(&E); // E = public exponent (public) + mbedtls_mpi_init(&N); // N = P * Q (public) + mbedtls_mpi_init(&P); // P = prime factor 1 (private) + mbedtls_mpi_init(&Q); // Q = prime factor 2 (private) + mbedtls_mpi_init(&D); // D = private exponent (private) + mbedtls_mpi DP, DQ, QP; + mbedtls_mpi_init(&DP); + mbedtls_mpi_init(&DQ); + mbedtls_mpi_init(&QP); + if(mbedtls_rsa_export(rsa, &N, &P, &Q, &D, &E) != 0 || + mbedtls_rsa_export_crt(rsa, &DP, &DQ, &QP) != 0) + { + puts(" could not export RSA parameters\n"); + return EXIT_FAILURE; + } + puts(" Core parameters:"); + if(mbedtls_mpi_write_file(" Exponent:\n E = 0x", &E, 16, NULL) != 0) + { + puts(" could not write MPI\n"); + return EXIT_FAILURE; + } + + if(mbedtls_mpi_write_file(" Modulus:\n N = 0x", &N, 16, NULL) != 0) + { + puts(" could not write MPI\n"); + return EXIT_FAILURE; + } + + if(mbedtls_mpi_cmp_mpi(&P, &Q) >= 0) + { + if(mbedtls_mpi_write_file(" Prime factors:\n P = 0x", &P, 16, NULL) != 0 || + mbedtls_mpi_write_file(" Q = 0x", &Q, 16, NULL) != 0) + { + puts(" could not write MPIs\n"); + return EXIT_FAILURE; + } + } + else + { + if(mbedtls_mpi_write_file(" Prime factors:\n Q = 0x", &Q, 16, NULL) != 0 || + mbedtls_mpi_write_file("\n P = 0x", &P, 16, NULL) != 0) + { + puts(" could not write MPIs\n"); + return EXIT_FAILURE; + } + } + + if(mbedtls_mpi_write_file(" Private exponent:\n D = 0x", &D, 16, NULL) != 0) + { + puts(" could not write MPI\n"); + return EXIT_FAILURE; + } + + mbedtls_mpi_free(&N); + mbedtls_mpi_free(&P); + mbedtls_mpi_free(&Q); + mbedtls_mpi_free(&D); + mbedtls_mpi_free(&E); + + puts(" CRT parameters:"); + if(mbedtls_mpi_write_file(" D mod (P-1):\n DP = 0x", &DP, 16, NULL) != 0 || + mbedtls_mpi_write_file(" D mod (Q-1):\n DQ = 0x", &DQ, 16, NULL) != 0 || + mbedtls_mpi_write_file(" Q^-1 mod P:\n QP = 0x", &QP, 16, NULL) != 0) + { + puts(" could not write MPIs\n"); + return EXIT_FAILURE; + } + + mbedtls_mpi_free(&DP); + mbedtls_mpi_free(&DQ); + mbedtls_mpi_free(&QP); + + } + else if(pk_type == MBEDTLS_PK_ECKEY) + { + mbedtls_ecp_keypair *ec = mbedtls_pk_ec(key); + mbedtls_ecp_curve_type ect = mbedtls_ecp_get_type(&ec->private_grp); + switch (ect) + { + case MBEDTLS_ECP_TYPE_NONE: + puts(" Curve type: Unknown"); + break; + case MBEDTLS_ECP_TYPE_SHORT_WEIERSTRASS: + puts(" Curve type: Short Weierstrass (y^2 = x^3 + a x + b)"); + break; + case MBEDTLS_ECP_TYPE_MONTGOMERY: + puts(" Curve type: Montgomery (y^2 = x^3 + a x^2 + x)"); + break; + } + const size_t bitlen = mbedtls_mpi_bitlen(&ec->private_d); + printf(" Bitlen: %zu bit\n", bitlen); + + mbedtls_mpi_write_file(" Private key:\n D = 0x", &ec->private_d, 16, NULL); + mbedtls_mpi_write_file(" Public key:\n X = 0x", &ec->MBEDTLS_PRIVATE(Q).MBEDTLS_PRIVATE(X), 16, NULL); + mbedtls_mpi_write_file(" Y = 0x", &ec->MBEDTLS_PRIVATE(Q).MBEDTLS_PRIVATE(Y), 16, NULL); + mbedtls_mpi_write_file(" Z = 0x", &ec->MBEDTLS_PRIVATE(Q).MBEDTLS_PRIVATE(Z), 16, NULL); + } + else + { + puts("Sorry, but FTL does not know how to print key information for this type\n"); + goto end; + } + + // Print private key in PEM format + mbedtls_pk_write_key_pem(&key, (unsigned char*)certinfo, BUFFER_SIZE); + puts("Private key (PEM):"); + puts(certinfo); + +end: + // Print public key in PEM format + mbedtls_pk_write_pubkey_pem(&key, (unsigned char*)certinfo, BUFFER_SIZE); + puts("Public key (PEM):"); + puts(certinfo); + + // Free resources + mbedtls_x509_crt_free(&crt); + mbedtls_pk_free(&key); + mbedtls_entropy_free(&entropy); + mbedtls_ctr_drbg_free(&ctr_drbg); + + return CERT_OKAY; +} diff --git a/src/webserver/x509.h b/src/webserver/x509.h index ad97537a4..e59ee1a79 100644 --- a/src/webserver/x509.h +++ b/src/webserver/x509.h @@ -13,6 +13,9 @@ #include #include +#include "enums.h" + bool generate_certificate(const char* certfile, bool rsa, const char *domain); +enum cert_check read_certificate(const char* certfile, const char *domain, const bool private_key); #endif // X509_H From 6498e6bf93279486e3b5de829958c17bf247f9fe Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 9 Nov 2023 16:00:26 +0100 Subject: [PATCH 031/221] Add logging to the Pi-hole diagnosis system when we detect a certificate domain mismatch Signed-off-by: DL6ER --- src/database/message-table.c | 53 ++++++++++++++++++++++++++++++++++++ src/database/message-table.h | 1 + src/enums.h | 1 + src/webserver/webserver.c | 22 +++++++++------ 4 files changed, 69 insertions(+), 8 deletions(-) diff --git a/src/database/message-table.c b/src/database/message-table.c index 72ac7b2f9..39b8e7f97 100644 --- a/src/database/message-table.c +++ b/src/database/message-table.c @@ -54,6 +54,8 @@ static const char *get_message_type_str(const enum message_type type) return "LIST"; case DISK_MESSAGE_EXTENDED: return "DISK_EXTENDED"; + case CERTIFICATE_DOMAIN_MISMATCH_MESSAGE: + return "CERTIFICATE_DOMAIN_MISMATCH"; case MAX_MESSAGE: default: return "UNKNOWN"; @@ -84,6 +86,8 @@ static enum message_type get_message_type_from_string(const char *typestr) return INACCESSIBLE_ADLIST_MESSAGE; else if (strcmp(typestr, "DISK_EXTENDED") == 0) return DISK_MESSAGE_EXTENDED; + else if (strcmp(typestr, "CERTIFICATE_DOMAIN_MISMATCH") == 0) + return CERTIFICATE_DOMAIN_MISMATCH_MESSAGE; else return MAX_MESSAGE; } @@ -653,6 +657,28 @@ static void format_inaccessible_adlist_message(char *plain, const int sizeof_pla free(escaped_address); } +static void format_certificate_domain_mismatch(char *plain, const int sizeof_plain, char *html, const int sizeof_html, + const char *certfile, const char*domain) +{ + if(snprintf(plain, sizeof_plain, "SSL/TLS certificate %s does not match domain %s!", certfile, domain) > sizeof_plain) + log_warn("format_certificate_domain_mismatch(): Buffer too small to hold plain message, warning truncated"); + + // Return early if HTML text is not required + if(sizeof_html < 1 || html == NULL) + return; + + char *escaped_certfile = escape_html(certfile); + char *escaped_domain = escape_html(domain); + + if(snprintf(html, sizeof_html, "SSL/TLS certificate %s does not match domain %s!", escaped_certfile, escaped_domain) > sizeof_html) + log_warn("format_certificate_domain_mismatch(): Buffer too small to hold HTML message, warning truncated"); + + if(escaped_certfile != NULL) + free(escaped_certfile); + if(escaped_domain != NULL) + free(escaped_domain); +} + int count_messages(const bool filter_dnsmasq_warnings) { int count = 0; @@ -876,6 +902,17 @@ bool format_messages(cJSON *array) break; } + + case CERTIFICATE_DOMAIN_MISMATCH_MESSAGE: + { + const char *certfile = (const char*)sqlite3_column_text(stmt, 3); + const char *domain = (const char*)sqlite3_column_text(stmt, 4); + + format_certificate_domain_mismatch(plain, sizeof(plain), html, sizeof(html), + certfile, domain); + + break; + } } // Add the plain message @@ -1095,3 +1132,19 @@ void logg_inaccessible_adlist(const int dbindex, const char *address) if(rowid == -1) log_err("logg_inaccessible_adlist(): Failed to add message to database"); } + +void log_certificate_domain_mismatch(const char *certfile, const char *domain) +{ + // Create message + char buf[2048]; + format_certificate_domain_mismatch(buf, sizeof(buf), NULL, 0, certfile, domain); + + // Log to FTL.log + log_warn("%s", buf); + + // Log to database + const int rowid = add_message(CERTIFICATE_DOMAIN_MISMATCH_MESSAGE, certfile, 1, domain); + + if(rowid == -1) + log_err("log_certificate_domain_mismatch(): Failed to add message to database"); +} diff --git a/src/database/message-table.h b/src/database/message-table.h index 52aff56c5..196b406ab 100644 --- a/src/database/message-table.h +++ b/src/database/message-table.h @@ -28,5 +28,6 @@ void logg_rate_limit_message(const char *clientIP, const unsigned int rate_limit void logg_warn_dnsmasq_message(char *message); void log_resource_shortage(const double load, const int nprocs, const int shmem, const int disk, const char *path, const char *msg); void logg_inaccessible_adlist(const int dbindex, const char *address); +void log_certificate_domain_mismatch(const char *certfile, const char *domain); #endif //MESSAGETABLE_H diff --git a/src/enums.h b/src/enums.h index c9294cdd9..65e481826 100644 --- a/src/enums.h +++ b/src/enums.h @@ -270,6 +270,7 @@ enum message_type { DISK_MESSAGE, INACCESSIBLE_ADLIST_MESSAGE, DISK_MESSAGE_EXTENDED, + CERTIFICATE_DOMAIN_MISMATCH_MESSAGE, MAX_MESSAGE, } __attribute__ ((packed)); diff --git a/src/webserver/webserver.c b/src/webserver/webserver.c index 59f6d93a6..95940e492 100644 --- a/src/webserver/webserver.c +++ b/src/webserver/webserver.c @@ -8,24 +8,26 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "../FTL.h" -#include "webserver.h" +#include "FTL.h" +#include "webserver/webserver.h" // api_handler() -#include "../api/api.h" +#include "api/api.h" // send_http() #include "http-common.h" // struct config -#include "../config/config.h" +#include "config/config.h" // log_web() -#include "../log.h" +#include "log.h" // get_nprocs() #include // file_readable() -#include "../files.h" +#include "files.h" // generate_certificate() -#include "x509.h" +#include "webserver/x509.h" // allocate_lua(), free_lua(), init_lua(), request_handler() -#include "lua_web.h" +#include "webserver/lua_web.h" +// log_certificate_domain_mismatch() +#include "database/message-table.h" // Server context handle static struct mg_context *ctx = NULL; @@ -341,6 +343,10 @@ void http_init(void) if(file_readable(config.webserver.tls.cert.v.s)) { + if(read_certificate(config.webserver.tls.cert.v.s, config.webserver.domain.v.s, false) != CERT_DOMAIN_MATCH) + { + log_certificate_domain_mismatch(config.webserver.tls.cert.v.s, config.webserver.domain.v.s); + } options[++next_option] = "ssl_certificate"; options[++next_option] = config.webserver.tls.cert.v.s; From 58b402be6f1a3283e172b12d1b9b4e5ae48e0355 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 9 Nov 2023 16:30:22 +0100 Subject: [PATCH 032/221] Add tests for --read-x509(-key) Signed-off-by: DL6ER --- src/webserver/x509.c | 12 +++++-- test/test_suite.bats | 83 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+), 2 deletions(-) diff --git a/src/webserver/x509.c b/src/webserver/x509.c index 219ab4f92..b4ed74771 100644 --- a/src/webserver/x509.c +++ b/src/webserver/x509.c @@ -359,8 +359,16 @@ enum cert_check read_certificate(const char* certfile, const char *domain, const // Also check against the common name (CN) field char subject[MBEDTLS_X509_MAX_DN_NAME_SIZE]; - if(mbedtls_x509_dn_gets(subject, sizeof(subject), &crt.subject) > 0 && strcasecmp(domain, subject) == 0) - found = true; + if(mbedtls_x509_dn_gets(subject, sizeof(subject), &crt.subject) > 0) + { + // Check subject == "CN=" + if(strlen(subject) > 3 && strncasecmp(subject, "CN=", 3) == 0 && strcasecmp(domain, subject + 3) == 0) + found = true; + // Check subject == "" + else if(strcasecmp(domain, subject) == 0) + found = true; + } + // Free resources mbedtls_x509_crt_free(&crt); diff --git a/test/test_suite.bats b/test/test_suite.bats index 9355df664..78f291bad 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -1356,6 +1356,89 @@ run bash -c 'curl -I --cacert /etc/pihole/test.crt --resolve pi.hole:443:127.0.0.1 https://pi.hole/' } +@test "X.509 certificate parser returns expected result" { + # We are getting the certificate from the config + run bash -c './pihole-FTL --read-x509' + printf "%s\n" "${lines[@]}" + [[ "${lines[0]}" == "Reading certificate from /etc/pihole/test.pem ..." ]] + [[ "${lines[1]}" == "Certificate (X.509):" ]] + [[ "${lines[2]}" == " cert. version : 3" ]] + [[ "${lines[3]}" == " serial number : 30:36:35:35:38:30:34:30:38:32:39:39:39:31:36" ]] + [[ "${lines[4]}" == " issuer name : CN=pi.hole" ]] + [[ "${lines[5]}" == " subject name : CN=pi.hole" ]] + [[ "${lines[6]}" == " issued on : 2001-01-01 00:00:00" ]] + [[ "${lines[7]}" == " expires on : 2030-12-31 23:59:59" ]] + [[ "${lines[8]}" == " signed using : ECDSA with SHA256" ]] + [[ "${lines[9]}" == " EC key size : 521 bits" ]] + [[ "${lines[10]}" == " basic constraints : CA=false" ]] + [[ "${lines[11]}" == "Public key (PEM):" ]] + [[ "${lines[12]}" == "-----BEGIN PUBLIC KEY-----" ]] + [[ "${lines[13]}" == "MIGbMBAGByqGSM49AgEGBSuBBAAjA4GGAAQBQ51HeOLjSap1Xr+pnFQJqvBZc92T" ]] + [[ "${lines[14]}" == "XyL4KwIZdpsHl95Pc0Xcn8Xzyox0cWhMyycQgcGbIw3nuefCZaXfc3CuU30BPDdb" ]] + [[ "${lines[15]}" == "91h+rDhV4+VkEkANPBbgKQ6kCiHNtMAdugyaeHxzFpqegGGvgQ2l4Vp98l4M7zBC" ]] + [[ "${lines[16]}" == "G6K/RbZDlDvNUCgwElE=" ]] + [[ "${lines[17]}" == "-----END PUBLIC KEY-----" ]] + [[ "${lines[18]}" == "" ]] +} + +@test "X.509 certificate parser returns expected result (with private key)" { + # We are explicitly specifying the certificate file here + run bash -c './pihole-FTL --read-x509-key /etc/pihole/test.pem' + printf "%s\n" "${lines[@]}" + [[ "${lines[0]}" == "Reading certificate from /etc/pihole/test.pem ..." ]] + [[ "${lines[1]}" == "Certificate (X.509):" ]] + [[ "${lines[2]}" == " cert. version : 3" ]] + [[ "${lines[3]}" == " serial number : 30:36:35:35:38:30:34:30:38:32:39:39:39:31:36" ]] + [[ "${lines[4]}" == " issuer name : CN=pi.hole" ]] + [[ "${lines[5]}" == " subject name : CN=pi.hole" ]] + [[ "${lines[6]}" == " issued on : 2001-01-01 00:00:00" ]] + [[ "${lines[7]}" == " expires on : 2030-12-31 23:59:59" ]] + [[ "${lines[8]}" == " signed using : ECDSA with SHA256" ]] + [[ "${lines[9]}" == " EC key size : 521 bits" ]] + [[ "${lines[10]}" == " basic constraints : CA=false" ]] + [[ "${lines[11]}" == "Private key:" ]] + [[ "${lines[12]}" == " Type: EC" ]] + [[ "${lines[13]}" == " Curve type: Short Weierstrass (y^2 = x^3 + a x + b)" ]] + [[ "${lines[14]}" == " Bitlen: 518 bit" ]] + [[ "${lines[15]}" == " Private key:" ]] + [[ "${lines[16]}" == " D = 0x2CBE6CF8A913B445F211165B0473B7037B5B06187C8685AEF4A58354C7061C388173E0B00374A55CEAC7BB5886159C9D54B3C020564355A0FA71A55559304156D8"* ]] + [[ "${lines[17]}" == " Public key:" ]] + [[ "${lines[18]}" == " X = 0x01439D4778E2E349AA755EBFA99C5409AAF05973DD935F22F82B0219769B0797DE4F7345DC9FC5F3CA8C7471684CCB271081C19B230DE7B9E7C265A5DF7370AE537D"* ]] + [[ "${lines[19]}" == " Y = 0x013C375BF7587EAC3855E3E56412400D3C16E0290EA40A21CDB4C01DBA0C9A787C73169A9E8061AF810DA5E15A7DF25E0CEF30421BA2BF45B643943BCD5028301251"* ]] + [[ "${lines[20]}" == " Z = 0x01"* ]] + [[ "${lines[21]}" == "Private key (PEM):" ]] + [[ "${lines[22]}" == "-----BEGIN EC PRIVATE KEY-----" ]] + [[ "${lines[23]}" == "MIHcAgEBBEIALL5s+KkTtEXyERZbBHO3A3tbBhh8hoWu9KWDVMcGHDiBc+CwA3Sl" ]] + [[ "${lines[24]}" == "XOrHu1iGFZydVLPAIFZDVaD6caVVWTBBVtigBwYFK4EEACOhgYkDgYYABAFDnUd4" ]] + [[ "${lines[25]}" == "4uNJqnVev6mcVAmq8Flz3ZNfIvgrAhl2mweX3k9zRdyfxfPKjHRxaEzLJxCBwZsj" ]] + [[ "${lines[26]}" == "Dee558Jlpd9zcK5TfQE8N1v3WH6sOFXj5WQSQA08FuApDqQKIc20wB26DJp4fHMW" ]] + [[ "${lines[27]}" == "mp6AYa+BDaXhWn3yXgzvMEIbor9FtkOUO81QKDASUQ==" ]] + [[ "${lines[28]}" == "-----END EC PRIVATE KEY-----" ]] + [[ "${lines[29]}" == "Public key (PEM):" ]] + [[ "${lines[30]}" == "-----BEGIN PUBLIC KEY-----" ]] + [[ "${lines[31]}" == "MIGbMBAGByqGSM49AgEGBSuBBAAjA4GGAAQBQ51HeOLjSap1Xr+pnFQJqvBZc92T" ]] + [[ "${lines[32]}" == "XyL4KwIZdpsHl95Pc0Xcn8Xzyox0cWhMyycQgcGbIw3nuefCZaXfc3CuU30BPDdb" ]] + [[ "${lines[33]}" == "91h+rDhV4+VkEkANPBbgKQ6kCiHNtMAdugyaeHxzFpqegGGvgQ2l4Vp98l4M7zBC" ]] + [[ "${lines[34]}" == "G6K/RbZDlDvNUCgwElE=" ]] + [[ "${lines[35]}" == "-----END PUBLIC KEY-----" ]] + [[ "${lines[36]}" == "" ]] +} + +@test "X.509 certificate parser can check if domain is included" { + run bash -c './pihole-FTL --read-x509-key /etc/pihole/test.pem pi.hole' + printf "%s\n" "${lines[@]}" + [[ "${lines[0]}" == "Reading certificate from /etc/pihole/test.pem ..." ]] + [[ "${lines[1]}" == "Certificate matches domain pi.hole" ]] + [[ "${lines[2]}" == "" ]] + [[ $status == 0 ]] + run bash -c './pihole-FTL --read-x509-key /etc/pihole/test.pem pi-hole.net' + printf "%s\n" "${lines[@]}" + [[ "${lines[0]}" == "Reading certificate from /etc/pihole/test.pem ..." ]] + [[ "${lines[1]}" == "Certificate does not match domain pi-hole.net" ]] + [[ "${lines[2]}" == "" ]] + [[ $status == 1 ]] +} + @test "Test embedded GZIP compressor" { run bash -c './pihole-FTL gzip test/pihole-FTL.db.sql' printf "Compression output:\n" From d310f8efb3d5fc85cd0521d7145b51aaf8d66d54 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 9 Nov 2023 17:50:07 +0100 Subject: [PATCH 033/221] Spellcheck fixes Signed-off-by: DL6ER --- src/webserver/x509.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/webserver/x509.c b/src/webserver/x509.c index b4ed74771..ca0b21403 100644 --- a/src/webserver/x509.c +++ b/src/webserver/x509.c @@ -324,7 +324,7 @@ enum cert_check read_certificate(const char* certfile, const char *domain, const rc = mbedtls_x509_crt_parse_file(&crt, certfile); if (rc != 0) { - log_err("Cannt parse certificate: Error code %d\n", rc); + log_err("Cannot parse certificate: Error code %d\n", rc); return CERT_CANNOT_PARSE_CERT; } @@ -472,8 +472,8 @@ enum cert_check read_certificate(const char* certfile, const char *domain, const else if(pk_type == MBEDTLS_PK_ECKEY) { mbedtls_ecp_keypair *ec = mbedtls_pk_ec(key); - mbedtls_ecp_curve_type ect = mbedtls_ecp_get_type(&ec->private_grp); - switch (ect) + mbedtls_ecp_curve_type ec_type = mbedtls_ecp_get_type(&ec->private_grp); + switch (ec_type) { case MBEDTLS_ECP_TYPE_NONE: puts(" Curve type: Unknown"); From 69db4c18e84d7e51156ae849cefd3053343c3ff4 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 9 Nov 2023 18:49:19 +0100 Subject: [PATCH 034/221] Run codespell on push Signed-off-by: DL6ER --- .github/workflows/codespell.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml index 777a4d7f0..874ee179d 100644 --- a/.github/workflows/codespell.yml +++ b/.github/workflows/codespell.yml @@ -1,5 +1,8 @@ name: Codespell on: + push: + branches: + - '**' pull_request: types: [opened, synchronize, reopened, ready_for_review] From 10c635b0189a6c560e0a470209902bdc903e9555 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 9 Nov 2023 19:05:03 +0100 Subject: [PATCH 035/221] Do not rewrite config file pihole.toml when the content did not change Signed-off-by: DL6ER --- src/config/toml_helper.c | 12 ++------ src/config/toml_helper.h | 2 +- src/config/toml_reader.c | 2 +- src/config/toml_writer.c | 60 +++++++++++++++++++++++++++++--------- src/files.c | 63 ++++++++++++++++++++++++++++++++++++++++ src/files.h | 1 + 6 files changed, 115 insertions(+), 25 deletions(-) diff --git a/src/config/toml_helper.c b/src/config/toml_helper.c index 7f40b683f..c91730d66 100644 --- a/src/config/toml_helper.c +++ b/src/config/toml_helper.c @@ -22,16 +22,10 @@ #include "config/password.h" // Open the TOML file for reading or writing -FILE * __attribute((malloc)) __attribute((nonnull(1))) openFTLtoml(const char *mode) +FILE * __attribute((malloc)) __attribute((nonnull(1,2))) openFTLtoml(const char *path, const char *mode) { - FILE *fp; - // Rotate config file, no rotation is done when the file is opened for - // reading (mode == "r") - if(mode[0] != 'r') - rotate_files(GLOBALTOMLPATH, NULL); - - // No readable local file found, try global file - fp = fopen(GLOBALTOMLPATH, mode); + // Try to open file in requested mode + FILE *fp = fopen(path, mode); // Return early if opening failed if(!fp) diff --git a/src/config/toml_helper.h b/src/config/toml_helper.h index 4a972f447..6ae9ca778 100644 --- a/src/config/toml_helper.h +++ b/src/config/toml_helper.h @@ -17,7 +17,7 @@ #include "tomlc99/toml.h" void indentTOML(FILE *fp, const unsigned int indent); -FILE *openFTLtoml(const char *mode) __attribute((malloc)) __attribute((nonnull(1))); +FILE *openFTLtoml(const char *path, const char *mode) __attribute((malloc)) __attribute((nonnull(1,2))); void closeFTLtoml(FILE *fp); void print_comment(FILE *fp, const char *str, const char *intro, const unsigned int width, const unsigned int indent); void print_toml_allowed_values(cJSON *allowed_values, FILE *fp, const unsigned int width, const unsigned int indent); diff --git a/src/config/toml_reader.c b/src/config/toml_reader.c index 19221549e..d25448579 100644 --- a/src/config/toml_reader.c +++ b/src/config/toml_reader.c @@ -138,7 +138,7 @@ static toml_table_t *parseTOML(void) { // Try to open default config file. Use fallback if not found FILE *fp; - if((fp = openFTLtoml("r")) == NULL) + if((fp = openFTLtoml(GLOBALTOMLPATH, "r")) == NULL) { log_warn("No config file available (%s), using defaults", strerror(errno)); diff --git a/src/config/toml_writer.c b/src/config/toml_writer.c index 5cf96822f..f7d4e13bb 100644 --- a/src/config/toml_writer.c +++ b/src/config/toml_writer.c @@ -19,27 +19,19 @@ #include "datastructure.h" // watch_config() #include "config/inotify.h" +// files_different() +#include "files.h" bool writeFTLtoml(const bool verbose) { - // Stop watching for changes in the config file - watch_config(false); - - // Try to open global config file + // Try to open a temporary config file for writing FILE *fp; - if((fp = openFTLtoml("w")) == NULL) + if((fp = openFTLtoml(GLOBALTOMLPATH".tmp", "w")) == NULL) { log_warn("Cannot write to FTL config file (%s), content not updated", strerror(errno)); - // Restart watching for changes in the config file - watch_config(true); return false; } - // Log that we are (re-)writing the config file if either in verbose or - // debug mode - if(verbose || config.debug.config.v.b) - log_info("Writing config file"); - // Write header fputs("# This file is managed by pihole-FTL\n#\n", fp); fputs("# Do not edit the file while FTL is\n", fp); @@ -119,8 +111,48 @@ bool writeFTLtoml(const bool verbose) // Close file and release exclusive lock closeFTLtoml(fp); - // Restart watching for changes in the config file - watch_config(true); + // Move temporary file to the final location if it is different + // We skip the first 8 lines as they contain the header and will always + // be different + if(files_different(GLOBALTOMLPATH".tmp", GLOBALTOMLPATH, 8)) + { + // Stop watching for changes in the config file + watch_config(false); + + // Rotate config file + rotate_files(GLOBALTOMLPATH, NULL); + + // Move file + if(rename(GLOBALTOMLPATH".tmp", GLOBALTOMLPATH) != 0) + { + log_warn("Cannot move temporary config file to final location (%s), content not updated", strerror(errno)); + // Restart watching for changes in the config file + watch_config(true); + return false; + } + + // Restart watching for changes in the config file + watch_config(true); + + // Log that we have written the config file if either in verbose or + // debug mode + if(verbose || config.debug.config.v.b) + log_info("Config file written to %s", GLOBALTOMLPATH); + } + else + { + // Remove temporary file + if(unlink(GLOBALTOMLPATH".tmp") != 0) + { + log_warn("Cannot remove temporary config file (%s), content not updated", strerror(errno)); + return false; + } + + // Log that we have written the config file if either in verbose or + // debug mode + if(verbose || config.debug.config.v.b) + log_info("Config file unchanged"); + } return true; } diff --git a/src/files.c b/src/files.c index 49247e4d2..525f57e3a 100644 --- a/src/files.c +++ b/src/files.c @@ -625,3 +625,66 @@ char * __attribute__((malloc)) get_hwmon_target(const char *path) return target; } + +// Returns true if the files have different contents +// from specifies from which line number the files should be compared +bool files_different(const char *pathA, const char* pathB, unsigned int from) +{ + // Check if both files exist + if(!file_exists(pathA) || !file_exists(pathB)) + return true; + + // Check if both files are identical + if(strcmp(pathA, pathB) == 0) + return false; + + // Open both files + FILE *fpA = fopen(pathA, "r"); + if(fpA == NULL) + { + log_warn("files_different(): Failed to open \"%s\" for reading: %s", pathA, strerror(errno)); + return true; + } + FILE *fpB = fopen(pathB, "r"); + if(fpB == NULL) + { + log_warn("files_different(): Failed to open \"%s\" for reading: %s", pathB, strerror(errno)); + fclose(fpA); + return true; + } + + // Compare both files line by line + char *lineA = NULL; + size_t lenA = 0; + ssize_t readA; + char *lineB = NULL; + size_t lenB = 0; + ssize_t readB; + bool different = false; + while((readA = getline(&lineA, &lenA, fpA)) != -1 && + (readB = getline(&lineB, &lenB, fpB)) != -1) + { + // Skip lines until we reach the requested line number + if(from > 0) + { + from--; + continue; + } + // Compare lines + if(strcmp(lineA, lineB) != 0) + { + different = true; + break; + } + } + + // Free memory + free(lineA); + free(lineB); + + // Close files + fclose(fpA); + fclose(fpB); + + return different; +} diff --git a/src/files.h b/src/files.h index 007df716a..329ce876e 100644 --- a/src/files.h +++ b/src/files.h @@ -30,6 +30,7 @@ unsigned int get_path_usage(const char *path, char buffer[64]); struct mntent *get_filesystem_details(const char *path); bool directory_exists(const char *path); void rotate_files(const char *path, char **first_file); +bool files_different(const char *pathA, const char* pathB, unsigned int from); int parse_line(char *line, char **key, char **value); From a3526b842fda57537a57473f3f683f6221a9cb0f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 9 Nov 2023 19:08:50 +0100 Subject: [PATCH 036/221] Do not rewrite config file pihole.conf (dnsmasq) when the content did not change Signed-off-by: DL6ER --- src/config/dnsmasq_config.c | 39 +++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/src/config/dnsmasq_config.c b/src/config/dnsmasq_config.c index e727e3a7a..d9af2c105 100644 --- a/src/config/dnsmasq_config.c +++ b/src/config/dnsmasq_config.c @@ -565,6 +565,13 @@ bool __attribute__((const)) write_dnsmasq_config(struct config *conf, bool test_ return false; } + // Close file + if(fclose(pihole_conf) != 0) + { + log_err("Cannot close dnsmasq config file: %s", strerror(errno)); + return false; + } + log_debug(DEBUG_CONFIG, "Testing "DNSMASQ_TEMP_CONF); if(test_config && !test_dnsmasq_config(errbuf)) { @@ -572,21 +579,29 @@ bool __attribute__((const)) write_dnsmasq_config(struct config *conf, bool test_ return false; } - // Rotate old config files - rotate_files(DNSMASQ_PH_CONFIG, NULL); - - log_debug(DEBUG_CONFIG, "Installing "DNSMASQ_TEMP_CONF" to "DNSMASQ_PH_CONFIG); - if(rename(DNSMASQ_TEMP_CONF, DNSMASQ_PH_CONFIG) != 0) + // Check if the new config file is different from the old one + // Skip the first 24 lines as they contain the header + if(files_different(DNSMASQ_TEMP_CONF, DNSMASQ_PH_CONFIG, 24)) { - log_err("Cannot install dnsmasq config file: %s", strerror(errno)); - return false; - } + // Rotate old config files + rotate_files(DNSMASQ_PH_CONFIG, NULL); - // Close file - if(fclose(pihole_conf) != 0) + log_debug(DEBUG_CONFIG, "Installing "DNSMASQ_TEMP_CONF" to "DNSMASQ_PH_CONFIG); + if(rename(DNSMASQ_TEMP_CONF, DNSMASQ_PH_CONFIG) != 0) + { + log_err("Cannot install dnsmasq config file: %s", strerror(errno)); + return false; + } + } + else { - log_err("Cannot close dnsmasq config file: %s", strerror(errno)); - return false; + log_debug(DEBUG_CONFIG, "dnsmasq config file unchanged"); + // Remove temporary config file + if(remove(DNSMASQ_TEMP_CONF) != 0) + { + log_err("Cannot remove temporary dnsmasq config file: %s", strerror(errno)); + return false; + } } return true; } From b6656808c8194c66c8fb60b55d7e2539872f37c8 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 9 Nov 2023 19:11:15 +0100 Subject: [PATCH 037/221] Do not rewrite config file custom.list (dnsmasq) when the content did not change Signed-off-by: DL6ER --- src/config/dnsmasq_config.c | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/src/config/dnsmasq_config.c b/src/config/dnsmasq_config.c index d9af2c105..a85abd0d4 100644 --- a/src/config/dnsmasq_config.c +++ b/src/config/dnsmasq_config.c @@ -797,11 +797,8 @@ bool read_legacy_custom_hosts_config(void) bool write_custom_list(void) { - // Rotate old hosts files - rotate_files(DNSMASQ_CUSTOM_LIST, NULL); - - log_debug(DEBUG_CONFIG, "Opening "DNSMASQ_CUSTOM_LIST" for writing"); - FILE *custom_list = fopen(DNSMASQ_CUSTOM_LIST, "w"); + log_debug(DEBUG_CONFIG, "Opening "DNSMASQ_CUSTOM_LIST".tmp for writing"); + FILE *custom_list = fopen(DNSMASQ_CUSTOM_LIST".tmp", "w"); // Return early if opening failed if(!custom_list) { @@ -853,5 +850,31 @@ bool write_custom_list(void) log_err("Cannot close custom.list: %s", strerror(errno)); return false; } + + // Check if the new config file is different from the old one + // Skip the first 24 lines as they contain the header + if(files_different(DNSMASQ_CUSTOM_LIST".tmp", DNSMASQ_CUSTOM_LIST, 24)) + { + // Rotate old hosts files + rotate_files(DNSMASQ_CUSTOM_LIST, NULL); + + log_debug(DEBUG_CONFIG, "Installing "DNSMASQ_CUSTOM_LIST".tmp to "DNSMASQ_CUSTOM_LIST); + if(rename(DNSMASQ_CUSTOM_LIST".tmp", DNSMASQ_CUSTOM_LIST) != 0) + { + log_err("Cannot install custom.list: %s", strerror(errno)); + return false; + } + } + else + { + log_debug(DEBUG_CONFIG, "custom.list unchanged"); + // Remove temporary config file + if(remove(DNSMASQ_CUSTOM_LIST".tmp") != 0) + { + log_err("Cannot remove temporary custom.list: %s", strerror(errno)); + return false; + } + } + return true; } From d340765b33ad01a17d32ce808eff3845018a2765 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 9 Nov 2023 19:11:47 +0100 Subject: [PATCH 038/221] Remove logging if unchanged config file if not in DEBUG_CONFIG mode Signed-off-by: DL6ER --- src/config/toml_writer.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/config/toml_writer.c b/src/config/toml_writer.c index f7d4e13bb..df1e78d33 100644 --- a/src/config/toml_writer.c +++ b/src/config/toml_writer.c @@ -148,10 +148,8 @@ bool writeFTLtoml(const bool verbose) return false; } - // Log that we have written the config file if either in verbose or - // debug mode - if(verbose || config.debug.config.v.b) - log_info("Config file unchanged"); + // Log that the config file has not changed if in debug mode + log_debug(DEBUG_CONFIG, "Config file unchanged"); } return true; From c2eca93af843aedfbd8b0b2a2ab3611c9e2705ec Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 9 Nov 2023 19:48:16 +0100 Subject: [PATCH 039/221] Add tests and unify success/error messages across the files we are writing Signed-off-by: DL6ER --- src/config/dnsmasq_config.c | 6 +++--- src/config/toml_writer.c | 2 +- test/run.sh | 2 +- test/test_suite.bats | 21 +++++++++++++++++++++ 4 files changed, 26 insertions(+), 5 deletions(-) diff --git a/src/config/dnsmasq_config.c b/src/config/dnsmasq_config.c index a85abd0d4..2b2d59b8b 100644 --- a/src/config/dnsmasq_config.c +++ b/src/config/dnsmasq_config.c @@ -586,16 +586,16 @@ bool __attribute__((const)) write_dnsmasq_config(struct config *conf, bool test_ // Rotate old config files rotate_files(DNSMASQ_PH_CONFIG, NULL); - log_debug(DEBUG_CONFIG, "Installing "DNSMASQ_TEMP_CONF" to "DNSMASQ_PH_CONFIG); if(rename(DNSMASQ_TEMP_CONF, DNSMASQ_PH_CONFIG) != 0) { log_err("Cannot install dnsmasq config file: %s", strerror(errno)); return false; } + log_debug(DEBUG_CONFIG, "Config file written to "DNSMASQ_PH_CONFIG); } else { - log_debug(DEBUG_CONFIG, "dnsmasq config file unchanged"); + log_debug(DEBUG_CONFIG, "dnsmasq.conf unchanged"); // Remove temporary config file if(remove(DNSMASQ_TEMP_CONF) != 0) { @@ -858,12 +858,12 @@ bool write_custom_list(void) // Rotate old hosts files rotate_files(DNSMASQ_CUSTOM_LIST, NULL); - log_debug(DEBUG_CONFIG, "Installing "DNSMASQ_CUSTOM_LIST".tmp to "DNSMASQ_CUSTOM_LIST); if(rename(DNSMASQ_CUSTOM_LIST".tmp", DNSMASQ_CUSTOM_LIST) != 0) { log_err("Cannot install custom.list: %s", strerror(errno)); return false; } + log_debug(DEBUG_CONFIG, "HOSTS file written to "DNSMASQ_CUSTOM_LIST); } else { diff --git a/src/config/toml_writer.c b/src/config/toml_writer.c index df1e78d33..563af3f3e 100644 --- a/src/config/toml_writer.c +++ b/src/config/toml_writer.c @@ -149,7 +149,7 @@ bool writeFTLtoml(const bool verbose) } // Log that the config file has not changed if in debug mode - log_debug(DEBUG_CONFIG, "Config file unchanged"); + log_debug(DEBUG_CONFIG, "pihole.toml unchanged"); } return true; diff --git a/test/run.sh b/test/run.sh index 4afa09992..88e8f7f2b 100755 --- a/test/run.sh +++ b/test/run.sh @@ -20,7 +20,7 @@ while pidof -s pihole-FTL > /dev/null; do done # Clean up possible old files from earlier test runs -rm -f /etc/pihole/gravity.db /etc/pihole/pihole-FTL.db /var/log/pihole/pihole.log /var/log/pihole/FTL.log /dev/shm/FTL-* +rm -rf /etc/pihole /var/log/pihole /dev/shm/FTL-* # Create necessary directories and files mkdir -p /home/pihole /etc/pihole /run/pihole /var/log/pihole diff --git a/test/test_suite.bats b/test/test_suite.bats index b5de84d88..9d1efc51b 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -1445,3 +1445,24 @@ [[ $status == 0 ]] run bash -c "rm ${filename}" } + +@test "Expected number of config file rotations" { + run bash -c 'grep -c "INFO: Config file written to /etc/pihole/pihole.toml" /var/log/pihole/FTL.log' + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == "3" ]] + run bash -c 'grep -c "DEBUG_CONFIG: pihole.toml unchanged" /var/log/pihole/FTL.log' + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == "3" ]] + run bash -c 'grep -c "DEBUG_CONFIG: Config file written to /etc/pihole/dnsmasq.conf" /var/log/pihole/FTL.log' + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == "1" ]] + run bash -c 'grep -c "DEBUG_CONFIG: dnsmasq.conf unchanged" /var/log/pihole/FTL.log' + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == "2" ]] + run bash -c 'grep -c "DEBUG_CONFIG: HOSTS file written to /etc/pihole/custom.list" /var/log/pihole/FTL.log' + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == "1" ]] + run bash -c 'grep -c "DEBUG_CONFIG: custom.list unchanged" /var/log/pihole/FTL.log' + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == "3" ]] +} From dcdca19caee87b32c086abb68629f243d267f3ff Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 10 Nov 2023 04:48:07 +0100 Subject: [PATCH 040/221] Move 2FA success message more human-readable and move it to debug printing Signed-off-by: DL6ER --- src/api/2fa.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/api/2fa.c b/src/api/2fa.c index 6988592f2..4e255df2f 100644 --- a/src/api/2fa.c +++ b/src/api/2fa.c @@ -230,7 +230,8 @@ enum totp_status verifyTOTP(const uint32_t incode) i, gencode, (unsigned long)(RFC6238_X - (now % RFC6238_X))); return TOTP_REUSED; } - log_info("2FA code verified successfully at %i", i); + const char *which = i == -1 ? "previous" : i == 0 ? "current" : "next"; + log_debug(DEBUG_API, "2FA code from %s time step is valid", which); last_code = gencode; return TOTP_CORRECT; } From 84ba6da4048c4cf68cc5320005ddca78cdbcc139 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 10 Nov 2023 20:05:06 +0100 Subject: [PATCH 041/221] max_sessions can only change across FTL restarts - not while it is running Signed-off-by: DL6ER --- src/api/auth.c | 23 +++++++++++++---------- src/database/session-table.c | 18 ++++++++++-------- src/database/session-table.h | 4 ++-- 3 files changed, 25 insertions(+), 20 deletions(-) diff --git a/src/api/auth.c b/src/api/auth.c index 8fa7d6a7f..d0f89128f 100644 --- a/src/api/auth.c +++ b/src/api/auth.c @@ -26,6 +26,7 @@ // database session functions #include "database/session-table.h" +static uint16_t max_sessions = 0; static struct session *auth_data = NULL; static void add_request_info(struct ftl_conn *api, const char *csrf) @@ -43,19 +44,21 @@ static void add_request_info(struct ftl_conn *api, const char *csrf) void init_api(void) { // Restore sessions from database - auth_data = calloc(config.webserver.api.max_sessions.v.u16, sizeof(struct session)); + max_sessions = config.webserver.api.max_sessions.v.u16; + auth_data = calloc(max_sessions, sizeof(struct session)); if(auth_data == NULL) { log_crit("Could not allocate memory for API sessions, check config value of webserver.api.max_sessions"); exit(EXIT_FAILURE); } - restore_db_sessions(auth_data); + restore_db_sessions(auth_data, max_sessions); } void free_api(void) { // Store sessions in database - backup_db_sessions(auth_data); + backup_db_sessions(auth_data, max_sessions); + max_sessions = 0; free(auth_data); auth_data = NULL; } @@ -195,7 +198,7 @@ int check_client_auth(struct ftl_conn *api, const bool is_api) } } - for(unsigned int i = 0; i < config.webserver.api.max_sessions.v.u16; i++) + for(unsigned int i = 0; i < max_sessions; i++) { if(auth_data[i].used && auth_data[i].valid_until >= now && @@ -261,7 +264,7 @@ static int get_all_sessions(struct ftl_conn *api, cJSON *json) { const time_t now = time(NULL); cJSON *sessions = JSON_NEW_ARRAY(); - for(int i = 0; i < config.webserver.api.max_sessions.v.u16; i++) + for(int i = 0; i < max_sessions; i++) { if(!auth_data[i].used) continue; @@ -324,7 +327,7 @@ static int get_session_object(struct ftl_conn *api, cJSON *json, const int user_ static void delete_session(const int user_id) { // Skip if nothing to be done here - if(user_id < 0 || user_id >= config.webserver.api.max_sessions.v.u16) + if(user_id < 0 || user_id >= max_sessions) return; // Zero out this session (also sets valid to false == 0) @@ -334,7 +337,7 @@ static void delete_session(const int user_id) void delete_all_sessions(void) { // Zero out all sessions without looping - memset(auth_data, 0, config.webserver.api.max_sessions.v.u16*sizeof(*auth_data)); + memset(auth_data, 0, max_sessions*sizeof(*auth_data)); } static int send_api_auth_status(struct ftl_conn *api, const int user_id, const time_t now) @@ -544,7 +547,7 @@ int api_auth(struct ftl_conn *api) } // Find unused authentication slot - for(unsigned int i = 0; i < config.webserver.api.max_sessions.v.u16; i++) + for(unsigned int i = 0; i < max_sessions; i++) { // Expired slow, mark as unused if(auth_data[i].used && @@ -603,7 +606,7 @@ int api_auth(struct ftl_conn *api) if(user_id == API_AUTH_UNAUTHORIZED) { log_warn("No free API seats available (webserver.api.max_sessions = %u), not authenticating client", - config.webserver.api.max_sessions.v.u16); + max_sessions); return send_json_error(api, 429, "api_seats_exceeded", @@ -644,7 +647,7 @@ int api_auth_session_delete(struct ftl_conn *api) return send_json_error(api, 400, "bad_request", "Missing or invalid session ID", NULL); // Check if session ID is valid - if(uid <= API_AUTH_UNAUTHORIZED || uid >= config.webserver.api.max_sessions.v.u16) + if(uid <= API_AUTH_UNAUTHORIZED || uid >= max_sessions) return send_json_error(api, 400, "bad_request", "Session ID out of bounds", NULL); // Check if session is used diff --git a/src/database/session-table.c b/src/database/session-table.c index 8c1847785..d185a55cd 100644 --- a/src/database/session-table.c +++ b/src/database/session-table.c @@ -64,7 +64,7 @@ bool add_session_app_column(sqlite3 *db) } // Store all session in database -bool backup_db_sessions(struct session *sessions) +bool backup_db_sessions(struct session *sessions, const uint16_t max_sessions) { if(!config.webserver.session.restore.v.b) { @@ -89,7 +89,7 @@ bool backup_db_sessions(struct session *sessions) } unsigned int api_sessions = 0; - for(unsigned int i = 0; i < config.webserver.api.max_sessions.v.u16; i++) + for(unsigned int i = 0; i < max_sessions; i++) { // Get session struct session *sess = &sessions[i]; @@ -198,8 +198,8 @@ bool backup_db_sessions(struct session *sessions) return false; } - log_info("Stored %u API session%s in the database", - api_sessions, api_sessions == 1 ? "" : "s"); + log_info("Stored %u/%u API session%s in the database", + api_sessions, max_sessions, max_sessions == 1 ? "" : "s"); // Close database connection dbclose(&db); @@ -208,7 +208,7 @@ bool backup_db_sessions(struct session *sessions) } // Restore all sessions found in the database -bool restore_db_sessions(struct session *sessions) +bool restore_db_sessions(struct session *sessions, const uint16_t max_sessions) { if(!config.webserver.session.restore.v.b) { @@ -237,7 +237,7 @@ bool restore_db_sessions(struct session *sessions) // Iterate over all still valid sessions unsigned int i = 0; - while(sqlite3_step(stmt) == SQLITE_ROW && i++ < config.webserver.api.max_sessions.v.u16) + while(sqlite3_step(stmt) == SQLITE_ROW && i < max_sessions) { // Allocate memory for new session struct session *sess = &sessions[i]; @@ -292,10 +292,12 @@ bool restore_db_sessions(struct session *sessions) // Mark session as used sess->used = true; + + i++; } - log_info("Restored %u API session%s from the database", - i, i == 1 ? "" : "s"); + log_info("Restored %u/%u API session%s from the database", + i, max_sessions, max_sessions == 1 ? "" : "s"); // Finalize statement if(sqlite3_finalize(stmt) != SQLITE_OK) diff --git a/src/database/session-table.h b/src/database/session-table.h index 7f7335779..2e9ff2967 100644 --- a/src/database/session-table.h +++ b/src/database/session-table.h @@ -16,7 +16,7 @@ bool create_session_table(sqlite3 *db); bool add_session_app_column(sqlite3 *db); -bool backup_db_sessions(struct session *sessions); -bool restore_db_sessions(struct session *sessions); +bool backup_db_sessions(struct session *sessions, const uint16_t max_sessions); +bool restore_db_sessions(struct session *sessions, const uint16_t max_sessions); #endif // SESSION_TABLE_PRIVATE_H From ec064d9c6ef565d9cc866d84dcc81952cad511eb Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 10 Nov 2023 20:47:33 +0100 Subject: [PATCH 042/221] Use correct type when manipulating domainlist entries Signed-off-by: DL6ER --- src/database/gravity-db.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/database/gravity-db.c b/src/database/gravity-db.c index e1ab5d3f2..802b57b4e 100644 --- a/src/database/gravity-db.c +++ b/src/database/gravity-db.c @@ -1663,7 +1663,7 @@ bool gravityDB_addToTable(const enum gravity_list_type listtype, tablerow *row, querystr = "INSERT INTO client (ip,comment) VALUES (:item,:comment) "\ "ON CONFLICT(ip) DO UPDATE SET comment = :comment;"; else // domainlist - querystr = "INSERT INTO domainlist (domain,type,enabled,comment) VALUES (:item,:type,:enabled,:comment) "\ + querystr = "INSERT INTO domainlist (domain,type,enabled,comment) VALUES (:item,:oldtype,:enabled,:comment) "\ "ON CONFLICT(domain,type) DO UPDATE SET type = :type, enabled = :enabled, comment = :comment;"; } @@ -1745,7 +1745,7 @@ bool gravityDB_addToTable(const enum gravity_list_type listtype, tablerow *row, else { if(strcasecmp("allow", row->type) == 0 && - strcasecmp("exact", row->kind) == 0) + strcasecmp("exact", row->kind) == 0) oldtype = 0; else if(strcasecmp("deny", row->type) == 0 && strcasecmp("exact", row->kind) == 0) From df2ba4c724bc66adba0ab7f2ea15edc6bd045d49 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 10 Nov 2023 21:03:28 +0100 Subject: [PATCH 043/221] Fix escape_html() crashing for NULL input Signed-off-by: DL6ER --- src/webserver/http-common.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/webserver/http-common.c b/src/webserver/http-common.c index f7521dac0..c1fdc2750 100644 --- a/src/webserver/http-common.c +++ b/src/webserver/http-common.c @@ -524,6 +524,10 @@ void read_and_parse_payload(struct ftl_conn *api) // See https://www.w3.org/International/questions/qa-escapes#use char *__attribute__((malloc)) escape_html(const char *string) { + // If the string is NULL, return NULL + if(string == NULL) + return NULL; + // Allocate memory for escaped string char *escaped = calloc(strlen(string) * 6 + 1, sizeof(char)); if(!escaped) From 1bcb7541dca712b04211ba5a3f70928c9e7c6115 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 10 Nov 2023 21:06:19 +0100 Subject: [PATCH 044/221] Add missing database type definition for CERTIFICATE_DOMAIN_MISMATCH_MESSAGE Signed-off-by: DL6ER --- src/database/message-table.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/database/message-table.c b/src/database/message-table.c index 39b8e7f97..b4fd43351 100644 --- a/src/database/message-table.c +++ b/src/database/message-table.c @@ -171,6 +171,14 @@ static unsigned char message_blob_types[MAX_MESSAGE][5] = SQLITE_TEXT, // File system type SQLITE_TEXT, // Directory mounted on SQLITE_NULL // not used + }, + { + // CERTIFICATE_DOMAIN_MISMATCH_MESSAGE: The message column contains the certificate file + SQLITE_TEXT, // domain + SQLITE_NULL, // not used + SQLITE_NULL, // not used + SQLITE_NULL, // not used + SQLITE_NULL // not used } }; // Create message table in the database From 5779db9ca5c5f54bb544236419dd93b6f202f2aa Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 10 Nov 2023 21:09:31 +0100 Subject: [PATCH 045/221] Warn if NULL is bound to a message row even when we actually want to bind a blob Signed-off-by: DL6ER --- src/database/message-table.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/database/message-table.c b/src/database/message-table.c index b4fd43351..ba1b75919 100644 --- a/src/database/message-table.c +++ b/src/database/message-table.c @@ -345,6 +345,8 @@ static int add_message(const enum message_type type, case SQLITE_NULL: /* Fall through */ default: + log_warn("add_message(type=%s, message=%s) - Excess property, binding NULL", + get_message_type_str(type), message); rc = sqlite3_bind_null(stmt, 3 + j); break; } From afefe4ca3011ccd5c5d20be97e9b0e61f1eff378 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 10 Nov 2023 22:17:56 +0100 Subject: [PATCH 046/221] Create dynamic validity period when generating X.509 certificate Signed-off-by: DL6ER --- src/webserver/x509.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/webserver/x509.c b/src/webserver/x509.c index ca0b21403..154e3c99a 100644 --- a/src/webserver/x509.c +++ b/src/webserver/x509.c @@ -145,6 +145,16 @@ bool generate_certificate(const char* certfile, bool rsa, const char *domain) serial[i] = '0' + (serial[i] % 10); serial[sizeof(serial) - 1] = '\0'; + // Create validity period + // Use YYYYMMDDHHMMSS as required by RFC 5280 + const time_t now = time(NULL); + struct tm *tm = localtime(&now); + char not_before[16] = { 0 }; + char not_after[16] = { 0 }; + strftime(not_before, sizeof(not_before), "%Y%m%d%H%M%S", tm); + tm->tm_year += 30; // 30 years from now + strftime(not_after, sizeof(not_after), "%Y%m%d%H%M%S", tm); + // Generate certificate printf("Generating new certificate with serial number %s...\n", serial); mbedtls_x509write_crt_set_version(&crt, MBEDTLS_X509_CRT_VERSION_3); @@ -154,7 +164,7 @@ bool generate_certificate(const char* certfile, bool rsa, const char *domain) mbedtls_x509write_crt_set_subject_key(&crt, &key); mbedtls_x509write_crt_set_issuer_key(&crt, &key); mbedtls_x509write_crt_set_issuer_name(&crt, "CN=pi.hole"); - mbedtls_x509write_crt_set_validity(&crt, "20010101000000", "20301231235959"); + mbedtls_x509write_crt_set_validity(&crt, not_before, not_after); mbedtls_x509write_crt_set_basic_constraints(&crt, 0, -1); mbedtls_x509write_crt_set_subject_key_identifier(&crt); mbedtls_x509write_crt_set_authority_key_identifier(&crt); From 4644843c9287bf04f31c50a4e94585761687998d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 11 Nov 2023 20:58:24 +0100 Subject: [PATCH 047/221] Use thread-safe variant of localtime() Signed-off-by: DL6ER --- src/webserver/x509.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/webserver/x509.c b/src/webserver/x509.c index 154e3c99a..cb154c4a2 100644 --- a/src/webserver/x509.c +++ b/src/webserver/x509.c @@ -148,7 +148,8 @@ bool generate_certificate(const char* certfile, bool rsa, const char *domain) // Create validity period // Use YYYYMMDDHHMMSS as required by RFC 5280 const time_t now = time(NULL); - struct tm *tm = localtime(&now); + struct tm tms = { 0 }; + struct tm *tm = localtime_r(&now, &tms); char not_before[16] = { 0 }; char not_after[16] = { 0 }; strftime(not_before, sizeof(not_before), "%Y%m%d%H%M%S", tm); From ea156f6d10801ff098d426ece0463df9b0d0c626 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 12 Nov 2023 20:59:41 +0100 Subject: [PATCH 048/221] Restart FTL on change of misc.etc_dnsmasq_d Co-authored-by: yubiuser Signed-off-by: DL6ER --- src/config/config.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config/config.c b/src/config/config.c index fb7a3af88..43976b9d1 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -1077,7 +1077,7 @@ void initConfig(struct config *conf) conf->misc.etc_dnsmasq_d.k = "misc.etc_dnsmasq_d"; conf->misc.etc_dnsmasq_d.h = "Should FTL load additional dnsmasq configuration files from /etc/dnsmasq.d/?"; conf->misc.etc_dnsmasq_d.t = CONF_BOOL; - conf->misc.etc_dnsmasq_d.f = FLAG_ADVANCED_SETTING; + conf->misc.etc_dnsmasq_d.f = FLAG_RESTART_FTL | FLAG_ADVANCED_SETTING; conf->misc.etc_dnsmasq_d.d.b = false; conf->misc.dnsmasq_lines.k = "misc.dnsmasq_lines"; From 1fec47907e022016156e83991e4590691ed21d02 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 12 Nov 2023 21:07:57 +0100 Subject: [PATCH 049/221] Add restart flag to config items where this is missing Signed-off-by: DL6ER --- src/config/config.c | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/config/config.c b/src/config/config.c index cc9257a22..a45081d6d 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -824,13 +824,14 @@ void initConfig(struct config *conf) conf->webserver.acl.k = "webserver.acl"; conf->webserver.acl.h = "Webserver access control list (ACL) allowing for restrictions to be put on the list of IP addresses which have access to the web server. The ACL is a comma separated list of IP subnets, where each subnet is prepended by either a - or a + sign. A plus sign means allow, where a minus sign means deny. If a subnet mask is omitted, such as -1.2.3.4, this means to deny only that single IP address. If this value is not set (empty string), all accesses are allowed. Otherwise, the default setting is to deny all accesses. On each request the full list is traversed, and the last (!) match wins. IPv6 addresses may be specified in CIDR-form [a:b::c]/64.\n\n Example 1: acl = \"+127.0.0.1,+[::1]\"\n ---> deny all access, except from 127.0.0.1 and ::1,\n Example 2: acl = \"+192.168.0.0/16\"\n ---> deny all accesses, except from the 192.168.0.0/16 subnet,\n Example 3: acl = \"+[::]/0\" ---> allow only IPv6 access."; conf->webserver.acl.a = cJSON_CreateStringReference(""); - conf->webserver.acl.f = FLAG_ADVANCED_SETTING; + conf->webserver.acl.f = FLAG_ADVANCED_SETTING | FLAG_RESTART_FTL; conf->webserver.acl.t = CONF_STRING; conf->webserver.acl.d.s = (char*)""; conf->webserver.port.k = "webserver.port"; conf->webserver.port.h = "Ports to be used by the webserver.\n Comma-separated list of ports to listen on. It is possible to specify an IP address to bind to. In this case, an IP address and a colon must be prepended to the port number. For example, to bind to the loopback interface on port 80 (IPv4) and to all interfaces port 8080 (IPv4), use \"127.0.0.1:80,8080\". \"[::]:80\" can be used to listen to IPv6 connections to port 80. IPv6 addresses of network interfaces can be specified as well, e.g. \"[::1]:80\" for the IPv6 loopback interface. [::]:80 will bind to port 80 IPv6 only.\n In order to use port 80 for all interfaces, both IPv4 and IPv6, use either the configuration \"80,[::]:80\" (create one socket for IPv4 and one for IPv6 only), or \"+80\" (create one socket for both, IPv4 and IPv6). The + notation to use IPv4 and IPv6 will only work if no network interface is specified. Depending on your operating system version and IPv6 network environment, some configurations might not work as expected, so you have to test to find the configuration most suitable for your needs. In case \"+80\" does not work for your environment, you need to use \"80,[::]:80\".\n If the port is TLS/SSL, a letter 's' must be appended, for example, \"80,443s\" will open port 80 and port 443, and connections on port 443 will be encrypted. For non-encrypted ports, it is allowed to append letter 'r' (as in redirect). Redirected ports will redirect all their traffic to the first configured SSL port. For example, if webserver.port is \"80r,443s\", then all HTTP traffic coming at port 80 will be redirected to HTTPS port 443."; conf->webserver.port.a = cJSON_CreateStringReference("comma-separated list of <[ip_address:]port>"); + conf->webserver.port.f = FLAG_ADVANCED_SETTING | FLAG_RESTART_FTL; conf->webserver.port.t = CONF_STRING; conf->webserver.port.d.s = (char*)"80,[::]:80,443s,[::]:443s"; @@ -862,14 +863,14 @@ void initConfig(struct config *conf) conf->webserver.paths.webroot.h = "Server root on the host"; conf->webserver.paths.webroot.a = cJSON_CreateStringReference(""); conf->webserver.paths.webroot.t = CONF_STRING; - conf->webserver.paths.webroot.f = FLAG_ADVANCED_SETTING; + conf->webserver.paths.webroot.f = FLAG_ADVANCED_SETTING | FLAG_RESTART_FTL; conf->webserver.paths.webroot.d.s = (char*)"/var/www/html"; conf->webserver.paths.webhome.k = "webserver.paths.webhome"; conf->webserver.paths.webhome.h = "Sub-directory of the root containing the web interface"; conf->webserver.paths.webhome.a = cJSON_CreateStringReference(", both slashes are needed!"); conf->webserver.paths.webhome.t = CONF_STRING; - conf->webserver.paths.webhome.f = FLAG_ADVANCED_SETTING; + conf->webserver.paths.webhome.f = FLAG_ADVANCED_SETTING | FLAG_RESTART_FTL; conf->webserver.paths.webhome.d.s = (char*)"/admin/"; // sub-struct interface @@ -991,7 +992,7 @@ void initConfig(struct config *conf) conf->files.pid.h = "The file which contains the PID of FTL's main process."; conf->files.pid.a = cJSON_CreateStringReference(""); conf->files.pid.t = CONF_STRING; - conf->files.pid.f = FLAG_ADVANCED_SETTING; + conf->files.pid.f = FLAG_ADVANCED_SETTING | FLAG_RESTART_FTL; conf->files.pid.d.s = (char*)"/run/pihole-FTL.pid"; conf->files.database.k = "files.database"; @@ -1005,7 +1006,7 @@ void initConfig(struct config *conf) conf->files.gravity.h = "The location of Pi-hole's gravity database"; conf->files.gravity.a = cJSON_CreateStringReference(""); conf->files.gravity.t = CONF_STRING; - conf->files.gravity.f = FLAG_ADVANCED_SETTING; + conf->files.gravity.f = FLAG_ADVANCED_SETTING | FLAG_RESTART_FTL; conf->files.gravity.d.s = (char*)"/etc/pihole/gravity.db"; conf->files.macvendor.k = "files.macvendor"; @@ -1026,14 +1027,14 @@ void initConfig(struct config *conf) conf->files.pcap.h = "An optional file containing a pcap capture of the network traffic. This file is used for debugging purposes only. If you don't know what this is, you don't need it.\n Setting this to an empty string disables pcap recording. The file must be writable by the user running FTL (typically pihole). Failure to write to this file will prevent the DNS resolver from starting. The file is appended to if it already exists."; conf->files.pcap.a = cJSON_CreateStringReference(""); conf->files.pcap.t = CONF_STRING; - conf->files.pcap.f = FLAG_ADVANCED_SETTING; + conf->files.pcap.f = FLAG_ADVANCED_SETTING | FLAG_RESTART_FTL; conf->files.pcap.d.s = (char*)""; conf->files.log.webserver.k = "files.log.webserver"; conf->files.log.webserver.h = "The log file used by the webserver"; conf->files.log.webserver.a = cJSON_CreateStringReference(""); conf->files.log.webserver.t = CONF_STRING; - conf->files.log.webserver.f = FLAG_ADVANCED_SETTING; + conf->files.log.webserver.f = FLAG_ADVANCED_SETTING | FLAG_RESTART_FTL; conf->files.log.webserver.d.s = (char*)"/var/log/pihole/webserver.log"; // sub-struct files.log @@ -1043,7 +1044,7 @@ void initConfig(struct config *conf) conf->files.log.dnsmasq.h = "The log file used by the embedded dnsmasq DNS server"; conf->files.log.dnsmasq.a = cJSON_CreateStringReference(""); conf->files.log.dnsmasq.t = CONF_STRING; - conf->files.log.dnsmasq.f = FLAG_ADVANCED_SETTING; + conf->files.log.dnsmasq.f = FLAG_ADVANCED_SETTING | FLAG_RESTART_FTL; conf->files.log.dnsmasq.d.s = (char*)"/var/log/pihole/pihole.log"; @@ -1071,7 +1072,7 @@ void initConfig(struct config *conf) conf->misc.nice.k = "misc.nice"; conf->misc.nice.h = "Set niceness of pihole-FTL. Defaults to -10 and can be disabled altogether by setting a value of -999. The nice value is an attribute that can be used to influence the CPU scheduler to favor or disfavor a process in scheduling decisions. The range of the nice value varies across UNIX systems. On modern Linux, the range is -20 (high priority = not very nice to other processes) to +19 (low priority)."; conf->misc.nice.t = CONF_INT; - conf->misc.nice.f = FLAG_ADVANCED_SETTING; + conf->misc.nice.f = FLAG_ADVANCED_SETTING | FLAG_RESTART_FTL; conf->misc.nice.d.i = -10; conf->misc.addr2line.k = "misc.addr2line"; @@ -1084,7 +1085,7 @@ void initConfig(struct config *conf) conf->misc.dnsmasq_lines.h = "Additional lines to inject into the generated dnsmasq configuration.\n Warning: This is an advanced setting and should only be used with care. Incorrectly formatted or duplicated lines as well as lines conflicting with the automatic configuration of Pi-hole can break the embedded dnsmasq and will stop DNS resolution from working.\n Use this option with extra care."; conf->misc.dnsmasq_lines.a = cJSON_CreateStringReference("array of valid dnsmasq config line options"); conf->misc.dnsmasq_lines.t = CONF_JSON_STRING_ARRAY; - conf->misc.dnsmasq_lines.f = FLAG_RESTART_FTL; + conf->misc.dnsmasq_lines.f = FLAG_ADVANCED_SETTING | FLAG_RESTART_FTL; conf->misc.dnsmasq_lines.d.json = cJSON_CreateArray(); // sub-struct misc.check From 19d37a6020cd6aa3171c04d6f79e54985f739504 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 12 Nov 2023 22:05:13 +0100 Subject: [PATCH 050/221] Fix hint of dns.cnameRecords Signed-off-by: DL6ER --- src/config/config.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config/config.c b/src/config/config.c index 8c8874e47..dbe2bcdbf 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -528,7 +528,7 @@ void initConfig(struct config *conf) conf->dns.cnameRecords.k = "dns.cnameRecords"; conf->dns.cnameRecords.h = "List of CNAME records which indicate that is really . If the is given, it overwrites the value of local-ttl"; - conf->dns.cnameRecords.a = cJSON_CreateStringReference("Array of static leases each on in one of the following forms: \",[,]\""); + conf->dns.cnameRecords.a = cJSON_CreateStringReference("Array of CNAMEs each on in one of the following forms: \",[,]\""); conf->dns.cnameRecords.t = CONF_JSON_STRING_ARRAY; conf->dns.cnameRecords.f = FLAG_RESTART_FTL | FLAG_ADVANCED_SETTING; conf->dns.cnameRecords.d.json = cJSON_CreateArray(); From 4d416031412e46d8a2b2446fe000746ace7ac6cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 12 Nov 2023 22:25:53 +0100 Subject: [PATCH 051/221] Impove readability of `dnsmasq.conf` by adding blank lines to group similar settings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- src/config/dnsmasq_config.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/config/dnsmasq_config.c b/src/config/dnsmasq_config.c index 4b6080056..366f876c3 100644 --- a/src/config/dnsmasq_config.c +++ b/src/config/dnsmasq_config.c @@ -248,6 +248,7 @@ bool __attribute__((const)) write_dnsmasq_config(struct config *conf, bool test_ fputs("\n", pihole_conf); fputs("# DNS port to be used\n", pihole_conf); fprintf(pihole_conf, "port=%u\n", conf->dns.port.v.u16); + fputs("\n", pihole_conf); if(cJSON_GetArraySize(conf->dns.upstreams.v.json) > 0) { fputs("# List of upstream DNS server\n", pihole_conf); @@ -279,12 +280,14 @@ bool __attribute__((const)) write_dnsmasq_config(struct config *conf, bool test_ fputs("# Enable query logging\n", pihole_conf); fputs("log-queries\n", pihole_conf); fputs("log-async\n", pihole_conf); + fputs("\n", pihole_conf); } else { fputs("# Disable query logging\n", pihole_conf); fputs("#log-queries\n", pihole_conf); fputs("#log-async\n", pihole_conf); + fputs("\n", pihole_conf); } if(strlen(conf->files.log.dnsmasq.v.s) > 0) @@ -335,12 +338,14 @@ bool __attribute__((const)) write_dnsmasq_config(struct config *conf, bool test_ { fputs("# Add A, AAAA and PTR records to the DNS\n", pihole_conf); fprintf(pihole_conf, "host-record=%s\n", conf->dns.hostRecord.v.s); + fputs("\n", pihole_conf); } if(conf->dns.cache.optimizer.v.ui > 0u) { fputs("# Use stale cache entries for a given number of seconds to optimize cache utilization\n", pihole_conf); fprintf(pihole_conf, "use-stale-cache=%u\n", conf->dns.cache.optimizer.v.ui); + fputs("\n", pihole_conf); } const char *interface = conf->dns.interface.v.s; @@ -508,6 +513,7 @@ bool __attribute__((const)) write_dnsmasq_config(struct config *conf, bool test_ fputs("# We do not include the \".local\" rule ourselves, see https://github.com/pi-hole/pi-hole/pull/4282#discussion_r689112972\n", pihole_conf); fputs("server=/bind/\n", pihole_conf); fputs("server=/onion/\n", pihole_conf); + fputs("\n", pihole_conf); if(directory_exists("/etc/dnsmasq.d") && conf->misc.etc_dnsmasq_d.v.b) { @@ -520,7 +526,8 @@ bool __attribute__((const)) write_dnsmasq_config(struct config *conf, bool test_ // Add option for caching all DNS records fputs("# Cache all DNS records\n", pihole_conf); - fputs("cache-rr=ANY\n\n", pihole_conf); + fputs("cache-rr=ANY\n", pihole_conf); + fputs("\n", pihole_conf); // Add option for PCAP file recording if(strlen(conf->files.pcap.v.s) > 0) From d43b0eed283546b071b141906f85c9fdb081e4db Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 13 Nov 2023 09:25:32 +0100 Subject: [PATCH 052/221] Add deprecation note for dhcp.domain Signed-off-by: DL6ER --- src/api/docs/content/specs/config.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/api/docs/content/specs/config.yaml b/src/api/docs/content/specs/config.yaml index 2bbafad0d..973a8b0d6 100644 --- a/src/api/docs/content/specs/config.yaml +++ b/src/api/docs/content/specs/config.yaml @@ -303,6 +303,8 @@ components: type: string domain: type: string + description: | + *Note:* This setting is deprecated and will be removed in a future release. Use dns.domain instead. leaseTime: type: string ipv6: From af4214d8f9087a7c38274bf724e5459f526f9b50 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 13 Nov 2023 09:52:45 +0100 Subject: [PATCH 053/221] Use hostsdir for custom.list to avoid cache flushing on changes Signed-off-by: DL6ER --- src/api/config.c | 8 -------- src/config/cli.c | 6 ++---- src/config/dnsmasq_config.c | 18 +++++++++++++----- src/config/dnsmasq_config.h | 4 +++- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/api/config.c b/src/api/config.c index 167763c49..2c2d0a5b4 100644 --- a/src/api/config.c +++ b/src/api/config.c @@ -778,11 +778,7 @@ static int api_config_patch(struct ftl_conn *api) // Rewrite HOSTS file if required if(rewrite_hosts) - { write_custom_list(); - // Reload HOSTS file - kill(main_pid(), SIGHUP); - } } else { @@ -976,11 +972,7 @@ static int api_config_put_delete(struct ftl_conn *api) // Rewrite HOSTS file if required if(rewrite_hosts) - { write_custom_list(); - // Reload HOSTS file - kill(main_pid(), SIGHUP); - } // Send empty reply with matching HTTP status code // 201 - Created or 204 - No content diff --git a/src/config/cli.c b/src/config/cli.c index 76c6dd07c..18bc9f9b1 100644 --- a/src/config/cli.c +++ b/src/config/cli.c @@ -441,10 +441,8 @@ int set_config_from_CLI(const char *key, const char *value) } else if(conf_item == &config.dns.hosts) { - // We need to rewrite the custom.list file but do not need to - // restart dnsmasq. If dnsmasq is going to be restarted anyway, - // this is not necessary as the file will be rewritten during - // the restart + // We need to rewrite the custom.list file but do not + // need to restart dnsmasq write_custom_list(); } diff --git a/src/config/dnsmasq_config.c b/src/config/dnsmasq_config.c index 366f876c3..a73f9c96d 100644 --- a/src/config/dnsmasq_config.c +++ b/src/config/dnsmasq_config.c @@ -241,7 +241,7 @@ bool __attribute__((const)) write_dnsmasq_config(struct config *conf, bool test_ write_config_header(pihole_conf, "Dnsmasq config for Pi-hole's FTLDNS"); fputs("addn-hosts=/etc/pihole/local.list\n", pihole_conf); - fputs("addn-hosts="DNSMASQ_CUSTOM_LIST"\n", pihole_conf); + fputs("hostsdir="DNSMASQ_HOSTSDIR"\n", pihole_conf); fputs("\n", pihole_conf); fputs("# Don't read /etc/resolv.conf. Get upstream servers only from the configuration\n", pihole_conf); fputs("no-resolv\n", pihole_conf); @@ -727,8 +727,8 @@ bool read_legacy_cnames_config(void) bool read_legacy_custom_hosts_config(void) { // Check if file exists, if not, there is nothing to do - const char *path = DNSMASQ_CUSTOM_LIST; - const char *target = DNSMASQ_CUSTOM_LIST".bck"; + const char *path = DNSMASQ_CUSTOM_LIST_LEGACY; + const char *target = DNSMASQ_CUSTOM_LIST_LEGACY".bck"; if(!file_exists(path)) return true; @@ -790,8 +790,16 @@ bool read_legacy_custom_hosts_config(void) bool write_custom_list(void) { - // Rotate old hosts files - rotate_files(DNSMASQ_CUSTOM_LIST, NULL); + // Ensure that the directory exists + if(!directory_exists(DNSMASQ_HOSTSDIR)) + { + log_debug(DEBUG_CONFIG, "Creating directory "DNSMASQ_HOSTSDIR); + if(mkdir(DNSMASQ_HOSTSDIR, 0755) != 0) + { + log_err("Cannot create directory "DNSMASQ_HOSTSDIR": %s", strerror(errno)); + return false; + } + } log_debug(DEBUG_CONFIG, "Opening "DNSMASQ_CUSTOM_LIST" for writing"); FILE *custom_list = fopen(DNSMASQ_CUSTOM_LIST, "w"); diff --git a/src/config/dnsmasq_config.h b/src/config/dnsmasq_config.h index 8552169c4..77b1405ce 100644 --- a/src/config/dnsmasq_config.h +++ b/src/config/dnsmasq_config.h @@ -26,7 +26,9 @@ bool write_custom_list(void); #define DNSMASQ_TEMP_CONF "/etc/pihole/dnsmasq.conf.temp" #define DNSMASQ_STATIC_LEASES "/etc/pihole/04-pihole-static-dhcp.conf" #define DNSMASQ_CNAMES "/etc/pihole/05-pihole-custom-cname.conf" -#define DNSMASQ_CUSTOM_LIST "/etc/pihole/custom.list" +#define DNSMASQ_HOSTSDIR "/etc/pihole/hosts" +#define DNSMASQ_CUSTOM_LIST DNSMASQ_HOSTSDIR"/custom.list" +#define DNSMASQ_CUSTOM_LIST_LEGACY "/etc/pihole/custom.list" #define DHCPLEASESFILE "/etc/pihole/dhcp.leases" #endif //DNSMASQ_CONFIG_H From 4b2b81f45fd02740663b2708998606574119a5d6 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 13 Nov 2023 10:09:42 +0100 Subject: [PATCH 054/221] Stop rotating dnsmasq.conf Signed-off-by: DL6ER --- src/config/dnsmasq_config.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/config/dnsmasq_config.c b/src/config/dnsmasq_config.c index a73f9c96d..cc6aaba0a 100644 --- a/src/config/dnsmasq_config.c +++ b/src/config/dnsmasq_config.c @@ -580,9 +580,6 @@ bool __attribute__((const)) write_dnsmasq_config(struct config *conf, bool test_ return false; } - // Rotate old config files - rotate_files(DNSMASQ_PH_CONFIG, NULL); - log_debug(DEBUG_CONFIG, "Installing "DNSMASQ_TEMP_CONF" to "DNSMASQ_PH_CONFIG); if(rename(DNSMASQ_TEMP_CONF, DNSMASQ_PH_CONFIG) != 0) { From 76d6fcab6e9bf58436aa1d24e0bf13748be665cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 13 Nov 2023 13:48:26 +0100 Subject: [PATCH 055/221] Remove additonal spaces in dnsmasq.conf MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- src/config/dnsmasq_config.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config/dnsmasq_config.c b/src/config/dnsmasq_config.c index 366f876c3..9cfe63c7d 100644 --- a/src/config/dnsmasq_config.c +++ b/src/config/dnsmasq_config.c @@ -507,7 +507,7 @@ bool __attribute__((const)) write_dnsmasq_config(struct config *conf, bool test_ fputs("# Pi-hole implements this via the dnsmasq option \"bogus-priv\" above\n", pihole_conf); fputs("# (if enabled!) as this option also covers IPv6.\n", pihole_conf); fputs("\n", pihole_conf); - fputs("# OpenWRT furthermore blocks bind, local, onion domains\n", pihole_conf); + fputs("# OpenWRT furthermore blocks bind, local, onion domains\n", pihole_conf); fputs("# see https://git.openwrt.org/?p=openwrt/openwrt.git;a=blob_plain;f=package/network/services/dnsmasq/files/rfc6761.conf;hb=HEAD\n", pihole_conf); fputs("# and https://www.iana.org/assignments/special-use-domain-names/special-use-domain-names.xhtml\n", pihole_conf); fputs("# We do not include the \".local\" rule ourselves, see https://github.com/pi-hole/pi-hole/pull/4282#discussion_r689112972\n", pihole_conf); From 8e043444fc950aa81c50ace08d1f3819a6057419 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 13 Nov 2023 23:18:18 +0100 Subject: [PATCH 056/221] Adjust tests Signed-off-by: DL6ER --- src/config/dnsmasq_config.c | 11 +++++++++++ test/test_suite.bats | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/config/dnsmasq_config.c b/src/config/dnsmasq_config.c index 14b8b81c5..c967abb7a 100644 --- a/src/config/dnsmasq_config.c +++ b/src/config/dnsmasq_config.c @@ -804,6 +804,17 @@ bool read_legacy_custom_hosts_config(void) bool write_custom_list(void) { + // Ensure that the directory exists + if(!directory_exists(DNSMASQ_HOSTSDIR)) + { + log_debug(DEBUG_CONFIG, "Creating directory "DNSMASQ_HOSTSDIR); + if(mkdir(DNSMASQ_HOSTSDIR, 0755) != 0) + { + log_err("Cannot create directory "DNSMASQ_HOSTSDIR": %s", strerror(errno)); + return false; + } + } + log_debug(DEBUG_CONFIG, "Opening "DNSMASQ_CUSTOM_LIST_LEGACY".tmp for writing"); FILE *custom_list = fopen(DNSMASQ_CUSTOM_LIST_LEGACY".tmp", "w"); // Return early if opening failed diff --git a/test/test_suite.bats b/test/test_suite.bats index 15f869bf7..d456543c3 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -1542,7 +1542,7 @@ run bash -c 'grep -c "DEBUG_CONFIG: dnsmasq.conf unchanged" /var/log/pihole/FTL.log' printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "2" ]] - run bash -c 'grep -c "DEBUG_CONFIG: HOSTS file written to /etc/pihole/custom.list" /var/log/pihole/FTL.log' + run bash -c 'grep -c "DEBUG_CONFIG: HOSTS file written to /etc/pihole/hosts/custom.list" /var/log/pihole/FTL.log' printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "1" ]] run bash -c 'grep -c "DEBUG_CONFIG: custom.list unchanged" /var/log/pihole/FTL.log' From ebe8e248d8b09203a8b57486d3f0b5066b162bd7 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 14 Nov 2023 10:26:19 +0100 Subject: [PATCH 057/221] Allow TLD blocking using ABP style Signed-off-by: DL6ER --- src/tools/gravity-parseList.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/tools/gravity-parseList.c b/src/tools/gravity-parseList.c index e3c1843d6..f519cd35e 100644 --- a/src/tools/gravity-parseList.c +++ b/src/tools/gravity-parseList.c @@ -40,7 +40,7 @@ static const char *false_positives[] = { #define MAX_INVALID_DOMAINS 5 // Validate domain name -static inline bool __attribute__((pure)) valid_domain(const char *domain, const size_t len) +static inline bool __attribute__((pure)) valid_domain(const char *domain, const size_t len, const bool abp) { // Domain must not be NULL or empty, and they should not be longer than // 255 characters @@ -84,8 +84,10 @@ static inline bool __attribute__((pure)) valid_domain(const char *domain, const // TLD checks // There must be at least two labels (i.e. one dot) - // e.g., "example.com" but not "localhost" - if(last_dot == -1) + // e.g., "example.com" but not "localhost" for exact domain + // We do not enforce this for ABP domains + // (see https://github.com/pi-hole/pi-hole/pull/5240) + if(last_dot == -1 && !abp) return false; // TLD must not start or end with a hyphen @@ -121,7 +123,7 @@ static inline bool __attribute__((pure)) valid_abp_domain(const char *line, cons return false; // Domain must be valid - return valid_domain(line+4, len-5); + return valid_domain(line+4, len-5, true); } else { @@ -138,7 +140,7 @@ static inline bool __attribute__((pure)) valid_abp_domain(const char *line, cons return false; // Domain must be valid - return valid_domain(line+2, len-3); + return valid_domain(line+2, len-3, true); } } @@ -279,7 +281,7 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis // Validate line if(line[0] != (antigravity ? '@' : '|') && // <- Not an ABP-style match - valid_domain(line, read)) + valid_domain(line, read, false)) { // Exact match found if(checkOnly) From a356cbd13baac00db0538cd71518d6c9e9fc142e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 15 Nov 2023 11:43:10 +0100 Subject: [PATCH 058/221] Add additional DHCP range tests Signed-off-by: DL6ER --- src/config/dnsmasq_config.c | 67 +++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/src/config/dnsmasq_config.c b/src/config/dnsmasq_config.c index 3a3894af1..8656bf6dc 100644 --- a/src/config/dnsmasq_config.c +++ b/src/config/dnsmasq_config.c @@ -221,6 +221,73 @@ static void write_config_header(FILE *fp, const char *description) bool __attribute__((const)) write_dnsmasq_config(struct config *conf, bool test_config, char errbuf[ERRBUF_SIZE]) { + // Early config checks + if(conf->dhcp.active.v.b) + { + // Check if the addresses are valid + // The addresses should neither be 0.0.0.0 nor 255.255.255.255 + if((ntohl(conf->dhcp.start.v.in_addr.s_addr) == 0) || + (ntohl(conf->dhcp.start.v.in_addr.s_addr) == 0xFFFFFFFF)) + { + strncpy(errbuf, "DHCP start address is not valid", ERRBUF_SIZE); + log_err("Unable to update dnsmasq configuration: %s", errbuf); + return false; + } + if((ntohl(conf->dhcp.end.v.in_addr.s_addr) == 0) || + (ntohl(conf->dhcp.end.v.in_addr.s_addr) == 0xFFFFFFFF)) + { + strncpy(errbuf, "DHCP end address is not valid", ERRBUF_SIZE); + log_err("Unable to update dnsmasq configuration: %s", errbuf); + return false; + } + if((ntohl(conf->dhcp.router.v.in_addr.s_addr) == 0) || + (ntohl(conf->dhcp.router.v.in_addr.s_addr) == 0xFFFFFFFF)) + { + strncpy(errbuf, "DHCP router address is not valid", ERRBUF_SIZE); + log_err("Unable to update dnsmasq configuration: %s", errbuf); + return false; + } + // The addresses should neither end in .0 or .255 in the last octet + if((ntohl(conf->dhcp.start.v.in_addr.s_addr) & 0xFF) == 0 || + (ntohl(conf->dhcp.start.v.in_addr.s_addr) & 0xFF) == 0xFF) + { + strncpy(errbuf, "DHCP start address is not valid", ERRBUF_SIZE); + log_err("Unable to update dnsmasq configuration: %s", errbuf); + return false; + } + if((ntohl(conf->dhcp.end.v.in_addr.s_addr) & 0xFF) == 0 || + (ntohl(conf->dhcp.end.v.in_addr.s_addr) & 0xFF) == 0xFF) + { + strncpy(errbuf, "DHCP end address is not valid", ERRBUF_SIZE); + log_err("Unable to update dnsmasq configuration: %s", errbuf); + return false; + } + if((ntohl(conf->dhcp.router.v.in_addr.s_addr) & 0xFF) == 0 || + (ntohl(conf->dhcp.router.v.in_addr.s_addr) & 0xFF) == 0xFF) + { + strncpy(errbuf, "DHCP router address is not valid", ERRBUF_SIZE); + log_err("Unable to update dnsmasq configuration: %s", errbuf); + return false; + } + + // Check if the DHCP range is valid (start needs to be smaller than end) + if(ntohl(conf->dhcp.start.v.in_addr.s_addr) >= ntohl(conf->dhcp.end.v.in_addr.s_addr)) + { + strncpy(errbuf, "DHCP range start address is larger than or equal to the end address", ERRBUF_SIZE); + log_err("Unable to update dnsmasq configuration: %s", errbuf); + return false; + } + + // Check if the router address is within the DHCP range + if(ntohl(conf->dhcp.router.v.in_addr.s_addr) >= ntohl(conf->dhcp.start.v.in_addr.s_addr) && + ntohl(conf->dhcp.router.v.in_addr.s_addr) <= ntohl(conf->dhcp.end.v.in_addr.s_addr)) + { + strncpy(errbuf, "DHCP router address should not be within DHCP range", ERRBUF_SIZE); + log_err("Unable to update dnsmasq configuration: %s", errbuf); + return false; + } + } + log_debug(DEBUG_CONFIG, "Opening "DNSMASQ_TEMP_CONF" for writing"); FILE *pihole_conf = fopen(DNSMASQ_TEMP_CONF, "w"); // Return early if opening failed From a69130585e0303064fe78da4b9a3a2edf0e3cba3 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 15 Nov 2023 11:49:03 +0100 Subject: [PATCH 059/221] Use package ipaddress for IP address validation in API tests Signed-off-by: DL6ER --- test/api/libs/responseVerifyer.py | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/test/api/libs/responseVerifyer.py b/test/api/libs/responseVerifyer.py index c09d154a4..dcfc8f6f6 100644 --- a/test/api/libs/responseVerifyer.py +++ b/test/api/libs/responseVerifyer.py @@ -223,24 +223,21 @@ def verify_teleporter_zip(self, teleporter_archive: bytes): # Check if a string is a valid IPv4 address def valid_ipv4(self, addr: str) -> bool: - octets = addr.split(".") # type: list[str] - if len(octets) != 4: - return False - for octet in octets: - if not octet.isdigit(): - return False - if int(octet) < 0 or int(octet) > 255: - return False - return True - + try: + if type(ipaddress.ip_address(addr)) is ipaddress.IPv4Address: + return True + except ValueError: + pass + return False # Check if a string is a valid IPv6 address def valid_ipv6(self, addr: str) -> bool: - # Split the address into parts - parts = addr.split(":") # type: list[str] - # Check if the address is a valid IPv6 address - if len(parts) != 8: - return False + try: + if type(ipaddress.ip_address(addr)) is ipaddress.IPv6Address: + return True + except ValueError: + pass + return False # Verify a single property's type From 2a465b5ea4d08803c24bc3e7c379424e1d9c3baf Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 15 Nov 2023 12:00:22 +0100 Subject: [PATCH 060/221] Don't call cJSON_free unconditionally on errors in api_list_write() Signed-off-by: DL6ER --- src/api/list.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/api/list.c b/src/api/list.c index 21efae430..3c0b6dc05 100644 --- a/src/api/list.c +++ b/src/api/list.c @@ -393,7 +393,8 @@ static int api_list_write(struct ftl_conn *api, strchr(it->valuestring, '\t') != NULL || strchr(it->valuestring, '\n') != NULL) { - cJSON_free(row.items); + if(allocated_json) + cJSON_free(row.items); return send_json_error(api, 400, // 400 Bad Request "bad_request", "Spaces, newlines and tabs are not allowed in domains and URLs", @@ -406,11 +407,12 @@ static int api_list_write(struct ftl_conn *api, if(!okay) { // Send error reply - cJSON_free(row.items); - return send_json_error(api, 400, // 400 Bad Request - "regex_error", - "Regex validation failed", - regex_msg); + if(allocated_json) + cJSON_free(row.items); + return send_json_error_free(api, 400, // 400 Bad Request + "regex_error", + "Regex validation failed", + regex_msg, true); } // Try to add item(s) to table From 67a630c790273393007d9fa9d296a6b1cc6d74b8 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 15 Nov 2023 12:06:08 +0100 Subject: [PATCH 061/221] Explicitly check hint being NULL before trying to free it in send_json_error_free() Signed-off-by: DL6ER --- src/webserver/http-common.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/webserver/http-common.c b/src/webserver/http-common.c index c1fdc2750..a8f9c1933 100644 --- a/src/webserver/http-common.c +++ b/src/webserver/http-common.c @@ -85,7 +85,7 @@ int send_json_error_free(struct ftl_conn *api, const int code, const char *key, const char* message, char *hint, bool free_hint) { - if(hint) + if(hint != NULL) log_warn("API: %s (%s)", message, hint); else log_warn("API: %s", message); @@ -94,7 +94,7 @@ int send_json_error_free(struct ftl_conn *api, const int code, JSON_REF_STR_IN_OBJECT(error, "key", key); JSON_REF_STR_IN_OBJECT(error, "message", message); JSON_COPY_STR_TO_OBJECT(error, "hint", hint); - if(free_hint) + if(free_hint && hint != NULL) free(hint); cJSON *json = JSON_NEW_OBJECT(); From 503c0538ed3c6bc6cfb1541d6be16e44e742c8b2 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 15 Nov 2023 22:09:03 +0100 Subject: [PATCH 062/221] IPv4 address 0.0.0.0 and IPv6 address :: correspond to empty strings in FTL settings Signed-off-by: DL6ER --- src/api/config.c | 35 +++++++++++++++++++++----- src/config/cli.c | 14 +++++++++-- src/config/config.c | 10 ++++---- src/config/toml_helper.c | 42 ++++++++++++++++++++++++++++--- test/api/libs/responseVerifyer.py | 10 ++++++-- 5 files changed, 92 insertions(+), 19 deletions(-) diff --git a/src/api/config.c b/src/api/config.c index 453052dac..168c558c7 100644 --- a/src/api/config.c +++ b/src/api/config.c @@ -128,12 +128,22 @@ static cJSON *addJSONvalue(const enum conf_type conf_type, union conf_value *val return cJSON_CreateStringReference(get_temp_unit_str(val->temp_unit)); case CONF_STRUCT_IN_ADDR: { + // Special case 0.0.0.0 -> return empty string + if(val->in_addr.s_addr == INADDR_ANY) + return cJSON_CreateStringReference(""); + + // else: normal address char addr4[INET_ADDRSTRLEN] = { 0 }; inet_ntop(AF_INET, &val->in_addr, addr4, INET_ADDRSTRLEN); return cJSON_CreateString(addr4); // Performs a copy } case CONF_STRUCT_IN6_ADDR: { + // Special case :: -> return empty string + if(memcmp(&val->in6_addr, &in6addr_any, sizeof(in6addr_any)) == 0) + return cJSON_CreateStringReference(""); + + // else: normal address char addr6[INET6_ADDRSTRLEN] = { 0 }; inet_ntop(AF_INET6, &val->in6_addr, addr6, INET6_ADDRSTRLEN); return cJSON_CreateString(addr6); // Performs a copy @@ -400,11 +410,19 @@ static const char *getJSONvalue(struct conf_item *conf_item, cJSON *elem, struct struct in_addr addr4 = { 0 }; if(!cJSON_IsString(elem)) return "not of type string"; - if(!inet_pton(AF_INET, elem->valuestring, &addr4)) + if(strlen(elem->valuestring) == 0) + { + // Special case: empty string -> 0.0.0.0 + conf_item->v.in_addr.s_addr = INADDR_ANY; + } + else if(inet_pton(AF_INET, elem->valuestring, &addr4)) + { + // Set item + memcpy(&conf_item->v.in_addr, &addr4, sizeof(addr4)); + } + else return "not a valid IPv4 address"; - // Set item - memcpy(&conf_item->v.in_addr, &addr4, sizeof(addr4)); - log_debug(DEBUG_CONFIG, "%s = %s", conf_item->k, elem->valuestring); + log_debug(DEBUG_CONFIG, "%s = \"%s\"", conf_item->k, elem->valuestring); break; } case CONF_STRUCT_IN6_ADDR: @@ -412,11 +430,16 @@ static const char *getJSONvalue(struct conf_item *conf_item, cJSON *elem, struct struct in6_addr addr6 = { 0 }; if(!cJSON_IsString(elem)) return "not of type string"; - if(!inet_pton(AF_INET6, elem->valuestring, &addr6)) + if(strlen(elem->valuestring) == 0) + { + // Special case: empty string -> :: + memcpy(&conf_item->v.in6_addr, &in6addr_any, sizeof(in6addr_any)); + } + else if(!inet_pton(AF_INET6, elem->valuestring, &addr6)) return "not a valid IPv6 address"; // Set item memcpy(&conf_item->v.in6_addr, &addr6, sizeof(addr6)); - log_debug(DEBUG_CONFIG, "%s = %s", conf_item->k, elem->valuestring); + log_debug(DEBUG_CONFIG, "%s = \"%s\"", conf_item->k, elem->valuestring); break; } case CONF_JSON_STRING_ARRAY: diff --git a/src/config/cli.c b/src/config/cli.c index ddc23db98..fab52d2eb 100644 --- a/src/config/cli.c +++ b/src/config/cli.c @@ -295,7 +295,12 @@ static bool readStringValue(struct conf_item *conf_item, const char *value, stru case CONF_STRUCT_IN_ADDR: { struct in_addr addr4 = { 0 }; - if(inet_pton(AF_INET, value, &addr4)) + if(strlen(value) == 0) + { + // Special case: empty string -> 0.0.0.0 + conf_item->v.in_addr.s_addr = INADDR_ANY; + } + else if(inet_pton(AF_INET, value, &addr4)) memcpy(&conf_item->v.in_addr, &addr4, sizeof(addr4)); else { @@ -307,7 +312,12 @@ static bool readStringValue(struct conf_item *conf_item, const char *value, stru case CONF_STRUCT_IN6_ADDR: { struct in6_addr addr6 = { 0 }; - if(inet_pton(AF_INET6, value, &addr6)) + if(strlen(value) == 0) + { + // Special case: empty string -> :: + memcpy(&conf_item->v.in6_addr, &in6addr_any, sizeof(in6addr_any)); + } + else if(inet_pton(AF_INET6, value, &addr6)) memcpy(&conf_item->v.in6_addr, &addr6, sizeof(addr6)); else { diff --git a/src/config/config.c b/src/config/config.c index 184b74f36..9bdd80b38 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -686,21 +686,21 @@ void initConfig(struct config *conf) conf->dhcp.start.k = "dhcp.start"; conf->dhcp.start.h = "Start address of the DHCP address pool"; - conf->dhcp.start.a = cJSON_CreateStringReference(", e.g., \"192.168.0.10\""); + conf->dhcp.start.a = cJSON_CreateStringReference(" or empty string (\"\"), e.g., \"192.168.0.10\""); conf->dhcp.start.t = CONF_STRUCT_IN_ADDR; conf->dhcp.start.f = FLAG_RESTART_FTL; memset(&conf->dhcp.start.d.in_addr, 0, sizeof(struct in_addr)); conf->dhcp.end.k = "dhcp.end"; conf->dhcp.end.h = "End address of the DHCP address pool"; - conf->dhcp.end.a = cJSON_CreateStringReference(", e.g., \"192.168.0.250\""); + conf->dhcp.end.a = cJSON_CreateStringReference(" or empty string (\"\"), e.g., \"192.168.0.250\""); conf->dhcp.end.t = CONF_STRUCT_IN_ADDR; conf->dhcp.end.f = FLAG_RESTART_FTL; memset(&conf->dhcp.end.d.in_addr, 0, sizeof(struct in_addr)); conf->dhcp.router.k = "dhcp.router"; conf->dhcp.router.h = "Address of the gateway to be used (typically the address of your router in a home installation)"; - conf->dhcp.router.a = cJSON_CreateStringReference(", e.g., \"192.168.0.1\""); + conf->dhcp.router.a = cJSON_CreateStringReference(" or empty string (\"\"), e.g., \"192.168.0.1\""); conf->dhcp.router.t = CONF_STRUCT_IN_ADDR; conf->dhcp.router.f = FLAG_RESTART_FTL; memset(&conf->dhcp.router.d.in_addr, 0, sizeof(struct in_addr)); @@ -713,8 +713,8 @@ void initConfig(struct config *conf) conf->dhcp.domain.d.s = (char*)"lan"; conf->dhcp.netmask.k = "dhcp.netmask"; - conf->dhcp.netmask.h = "The netmask used by your Pi-hole. For directly connected networks (i.e., networks on which the machine running Pi-hole has an interface) the netmask is optional and may be set to \"0.0.0.0\": it will then be determined from the interface configuration itself. For networks which receive DHCP service via a relay agent, we cannot determine the netmask itself, so it should explicitly be specified, otherwise Pi-hole guesses based on the class (A, B or C) of the network address."; - conf->dhcp.netmask.a = cJSON_CreateStringReference(", e.g., \"255.255.255.0\" or \"0.0.0.0\" for auto-discovery"); + conf->dhcp.netmask.h = "The netmask used by your Pi-hole. For directly connected networks (i.e., networks on which the machine running Pi-hole has an interface) the netmask is optional and may be set to an empty string (\"\"): it will then be determined from the interface configuration itself. For networks which receive DHCP service via a relay agent, we cannot determine the netmask itself, so it should explicitly be specified, otherwise Pi-hole guesses based on the class (A, B or C) of the network address."; + conf->dhcp.netmask.a = cJSON_CreateStringReference(" (e.g., \"255.255.255.0\") or empty string (\"\") for auto-discovery"); conf->dhcp.netmask.t = CONF_STRUCT_IN_ADDR; conf->dhcp.netmask.f = FLAG_RESTART_FTL | FLAG_ADVANCED_SETTING; memset(&conf->dhcp.netmask.d.in_addr, 0, sizeof(struct in_addr)); diff --git a/src/config/toml_helper.c b/src/config/toml_helper.c index 7f40b683f..2dffff6a5 100644 --- a/src/config/toml_helper.c +++ b/src/config/toml_helper.c @@ -364,6 +364,13 @@ void writeTOMLvalue(FILE * fp, const int indent, const enum conf_type t, union c break; case CONF_STRUCT_IN_ADDR: { + // Special case: 0.0.0.0 -> return empty string + if(v->in_addr.s_addr == INADDR_ANY) + { + printTOMLstring(fp, "", toml); + break; + } + // else: normal address char addr4[INET_ADDRSTRLEN] = { 0 }; inet_ntop(AF_INET, &v->in_addr, addr4, INET_ADDRSTRLEN); printTOMLstring(fp, addr4, toml); @@ -371,6 +378,13 @@ void writeTOMLvalue(FILE * fp, const int indent, const enum conf_type t, union c } case CONF_STRUCT_IN6_ADDR: { + // Special case: :: -> return empty string + if(memcmp(&v->in6_addr, &in6addr_any, sizeof(in6addr_any)) == 0) + { + printTOMLstring(fp, "", toml); + break; + } + // else: normal address char addr6[INET6_ADDRSTRLEN] = { 0 }; inet_ntop(AF_INET6, &v->in6_addr, addr6, INET6_ADDRSTRLEN); printTOMLstring(fp, addr6, toml); @@ -654,7 +668,12 @@ void readTOMLvalue(struct conf_item *conf_item, const char* key, toml_table_t *t const toml_datum_t val = toml_string_in(toml, key); if(val.ok) { - if(inet_pton(AF_INET, val.u.s, &addr4)) + if(strlen(val.u.s) == 0) + { + // Special case: empty string -> 0.0.0.0 + conf_item->v.in_addr.s_addr = INADDR_ANY; + } + else if(inet_pton(AF_INET, val.u.s, &addr4)) memcpy(&conf_item->v.in_addr, &addr4, sizeof(addr4)); else log_warn("Config %s is invalid (not of type IPv4 address)", conf_item->k); @@ -670,7 +689,12 @@ void readTOMLvalue(struct conf_item *conf_item, const char* key, toml_table_t *t const toml_datum_t val = toml_string_in(toml, key); if(val.ok) { - if(inet_pton(AF_INET6, val.u.s, &addr6)) + if(strlen(val.u.s) == 0) + { + // Special case: empty string -> :: + memcpy(&conf_item->v.in6_addr, &in6addr_any, sizeof(in6addr_any)); + } + else if(inet_pton(AF_INET6, val.u.s, &addr6)) memcpy(&conf_item->v.in6_addr, &addr6, sizeof(addr6)); else log_warn("Config %s is invalid (not of type IPv6 address)", conf_item->k); @@ -910,7 +934,12 @@ bool readEnvValue(struct conf_item *conf_item, struct config *newconf) case CONF_STRUCT_IN_ADDR: { struct in_addr addr4 = { 0 }; - if(inet_pton(AF_INET, envvar, &addr4)) + if(strlen(envvar) == 0) + { + // Special case: empty string -> 0.0.0.0 + conf_item->v.in_addr.s_addr = INADDR_ANY; + } + else if(inet_pton(AF_INET, envvar, &addr4)) memcpy(&conf_item->v.in_addr, &addr4, sizeof(addr4)); else log_warn("ENV %s is invalid (not of type IPv4 address)", envkey); @@ -919,7 +948,12 @@ bool readEnvValue(struct conf_item *conf_item, struct config *newconf) case CONF_STRUCT_IN6_ADDR: { struct in6_addr addr6 = { 0 }; - if(inet_pton(AF_INET6, envvar, &addr6)) + if(strlen(envvar) == 0) + { + // Special case: empty string -> :: + memcpy(&conf_item->v.in6_addr, &in6addr_any, sizeof(in6addr_any)); + } + else if(inet_pton(AF_INET6, envvar, &addr6)) memcpy(&conf_item->v.in6_addr, &addr6, sizeof(addr6)); else log_warn("ENV %s is invalid (not of type IPv6 address)", envkey); diff --git a/test/api/libs/responseVerifyer.py b/test/api/libs/responseVerifyer.py index dcfc8f6f6..b527c682f 100644 --- a/test/api/libs/responseVerifyer.py +++ b/test/api/libs/responseVerifyer.py @@ -223,6 +223,9 @@ def verify_teleporter_zip(self, teleporter_archive: bytes): # Check if a string is a valid IPv4 address def valid_ipv4(self, addr: str) -> bool: + # Empty string is valid (0.0.0.0) + if len(addr) == 0: + return True try: if type(ipaddress.ip_address(addr)) is ipaddress.IPv4Address: return True @@ -232,6 +235,9 @@ def valid_ipv4(self, addr: str) -> bool: # Check if a string is a valid IPv6 address def valid_ipv6(self, addr: str) -> bool: + # Empty string is valid (::) + if len(addr) == 0: + return True try: if type(ipaddress.ip_address(addr)) is ipaddress.IPv6Address: return True @@ -253,10 +259,10 @@ def verify_type(self, prop: any, yaml_type: str, yaml_nullable: bool, yaml_forma return False if yaml_format is not None: # Check if the format is correct - if yaml_format == "ipv4" and not type(ipaddress.ip_address(prop)) is ipaddress.IPv4Address: + if yaml_format == "ipv4" and not self.valid_ipv4(prop): self.errors.append("Property \"" + str(prop) + "\" is not a valid IPv4 address") return False - elif yaml_format == "ipv6" and not type(ipaddress.ip_address(prop)) is ipaddress.IPv6Address: + elif yaml_format == "ipv6" and not self.valid_ipv6(prop): self.errors.append("Property \"" + str(prop) + "\" is not a valid IPv6 address") return False return prop_type in self.YAML_TYPES[yaml_type] From d0345cb4fa39ae70feac68f1a124534db2338c7c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 18 Nov 2023 12:48:47 +0100 Subject: [PATCH 063/221] Validate domains before adding the to the database via /api/list Signed-off-by: DL6ER --- src/api/list.c | 15 +++++++++++++++ src/tools/gravity-parseList.c | 2 +- src/tools/gravity-parseList.h | 1 + 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/api/list.c b/src/api/list.c index 3c0b6dc05..276cea6d7 100644 --- a/src/api/list.c +++ b/src/api/list.c @@ -17,6 +17,8 @@ #include "shmem.h" // getNameFromIP() #include "database/network-table.h" +// valid_domain() +#include "tools/gravity-parseList.h" static int api_list_read(struct ftl_conn *api, const int code, @@ -400,6 +402,19 @@ static int api_list_write(struct ftl_conn *api, "Spaces, newlines and tabs are not allowed in domains and URLs", it->valuestring); } + + // Validate domains + if((listtype == GRAVITY_DOMAINLIST_ALLOW_EXACT || + listtype == GRAVITY_DOMAINLIST_DENY_EXACT) && + !valid_domain(it->valuestring, strlen(it->valuestring), false)) + { + if(allocated_json) + cJSON_free(row.items); + return send_json_error(api, 400, // 400 Bad Request + "bad_request", + "Invalid domain", + it->valuestring); + } } } diff --git a/src/tools/gravity-parseList.c b/src/tools/gravity-parseList.c index f519cd35e..0af753454 100644 --- a/src/tools/gravity-parseList.c +++ b/src/tools/gravity-parseList.c @@ -40,7 +40,7 @@ static const char *false_positives[] = { #define MAX_INVALID_DOMAINS 5 // Validate domain name -static inline bool __attribute__((pure)) valid_domain(const char *domain, const size_t len, const bool abp) +inline bool __attribute__((pure)) valid_domain(const char *domain, const size_t len, const bool abp) { // Domain must not be NULL or empty, and they should not be longer than // 255 characters diff --git a/src/tools/gravity-parseList.h b/src/tools/gravity-parseList.h index 93e6b4882..560fbd9bd 100644 --- a/src/tools/gravity-parseList.h +++ b/src/tools/gravity-parseList.h @@ -11,3 +11,4 @@ #include "FTL.h" int gravity_parseList(const char *infile, const char *outfile, const char *adlistID, const bool checkOnly, const bool antigravity); +bool __attribute__((pure)) valid_domain(const char *domain, const size_t len, const bool abp); From 04742405abaf7fa9e195cee15354185e794808d4 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 18 Nov 2023 16:56:31 +0100 Subject: [PATCH 064/221] Hostnames (non-FQDN) are valid domains in the context of allow/deny domains, too Signed-off-by: DL6ER --- src/api/list.c | 6 +++--- src/tools/gravity-parseList.c | 10 +++++----- src/tools/gravity-parseList.h | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/api/list.c b/src/api/list.c index 276cea6d7..780a0754f 100644 --- a/src/api/list.c +++ b/src/api/list.c @@ -398,9 +398,9 @@ static int api_list_write(struct ftl_conn *api, if(allocated_json) cJSON_free(row.items); return send_json_error(api, 400, // 400 Bad Request - "bad_request", - "Spaces, newlines and tabs are not allowed in domains and URLs", - it->valuestring); + "bad_request", + "Spaces, newlines and tabs are not allowed in domains and URLs", + it->valuestring); } // Validate domains diff --git a/src/tools/gravity-parseList.c b/src/tools/gravity-parseList.c index 0af753454..8f7ad6838 100644 --- a/src/tools/gravity-parseList.c +++ b/src/tools/gravity-parseList.c @@ -40,7 +40,7 @@ static const char *false_positives[] = { #define MAX_INVALID_DOMAINS 5 // Validate domain name -inline bool __attribute__((pure)) valid_domain(const char *domain, const size_t len, const bool abp) +inline bool __attribute__((pure)) valid_domain(const char *domain, const size_t len, const bool fqdn_only) { // Domain must not be NULL or empty, and they should not be longer than // 255 characters @@ -87,7 +87,7 @@ inline bool __attribute__((pure)) valid_domain(const char *domain, const size_t // e.g., "example.com" but not "localhost" for exact domain // We do not enforce this for ABP domains // (see https://github.com/pi-hole/pi-hole/pull/5240) - if(last_dot == -1 && !abp) + if(last_dot == -1 && fqdn_only) return false; // TLD must not start or end with a hyphen @@ -123,7 +123,7 @@ static inline bool __attribute__((pure)) valid_abp_domain(const char *line, cons return false; // Domain must be valid - return valid_domain(line+4, len-5, true); + return valid_domain(line+4, len-5, false); } else { @@ -140,7 +140,7 @@ static inline bool __attribute__((pure)) valid_abp_domain(const char *line, cons return false; // Domain must be valid - return valid_domain(line+2, len-3, true); + return valid_domain(line+2, len-3, false); } } @@ -281,7 +281,7 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis // Validate line if(line[0] != (antigravity ? '@' : '|') && // <- Not an ABP-style match - valid_domain(line, read, false)) + valid_domain(line, read, true)) { // Exact match found if(checkOnly) diff --git a/src/tools/gravity-parseList.h b/src/tools/gravity-parseList.h index 560fbd9bd..e9e54ac4c 100644 --- a/src/tools/gravity-parseList.h +++ b/src/tools/gravity-parseList.h @@ -11,4 +11,4 @@ #include "FTL.h" int gravity_parseList(const char *infile, const char *outfile, const char *adlistID, const bool checkOnly, const bool antigravity); -bool __attribute__((pure)) valid_domain(const char *domain, const size_t len, const bool abp); +bool __attribute__((pure)) valid_domain(const char *domain, const size_t len, const bool fqdn_only); From ef679f2edb998fa5d5a877a9b5e6a5f5e9cf0d63 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 18 Nov 2023 21:57:04 +0100 Subject: [PATCH 065/221] Update src/config/config.c Co-authored-by: yubiuser Signed-off-by: DL6ER --- src/config/config.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config/config.c b/src/config/config.c index 9af0b7cd7..9b086908a 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -1346,7 +1346,7 @@ void readFTLconf(struct config *conf, const bool rewrite) // the most recent one and going back in time until we find a valid config for(unsigned int i = 0; i < PLAIN_ROTATIONS; i++) { - if(readFTLtoml(NULL, conf, NULL, rewrite, NULL, 0)) + if(readFTLtoml(NULL, conf, NULL, rewrite, NULL, i)) { // If successful, we write the config file back to disk // to ensure that all options are present and comments From d8a452f8f3af3d1b8d75e2e76b39f35b3a310e94 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 18 Nov 2023 21:58:06 +0100 Subject: [PATCH 066/221] Update src/tools/gravity-parseList.c Co-authored-by: yubiuser Signed-off-by: DL6ER --- src/tools/gravity-parseList.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tools/gravity-parseList.c b/src/tools/gravity-parseList.c index 8f7ad6838..912ad358f 100644 --- a/src/tools/gravity-parseList.c +++ b/src/tools/gravity-parseList.c @@ -85,7 +85,7 @@ inline bool __attribute__((pure)) valid_domain(const char *domain, const size_t // There must be at least two labels (i.e. one dot) // e.g., "example.com" but not "localhost" for exact domain - // We do not enforce this for ABP domains + // We do not enforce this for ABP domains and domainlist input // (see https://github.com/pi-hole/pi-hole/pull/5240) if(last_dot == -1 && fqdn_only) return false; From ad5d07819376e363ec718b1442dd2b4d125dec32 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 19 Nov 2023 07:46:51 +0100 Subject: [PATCH 067/221] The group table's column is called "description" but we expose it in the API as "comment". Adjust internally used SQL to translate between them (this was already implemented and working for comment editing) Signed-off-by: DL6ER --- src/database/gravity-db.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/database/gravity-db.c b/src/database/gravity-db.c index 802b57b4e..7e66df2e6 100644 --- a/src/database/gravity-db.c +++ b/src/database/gravity-db.c @@ -1626,7 +1626,7 @@ bool gravityDB_addToTable(const enum gravity_list_type listtype, tablerow *row, // The item is the item for all POST requests if(listtype == GRAVITY_GROUPS) { - querystr = "INSERT INTO \"group\" (name,enabled,description) VALUES (:item,:enabled,:description);"; + querystr = "INSERT INTO \"group\" (name,enabled,description) VALUES (:item,:enabled,:comment);"; } else if(listtype == GRAVITY_ADLISTS) { @@ -1648,8 +1648,8 @@ bool gravityDB_addToTable(const enum gravity_list_type listtype, tablerow *row, if(row->name == NULL) { // Name is not to be changed - querystr = "INSERT INTO \"group\" (name,enabled,description) VALUES (:item,:enabled,:description) " - "ON CONFLICT(name) DO UPDATE SET enabled = :enabled, description = :description;"; + querystr = "INSERT INTO \"group\" (name,enabled,description) VALUES (:item,:enabled,:comment) " + "ON CONFLICT(name) DO UPDATE SET enabled = :enabled, description = :comment;"; } else { From 5926c5b24604696a53f3c724dbb60e84514fd3bb Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 19 Nov 2023 09:33:36 +0100 Subject: [PATCH 068/221] Set civetweb's authentication_domain to config.webserver.domain Signed-off-by: DL6ER --- src/webserver/webserver.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webserver/webserver.c b/src/webserver/webserver.c index 95940e492..b5c3c6bf7 100644 --- a/src/webserver/webserver.c +++ b/src/webserver/webserver.c @@ -304,13 +304,13 @@ void http_init(void) "decode_url", "yes", "enable_directory_listing", "no", "num_threads", num_threads, + "authentication_domain", config.webserver.domain.v.s, "additional_header", "Content-Security-Policy: default-src 'self' 'unsafe-inline';\r\n" "X-Frame-Options: DENY\r\n" "X-XSS-Protection: 0\r\n" "X-Content-Type-Options: nosniff\r\n" "Referrer-Policy: strict-origin-when-cross-origin", "index_files", "index.html,index.htm,index.lp", - "enable_auth_domain_check", "no", NULL, NULL, NULL, NULL, // Leave slots for access control list (ACL) and TLS configuration at the end NULL From 262965375adea579113d8c81aa986099a20a5aff Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 19 Nov 2023 09:34:38 +0100 Subject: [PATCH 069/221] Require restarting after domain change - this also re-reads the TLS certificate file Signed-off-by: DL6ER --- src/config/config.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/config/config.c b/src/config/config.c index 3d1cf57c4..85578a95f 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -833,6 +833,7 @@ void initConfig(struct config *conf) conf->webserver.domain.h = "On which domain is the web interface served?"; conf->webserver.domain.a = cJSON_CreateStringReference(""); conf->webserver.domain.t = CONF_STRING; + conf->webserver.domain.f = FLAG_ADVANCED_SETTING | FLAG_RESTART_FTL; conf->webserver.domain.d.s = (char*)"pi.hole"; conf->webserver.acl.k = "webserver.acl"; From c97239acedb92b64930357e51f17a460bef6f9c6 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 19 Nov 2023 11:20:40 +0100 Subject: [PATCH 070/221] Remove CivetWeb patch which is not needed when authentication_domain is set Signed-off-by: DL6ER --- patch/civetweb.sh | 1 - ...ess-server-hostname-in-Civetweb-when.patch | 29 ------------------- src/webserver/civetweb/civetweb.c | 4 +-- 3 files changed, 1 insertion(+), 33 deletions(-) delete mode 100644 patch/civetweb/0001-Do-not-try-to-guess-server-hostname-in-Civetweb-when.patch diff --git a/patch/civetweb.sh b/patch/civetweb.sh index e66845157..f33dc8de0 100644 --- a/patch/civetweb.sh +++ b/patch/civetweb.sh @@ -7,7 +7,6 @@ patch -p1 < patch/civetweb/0001-Always-Kepler-syntax-for-Lua-server-pages.patch patch -p1 < patch/civetweb/0001-Add-FTL-URI-rewriting-changes-to-CivetWeb.patch patch -p1 < patch/civetweb/0001-Add-mbedTLS-debug-logging-hook.patch patch -p1 < patch/civetweb/0001-Register-CSRF-token-in-conn-request_info.patch -patch -p1 < patch/civetweb/0001-Do-not-try-to-guess-server-hostname-in-Civetweb-when.patch patch -p1 < patch/civetweb/0001-Log-debug-messages-to-webserver.log-when-debug.webse.patch patch -p1 < patch/civetweb/0001-Allow-extended-ASCII-characters-in-URIs.patch diff --git a/patch/civetweb/0001-Do-not-try-to-guess-server-hostname-in-Civetweb-when.patch b/patch/civetweb/0001-Do-not-try-to-guess-server-hostname-in-Civetweb-when.patch deleted file mode 100644 index 862cc4215..000000000 --- a/patch/civetweb/0001-Do-not-try-to-guess-server-hostname-in-Civetweb-when.patch +++ /dev/null @@ -1,29 +0,0 @@ -From e41d902b5b01896360e1235aebabd3eb352158aa Mon Sep 17 00:00:00 2001 -From: DL6ER -Date: Sun, 8 Oct 2023 14:31:20 +0200 -Subject: [PATCH] Do not try to guess server hostname in Civetweb when - redirecting directory URIs to end with a slash - -Signed-off-by: DL6ER ---- - src/webserver/civetweb/civetweb.c | 4 +++- - 1 file changed, 3 insertions(+), 1 deletion(-) - -diff --git a/src/webserver/civetweb/civetweb.c b/src/webserver/civetweb/civetweb.c -index f44b17ba..3df8eab9 100644 ---- a/src/webserver/civetweb/civetweb.c -+++ b/src/webserver/civetweb/civetweb.c -@@ -15306,7 +15306,9 @@ handle_request(struct mg_connection *conn) - if (!new_path) { - mg_send_http_error(conn, 500, "out or memory"); - } else { -- mg_get_request_link(conn, new_path, buflen - 1); -+ /* Pi-hole modification */ -+ //mg_get_request_link(conn, new_path, buflen - 1); -+ strcpy(new_path, ri->local_uri_raw); - strcat(new_path, "/"); - if (ri->query_string) { - /* Append ? and query string */ --- -2.34.1 - diff --git a/src/webserver/civetweb/civetweb.c b/src/webserver/civetweb/civetweb.c index 5320c4d44..367e19ae8 100644 --- a/src/webserver/civetweb/civetweb.c +++ b/src/webserver/civetweb/civetweb.c @@ -15307,9 +15307,7 @@ handle_request(struct mg_connection *conn) if (!new_path) { mg_send_http_error(conn, 500, "out or memory"); } else { - /* Pi-hole modification */ - //mg_get_request_link(conn, new_path, buflen - 1); - strcpy(new_path, ri->local_uri_raw); + mg_get_request_link(conn, new_path, buflen - 1); strcat(new_path, "/"); if (ri->query_string) { /* Append ? and query string */ From 00a9bc8d17652ad6182fd451906272f02c441514 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 19 Nov 2023 18:43:08 +0100 Subject: [PATCH 071/221] Generalize upload handing scripts to possibly accept other files than ZIP archives Signed-off-by: DL6ER --- src/api/teleporter.c | 78 ++++++++++++++++++++------------------------ 1 file changed, 35 insertions(+), 43 deletions(-) diff --git a/src/api/teleporter.c b/src/api/teleporter.c index 87688f623..470829f41 100644 --- a/src/api/teleporter.c +++ b/src/api/teleporter.c @@ -16,7 +16,7 @@ // ERRBUF_SIZE #include "config/dnsmasq_config.h" -#define MAXZIPSIZE (50u*1024*1024) +#define MAXFILESIZE (50u*1024*1024) static int api_teleporter_GET(struct ftl_conn *api) { @@ -58,9 +58,9 @@ static int api_teleporter_GET(struct ftl_conn *api) struct upload_data { bool too_large; char *sid; - char *zip_data; - char *zip_filename; - size_t zip_size; + char *data; + char *filename; + size_t filesize; }; // Callback function for CivetWeb to determine which fields we want to receive @@ -79,7 +79,7 @@ static int field_found(const char *key, is_sid = false; if(strcasecmp(key, "file") == 0 && filename && *filename) { - data->zip_filename = strdup(filename); + data->filename = strdup(filename); is_file = true; return MG_FORM_FIELD_STORAGE_GET; } @@ -103,21 +103,21 @@ static int field_get(const char *key, const char *value, size_t valuelen, void * if(is_file) { - if(data->zip_size + valuelen > MAXZIPSIZE) + if(data->filesize + valuelen > MAXFILESIZE) { - log_warn("Uploaded Teleporter ZIP archive is too large (limit is %u bytes)", - MAXZIPSIZE); + log_warn("Uploaded Teleporter file is too large (limit is %u bytes)", + MAXFILESIZE); data->too_large = true; return MG_FORM_FIELD_HANDLE_ABORT; } - // Allocate memory for the raw ZIP archive data - data->zip_data = realloc(data->zip_data, data->zip_size + valuelen); - // Copy the raw ZIP archive data - memcpy(data->zip_data + data->zip_size, value, valuelen); - // Store the size of the ZIP archive raw data - data->zip_size += valuelen; - log_debug(DEBUG_API, "Received ZIP archive (%zu bytes, buffer is now %zu bytes)", - valuelen, data->zip_size); + // Allocate memory for the raw file data + data->data = realloc(data->data, data->filesize + valuelen); + // Copy the raw file data + memcpy(data->data + data->filesize, value, valuelen); + // Store the size of the file raw data + data->filesize += valuelen; + log_debug(DEBUG_API, "Received file (%zu bytes, buffer is now %zu bytes)", + valuelen, data->filesize); } else if(is_sid) { @@ -143,24 +143,27 @@ static int field_stored(const char *path, long long file_size, void *user_data) static int free_upload_data(struct upload_data *data) { // Free allocated memory - if(data->zip_filename) + if(data->filename) { - free(data->zip_filename); - data->zip_filename = NULL; + free(data->filename); + data->filename = NULL; } if(data->sid) { free(data->sid); data->sid = NULL; } - if(data->zip_data) + if(data->data) { - free(data->zip_data); - data->zip_data = NULL; + free(data->data); + data->data = NULL; } return 0; } +// Private function prototypes +static int process_received_zip(struct ftl_conn *api, struct upload_data *data); + static int api_teleporter_POST(struct ftl_conn *api) { struct upload_data data; @@ -170,7 +173,7 @@ static int api_teleporter_POST(struct ftl_conn *api) // Disallow large ZIP archives (> 50 MB) to prevent DoS attacks. // Typically, the ZIP archive size should be around 30-100 kB. - if(req_info->content_length > MAXZIPSIZE) + if(req_info->content_length > MAXFILESIZE) { free_upload_data(&data); return send_json_error(api, 400, @@ -191,7 +194,7 @@ static int api_teleporter_POST(struct ftl_conn *api) } // Check if we received something we consider being a file - if(data.zip_data == NULL || data.zip_size == 0) + if(data.data == NULL || data.filesize == 0) { free_upload_data(&data); return send_json_error(api, 400, @@ -209,28 +212,17 @@ static int api_teleporter_POST(struct ftl_conn *api) "ZIP archive too large", NULL); } -/* - // Set the payload to the SID we received (if available) - if(data.sid != NULL) - { - const size_t bufsize = strlen(data.sid) + 5; - api->payload.raw = calloc(bufsize, sizeof(char)); - strncpy(api->payload.raw, "sid=", 5); - strncat(api->payload.raw, data.sid, bufsize - 4); - } - // Check if the client is authorized to use this API endpoint - if(check_client_auth(api) == API_AUTH_UNAUTHORIZED) - { - free_upload_data(&data); - return send_json_unauthorized(api); - } -*/ // Process what we received + return process_received_zip(api, &data); +} + +static int process_received_zip(struct ftl_conn *api, struct upload_data *data) +{ char hint[ERRBUF_SIZE]; memset(hint, 0, sizeof(hint)); cJSON *json_files = JSON_NEW_ARRAY(); - const char *error = read_teleporter_zip(data.zip_data, data.zip_size, hint, json_files); + const char *error = read_teleporter_zip(data->data, data->filesize, hint, json_files); if(error != NULL) { const size_t msglen = strlen(error) + strlen(hint) + 4; @@ -242,7 +234,7 @@ static int api_teleporter_POST(struct ftl_conn *api) strcat(msg, ": "); strcat(msg, hint); } - free_upload_data(&data); + free_upload_data(data); return send_json_error_free(api, 400, "bad_request", "Invalid ZIP archive", @@ -250,7 +242,7 @@ static int api_teleporter_POST(struct ftl_conn *api) } // Free allocated memory - free_upload_data(&data); + free_upload_data(data); // Send response cJSON *json = JSON_NEW_OBJECT(); From b9fa29c18ce84b752eefa1f65d28053fa00a363b Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 19 Nov 2023 19:04:00 +0100 Subject: [PATCH 072/221] Add some rule along which we will decide the user supplied somethings that (superficially) looks like a Teleporter v6 ZIP file Signed-off-by: DL6ER --- src/api/teleporter.c | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/src/api/teleporter.c b/src/api/teleporter.c index 470829f41..856da89c6 100644 --- a/src/api/teleporter.c +++ b/src/api/teleporter.c @@ -213,8 +213,32 @@ static int api_teleporter_POST(struct ftl_conn *api) NULL); } - // Process what we received - return process_received_zip(api, &data); + // Check if we received something that claims to be a ZIP archive + // - filename + // - shoud be at least 12 characters long, + // - should start in "pi-hole_", + // - have "_teleporter_" in the middle, and + // - end in ".zip" + // - the data itself + // - should be at least 40 bytes long + // - start with 0x04034b50 (local file header signature, see https://pkware.cachefly.net/webdocs/APPNOTE/APPNOTE-6.3.9.TXT) + if(strlen(data.filename) >= 12 && + strncmp(data.filename, "pi-hole_", 8) == 0 && + strstr(data.filename, "_teleporter_") != NULL && + strcmp(data.filename + strlen(data.filename) - 4, ".zip") == 0 && + data.filesize >= 40 && + memcmp(data.data, "\x50\x4b\x03\x04", 4) == 0) + { + return process_received_zip(api, &data); + } + else + { + free_upload_data(&data); + return send_json_error(api, 400, + "bad_request", + "Invalid file", + "The uploaded file does not appear to be a valid Pi-hole Teleporter archive"); + } } static int process_received_zip(struct ftl_conn *api, struct upload_data *data) From 50a72afcef99d980dc9fbb4e39a4c82f61ba564b Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 19 Nov 2023 22:21:48 +0100 Subject: [PATCH 073/221] Add TAR routines for efficient parsing of a tar archive in memory Signed-off-by: DL6ER --- src/api/teleporter.c | 72 ++++++++++++++++++++--- src/zip/CMakeLists.txt | 2 + src/zip/gzip.c | 5 +- src/zip/gzip.h | 4 ++ src/zip/tar.c | 129 +++++++++++++++++++++++++++++++++++++++++ src/zip/tar.h | 19 ++++++ src/zip/teleporter.c | 2 +- src/zip/teleporter.h | 2 +- 8 files changed, 223 insertions(+), 12 deletions(-) create mode 100644 src/zip/tar.c create mode 100644 src/zip/tar.h diff --git a/src/api/teleporter.c b/src/api/teleporter.c index 856da89c6..b82cd535f 100644 --- a/src/api/teleporter.c +++ b/src/api/teleporter.c @@ -15,6 +15,10 @@ #include "api/api.h" // ERRBUF_SIZE #include "config/dnsmasq_config.h" +// inflate_buffer() +#include "zip/gzip.h" +// find_file_in_tar() +#include "zip/tar.h" #define MAXFILESIZE (50u*1024*1024) @@ -58,7 +62,7 @@ static int api_teleporter_GET(struct ftl_conn *api) struct upload_data { bool too_large; char *sid; - char *data; + uint8_t *data; char *filename; size_t filesize; }; @@ -163,6 +167,7 @@ static int free_upload_data(struct upload_data *data) // Private function prototypes static int process_received_zip(struct ftl_conn *api, struct upload_data *data); +static int process_received_tar_gz(struct ftl_conn *api, struct upload_data *data); static int api_teleporter_POST(struct ftl_conn *api) { @@ -231,14 +236,31 @@ static int api_teleporter_POST(struct ftl_conn *api) { return process_received_zip(api, &data); } - else + // Check if we received something that claims to be a TAR.GZ archive + // - filename + // - shoud be at least 12 characters long, + // - should start in "pi-hole-", + // - have "-teleporter_" in the middle, and + // - end in ".tar.gz" + // - the data itself + // - should be at least 40 bytes long + // - start with 0x8b1f (local file header signature, see https://www.ietf.org/rfc/rfc1952.txt) + else if(strlen(data.filename) >= 12 && + strncmp(data.filename, "pi-hole-", 8) == 0 && + strstr(data.filename, "-teleporter_") != NULL && + strcmp(data.filename + strlen(data.filename) - 7, ".tar.gz") == 0 && + data.filesize >= 40 && + memcmp(data.data, "\x1f\x8b", 2) == 0) { - free_upload_data(&data); - return send_json_error(api, 400, - "bad_request", - "Invalid file", - "The uploaded file does not appear to be a valid Pi-hole Teleporter archive"); + return process_received_tar_gz(api, &data); } + + // else: invalid file + free_upload_data(&data); + return send_json_error(api, 400, + "bad_request", + "Invalid file", + "The uploaded file does not appear to be a valid Pi-hole Teleporter archive"); } static int process_received_zip(struct ftl_conn *api, struct upload_data *data) @@ -274,6 +296,42 @@ static int process_received_zip(struct ftl_conn *api, struct upload_data *data) JSON_SEND_OBJECT(json); } +static int process_received_tar_gz(struct ftl_conn *api, struct upload_data *data) +{ + // Try to decompress the received data + uint8_t *archive = NULL; + mz_ulong archive_size = 0u; + if(!inflate_buffer(data->data, data->filesize, &archive, &archive_size)) + { + free_upload_data(data); + return send_json_error(api, 400, + "bad_request", + "Invalid GZIP archive", + "The uploaded file does not appear to be a valid gzip archive - decompression failed"); + } + + // Check if the decompressed data is a valid TAR archive + cJSON *json_files = list_files_in_tar(archive, archive_size); + + // Print all files in the TAR archive + cJSON *file = NULL; + cJSON_ArrayForEach(file, json_files) + { + cJSON *name = cJSON_GetObjectItemCaseSensitive(file, "name"); + cJSON *size = cJSON_GetObjectItemCaseSensitive(file, "size"); + log_info("Found file in TAR archive: \"%s\" (%d bytes)", + name->valuestring, size->valueint); + } + + // Free allocated memory + free_upload_data(data); + + // Send response + cJSON *json = JSON_NEW_OBJECT(); + JSON_ADD_ITEM_TO_OBJECT(json, "files", json_files); + JSON_SEND_OBJECT(json); +} + int api_teleporter(struct ftl_conn *api) { if(api->method == HTTP_GET) diff --git a/src/zip/CMakeLists.txt b/src/zip/CMakeLists.txt index 8042ed5eb..3abeb7ad7 100644 --- a/src/zip/CMakeLists.txt +++ b/src/zip/CMakeLists.txt @@ -11,6 +11,8 @@ set(sources gzip.c gzip.h + tar.c + tar.h teleporter.c teleporter.h ) diff --git a/src/zip/gzip.c b/src/zip/gzip.c index bf15ea862..f7fbc5716 100644 --- a/src/zip/gzip.c +++ b/src/zip/gzip.c @@ -14,7 +14,6 @@ #include // le32toh and friends #include -#include "miniz/miniz.h" #include "gzip.h" #include "log.h" @@ -103,8 +102,8 @@ static bool deflate_buffer(const unsigned char *buffer_uncompressed, const mz_ul return true; } -static bool inflate_buffer(unsigned char *buffer_compressed, mz_ulong size_compressed, - unsigned char **buffer_uncompressed, mz_ulong *size_uncompressed) +bool inflate_buffer(unsigned char *buffer_compressed, mz_ulong size_compressed, + unsigned char **buffer_uncompressed, mz_ulong *size_uncompressed) { // Check GZIP header (magic byte 1F 8B and compression algorithm deflate 08) if(buffer_compressed[0] != 0x1F || buffer_compressed[1] != 0x8B) diff --git a/src/zip/gzip.h b/src/zip/gzip.h index 7839602c6..b0c916b6a 100644 --- a/src/zip/gzip.h +++ b/src/zip/gzip.h @@ -11,6 +11,10 @@ #define GZIP_H #include +#include "miniz/miniz.h" + +bool inflate_buffer(unsigned char *buffer_compressed, mz_ulong size_compressed, + unsigned char **buffer_uncompressed, mz_ulong *size_uncompressed); bool deflate_file(const char *in, const char *out, bool verbose); bool inflate_file(const char *infile, const char *outfile, bool verbose); diff --git a/src/zip/tar.c b/src/zip/tar.c new file mode 100644 index 000000000..639af2319 --- /dev/null +++ b/src/zip/tar.c @@ -0,0 +1,129 @@ +/* Pi-hole: A black hole for Internet advertisements + * (c) 2023 Pi-hole, LLC (https://pi-hole.net) + * Network-wide ad blocking via your own hardware. + * + * FTL Engine + * In-memory tar reading routines + * + * This file is copyright under the latest version of the EUPL. + * Please see LICENSE file for your rights under this license. */ + +#include "zip/tar.h" +#include "log.h" + +// TAR offsets +#define TAR_NAME_OFFSET 0 +#define TAR_SIZE_OFFSET 124 +#define TAR_MAGIC_OFFSET 257 + +// TAR constants +#define TAR_BLOCK_SIZE 512 +#define TAR_NAME_SIZE 100 +#define TAR_SIZE_SIZE 12 +#define TAR_MAGIC_SIZE 5 + +static const char MAGIC_CONST[] = "ustar"; // Modern GNU tar's magic const */ + +/** + * Find a file in a TAR archive + * @param tarData Pointer to the TAR archive in memory + * @param tarSize Size of the TAR archive in memory in bytes + * @param fileName Name of the file to find + * @param fileSize Pointer to a size_t variable to store the file size in + * @return Pointer to the file data or NULL if not found + */ +const uint8_t *find_file_in_tar(const uint8_t *tarData, const size_t tarSize, + const char *fileName, size_t *fileSize) +{ + bool found = false; + size_t size, p = 0, newOffset = 0; + + // Convert to char * to be able to do pointer arithmetic more easily + const char *tar = (const char *)tarData; + + // Initialize fileSize to 0 + *fileSize = 0; + + // Loop through TAR file + do + { + // "Load" data from tar - just point to passed memory + const char *name = tar + TAR_NAME_OFFSET + p + newOffset; + const char *sz = tar + TAR_SIZE_OFFSET + p + newOffset; // size str + p += newOffset; // pointer to current file's data in TAR + + // Check for supported TAR version or end of TAR + for (size_t i = 0; i < TAR_MAGIC_SIZE; i++) + if (tar[i + TAR_MAGIC_OFFSET + p] != MAGIC_CONST[i]) + return NULL; + + // Convert file size from string into integer + size = 0; + for (ssize_t i = TAR_SIZE_SIZE - 2, mul = 1; i >= 0; mul *= 8, i--) // Octal str to int + if ((sz[i] >= '1') && (sz[i] <= '9')) + size += (sz[i] - '0') * mul; + + //Offset size in bytes. Depends on file size and TAR block size + newOffset = (1 + size / TAR_BLOCK_SIZE) * TAR_BLOCK_SIZE; //trim by block + if ((size % TAR_BLOCK_SIZE) > 0) + newOffset += TAR_BLOCK_SIZE; + + found = strncmp(name, fileName, TAR_NAME_SIZE) == 0; + } while (!found && (p + newOffset + TAR_BLOCK_SIZE <= tarSize)); + + if (!found) + return NULL; // No file found in TAR - return NULL + + // File found in TAR - return pointer to file data and set fileSize + *fileSize = size; + return tarData + p + TAR_BLOCK_SIZE; +} + +/** + * List all files in a TAR archive + * @param tarData Pointer to the TAR archive in memory + * @param tarSize Size of the TAR archive in memory in bytes + * @return Pointer to a cJSON array containing all file names with file size + */ +cJSON *list_files_in_tar(const uint8_t *tarData, const size_t tarSize) +{ + cJSON *files = cJSON_CreateArray(); + size_t size, p = 0, newOffset = 0; + + // Convert to char * to be able to do pointer arithmetic more easily + const char *tar = (const char *)tarData; + + // Loop through TAR file + do + { + // "Load" data from tar - just point to passed memory + const char *name = tar + TAR_NAME_OFFSET + p + newOffset; + const char *sz = tar + TAR_SIZE_OFFSET + p + newOffset; // size str + p += newOffset; // pointer to current file's data in TAR + + // Check for supported TAR version or end of TAR + for (size_t i = 0; i < TAR_MAGIC_SIZE; i++) + if (tar[i + TAR_MAGIC_OFFSET + p] != MAGIC_CONST[i]) + return files; + + // Convert file size from string into integer + size = 0; + for (ssize_t i = TAR_SIZE_SIZE - 2, mul = 1; i >= 0; mul *= 8, i--) // Octal str to int + if ((sz[i] >= '1') && (sz[i] <= '9')) + size += (sz[i] - '0') * mul; + + //Offset size in bytes. Depends on file size and TAR block size + newOffset = (1 + size / TAR_BLOCK_SIZE) * TAR_BLOCK_SIZE; //trim by block + if ((size % TAR_BLOCK_SIZE) > 0) + newOffset += TAR_BLOCK_SIZE; + + // Add file name to cJSON array + log_info("Found file '%s' with size %zu", name, size); + cJSON *file = cJSON_CreateObject(); + cJSON_AddItemToObject(file, "name", cJSON_CreateString(name)); + cJSON_AddItemToObject(file, "size", cJSON_CreateNumber(size)); + cJSON_AddItemToArray(files, file); + } while (p + newOffset + TAR_BLOCK_SIZE <= tarSize); + + return files; +} \ No newline at end of file diff --git a/src/zip/tar.h b/src/zip/tar.h new file mode 100644 index 000000000..426a4037a --- /dev/null +++ b/src/zip/tar.h @@ -0,0 +1,19 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2023 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* TAR reading routines +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ +#ifndef TAR_H +#define TAR_H + +#include "FTL.h" +#include "webserver/cJSON/cJSON.h" + +const uint8_t *find_file_in_tar(const uint8_t *tar, const size_t tarSize, const char *fileName, size_t *fileSize); +cJSON *list_files_in_tar(const uint8_t *tarData, const size_t tarSize); + +#endif // TAR_H \ No newline at end of file diff --git a/src/zip/teleporter.c b/src/zip/teleporter.c index 62281d897..5935b6de1 100644 --- a/src/zip/teleporter.c +++ b/src/zip/teleporter.c @@ -524,7 +524,7 @@ static const char *test_and_import_database(void *ptr, size_t size, const char * return NULL; } -const char *read_teleporter_zip(char *buffer, const size_t buflen, char * const hint, cJSON *imported_files) +const char *read_teleporter_zip(uint8_t *buffer, const size_t buflen, char * const hint, cJSON *imported_files) { // Initialize ZIP archive mz_zip_archive zip = { 0 }; diff --git a/src/zip/teleporter.h b/src/zip/teleporter.h index b20567b99..a5743028e 100644 --- a/src/zip/teleporter.h +++ b/src/zip/teleporter.h @@ -15,7 +15,7 @@ const char *generate_teleporter_zip(mz_zip_archive *zip, char filename[128], void **ptr, size_t *size); bool free_teleporter_zip(mz_zip_archive *zip); -const char *read_teleporter_zip(char *buffer, const size_t buflen, char *hint, cJSON *json_files); +const char *read_teleporter_zip(uint8_t *buffer, const size_t buflen, char *hint, cJSON *json_files); bool write_teleporter_zip_to_disk(void); bool read_teleporter_zip_from_disk(const char *filename); From abbcbdaf5add889fc894a5e2baae531106cb85a2 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 19 Nov 2023 22:34:43 +0100 Subject: [PATCH 074/221] Convert IDNs to punycode before validation Signed-off-by: DL6ER --- src/api/list.c | 47 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 10 deletions(-) diff --git a/src/api/list.c b/src/api/list.c index 780a0754f..8e1e7c111 100644 --- a/src/api/list.c +++ b/src/api/list.c @@ -20,6 +20,8 @@ // valid_domain() #include "tools/gravity-parseList.h" +#include + static int api_list_read(struct ftl_conn *api, const int code, const enum gravity_list_type listtype, @@ -403,17 +405,42 @@ static int api_list_write(struct ftl_conn *api, it->valuestring); } - // Validate domains - if((listtype == GRAVITY_DOMAINLIST_ALLOW_EXACT || - listtype == GRAVITY_DOMAINLIST_DENY_EXACT) && - !valid_domain(it->valuestring, strlen(it->valuestring), false)) + if(listtype == GRAVITY_DOMAINLIST_ALLOW_EXACT || + listtype == GRAVITY_DOMAINLIST_DENY_EXACT) { - if(allocated_json) - cJSON_free(row.items); - return send_json_error(api, 400, // 400 Bad Request - "bad_request", - "Invalid domain", - it->valuestring); + char *punycode = NULL; + const Idna_rc rc = idna_to_ascii_lz(it->valuestring, &punycode, 0); + if (rc != IDNA_SUCCESS) + { + // Invalid domain name + return send_json_error(api, 400, + "bad_request", + "Invalid request: Invalid domain name", + idna_strerror(rc)); + } + // Convert punycode domain to lowercase + for(unsigned int i = 0u; i < strlen(punycode); i++) + punycode[i] = tolower(punycode[i]); + + // Validate punycode domain + // This will reject domains like äöü{{{.com + // which convert to xn--{{{-pla4gpb.com + if(!valid_domain(punycode, strlen(punycode), false)) + { + if(allocated_json) + cJSON_free(row.items); + return send_json_error(api, 400, // 400 Bad Request + "bad_request", + "Invalid domain", + it->valuestring); + } + + // Replace domain with punycode version + if(!(it->type & cJSON_IsReference)) + free(it->valuestring); + it->valuestring = punycode; + // Remove reference flag + it->type &= ~cJSON_IsReference; } } } From 7f0a0a77da263c47e6388fa9ec8951a4a79d37a6 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 19 Nov 2023 22:54:19 +0100 Subject: [PATCH 075/221] Add "unicode" field to API response for GET /api/domain Signed-off-by: DL6ER --- src/api/docs/content/specs/domains.yaml | 12 +++++++++++- src/api/list.c | 8 ++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/api/docs/content/specs/domains.yaml b/src/api/docs/content/specs/domains.yaml index c63149c6b..773cbeba3 100644 --- a/src/api/docs/content/specs/domains.yaml +++ b/src/api/docs/content/specs/domains.yaml @@ -223,6 +223,7 @@ components: items: allOf: - $ref: 'domains.yaml#/components/schemas/domain' + - $ref: 'domains.yaml#/components/schemas/unicode' - $ref: 'domains.yaml#/components/schemas/type' - $ref: 'domains.yaml#/components/schemas/kind' - $ref: 'domains.yaml#/components/schemas/comment' @@ -249,6 +250,13 @@ components: description: Domain type: string example: testdomain.com + unicode: + type: object + properties: + unicode: + description: Unicode domain (may be different from `domain` if punycode-encoding is used) + type: string + example: "äbc.com" domain_array: type: object properties: @@ -368,6 +376,7 @@ components: value: domains: - domain: "allowed.com" + unicode: "allowed.com" type: allow kind: exact comment: null @@ -377,7 +386,8 @@ components: id: 299 date_added: 1611239095 date_modified: 1612163756 - - domain: "allowed2.comm" + - domain: "xn--4ca.com" + unicode: "ä.com" type: allow kind: regex comment: "Some text" diff --git a/src/api/list.c b/src/api/list.c index 8e1e7c111..0502f5ec8 100644 --- a/src/api/list.c +++ b/src/api/list.c @@ -74,10 +74,18 @@ static int api_list_read(struct ftl_conn *api, } else // domainlists { + char *unicode = NULL; + const Idna_rc rc = idna_to_unicode_lzlz(table.domain, &unicode, 0); JSON_COPY_STR_TO_OBJECT(row, "domain", table.domain); + if(rc == IDNA_SUCCESS) + JSON_COPY_STR_TO_OBJECT(row, "unicode", unicode); + else + JSON_COPY_STR_TO_OBJECT(row, "unicode", table.domain); JSON_REF_STR_IN_OBJECT(row, "type", table.type); JSON_REF_STR_IN_OBJECT(row, "kind", table.kind); JSON_COPY_STR_TO_OBJECT(row, "comment", table.comment); + if(unicode != NULL) + free(unicode); } // Groups don't have the groups property From 98f7ff8e626b7aa75a5abb60e835ffd2c3ccc1ed Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 19 Nov 2023 22:38:45 +0100 Subject: [PATCH 076/221] Do not compress rotated files - they are not expected to be large Signed-off-by: DL6ER --- src/config/config.c | 2 +- src/files.c | 45 --------------------------------------------- src/files.h | 1 - 3 files changed, 1 insertion(+), 47 deletions(-) diff --git a/src/config/config.c b/src/config/config.c index 9b086908a..fc0ce3cec 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -1344,7 +1344,7 @@ void readFTLconf(struct config *conf, const bool rewrite) // If we cannot parse /etc/pihole.toml (due to missing or invalid syntax), // we try to read the rotated files in /etc/pihole/config_backup starting at // the most recent one and going back in time until we find a valid config - for(unsigned int i = 0; i < PLAIN_ROTATIONS; i++) + for(unsigned int i = 0; i < MAX_ROTATIONS; i++) { if(readFTLtoml(NULL, conf, NULL, rewrite, NULL, i)) { diff --git a/src/files.c b/src/files.c index f011b5850..3c15891b3 100644 --- a/src/files.c +++ b/src/files.c @@ -26,8 +26,6 @@ #include // dirname() #include -// compression functions -#include "zip/gzip.h" // sendfile() #include #include @@ -463,14 +461,6 @@ void rotate_files(const char *path, char **first_file) if(i == 1 && first_file != NULL) *first_file = strdup(new_path); - size_t old_path_len = strlen(old_path) + 4; - char *old_path_compressed = calloc(old_path_len, sizeof(char)); - snprintf(old_path_compressed, old_path_len, "%s.gz", old_path); - - size_t new_path_len = strlen(new_path) + 4; - char *new_path_compressed = calloc(new_path_len, sizeof(char)); - snprintf(new_path_compressed, new_path_len, "%s.gz", new_path); - if(file_exists(old_path)) { // Copy file to backup directory @@ -505,46 +495,11 @@ void rotate_files(const char *path, char **first_file) // Change ownership of file to pihole user chown_pihole(new_path); - - // Compress file if we are rotating a sufficiently old file - if(i > PLAIN_ROTATIONS) - { - log_debug(DEBUG_CONFIG, "Compressing %s -> %s", - new_path, new_path_compressed); - if(deflate_file(new_path, new_path_compressed, false)) - { - // On success, we remove the uncompressed file - remove(new_path); - } - - // Change ownership of file to pihole user - chown_pihole(new_path_compressed); - } - } - else if(file_exists(old_path_compressed)) - { - // Rename file - if(rename(old_path_compressed, new_path_compressed) < 0) - { - log_warn("Rotation %s -(MOVE)> %s failed: %s", - old_path_compressed, new_path_compressed, strerror(errno)); - } - else - { - // Log success if debug is enabled - log_debug(DEBUG_CONFIG, "Rotated %s -> %s", - old_path_compressed, new_path_compressed); - } - - // Change ownership of file to pihole user - chown_pihole(new_path_compressed); } // Free memory free(old_path); free(new_path); - free(old_path_compressed); - free(new_path_compressed); } } diff --git a/src/files.h b/src/files.h index 0482cca4e..555e5ac04 100644 --- a/src/files.h +++ b/src/files.h @@ -15,7 +15,6 @@ // setmntent() #include -#define PLAIN_ROTATIONS 3 #define MAX_ROTATIONS 15 #define BACKUP_DIR "/etc/pihole/config_backups" From 68d6f4ab9416bb9c8db10c76514830c145e89014 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 19 Nov 2023 23:33:53 +0100 Subject: [PATCH 077/221] Test parse adlist.json Signed-off-by: DL6ER --- src/api/teleporter.c | 20 ++++++++++++++++++++ src/zip/tar.c | 4 ++-- src/zip/tar.h | 2 +- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/src/api/teleporter.c b/src/api/teleporter.c index b82cd535f..51183ca1b 100644 --- a/src/api/teleporter.c +++ b/src/api/teleporter.c @@ -323,6 +323,26 @@ static int process_received_tar_gz(struct ftl_conn *api, struct upload_data *dat name->valuestring, size->valueint); } + // Parse adlist.json + size_t fileSize = 0u; + const char *adlist_json = find_file_in_tar(archive, archive_size, "adlist.json", &fileSize); + if(adlist_json != NULL) + { + cJSON *adlists = cJSON_ParseWithLength(adlist_json, fileSize); + if(adlists != NULL) + { + cJSON *adlist = NULL; + cJSON_ArrayForEach(adlist, adlists) + { + cJSON *address = cJSON_GetObjectItemCaseSensitive(adlist, "address"); + cJSON *comment = cJSON_GetObjectItemCaseSensitive(adlist, "comment"); + log_info("Found adlist in TAR archive: \"%s\" (%s)", + address->valuestring, comment->valuestring); + } + cJSON_Delete(adlists); + } + } + // Free allocated memory free_upload_data(data); diff --git a/src/zip/tar.c b/src/zip/tar.c index 639af2319..ee64b9d03 100644 --- a/src/zip/tar.c +++ b/src/zip/tar.c @@ -32,7 +32,7 @@ static const char MAGIC_CONST[] = "ustar"; // Modern GNU tar's magic const */ * @param fileSize Pointer to a size_t variable to store the file size in * @return Pointer to the file data or NULL if not found */ -const uint8_t *find_file_in_tar(const uint8_t *tarData, const size_t tarSize, +const char *find_file_in_tar(const uint8_t *tarData, const size_t tarSize, const char *fileName, size_t *fileSize) { bool found = false; @@ -76,7 +76,7 @@ const uint8_t *find_file_in_tar(const uint8_t *tarData, const size_t tarSize, // File found in TAR - return pointer to file data and set fileSize *fileSize = size; - return tarData + p + TAR_BLOCK_SIZE; + return tar + p + TAR_BLOCK_SIZE; } /** diff --git a/src/zip/tar.h b/src/zip/tar.h index 426a4037a..0e23625f0 100644 --- a/src/zip/tar.h +++ b/src/zip/tar.h @@ -13,7 +13,7 @@ #include "FTL.h" #include "webserver/cJSON/cJSON.h" -const uint8_t *find_file_in_tar(const uint8_t *tar, const size_t tarSize, const char *fileName, size_t *fileSize); +const char *find_file_in_tar(const uint8_t *tar, const size_t tarSize, const char *fileName, size_t *fileSize); cJSON *list_files_in_tar(const uint8_t *tarData, const size_t tarSize); #endif // TAR_H \ No newline at end of file From 07f040ea0baf97bceddc8da01c21b49ea4e143f9 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 20 Nov 2023 08:58:48 +0100 Subject: [PATCH 078/221] Switch from libidn to libidn2 to get IDN conversion conforming to IDNA2008 + TR46 specifications (RFC 5890, RFC 5891, RFC 5892, RFC 5893, TR 46) Signed-off-by: DL6ER --- src/CMakeLists.txt | 7 +++++-- src/api/list.c | 13 ++++++------- src/api/search.c | 10 +++++----- src/dnsmasq/config.h | 2 +- 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4b02c2e92..3428cedb2 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -283,9 +283,12 @@ find_package(Threads REQUIRED) find_library(LIBHOGWEED NAMES libhogweed${CMAKE_STATIC_LIBRARY_SUFFIX} hogweed HINTS /usr/local/lib64) find_library(LIBGMP NAMES libgmp${CMAKE_STATIC_LIBRARY_SUFFIX} gmp) find_library(LIBNETTLE NAMES libnettle${CMAKE_STATIC_LIBRARY_SUFFIX} nettle HINTS /usr/local/lib64) -find_library(LIBIDN NAMES libidn${CMAKE_STATIC_LIBRARY_SUFFIX} idn) -target_link_libraries(pihole-FTL rt Threads::Threads ${LIBHOGWEED} ${LIBGMP} ${LIBNETTLE} ${LIBIDN}) +# for IDN2 we need the idn2 library which in turn depends on the unistring library +find_library(LIBIDN2 NAMES libidn2${CMAKE_STATIC_LIBRARY_SUFFIX} idn) +find_library(LIBUNISTRING NAMES libunistring${CMAKE_STATIC_LIBRARY_SUFFIX} unistring) + +target_link_libraries(pihole-FTL rt Threads::Threads ${LIBHOGWEED} ${LIBGMP} ${LIBNETTLE} ${LIBIDN2} ${LIBUNISTRING}) if(LUA_DL STREQUAL "true") find_library(LIBDL dl) diff --git a/src/api/list.c b/src/api/list.c index 0502f5ec8..42f71441b 100644 --- a/src/api/list.c +++ b/src/api/list.c @@ -19,8 +19,7 @@ #include "database/network-table.h" // valid_domain() #include "tools/gravity-parseList.h" - -#include +#include static int api_list_read(struct ftl_conn *api, const int code, @@ -75,9 +74,9 @@ static int api_list_read(struct ftl_conn *api, else // domainlists { char *unicode = NULL; - const Idna_rc rc = idna_to_unicode_lzlz(table.domain, &unicode, 0); + const int rc = idn2_to_unicode_lzlz(table.domain, &unicode, IDN2_NONTRANSITIONAL); JSON_COPY_STR_TO_OBJECT(row, "domain", table.domain); - if(rc == IDNA_SUCCESS) + if(rc == IDN2_OK) JSON_COPY_STR_TO_OBJECT(row, "unicode", unicode); else JSON_COPY_STR_TO_OBJECT(row, "unicode", table.domain); @@ -417,14 +416,14 @@ static int api_list_write(struct ftl_conn *api, listtype == GRAVITY_DOMAINLIST_DENY_EXACT) { char *punycode = NULL; - const Idna_rc rc = idna_to_ascii_lz(it->valuestring, &punycode, 0); - if (rc != IDNA_SUCCESS) + const int rc = idn2_to_ascii_lz(it->valuestring, &punycode, IDN2_NONTRANSITIONAL); + if (rc != IDN2_OK) { // Invalid domain name return send_json_error(api, 400, "bad_request", "Invalid request: Invalid domain name", - idna_strerror(rc)); + idn2_strerror(rc)); } // Convert punycode domain to lowercase for(unsigned int i = 0u; i < strlen(punycode); i++) diff --git a/src/api/search.c b/src/api/search.c index 0ed5a3bf8..511d9c97f 100644 --- a/src/api/search.c +++ b/src/api/search.c @@ -15,7 +15,7 @@ #include "database/gravity-db.h" // match_regex() #include "regex_r.h" -#include +#include #define MAX_SEARCH_RESULTS 10000u @@ -182,18 +182,18 @@ int api_search(struct ftl_conn *api) // use characters drawn from a large repertoire (Unicode), but IDNA // allows the non-ASCII characters to be represented using only the // ASCII characters already allowed in so-called host names today. - // idna_to_ascii_lz() convert domain name in the locale’s encoding to an + // idn2_to_ascii_lz() convert domain name in the locale’s encoding to an // ASCII string. The domain name may contain several labels, separated // by dots. The output buffer must be deallocated by the caller. char *punycode = NULL; - const Idna_rc rc = idna_to_ascii_lz(domain, &punycode, 0); - if (rc != IDNA_SUCCESS) + const int rc = idn2_to_ascii_lz(domain, &punycode, IDN2_NONTRANSITIONAL); + if (rc != IDN2_OK) { // Invalid domain name return send_json_error(api, 400, "bad_request", "Invalid request: Invalid domain name", - idna_strerror(rc)); + idn2_strerror(rc)); } // Convert punycode domain to lowercase diff --git a/src/dnsmasq/config.h b/src/dnsmasq/config.h index df302fbd1..d73359bed 100644 --- a/src/dnsmasq/config.h +++ b/src/dnsmasq/config.h @@ -205,7 +205,7 @@ RESOLVFILE /* Pi-hole definitions */ #define HAVE_LUASCRIPT -#define HAVE_IDN +#define HAVE_LIBIDN2 #define HAVE_DNSSEC #ifdef DNSMASQ_ALL_OPTS #define HAVE_DBUS From b85f85d73cfebf3a4c771b5b155fdcc19ad36216 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 20 Nov 2023 09:11:42 +0100 Subject: [PATCH 079/221] Add CLI IDN2 conversion interface Signed-off-by: DL6ER --- src/args.c | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/src/args.c b/src/args.c index 4696b5376..972ba6572 100644 --- a/src/args.c +++ b/src/args.c @@ -60,6 +60,8 @@ #include "tools/arp-scan.h" // run_performance_test() #include "config/password.h" +// idn2_to_ascii_lz() +#include // defined in dnsmasq.c extern void print_dnsmasq_version(const char *yellow, const char *green, const char *bold, const char *normal); @@ -427,6 +429,51 @@ void parse_args(int argc, char* argv[]) exit(run_arp_scan(scan_all, extreme_mode)); } + // IDN2 conversion mode + if(argc > 1 && strcmp(argv[1], "idn2") == 0) + { + // Enable stdout printing + cli_mode = true; + if(argc == 3) + { + char *punycode = NULL; + const int rc = idn2_to_ascii_lz(argv[2], &punycode, IDN2_NFC_INPUT | IDN2_NONTRANSITIONAL); + if (rc != IDN2_OK) + { + // Invalid domain name + printf("Invalid domain name: %s\n", argv[2]); + exit(EXIT_FAILURE); + } + + // Convert punycode domain to lowercase + for(unsigned int i = 0u; i < strlen(punycode); i++) + punycode[i] = tolower(punycode[i]); + + printf("%s\n", punycode); + exit(EXIT_SUCCESS); + + } + else if(argc == 4 && (strcmp(argv[2], "-d") == 0 || strcmp(argv[2], "--decode") == 0)) + { + char *unicode = NULL; + const int rc = idn2_to_unicode_lzlz(argv[3], &unicode, IDN2_NFC_INPUT | IDN2_NONTRANSITIONAL); + if (rc != IDN2_OK) + { + // Invalid domain name + printf("Invalid domain name: %s\n", argv[3]); + exit(EXIT_FAILURE); + } + + printf("%s\n", unicode); + exit(EXIT_SUCCESS); + } + else + { + printf("Usage: %s idn2 \n", argv[0]); + exit(EXIT_FAILURE); + } + } + // start from 1, as argv[0] is the executable name for(int i = 1; i < argc; i++) { @@ -878,6 +925,12 @@ void parse_args(int argc, char* argv[]) printf(" per line (no HOSTS lists, etc.)\n\n"); printf(" Usage: %spihole-FTL gravity checkList %sinfile%s\n\n", green, cyan, normal); + printf("%sIDN2 conversion:%s\n", yellow, normal); + printf(" Convert a given internationalized domain name (IDN) to\n"); + printf(" punycode or vice versa.\n\n"); + printf(" Encoding: %spihole-FTL idn2 %sdomain%s\n", green, cyan, normal); + printf(" Decoding: %spihole-FTL idn2 -d %spunycode%s\n\n", green, cyan, normal); + printf("%sOther:%s\n", yellow, normal); printf("\t%sdhcp-discover%s Discover DHCP servers in the local\n", green, normal); printf("\t network\n"); From 93d0c557e8033d11c81665029b357d44eaf67034 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 20 Nov 2023 09:15:25 +0100 Subject: [PATCH 080/221] Add CI test for IDN2 conversion Signed-off-by: DL6ER --- src/api/list.c | 2 +- src/api/search.c | 5 ++++- src/args.c | 4 +++- test/test_suite.bats | 25 ++++++++++++++++++------- 4 files changed, 26 insertions(+), 10 deletions(-) diff --git a/src/api/list.c b/src/api/list.c index 42f71441b..c584a334c 100644 --- a/src/api/list.c +++ b/src/api/list.c @@ -416,7 +416,7 @@ static int api_list_write(struct ftl_conn *api, listtype == GRAVITY_DOMAINLIST_DENY_EXACT) { char *punycode = NULL; - const int rc = idn2_to_ascii_lz(it->valuestring, &punycode, IDN2_NONTRANSITIONAL); + const int rc = idn2_to_ascii_lz(it->valuestring, &punycode, IDN2_NFC_INPUT | IDN2_NONTRANSITIONAL); if (rc != IDN2_OK) { // Invalid domain name diff --git a/src/api/search.c b/src/api/search.c index 511d9c97f..7358a612d 100644 --- a/src/api/search.c +++ b/src/api/search.c @@ -185,8 +185,11 @@ int api_search(struct ftl_conn *api) // idn2_to_ascii_lz() convert domain name in the locale’s encoding to an // ASCII string. The domain name may contain several labels, separated // by dots. The output buffer must be deallocated by the caller. + // Used flags: + // - IDN2_NFC_INPUT: Input is in Unicode Normalization Form C (NFC) + // - IDN2_NONTRANSITIONAL: Use Unicode TR46 non-transitional processing char *punycode = NULL; - const int rc = idn2_to_ascii_lz(domain, &punycode, IDN2_NONTRANSITIONAL); + const int rc = idn2_to_ascii_lz(domain, &punycode, IDN2_NFC_INPUT | IDN2_NONTRANSITIONAL); if (rc != IDN2_OK) { // Invalid domain name diff --git a/src/args.c b/src/args.c index 972ba6572..57550e52c 100644 --- a/src/args.c +++ b/src/args.c @@ -436,6 +436,7 @@ void parse_args(int argc, char* argv[]) cli_mode = true; if(argc == 3) { + // Convert unicode domain to punycode char *punycode = NULL; const int rc = idn2_to_ascii_lz(argv[2], &punycode, IDN2_NFC_INPUT | IDN2_NONTRANSITIONAL); if (rc != IDN2_OK) @@ -455,6 +456,7 @@ void parse_args(int argc, char* argv[]) } else if(argc == 4 && (strcmp(argv[2], "-d") == 0 || strcmp(argv[2], "--decode") == 0)) { + // Convert punycode domain to unicode char *unicode = NULL; const int rc = idn2_to_unicode_lzlz(argv[3], &unicode, IDN2_NFC_INPUT | IDN2_NONTRANSITIONAL); if (rc != IDN2_OK) @@ -469,7 +471,7 @@ void parse_args(int argc, char* argv[]) } else { - printf("Usage: %s idn2 \n", argv[0]); + printf("Usage: %s idn2 [--decode] \n", argv[0]); exit(EXIT_FAILURE); } } diff --git a/test/test_suite.bats b/test/test_suite.bats index 3ff730438..0fa48c678 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -35,11 +35,7 @@ @test "dnsmasq options as expected" { run bash -c './pihole-FTL -vv | grep "cryptohash"' printf "%s\n" "${lines[@]}" - if [[ "${CI_ARCH}" == "x86_64_full" ]]; then - [[ ${lines[0]} == "Features: IPv6 GNU-getopt DBus no-UBus no-i18n IDN DHCP DHCPv6 Lua TFTP conntrack ipset nftset auth cryptohash DNSSEC loop-detect inotify dumpfile" ]] - else - [[ ${lines[0]} == "Features: IPv6 GNU-getopt no-DBus no-UBus no-i18n IDN DHCP DHCPv6 Lua TFTP no-conntrack ipset no-nftset auth cryptohash DNSSEC loop-detect inotify dumpfile" ]] - fi + [[ ${lines[0]} == "Features: IPv6 GNU-getopt no-DBus no-UBus no-i18n IDN2 DHCP DHCPv6 Lua TFTP no-conntrack ipset no-nftset auth cryptohash DNSSEC loop-detect inotify dumpfile" ]] [[ ${lines[1]} == "" ]] } @@ -1262,7 +1258,7 @@ [[ "${lines[0]}" == "1.1.1.1" ]] } -@test "Custom DNS records: International domains are converted to IDNA form" { +@test "Custom DNS records: International domains are converted to IDN form" { # äste.com ---> xn--ste-pla.com run bash -c "dig A xn--ste-pla.com +short @127.0.0.1" printf "%s\n" "${lines[@]}" @@ -1273,7 +1269,7 @@ [[ "${lines[0]}" == "2.2.2.2" ]] } -@test "Local CNAME records: International domains are converted to IDNA form" { +@test "Local CNAME records: International domains are converted to IDN form" { # brücke.com ---> xn--brcke-lva.com run bash -c "dig A xn--brcke-lva.com +short @127.0.0.1" printf "%s\n" "${lines[@]}" @@ -1282,6 +1278,21 @@ [[ "${lines[1]}" == "2.2.2.2" ]] } +@test "IDN2 CLI interface correctly encodes/decodes domain according to IDNA2008 + TR46" { + run bash -c './pihole-FTL idn2 äste.com' + printf "%s\n" "${lines[@]}" + [[ "${lines[0]}" == "xn--ste-pla.com" ]] + run bash -c './pihole-FTL idn2 -d xn--ste-pla.com' + printf "%s\n" "${lines[@]}" + [[ "${lines[0]}" == "äste.com" ]] + run bash -c './pihole-FTL idn2 ß.de' + printf "%s\n" "${lines[@]}" + [[ "${lines[0]}" == "xn--zca.de" ]] + run bash -c './pihole-FTL idn2 -d xn--zca.de' + printf "%s\n" "${lines[@]}" + [[ "${lines[0]}" == "ß.de" ]] +} + @test "Environmental variable is favored over config file" { # The config file has -10 but we set FTLCONF_misc_nice="-11" run bash -c 'grep -B1 "nice = -11" /etc/pihole/pihole.toml' From 6c62d3e3f95eeb374dae75497b49795fd8c20158 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 20 Nov 2023 14:06:44 +0100 Subject: [PATCH 081/221] Add import_json_table() routine that can read, parse, and import .json files from Pi-hole v5.x Teleporter archives Signed-off-by: DL6ER --- src/api/list.c | 2 +- src/api/teleporter.c | 354 ++++++++++++++++++++++++++++++++++++++++--- src/zip/tar.c | 6 +- src/zip/tar.h | 4 +- 4 files changed, 336 insertions(+), 30 deletions(-) diff --git a/src/api/list.c b/src/api/list.c index 3c0b6dc05..2d9931c20 100644 --- a/src/api/list.c +++ b/src/api/list.c @@ -524,7 +524,7 @@ int api_list(struct ftl_conn *api) } else if((api->item = startsWith("/api/domains/allow", api)) != NULL) { - listtype = GRAVITY_DOMAINLIST_ALLOW_ALL; + listtype = GRAVITY_DOMAINLIST_ALLOW_ALL; } else if((api->item = startsWith("/api/domains/deny/exact", api)) != NULL) { diff --git a/src/api/teleporter.c b/src/api/teleporter.c index 51183ca1b..6cda1d529 100644 --- a/src/api/teleporter.c +++ b/src/api/teleporter.c @@ -19,6 +19,10 @@ #include "zip/gzip.h" // find_file_in_tar() #include "zip/tar.h" +// sqlite3_open_v2() +#include "database/sqlite3.h" +// dbquery() +#include "database/common.h" #define MAXFILESIZE (50u*1024*1024) @@ -296,6 +300,306 @@ static int process_received_zip(struct ftl_conn *api, struct upload_data *data) JSON_SEND_OBJECT(json); } +static struct teleporter_files { + const char *filename; // Filename of the file in the archive + const char *table_name; // Name of the table in the database + const int listtype; // Type of list (only used for domainlist table) + const size_t num_columns; // Number of columns in the table + const char *columns[10]; // List of columns in the table +} teleporter_v5_files[] = { + { + .filename = "adlist.json", + .table_name = "adlist", + .listtype = -1, + .num_columns = 10, + .columns = { "id", "address", "enabled", "date_added", "date_modified", "comment", "date_updated", "number", "invalid_domains", "status" } // abp_entries and type are not defined in Pi-hole v5.x + },{ + .filename = "adlist_by_group.json", + .table_name = "adlist_by_group", + .listtype = -1, + .num_columns = 2, + .columns = { "group_id", "adlist_id" } + },{ + .filename = "blacklist.exact.json", + .table_name = "domainlist", + .listtype = 1, // GRAVITY_DOMAINLIST_DENY_EXACT + .num_columns = 6, + .columns = { "id", "domain", "enabled", "date_added", "date_modified", "comment", "type" } + },{ + .filename = "blacklist.regex.json", + .table_name = "domainlist", + .listtype = 3, // GRAVITY_DOMAINLIST_DENY_REGEX + .num_columns = 6, + .columns = { "id", "domain", "enabled", "date_added", "date_modified", "comment", "type" } + },{ + .filename = "client.json", + .table_name = "client", + .listtype = -1, + .num_columns = 5, + .columns = { "id", "ip", "date_added", "date_modified", "comment" } + },{ + .filename = "client_by_group.json", + .table_name = "client_by_group", + .listtype = -1, + .num_columns = 2, + .columns = { "group_id", "client_id" } + },{ + .filename = "domainlist_by_group.json", + .table_name = "domainlist_by_group", + .listtype = -1, + .num_columns = 2, + .columns = { "group_id", "domainlist_id" } + },{ + .filename = "group.json", + .table_name = "group", + .listtype = -1, + .num_columns = 6, + .columns = { "id", "enabled", "name", "date_added", "date_modified", "description" } + },{ + .filename = "whitelist.exact.json", + .table_name = "domainlist", + .listtype = 0, // GRAVITY_DOMAINLIST_ALLOW_EXACT + .num_columns = 6, + .columns = { "id", "domain", "enabled", "date_added", "date_modified", "comment", "type" } + },{ + .filename = "whitelist.regex.json", + .table_name = "domainlist", + .listtype = 2, // GRAVITY_DOMAINLIST_ALLOW_REGEX + .num_columns = 6, + .columns = { "id", "domain", "enabled", "date_added", "date_modified", "comment", "type" } + } +}; + +static bool import_json_table(cJSON *json, struct teleporter_files *file) +{ + // Check if the JSON object is an array + if(!cJSON_IsArray(json)) + { + log_err("import_json_table(%s): JSON object is not an array", file->filename); + return false; + } + + // Check if the JSON array is empty, if so, we can return early + const int num_entries = cJSON_GetArraySize(json); + if(num_entries == 0) + { + log_info("import_json_table(%s): JSON array is empty", file->filename); + return true; + } + + // Check if all the JSON entries contain all the expected columns + cJSON *json_object = NULL; + cJSON_ArrayForEach(json_object, json) + { + if(!cJSON_IsObject(json_object)) + { + log_err("import_json_table(%s): JSON array does not contain objects", file->filename); + return false; + } + + // If this is a record for the domainlist table, add type/kind + if(strcmp(file->table_name, "domainlist") == 0) + { + // Add type/kind to the JSON object + cJSON_AddNumberToObject(json_object, "type", file->listtype); + } + + // Check if the JSON object contains the expected columns + for(size_t i = 0; i < file->num_columns; i++) + { + if(cJSON_GetObjectItemCaseSensitive(json_object, file->columns[i]) == NULL) + { + log_err("import_json_table(%s): JSON object does not contain column \"%s\"", file->filename, file->columns[i]); + return false; + } + } + } + + log_info("import_json_table(%s): JSON array contains %d entr%s", file->filename, num_entries, num_entries == 1 ? "y" : "ies"); + + // Open database connection + sqlite3 *db = NULL; + if(sqlite3_open_v2(config.files.gravity.v.s, &db, SQLITE_OPEN_READWRITE, NULL) != SQLITE_OK) + { + log_err("import_json_table(%s): Unable to open database file \"%s\": %s", + file->filename, config.files.database.v.s, sqlite3_errmsg(db)); + sqlite3_close(db); + return false; + } + + // Disable foreign key constraints + if(sqlite3_exec(db, "PRAGMA foreign_keys = OFF;", NULL, NULL, NULL) != SQLITE_OK) + { + log_err("import_json_table(%s): Unable to disable foreign key constraints: %s", file->filename, sqlite3_errmsg(db)); + sqlite3_close(db); + return false; + } + + // Start transaction + if(sqlite3_exec(db, "BEGIN TRANSACTION;", NULL, NULL, NULL) != SQLITE_OK) + { + log_err("import_json_table(%s): Unable to start transaction: %s", file->filename, sqlite3_errmsg(db)); + sqlite3_close(db); + return false; + } + + // Clear existing table entries + if(file->listtype < 0) + { + // Delete all entries in the table + if(dbquery(db, "DELETE FROM \"%s\";", file->table_name) != SQLITE_OK) + { + log_err("import_json_table(%s): Unable to delete entries from table \"%s\": %s", + file->filename, file->table_name, sqlite3_errmsg(db)); + sqlite3_close(db); + return false; + } + } + else + { + // Delete all entries in the table of the same type + if(dbquery(db, "DELETE FROM \"%s\" WHERE type = %d;", file->table_name, file->listtype) != SQLITE_OK) + { + log_err("import_json_table(%s): Unable to delete entries from table \"%s\": %s", + file->filename, file->table_name, sqlite3_errmsg(db)); + sqlite3_close(db); + return false; + } + } + + // Build dynamic SQL insertion statement + // "INSERT OR IGNORE INTO table (column1, column2, ...) VALUES (?, ?, ...);" + char *sql = sqlite3_mprintf("INSERT OR IGNORE INTO \"%s\" (", file->table_name); + for(size_t i = 0; i < file->num_columns; i++) + { + char *sql2 = sqlite3_mprintf("%s%s", sql, file->columns[i]); + sqlite3_free(sql); + sql = NULL; + if(i < file->num_columns - 1) + { + sql = sqlite3_mprintf("%s, ", sql2); + sqlite3_free(sql2); + sql2 = NULL; + } + else + { + sql = sqlite3_mprintf("%s) VALUES (", sql2); + sqlite3_free(sql2); + sql2 = NULL; + } + } + for(size_t i = 0; i < file->num_columns; i++) + { + char *sql2 = sqlite3_mprintf("%s?", sql); + sqlite3_free(sql); + sql = NULL; + if(i < file->num_columns - 1) + { + sql = sqlite3_mprintf("%s, ", sql2); + sqlite3_free(sql2); + sql2 = NULL; + } + else + { + sql = sqlite3_mprintf("%s);", sql2); + sqlite3_free(sql2); + sql2 = NULL; + } + } + + // Prepare SQL statement + sqlite3_stmt *stmt = NULL; + if(sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) != SQLITE_OK) + { + log_err("Unable to prepare SQL statement: %s", sqlite3_errmsg(db)); + sqlite3_free(sql); + sqlite3_close(db); + return false; + } + + // Free allocated memory + sqlite3_free(sql); + sql = NULL; + + // Iterate over all JSON objects + cJSON_ArrayForEach(json_object, json) + { + // Bind values to SQL statement + for(size_t i = 0; i < file->num_columns; i++) + { + cJSON *json_value = cJSON_GetObjectItemCaseSensitive(json_object, file->columns[i]); + if(cJSON_IsString(json_value)) + { + // Bind string value + if(sqlite3_bind_text(stmt, i + 1, json_value->valuestring, -1, SQLITE_STATIC) != SQLITE_OK) + { + log_err("Unable to bind text value to SQL statement: %s", sqlite3_errmsg(db)); + sqlite3_finalize(stmt); + sqlite3_close(db); + return false; + } + } + else if(cJSON_IsNumber(json_value)) + { + // Bind integer value + if(sqlite3_bind_int(stmt, i + 1, json_value->valueint) != SQLITE_OK) + { + log_err("Unable to bind integer value to SQL statement: %s", sqlite3_errmsg(db)); + sqlite3_finalize(stmt); + sqlite3_close(db); + return false; + } + } + else + { + log_err("Unable to bind value to SQL statement: %s", sqlite3_errmsg(db)); + sqlite3_finalize(stmt); + sqlite3_close(db); + return false; + } + } + + // Execute SQL statement + if(sqlite3_step(stmt) != SQLITE_DONE) + { + log_err("Unable to execute SQL statement: %s", sqlite3_errmsg(db)); + sqlite3_finalize(stmt); + sqlite3_close(db); + return false; + } + + // Reset SQL statement + if(sqlite3_reset(stmt) != SQLITE_OK) + { + log_err("Unable to reset SQL statement: %s", sqlite3_errmsg(db)); + sqlite3_finalize(stmt); + sqlite3_close(db); + return false; + } + } + + // Finalize SQL statement + if(sqlite3_finalize(stmt) != SQLITE_OK) + { + log_err("Unable to finalize SQL statement: %s", sqlite3_errmsg(db)); + sqlite3_close(db); + return false; + } + + // Commit transaction + if(sqlite3_exec(db, "COMMIT;", NULL, NULL, NULL) != SQLITE_OK) + { + log_err("Unable to commit transaction: %s", sqlite3_errmsg(db)); + sqlite3_close(db); + return false; + } + + // Close database connection + sqlite3_close(db); + + return true; +} + static int process_received_tar_gz(struct ftl_conn *api, struct upload_data *data) { // Try to decompress the received data @@ -313,36 +617,38 @@ static int process_received_tar_gz(struct ftl_conn *api, struct upload_data *dat // Check if the decompressed data is a valid TAR archive cJSON *json_files = list_files_in_tar(archive, archive_size); - // Print all files in the TAR archive - cJSON *file = NULL; - cJSON_ArrayForEach(file, json_files) + // Print all files in the TAR archive if in debug mode + if(config.debug.api.v.b) { - cJSON *name = cJSON_GetObjectItemCaseSensitive(file, "name"); - cJSON *size = cJSON_GetObjectItemCaseSensitive(file, "size"); - log_info("Found file in TAR archive: \"%s\" (%d bytes)", - name->valuestring, size->valueint); + cJSON *file = NULL; + cJSON_ArrayForEach(file, json_files) + { + const cJSON *name = cJSON_GetObjectItemCaseSensitive(file, "name"); + const cJSON *size = cJSON_GetObjectItemCaseSensitive(file, "size"); + if(name == NULL || size == NULL) + continue; + + log_debug(DEBUG_API, "Found file in TAR archive: \"%s\" (%d bytes)", + name->valuestring, size->valueint); + } } - // Parse adlist.json - size_t fileSize = 0u; - const char *adlist_json = find_file_in_tar(archive, archive_size, "adlist.json", &fileSize); - if(adlist_json != NULL) + // Parse JSON files in the TAR archive + for(size_t i = 0; i < sizeof(teleporter_v5_files) / sizeof(struct teleporter_files); i++) { - cJSON *adlists = cJSON_ParseWithLength(adlist_json, fileSize); - if(adlists != NULL) - { - cJSON *adlist = NULL; - cJSON_ArrayForEach(adlist, adlists) - { - cJSON *address = cJSON_GetObjectItemCaseSensitive(adlist, "address"); - cJSON *comment = cJSON_GetObjectItemCaseSensitive(adlist, "comment"); - log_info("Found adlist in TAR archive: \"%s\" (%s)", - address->valuestring, comment->valuestring); - } - cJSON_Delete(adlists); - } + size_t fileSize = 0u; + cJSON *json = NULL; + const char *file = find_file_in_tar(archive, archive_size, teleporter_v5_files[i].filename, &fileSize); + if(file != NULL && fileSize > 0u && (json = cJSON_ParseWithLength(file, fileSize)) != NULL) + import_json_table(json, &teleporter_v5_files[i]); } + // Further files to process if present: + // custom.list + // dhcp.leases + // pihole-FTL.conf + // setupVars.conf + // Free allocated memory free_upload_data(data); diff --git a/src/zip/tar.c b/src/zip/tar.c index ee64b9d03..be9a9e515 100644 --- a/src/zip/tar.c +++ b/src/zip/tar.c @@ -32,8 +32,8 @@ static const char MAGIC_CONST[] = "ustar"; // Modern GNU tar's magic const */ * @param fileSize Pointer to a size_t variable to store the file size in * @return Pointer to the file data or NULL if not found */ -const char *find_file_in_tar(const uint8_t *tarData, const size_t tarSize, - const char *fileName, size_t *fileSize) +const char * __attribute__((nonnull (1,3,4))) find_file_in_tar(const uint8_t *tarData, const size_t tarSize, + const char *fileName, size_t *fileSize) { bool found = false; size_t size, p = 0, newOffset = 0; @@ -85,7 +85,7 @@ const char *find_file_in_tar(const uint8_t *tarData, const size_t tarSize, * @param tarSize Size of the TAR archive in memory in bytes * @return Pointer to a cJSON array containing all file names with file size */ -cJSON *list_files_in_tar(const uint8_t *tarData, const size_t tarSize) +cJSON * __attribute__((nonnull (1))) list_files_in_tar(const uint8_t *tarData, const size_t tarSize) { cJSON *files = cJSON_CreateArray(); size_t size, p = 0, newOffset = 0; diff --git a/src/zip/tar.h b/src/zip/tar.h index 0e23625f0..11f5e200e 100644 --- a/src/zip/tar.h +++ b/src/zip/tar.h @@ -13,7 +13,7 @@ #include "FTL.h" #include "webserver/cJSON/cJSON.h" -const char *find_file_in_tar(const uint8_t *tar, const size_t tarSize, const char *fileName, size_t *fileSize); -cJSON *list_files_in_tar(const uint8_t *tarData, const size_t tarSize); +const char *find_file_in_tar(const uint8_t *tar, const size_t tarSize, const char *fileName, size_t *fileSize) __attribute__((nonnull (1,3,4))); +cJSON *list_files_in_tar(const uint8_t *tarData, const size_t tarSize) __attribute__((nonnull (1))); #endif // TAR_H \ No newline at end of file From 40e2e97259f54c015f5434fe8d7c5d244c990233 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 20 Nov 2023 15:31:22 +0100 Subject: [PATCH 082/221] Install also remaining files Signed-off-by: DL6ER --- src/api/teleporter.c | 64 ++++++++++++++++++++++++++++++++------ src/config/config.h | 3 ++ src/config/legacy_reader.c | 2 +- 3 files changed, 59 insertions(+), 10 deletions(-) diff --git a/src/api/teleporter.c b/src/api/teleporter.c index 6cda1d529..4d7c8e869 100644 --- a/src/api/teleporter.c +++ b/src/api/teleporter.c @@ -224,7 +224,7 @@ static int api_teleporter_POST(struct ftl_conn *api) // Check if we received something that claims to be a ZIP archive // - filename - // - shoud be at least 12 characters long, + // - should be at least 12 characters long, // - should start in "pi-hole_", // - have "_teleporter_" in the middle, and // - end in ".zip" @@ -242,7 +242,7 @@ static int api_teleporter_POST(struct ftl_conn *api) } // Check if we received something that claims to be a TAR.GZ archive // - filename - // - shoud be at least 12 characters long, + // - should be at least 12 characters long, // - should start in "pi-hole-", // - have "-teleporter_" in the middle, and // - end in ".tar.gz" @@ -634,27 +634,73 @@ static int process_received_tar_gz(struct ftl_conn *api, struct upload_data *dat } // Parse JSON files in the TAR archive + cJSON *imported_files = JSON_NEW_ARRAY(); for(size_t i = 0; i < sizeof(teleporter_v5_files) / sizeof(struct teleporter_files); i++) { size_t fileSize = 0u; cJSON *json = NULL; const char *file = find_file_in_tar(archive, archive_size, teleporter_v5_files[i].filename, &fileSize); if(file != NULL && fileSize > 0u && (json = cJSON_ParseWithLength(file, fileSize)) != NULL) - import_json_table(json, &teleporter_v5_files[i]); + if(import_json_table(json, &teleporter_v5_files[i])) + JSON_COPY_STR_TO_ARRAY(imported_files, teleporter_v5_files[i].filename); } - // Further files to process if present: - // custom.list - // dhcp.leases - // pihole-FTL.conf - // setupVars.conf + // Temporarily write further files to to disk so we can import them on restart + struct { + const char *archive_name; + const char *destination; + } extract_files[] = { + { + .archive_name = "custom.list", + .destination = DNSMASQ_CUSTOM_LIST_LEGACY + },{ + .archive_name = "dhcp.leases", + .destination = DHCPLEASESFILE + },{ + .archive_name = "pihole-FTL.conf", + .destination = GLOBALCONFFILE_LEGACY + },{ + .archive_name = "setupVars.conf", + .destination = config.files.setupVars.v.s + } + }; + for(size_t i = 0; i < sizeof(extract_files) / sizeof(*extract_files); i++) + { + size_t fileSize = 0u; + const char *file = find_file_in_tar(archive, archive_size, extract_files[i].archive_name, &fileSize); + if(file != NULL && fileSize > 0u) + { + // Write file to disk + FILE *fp = fopen(extract_files[i].destination, "wb"); + if(fp == NULL) + { + log_err("Unable to open file \"%s\" for writing: %s", extract_files[i].destination, strerror(errno)); + continue; + } + if(fwrite(file, fileSize, 1, fp) != 1) + { + log_err("Unable to write file \"%s\": %s", extract_files[i].destination, strerror(errno)); + fclose(fp); + continue; + } + fclose(fp); + JSON_COPY_STR_TO_ARRAY(imported_files, extract_files[i].destination); + } + } + + // Remove pihole.toml to prevent it from being imported on restart + if(remove(GLOBALTOMLPATH) != 0) + log_err("Unable to remove file \"%s\": %s", GLOBALTOMLPATH, strerror(errno)); // Free allocated memory free_upload_data(data); + // Signal FTL we want to restart for re-import + api->ftl.restart = true; + // Send response cJSON *json = JSON_NEW_OBJECT(); - JSON_ADD_ITEM_TO_OBJECT(json, "files", json_files); + JSON_ADD_ITEM_TO_OBJECT(json, "files", imported_files); JSON_SEND_OBJECT(json); } diff --git a/src/config/config.h b/src/config/config.h index dc4937396..18fa6e14d 100644 --- a/src/config/config.h +++ b/src/config/config.h @@ -38,6 +38,9 @@ // characters will be replaced by their UTF-8 escape sequences (UCS-2) #define TOML_UTF8 +// Location of the legacy (pre-v6.0) config file +#define GLOBALCONFFILE_LEGACY "/etc/pihole/pihole-FTL.conf" + union conf_value { bool b; // boolean value int i; // integer value diff --git a/src/config/legacy_reader.c b/src/config/legacy_reader.c index bf4600f84..ce09f3df0 100644 --- a/src/config/legacy_reader.c +++ b/src/config/legacy_reader.c @@ -43,7 +43,7 @@ static FILE * __attribute__((nonnull(1), malloc, warn_unused_result)) openFTLcon return fp; // Local file not present, try system file - *path = "/etc/pihole/pihole-FTL.conf"; + *path = GLOBALCONFFILE_LEGACY; fp = fopen(*path, "r"); return fp; From d901dae526659b9349157d4c38cd697b9ef993ca Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 20 Nov 2023 15:38:45 +0100 Subject: [PATCH 083/221] Use new ftl-build container v2.4.1 Signed-off-by: DL6ER --- .devcontainer/devcontainer.json | 2 +- .github/Dockerfile | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 2710e65a0..6c42c2d0d 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,6 +1,6 @@ { "name": "FTL x86_64 Build Env", - "image": "ghcr.io/pi-hole/ftl-build:v2.3-alpine", + "image": "ghcr.io/pi-hole/ftl-build:v2.4.1", "runArgs": [ "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined" ], "extensions": [ "jetmartin.bats", diff --git a/.github/Dockerfile b/.github/Dockerfile index d857ec408..c0f6e7b6d 100644 --- a/.github/Dockerfile +++ b/.github/Dockerfile @@ -1,5 +1,4 @@ -ARG BUILDER="alpine" -FROM ghcr.io/pi-hole/ftl-build:v2.3-${BUILDER} AS builder +FROM ghcr.io/pi-hole/ftl-build:v2.4.1 AS builder WORKDIR /app From 41f01ae4c9fabdb86ab8008d3988ca81bd2d4449 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 20 Nov 2023 15:44:40 +0100 Subject: [PATCH 084/221] Relax filename constraints in archive type detection Signed-off-by: DL6ER --- src/api/teleporter.c | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/src/api/teleporter.c b/src/api/teleporter.c index 4d7c8e869..f0916340a 100644 --- a/src/api/teleporter.c +++ b/src/api/teleporter.c @@ -223,17 +223,11 @@ static int api_teleporter_POST(struct ftl_conn *api) } // Check if we received something that claims to be a ZIP archive - // - filename - // - should be at least 12 characters long, - // - should start in "pi-hole_", - // - have "_teleporter_" in the middle, and - // - end in ".zip" + // - filename should end in ".zip" // - the data itself // - should be at least 40 bytes long // - start with 0x04034b50 (local file header signature, see https://pkware.cachefly.net/webdocs/APPNOTE/APPNOTE-6.3.9.TXT) - if(strlen(data.filename) >= 12 && - strncmp(data.filename, "pi-hole_", 8) == 0 && - strstr(data.filename, "_teleporter_") != NULL && + if(strlen(data.filename) > 4 && strcmp(data.filename + strlen(data.filename) - 4, ".zip") == 0 && data.filesize >= 40 && memcmp(data.data, "\x50\x4b\x03\x04", 4) == 0) @@ -241,17 +235,11 @@ static int api_teleporter_POST(struct ftl_conn *api) return process_received_zip(api, &data); } // Check if we received something that claims to be a TAR.GZ archive - // - filename - // - should be at least 12 characters long, - // - should start in "pi-hole-", - // - have "-teleporter_" in the middle, and - // - end in ".tar.gz" + // - filename should end in ".tar.gz" // - the data itself // - should be at least 40 bytes long // - start with 0x8b1f (local file header signature, see https://www.ietf.org/rfc/rfc1952.txt) - else if(strlen(data.filename) >= 12 && - strncmp(data.filename, "pi-hole-", 8) == 0 && - strstr(data.filename, "-teleporter_") != NULL && + else if(strlen(data.filename) > 7 && strcmp(data.filename + strlen(data.filename) - 7, ".tar.gz") == 0 && data.filesize >= 40 && memcmp(data.data, "\x1f\x8b", 2) == 0) From f30c1e89a8fa2dab6efcc4bf618cf0b1bf839b79 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 20 Nov 2023 15:56:13 +0100 Subject: [PATCH 085/221] Also remove all rotated files in light of the upcoming https://github.com/pi-hole/FTL/pull/1738 Signed-off-by: DL6ER --- src/api/teleporter.c | 18 ++++++++++++++++++ src/files.c | 2 -- src/files.h | 1 + 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/api/teleporter.c b/src/api/teleporter.c index f0916340a..6279c5c3f 100644 --- a/src/api/teleporter.c +++ b/src/api/teleporter.c @@ -23,6 +23,8 @@ #include "database/sqlite3.h" // dbquery() #include "database/common.h" +// MAX_ROTATIONS +#include "files.h" #define MAXFILESIZE (50u*1024*1024) @@ -680,6 +682,22 @@ static int process_received_tar_gz(struct ftl_conn *api, struct upload_data *dat if(remove(GLOBALTOMLPATH) != 0) log_err("Unable to remove file \"%s\": %s", GLOBALTOMLPATH, strerror(errno)); + // Remove all rotated pihole.toml files to avoid automatic config + // restore on restart + for(unsigned int i = MAX_ROTATIONS; i > 0; i--) + { + const char *fname = GLOBALTOMLPATH; + const char *filename = basename(fname); + // extra 6 bytes is enough space for up to 999 rotations ("/", ".", "\0", "999") + const size_t buflen = strlen(filename) + strlen(BACKUP_DIR) + 6; + char *path = calloc(buflen, sizeof(char)); + snprintf(path, buflen, BACKUP_DIR"/%s.%u", filename, i); + + // Remove file (if it exists) + if(remove(path) != 0 && errno != ENOENT) + log_err("Unable to remove file \"%s\": %s", path, strerror(errno)); + } + // Free allocated memory free_upload_data(data); diff --git a/src/files.c b/src/files.c index 525f57e3a..bce995c32 100644 --- a/src/files.c +++ b/src/files.c @@ -32,8 +32,6 @@ #include #include -#define BACKUP_DIR "/etc/pihole/config_backups" - // chmod_file() changes the file mode bits of a given file (relative // to the directory file descriptor) according to mode. mode is an // octal number representing the bit pattern for the new mode bits diff --git a/src/files.h b/src/files.h index 329ce876e..2e99b6ee8 100644 --- a/src/files.h +++ b/src/files.h @@ -17,6 +17,7 @@ #define ZIP_ROTATIONS 3 #define MAX_ROTATIONS 15 +#define BACKUP_DIR "/etc/pihole/config_backups" bool chmod_file(const char *filename, const mode_t mode); bool file_exists(const char *filename); From 65aef156cd06575f76ed4516a0b0c8ab6339f11b Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 20 Nov 2023 19:38:59 +0100 Subject: [PATCH 086/221] Add parsing support for NULL-values Signed-off-by: DL6ER --- .devcontainer/devcontainer.json | 19 +++++++++++------- src/api/teleporter.c | 35 +++++++++++++++++++++------------ src/zip/tar.c | 1 - 3 files changed, 34 insertions(+), 21 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 2710e65a0..8c2469a67 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -2,14 +2,19 @@ "name": "FTL x86_64 Build Env", "image": "ghcr.io/pi-hole/ftl-build:v2.3-alpine", "runArgs": [ "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined" ], - "extensions": [ - "jetmartin.bats", - "ms-vscode.cpptools", - "ms-vscode.cmake-tools", - "eamodio.gitlens" - ], + "customizations": { + "vscode": { + "extensions": [ + "jetmartin.bats", + "ms-vscode.cpptools", + "ms-vscode.cmake-tools", + "eamodio.gitlens" + ] + } + }, "mounts": [ - "type=bind,source=/home/${localEnv:USER}/.ssh,target=/root/.ssh,readonly" + "type=bind,source=/home/${localEnv:USER}/.ssh,target=/root/.ssh,readonly", + "type=bind,source=/var/www/html,target=/var/www/html,readonly" ] } diff --git a/src/api/teleporter.c b/src/api/teleporter.c index 6279c5c3f..f726318b2 100644 --- a/src/api/teleporter.c +++ b/src/api/teleporter.c @@ -313,13 +313,13 @@ static struct teleporter_files { .filename = "blacklist.exact.json", .table_name = "domainlist", .listtype = 1, // GRAVITY_DOMAINLIST_DENY_EXACT - .num_columns = 6, + .num_columns = 7, .columns = { "id", "domain", "enabled", "date_added", "date_modified", "comment", "type" } },{ .filename = "blacklist.regex.json", .table_name = "domainlist", .listtype = 3, // GRAVITY_DOMAINLIST_DENY_REGEX - .num_columns = 6, + .num_columns = 7, .columns = { "id", "domain", "enabled", "date_added", "date_modified", "comment", "type" } },{ .filename = "client.json", @@ -349,13 +349,13 @@ static struct teleporter_files { .filename = "whitelist.exact.json", .table_name = "domainlist", .listtype = 0, // GRAVITY_DOMAINLIST_ALLOW_EXACT - .num_columns = 6, + .num_columns = 7, .columns = { "id", "domain", "enabled", "date_added", "date_modified", "comment", "type" } },{ .filename = "whitelist.regex.json", .table_name = "domainlist", .listtype = 2, // GRAVITY_DOMAINLIST_ALLOW_REGEX - .num_columns = 6, + .num_columns = 7, .columns = { "id", "domain", "enabled", "date_added", "date_modified", "comment", "type" } } }; @@ -371,11 +371,6 @@ static bool import_json_table(cJSON *json, struct teleporter_files *file) // Check if the JSON array is empty, if so, we can return early const int num_entries = cJSON_GetArraySize(json); - if(num_entries == 0) - { - log_info("import_json_table(%s): JSON array is empty", file->filename); - return true; - } // Check if all the JSON entries contain all the expected columns cJSON *json_object = NULL; @@ -437,6 +432,7 @@ static bool import_json_table(cJSON *json, struct teleporter_files *file) if(file->listtype < 0) { // Delete all entries in the table + log_debug(DEBUG_API, "import_json_table(%s): Deleting all entries from table \"%s\"", file->filename, file->table_name); if(dbquery(db, "DELETE FROM \"%s\";", file->table_name) != SQLITE_OK) { log_err("import_json_table(%s): Unable to delete entries from table \"%s\": %s", @@ -448,6 +444,7 @@ static bool import_json_table(cJSON *json, struct teleporter_files *file) else { // Delete all entries in the table of the same type + log_debug(DEBUG_API, "import_json_table(%s): Deleting all entries from table \"%s\" of type %d", file->filename, file->table_name, file->listtype); if(dbquery(db, "DELETE FROM \"%s\" WHERE type = %d;", file->table_name, file->listtype) != SQLITE_OK) { log_err("import_json_table(%s): Unable to delete entries from table \"%s\": %s", @@ -540,9 +537,20 @@ static bool import_json_table(cJSON *json, struct teleporter_files *file) return false; } } + else if(cJSON_IsNull(json_value)) + { + // Bind NULL value + if(sqlite3_bind_null(stmt, i + 1) != SQLITE_OK) + { + log_err("Unable to bind NULL value to SQL statement: %s", sqlite3_errmsg(db)); + sqlite3_finalize(stmt); + sqlite3_close(db); + return false; + } + } else { - log_err("Unable to bind value to SQL statement: %s", sqlite3_errmsg(db)); + log_err("Unable to bind value to SQL statement: type = %X", (unsigned int)json_value->type & 0xFF); sqlite3_finalize(stmt); sqlite3_close(db); return false; @@ -604,12 +612,11 @@ static int process_received_tar_gz(struct ftl_conn *api, struct upload_data *dat "The uploaded file does not appear to be a valid gzip archive - decompression failed"); } - // Check if the decompressed data is a valid TAR archive - cJSON *json_files = list_files_in_tar(archive, archive_size); - // Print all files in the TAR archive if in debug mode if(config.debug.api.v.b) { + cJSON *json_files = list_files_in_tar(archive, archive_size); + cJSON *file = NULL; cJSON_ArrayForEach(file, json_files) { @@ -661,6 +668,8 @@ static int process_received_tar_gz(struct ftl_conn *api, struct upload_data *dat if(file != NULL && fileSize > 0u) { // Write file to disk + log_debug(DEBUG_API, "Writing file \"%s\" (%zu bytes) to \"%s\"", + extract_files[i].archive_name, fileSize, extract_files[i].destination); FILE *fp = fopen(extract_files[i].destination, "wb"); if(fp == NULL) { diff --git a/src/zip/tar.c b/src/zip/tar.c index be9a9e515..5e497622d 100644 --- a/src/zip/tar.c +++ b/src/zip/tar.c @@ -118,7 +118,6 @@ cJSON * __attribute__((nonnull (1))) list_files_in_tar(const uint8_t *tarData, c newOffset += TAR_BLOCK_SIZE; // Add file name to cJSON array - log_info("Found file '%s' with size %zu", name, size); cJSON *file = cJSON_CreateObject(); cJSON_AddItemToObject(file, "name", cJSON_CreateString(name)); cJSON_AddItemToObject(file, "size", cJSON_CreateNumber(size)); From d6f30b393185a98102ffbdf8859d3da486f83168 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 20 Nov 2023 22:26:44 +0100 Subject: [PATCH 087/221] Install safe-guards for string-related functions to not crash when accidentially supplied with a NULL pointer Signed-off-by: DL6ER --- src/FTL.h | 16 +++ src/database/gravity-db.c | 4 +- src/syscalls/CMakeLists.txt | 1 + src/syscalls/string.c | 251 ++++++++++++++++++++++++++++++++++++ src/syscalls/syscalls.h | 18 +++ 5 files changed, 288 insertions(+), 2 deletions(-) create mode 100644 src/syscalls/string.c diff --git a/src/FTL.h b/src/FTL.h index 2b11b32e9..03fe34b0b 100644 --- a/src/FTL.h +++ b/src/FTL.h @@ -161,6 +161,22 @@ #define pthread_mutex_lock(mutex) FTLpthread_mutex_lock(mutex, __FILE__, __FUNCTION__, __LINE__) #define fopen(pathname, mode) FTLfopen(pathname, mode, __FILE__, __FUNCTION__, __LINE__) #define ftlallocate(fd, offset, len) FTLfallocate(fd, offset, len, __FILE__, __FUNCTION__, __LINE__) +#define strlen(str) FTLstrlen(str, __FILE__, __FUNCTION__, __LINE__) +#define strnlen(str, maxlen) FTLstrnlen(str, maxlen, __FILE__, __FUNCTION__, __LINE__) +#define strcpy(dest, src) FTLstrcpy(dest, src, __FILE__, __FUNCTION__, __LINE__) +#define strncpy(dest, src, n) FTLstrncpy(dest, src, n, __FILE__, __FUNCTION__, __LINE__) +#define memset(s, c, n) FTLmemset(s, c, n, __FILE__, __FUNCTION__, __LINE__) +#define memcpy(dest, src, n) FTLmemcpy(dest, src, n, __FILE__, __FUNCTION__, __LINE__) +#define memmove(dest, src, n) FTLmemmove(dest, src, n, __FILE__, __FUNCTION__, __LINE__) +#define strstr(haystack, needle) FTLstrstr(haystack, needle, __FILE__, __FUNCTION__, __LINE__) +#define strcmp(s1, s2) FTLstrcmp(s1, s2, __FILE__, __FUNCTION__, __LINE__) +#define strncmp(s1, s2, n) FTLstrncmp(s1, s2, n, __FILE__, __FUNCTION__, __LINE__) +#define strcasecmp(s1, s2) FTLstrcasecmp(s1, s2, __FILE__, __FUNCTION__, __LINE__) +#define strncasecmp(s1, s2, n) FTLstrncasecmp(s1, s2, n, __FILE__, __FUNCTION__, __LINE__) +#define strcat(dest, src) FTLstrcat(dest, src, __FILE__, __FUNCTION__, __LINE__) +#define strncat(dest, src, n) FTLstrncat(dest, src, n, __FILE__, __FUNCTION__, __LINE__) +#define memcmp(s1, s2, n) FTLmemcmp(s1, s2, n, __FILE__, __FUNCTION__, __LINE__) +#define memmem(haystack, haystacklen, needle, needlelen) FTLmemmem(haystack, haystacklen, needle, needlelen, __FILE__, __FUNCTION__, __LINE__) #include "syscalls/syscalls.h" // Preprocessor help functions diff --git a/src/database/gravity-db.c b/src/database/gravity-db.c index 7e66df2e6..b2dec109d 100644 --- a/src/database/gravity-db.c +++ b/src/database/gravity-db.c @@ -259,8 +259,8 @@ static char* get_client_querystr(const char *table, const char *column, const ch } // Determine whether to show IP or hardware address -static inline const char *show_client_string(const char *hwaddr, const char *hostname, - const char *ip) +static const char *show_client_string(const char *hwaddr, const char *hostname, + const char *ip) { if(hostname != NULL && strlen(hostname) > 0) { diff --git a/src/syscalls/CMakeLists.txt b/src/syscalls/CMakeLists.txt index 8582ac514..101030943 100644 --- a/src/syscalls/CMakeLists.txt +++ b/src/syscalls/CMakeLists.txt @@ -25,6 +25,7 @@ set(sources snprintf.c sprintf.c strdup.c + string.c syscalls.h vasprintf.c vfprintf.c diff --git a/src/syscalls/string.c b/src/syscalls/string.c new file mode 100644 index 000000000..88254e358 --- /dev/null +++ b/src/syscalls/string.c @@ -0,0 +1,251 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2023 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Pi-hole syscall implementation for string-related functions +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#include "../FTL.h" +//#include "syscalls.h" is implicitly done in FTL.h +#include "../log.h" + +#undef strlen +size_t FTLstrlen(const char *s, const char *file, const char *func, const int line) +{ + // The strlen() function calculates the length of the string s, not + // including the terminating '\0' character. + if(s == NULL) + { + log_err("Trying to get the length of a NULL string in %s() (%s:%i)", func, file, line); + return 0; + } + return strlen(s); +} + +#undef strnlen +size_t FTLstrnlen(const char *s, const size_t maxlen, const char *file, const char *func, const int line) +{ + // The strnlen() function returns the number of characters in the string s, + // not including the terminating '\0' character, but at most maxlen. In + // doing this, strnlen() looks only at the first maxlen characters at s and + // never beyond s+maxlen. + if(s == NULL) + { + log_err("Trying to get the length of a NULL string in %s() (%s:%i)", func, file, line); + return 0; + } + return strnlen(s, maxlen); +} + +#undef strstr +char *FTLstrstr(const char *haystack, const char *needle, const char *file, const char *func, const int line) +{ + // The strstr() function finds the first occurrence of the substring needle + // in the string haystack. The terminating '\0' characters are not + // compared. + if(haystack == NULL || needle == NULL) + { + log_err("Trying to find a NULL (%s%s) string in %s() (%s:%i)", + haystack == NULL ? "L" : "", needle == NULL ? "R" : "", func, file, line); + return NULL; + } + return strstr(haystack, needle); +} + +#undef strcmp +int FTLstrcmp(const char *s1, const char *s2, const char *file, const char *func, const int line) +{ + // The strcmp() function compares the two strings s1 and s2. It returns an + // integer less than, equal to, or greater than zero if s1 is found, + // respectively, to be less than, to match, or be greater than s2. + if(s1 == NULL || s2 == NULL) + { + log_err("Trying to compare a NULL (%s%s) string in %s() (%s:%i)", + s1 == NULL ? "L" : "", s2 == NULL ? "R" : "", func, file, line); + return -1; + } + return strcmp(s1, s2); +} + +#undef strncmp +int FTLstrncmp(const char *s1, const char *s2, const size_t n, const char *file, const char *func, const int line) +{ + // The strncmp() function is similar, except it compares only the first (at + // most) n bytes of s1 and s2. + if(s1 == NULL || s2 == NULL) + { + log_err("Trying to compare a NULL (%s%s) string in %s() (%s:%i)", + s1 == NULL ? "L" : "", s2 == NULL ? "R" : "", func, file, line); + return -1; + } + return strncmp(s1, s2, n); +} + +#undef strcasecmp +int FTLstrcasecmp(const char *s1, const char *s2, const char *file, const char *func, const int line) +{ + // The strcasecmp() function performs a byte-by-byte comparison of the + // strings s1 and s2, ignoring the case of the characters. It returns an + // integer less than, equal to, or greater than zero if s1 is found, + // respectively, to be less than, to match, or be greater than s2. + if(s1 == NULL || s2 == NULL) + { + log_err("Trying to compare a NULL (%s%s) string in %s() (%s:%i)", + s1 == NULL ? "L" : "", s2 == NULL ? "R" : "", func, file, line); + return -1; + } + return strcasecmp(s1, s2); +} + +#undef strncasecmp +int FTLstrncasecmp(const char *s1, const char *s2, const size_t n, const char *file, const char *func, const int line) +{ + // The strncasecmp() function is similar, except it compares only the first + // (at most) n bytes of s1 and s2. + if(s1 == NULL || s2 == NULL) + { + log_err("Trying to compare a NULL (%s%s) string in %s() (%s:%i)", + s1 == NULL ? "L" : "", s2 == NULL ? "R" : "", func, file, line); + return -1; + } + return strncasecmp(s1, s2, n); +} + +#undef strcat +char *FTLstrcat(char *dest, const char *src, const char *file, const char *func, const int line) +{ + // The strcat() function appends the src string to the dest string, + // overwriting the terminating null byte ('\0') at the end of dest, and then + // adds a terminating null byte. The strings may not overlap, and the dest + // string must have enough space for the result. If dest is not large enough, + // program behavior is unpredictable; buffer overruns are a favorite avenue + // for attacking secure programs. + if(dest == NULL || src == NULL) + { + log_err("Trying to concatenate a NULL (%s%s) string in %s() (%s:%i)", + dest == NULL ? "L" : "", src == NULL ? "R" : "", func, file, line); + return NULL; + } + return strcat(dest, src); +} + +#undef strncat +char *FTLstrncat(char *dest, const char *src, const size_t n, const char *file, const char *func, const int line) +{ + // The strncat() function is similar, except that it will use at most n bytes + // from src; and src does not need to be null-terminated if it contains n or + // more bytes. + if(dest == NULL || src == NULL) + { + log_err("Trying to concatenate a NULL (%s%s) string in %s() (%s:%i)", + dest == NULL ? "L" : "", src == NULL ? "R" : "", func, file, line); + return NULL; + } + return strncat(dest, src, n); +} + +#undef strcpy +char *FTLstrcpy(char *dest, const char *src, const char *file, const char *func, const int line) +{ + // The strcpy() function copies the string src to dest (including the + // terminating '\0' character.) + if(dest == NULL || src == NULL) + { + log_err("Trying to copy a NULL (%s%s) string in %s() (%s:%i)", + dest == NULL ? "L" : "", src == NULL ? "R" : "", func, file, line); + return NULL; + } + return strcpy(dest, src); +} + +#undef strncpy +char *FTLstrncpy(char *dest, const char *src, const size_t n, const char *file, const char *func, const int line) +{ + // The strncpy() function is similar, except that at most n bytes of src are + // copied. Warning: If there is no null byte among the first n bytes of src, + // the string placed in dest will not be null-terminated. + if(dest == NULL || src == NULL) + { + log_err("Trying to copy a NULL (%s%s) string in %s() (%s:%i)", + dest == NULL ? "L" : "", src == NULL ? "R" : "", func, file, line); + return NULL; + } + return strncpy(dest, src, n); +} + +#undef memset +void *FTLmemset(void *s, const int c, const size_t n, const char *file, const char *func, const int line) +{ + // The memset() function fills the first n bytes of the memory area pointed + // to by s with the constant byte c. + if(s == NULL) + { + log_err("Trying to fill a NULL memory area in %s() (%s:%i)", func, file, line); + return NULL; + } + return memset(s, c, n); +} + +#undef memcpy +void *FTLmemcpy(void *dest, const void *src, const size_t n, const char *file, const char *func, const int line) +{ + // The memcpy() function copies n bytes from memory area src to memory area + // dest. The memory areas must not overlap. Use memmove(3) if the memory + // areas do overlap. + if(dest == NULL || src == NULL) + { + log_err("Trying to copy a NULL (%s%s) memory area in %s() (%s:%i)", + dest == NULL ? "L" : "", src == NULL ? "R" : "", func, file, line); + return NULL; + } + return memcpy(dest, src, n); +} + +#undef memmove +void *FTLmemmove(void *dest, const void *src, const size_t n, const char *file, const char *func, const int line) +{ + // The memmove() function copies n bytes from memory area src to memory area + // dest. The memory areas may overlap: copying takes place as though the + // bytes in src are first copied into a temporary array that does not + // overlap src or dest, and the bytes are then copied from the temporary + // array to dest. + if(dest == NULL || src == NULL) + { + log_err("Trying to move a NULL (%s%s) memory area in %s() (%s:%i)", + dest == NULL ? "L" : "", src == NULL ? "R" : "", func, file, line); + return NULL; + } + return memmove(dest, src, n); +} + +#undef memcmp +int FTLmemcmp(const void *s1, const void *s2, const size_t n, const char *file, const char *func, const int line) +{ + // The memcmp() function compares the first n bytes (each interpreted as + // unsigned char) of the memory areas s1 and s2. + if(s1 == NULL || s2 == NULL) + { + log_err("Trying to compare a NULL (%s%s) memory area in %s() (%s:%i)", + s1 == NULL ? "L" : "", s2 == NULL ? "R" : "", func, file, line); + return -1; + } + return memcmp(s1, s2, n); +} + +#undef memmem +void *FTLmemmem(const void *haystack, const size_t haystacklen, const void *needle, const size_t needlelen, const char *file, const char *func, const int line) +{ + // The memmem() function finds the start of the first occurrence of the + // substring needle of length needlelen in the memory area haystack of + // length haystacklen. + if(haystack == NULL || needle == NULL) + { + log_err("Trying to find a NULL (%s%s) memory area in %s() (%s:%i)", + haystack == NULL ? "L" : "", needle == NULL ? "R" : "", func, file, line); + return NULL; + } + return memmem(haystack, haystacklen, needle, needlelen); +} diff --git a/src/syscalls/syscalls.h b/src/syscalls/syscalls.h index dfe510035..3d77ebe7e 100644 --- a/src/syscalls/syscalls.h +++ b/src/syscalls/syscalls.h @@ -50,4 +50,22 @@ FILE *FTLfopen(const char *pathname, const char *mode, const char *file, const c // Syscall helpers void syscalls_report_error(const char *error, FILE *stream, const int _errno, const char *format, const char *func, const char *file, const int line); +// String-related functions +size_t FTLstrlen(const char *s, const char *file, const char *func, const int line); +size_t FTLstrnlen(const char *s, const size_t maxlen, const char *file, const char *func, const int line); +char *FTLstrcpy(char *dest, const char *src, const char *file, const char *func, const int line); +char *FTLstrncpy(char *dest, const char *src, const size_t n, const char *file, const char *func, const int line); +void *FTLmemset(void *s, const int c, const size_t n, const char *file, const char *func, const int line); +void *FTLmemcpy(void *dest, const void *src, const size_t n, const char *file, const char *func, const int line); +void *FTLmemmove(void *dest, const void *src, const size_t n, const char *file, const char *func, const int line); +char *FTLstrstr(const char *haystack, const char *needle, const char *file, const char *func, const int line); +int FTLstrcmp(const char *s1, const char *s2, const char *file, const char *func, const int line); +int FTLstrncmp(const char *s1, const char *s2, const size_t n, const char *file, const char *func, const int line); +int FTLstrcasecmp(const char *s1, const char *s2, const char *file, const char *func, const int line); +int FTLstrncasecmp(const char *s1, const char *s2, const size_t n, const char *file, const char *func, const int line); +char *FTLstrcat(char *dest, const char *src, const char *file, const char *func, const int line); +char *FTLstrncat(char *dest, const char *src, const size_t n, const char *file, const char *func, const int line); +int FTLmemcmp(const void *s1, const void *s2, const size_t n, const char *file, const char *func, const int line); +void *FTLmemmem(const void *haystack, const size_t haystacklen, const void *needle, const size_t needlelen, const char *file, const char *func, const int line); + #endif //SYSCALLS_H From 5320dc3a848ce3d19541203bc0a6fcfdccb2d35e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 20 Nov 2023 22:30:33 +0100 Subject: [PATCH 088/221] Verify we have no default string pointers to NULL Signed-off-by: DL6ER --- src/config/config.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/config/config.c b/src/config/config.c index 85578a95f..7de3a7c42 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -1333,6 +1333,13 @@ void initConfig(struct config *conf) log_err("Config option %s has no type!", conf_item->k); continue; } + + // Verify we have no default string pointers to NULL + if((conf_item->t == CONF_STRING || conf_item->t == CONF_STRING_ALLOCATED) && conf_item->d.s == NULL) + { + log_err("Config option %s has NULL default string!", conf_item->k); + continue; + } } } From 015708226eeb40b12affabcc926c3bd4b47ff680 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 20 Nov 2023 22:35:30 +0100 Subject: [PATCH 089/221] Add dedicated function to reset config values to their defaults Signed-off-by: DL6ER --- src/config/config.c | 48 +++++++++++++++++++++++++++++++++------------ src/config/config.h | 1 + 2 files changed, 37 insertions(+), 12 deletions(-) diff --git a/src/config/config.c b/src/config/config.c index 7de3a7c42..7d20c57cb 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -1304,18 +1304,7 @@ void initConfig(struct config *conf) // Initialize config value with default one for all *except* the log file path if(conf_item != &conf->files.log.ftl) - { - if(conf_item->t == CONF_JSON_STRING_ARRAY) - // JSON objects really need to be duplicated as the config - // structure stores only a pointer to memory somewhere else - conf_item->v.json = cJSON_Duplicate(conf_item->d.json, true); - else if(conf_item->t == CONF_STRING_ALLOCATED) - // Allocated string: Make our own copy - conf_item->v.s = strdup(conf_item->d.s); - else - // Ordinary value: Simply copy the union over - memcpy(&conf_item->v, &conf_item->d, sizeof(conf_item->d)); - } + reset_config(conf_item); // Parse and split paths conf_item->p = gen_config_path(conf_item->k, '.'); @@ -1340,6 +1329,41 @@ void initConfig(struct config *conf) log_err("Config option %s has NULL default string!", conf_item->k); continue; } + + // Verify we have no default JSON pointers to NULL + if(conf_item->t == CONF_JSON_STRING_ARRAY && conf_item->d.json == NULL) + { + log_err("Config option %s has NULL default JSON array!", conf_item->k); + continue; + } + } +} + +void reset_config(struct conf_item *conf_item) +{ + if(conf_item->t == CONF_JSON_STRING_ARRAY) + { + // Free allocated memory (if any) + if(conf_item->v.json != NULL) + cJSON_Delete(conf_item->v.json); + + // JSON objects really need to be duplicated as the config + // structure stores only a pointer to memory somewhere else + conf_item->v.json = cJSON_Duplicate(conf_item->d.json, true); + } + else if(conf_item->t == CONF_STRING_ALLOCATED) + { + // Free allocated memory (if any) + if(conf_item->v.s != NULL) + free(conf_item->v.s); + + // Allocated string: Make our own copy + conf_item->v.s = strdup(conf_item->d.s); + } + else + { + // Ordinary value: Simply copy the union over + memcpy(&conf_item->v, &conf_item->d, sizeof(conf_item->d)); } } diff --git a/src/config/config.h b/src/config/config.h index dc4937396..a66d8c3e4 100644 --- a/src/config/config.h +++ b/src/config/config.h @@ -323,6 +323,7 @@ extern struct config config; void set_debug_flags(struct config *conf); void set_all_debug(struct config *conf, const bool status); void initConfig(struct config *conf); +void reset_config(struct conf_item *conf_item); void readFTLconf(struct config *conf, const bool rewrite); bool getLogFilePath(void); struct conf_item *get_conf_item(struct config *conf, const unsigned int n); From e2f35017c04bafd882f17a095755f882ad0d54ba Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 21 Nov 2023 09:30:35 +0100 Subject: [PATCH 090/221] Fix legacy settings parsing: All the DHCP-related IP addresses are not of type INADDR, not STRING! Signed-off-by: DL6ER --- src/config/legacy_reader.c | 12 +++---- src/setupVars.c | 69 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 72 insertions(+), 9 deletions(-) diff --git a/src/config/legacy_reader.c b/src/config/legacy_reader.c index bf4600f84..9ae6e664b 100644 --- a/src/config/legacy_reader.c +++ b/src/config/legacy_reader.c @@ -401,7 +401,7 @@ const char *readFTLlegacy(struct config *conf) conf->dns.reply.host.force4.v.b = false; conf->dns.reply.host.v4.v.in_addr.s_addr = 0; buffer = parseFTLconf(fp, "LOCAL_IPV4"); - if(buffer != NULL && inet_pton(AF_INET, buffer, &conf->dns.reply.host.v4.v.in_addr)) + if(buffer != NULL && strlen(buffer) > 0 && inet_pton(AF_INET, buffer, &conf->dns.reply.host.v4.v.in_addr)) conf->dns.reply.host.force4.v.b = true; // LOCAL_IPV6 @@ -411,7 +411,7 @@ const char *readFTLlegacy(struct config *conf) conf->dns.reply.host.force6.v.b = false; memset(&conf->dns.reply.host.v6.v.in6_addr, 0, sizeof(conf->dns.reply.host.v6.v.in6_addr)); buffer = parseFTLconf(fp, "LOCAL_IPV6"); - if(buffer != NULL && inet_pton(AF_INET6, buffer, &conf->dns.reply.host.v6.v.in6_addr)) + if(buffer != NULL && strlen(buffer) > 0 && inet_pton(AF_INET6, buffer, &conf->dns.reply.host.v6.v.in6_addr)) conf->dns.reply.host.force6.v.b = true; // BLOCK_IPV4 @@ -420,7 +420,7 @@ const char *readFTLlegacy(struct config *conf) conf->dns.reply.blocking.force4.v.b = false; conf->dns.reply.blocking.v4.v.in_addr.s_addr = 0; buffer = parseFTLconf(fp, "BLOCK_IPV4"); - if(buffer != NULL && inet_pton(AF_INET, buffer, &conf->dns.reply.blocking.v4.v.in_addr)) + if(buffer != NULL && strlen(buffer) > 0 && inet_pton(AF_INET, buffer, &conf->dns.reply.blocking.v4.v.in_addr)) conf->dns.reply.blocking.force4.v.b = true; // BLOCK_IPV6 @@ -429,7 +429,7 @@ const char *readFTLlegacy(struct config *conf) conf->dns.reply.blocking.force6.v.b = false; memset(&conf->dns.reply.blocking.v6.v.in6_addr, 0, sizeof(conf->dns.reply.host.v6.v.in6_addr)); buffer = parseFTLconf(fp, "BLOCK_IPV6"); - if(buffer != NULL && inet_pton(AF_INET6, buffer, &conf->dns.reply.blocking.v6.v.in6_addr)) + if(buffer != NULL && strlen(buffer) > 0 && inet_pton(AF_INET6, buffer, &conf->dns.reply.blocking.v6.v.in6_addr)) conf->dns.reply.blocking.force6.v.b = true; // REPLY_ADDR4 (deprecated setting) @@ -438,7 +438,7 @@ const char *readFTLlegacy(struct config *conf) // defaults to: not set struct in_addr reply_addr4; buffer = parseFTLconf(fp, "REPLY_ADDR4"); - if(buffer != NULL && inet_pton(AF_INET, buffer, &reply_addr4)) + if(buffer != NULL && strlen(buffer) > 0 && inet_pton(AF_INET, buffer, &reply_addr4)) { if(conf->dns.reply.host.force4.v.b || conf->dns.reply.blocking.force4.v.b) { @@ -459,7 +459,7 @@ const char *readFTLlegacy(struct config *conf) // defaults to: not set struct in6_addr reply_addr6; buffer = parseFTLconf(fp, "REPLY_ADDR6"); - if(buffer != NULL && inet_pton(AF_INET, buffer, &reply_addr6)) + if(buffer != NULL && strlen(buffer) > 0 && inet_pton(AF_INET, buffer, &reply_addr6)) { if(conf->dns.reply.host.force6.v.b || conf->dns.reply.blocking.force6.v.b) { diff --git a/src/setupVars.c b/src/setupVars.c index ca069d5f4..e00405717 100644 --- a/src/setupVars.c +++ b/src/setupVars.c @@ -19,6 +19,13 @@ char ** setupVarsArray = NULL; static void get_conf_string_from_setupVars(const char *key, struct conf_item *conf_item) { + // Verify we are allowed to use this function + if(conf_item->t != CONF_STRING && conf_item->t != CONF_STRING_ALLOCATED) + { + log_err("get_conf_string_from_setupVars(%s) failed: conf_item->t is neither CONF_STRING nor CONF_STRING_ALLOCATED", key); + return; + } + const char *setupVarsValue = read_setupVarsconf(key); if(setupVarsValue == NULL) { @@ -43,8 +50,50 @@ static void get_conf_string_from_setupVars(const char *key, struct conf_item *co log_debug(DEBUG_CONFIG, "setupVars.conf:%s -> Setting %s to %s", key, conf_item->k, conf_item->v.s); } +static void get_conf_ipv4_from_setupVars(const char *key, struct conf_item *conf_item) +{ + // Verify we are allowed to use this function + if(conf_item->t != CONF_STRUCT_IN_ADDR) + { + log_err("get_conf_ipv4_from_setupVars(%s) failed: conf_item->t != CONF_STRUCT_IN_ADDR", key); + return; + } + + const char *setupVarsValue = read_setupVarsconf(key); + if(setupVarsValue == NULL) + { + // Do not change default value, this value is not set in setupVars.conf + log_debug(DEBUG_CONFIG, "setupVars.conf:%s -> Not set", key); + + // Free memory, harmless to call if read_setupVarsconf() didn't return a result + clearSetupVarsArray(); + return; + } + + if(strlen(setupVarsValue) == 0) + memset(&conf_item->v.in_addr, 0, sizeof(struct in_addr)); + else if(inet_pton(AF_INET, setupVarsValue, &conf_item->v.in_addr) != 1) + { + log_debug(DEBUG_CONFIG, "setupVars.conf:%s -> Invalid IPv4 address: %s", key, setupVarsValue); + memset(&conf_item->v.in_addr, 0, sizeof(struct in_addr)); + } + + // Free memory, harmless to call if read_setupVarsconf() didn't return a result + clearSetupVarsArray(); + + // Parameter present in setupVars.conf + log_debug(DEBUG_CONFIG, "setupVars.conf:%s -> Setting %s to %s", key, conf_item->k, inet_ntoa(conf_item->v.in_addr)); +} + static void get_conf_bool_from_setupVars(const char *key, struct conf_item *conf_item) { + // Verify we are allowed to use this function + if(conf_item->t != CONF_BOOL) + { + log_err("get_conf_bool_from_setupVars(%s) failed: conf_item->t != CONF_BOOL", key); + return; + } + const char *boolean = read_setupVarsconf(key); if(boolean == NULL) @@ -72,6 +121,13 @@ static void get_conf_bool_from_setupVars(const char *key, struct conf_item *conf static void get_conf_string_array_from_setupVars(const char *key, struct conf_item *conf_item) { + // Verify we are allowed to use this function + if(conf_item->t != CONF_JSON_STRING_ARRAY) + { + log_err("get_conf_string_array_from_setupVars(%s) failed: conf_item->t != CONF_JSON_STRING_ARRAY", key); + return; + } + // Get clients which the user doesn't want to see const char *array = read_setupVarsconf(key); @@ -95,6 +151,13 @@ static void get_conf_string_array_from_setupVars(const char *key, struct conf_it static void get_conf_upstream_servers_from_setupVars(struct conf_item *conf_item) { + // Verify we are allowed to use this function + if(conf_item->t != CONF_JSON_STRING_ARRAY) + { + log_err("get_conf_upstream_servers_from_setupVars() failed: conf_item->t != CONF_JSON_STRING_ARRAY"); + return; + } + // Try to import up to 50 servers... #define MAX_SERVERS 50 for(unsigned int j = 0; j < MAX_SERVERS; j++) @@ -360,9 +423,9 @@ void importsetupVarsConf(void) // Try to obtain DHCP settings get_conf_bool_from_setupVars("DHCP_ACTIVE", &config.dhcp.active); - get_conf_string_from_setupVars("DHCP_START", &config.dhcp.start); - get_conf_string_from_setupVars("DHCP_END", &config.dhcp.end); - get_conf_string_from_setupVars("DHCP_ROUTER", &config.dhcp.router); + get_conf_ipv4_from_setupVars("DHCP_START", &config.dhcp.start); + get_conf_ipv4_from_setupVars("DHCP_END", &config.dhcp.end); + get_conf_ipv4_from_setupVars("DHCP_ROUTER", &config.dhcp.router); get_conf_string_from_setupVars("DHCP_LEASETIME", &config.dhcp.leaseTime); // If the DHCP lease time is set to "24", it is interpreted as "24h". From 46eca50db033e5720934a828002c796f9ce33312 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 21 Nov 2023 09:43:08 +0100 Subject: [PATCH 091/221] Fix logging when reading the TLS certificate Signed-off-by: DL6ER --- src/webserver/x509.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/webserver/x509.c b/src/webserver/x509.c index cb154c4a2..e4d224fec 100644 --- a/src/webserver/x509.c +++ b/src/webserver/x509.c @@ -316,26 +316,26 @@ enum cert_check read_certificate(const char* certfile, const char *domain, const mbedtls_entropy_init(&entropy); mbedtls_ctr_drbg_init(&ctr_drbg); - printf("Reading certificate from %s ...\n\n", certfile); + log_info("Reading certificate from %s ...", certfile); // Check if the file exists and is readable if(access(certfile, R_OK) != 0) { - log_err("Could not read certificate file: %s\n", strerror(errno)); + log_err("Could not read certificate file: %s", strerror(errno)); return CERT_FILE_NOT_FOUND; } int rc = mbedtls_pk_parse_keyfile(&key, certfile, NULL, mbedtls_ctr_drbg_random, &ctr_drbg); if (rc != 0) { - log_err("Cannot parse key: Error code %d\n", rc); + log_err("Cannot parse key: Error code %d", rc); return CERT_CANNOT_PARSE_KEY; } rc = mbedtls_x509_crt_parse_file(&crt, certfile); if (rc != 0) { - log_err("Cannot parse certificate: Error code %d\n", rc); + log_err("Cannot parse certificate: Error code %d", rc); return CERT_CANNOT_PARSE_CERT; } From 2c765c94bb8a8dd06ac87310d86438ee43228dd4 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 21 Nov 2023 12:04:48 +0100 Subject: [PATCH 092/221] Add WEB_PORTS to setupVars.conf when importing v5 Teleporter files Signed-off-by: DL6ER --- src/api/teleporter.c | 10 ++++++++ src/config/config.c | 59 +++++++++++++++++++++++--------------------- src/config/config.h | 1 + src/main.c | 2 +- src/setupVars.c | 4 +++ 5 files changed, 47 insertions(+), 29 deletions(-) diff --git a/src/api/teleporter.c b/src/api/teleporter.c index f726318b2..2350eb28e 100644 --- a/src/api/teleporter.c +++ b/src/api/teleporter.c @@ -687,6 +687,16 @@ static int process_received_tar_gz(struct ftl_conn *api, struct upload_data *dat } } + // Append WEB_PORTS to setupVars.conf + FILE *fp = fopen(config.files.setupVars.v.s, "a"); + if(fp == NULL) + log_err("Unable to open file \"%s\" for appending: %s", config.files.setupVars.v.s, strerror(errno)); + else + { + fprintf(fp, "WEB_PORT=%s\n", config.webserver.port.v.s); + fclose(fp); + } + // Remove pihole.toml to prevent it from being imported on restart if(remove(GLOBALTOMLPATH) != 0) log_err("Unable to remove file \"%s\": %s", GLOBALTOMLPATH, strerror(errno)); diff --git a/src/config/config.c b/src/config/config.c index 3d1cf57c4..d2a133e84 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -1385,36 +1385,39 @@ void readFTLconf(struct config *conf, const bool rewrite) rename(GLOBALTOMLPATH, new_name); } - // Determine default webserver ports - // Check if ports 80/TCP and 443/TCP are already in use - const in_port_t http_port = port_in_use(80) ? 8080 : 80; - const in_port_t https_port = port_in_use(443) ? 8443 : 443; - - // Create a string with the default ports - // Allocate memory for the string - char *ports = calloc(32, sizeof(char)); - if(ports == NULL) + // Determine default webserver ports if not imported from setupVars.conf + if(!(config.webserver.port.f & FLAG_CONF_IMPORTED)) { - log_err("Unable to allocate memory for default ports string"); - return; + // Check if ports 80/TCP and 443/TCP are already in use + const in_port_t http_port = port_in_use(80) ? 8080 : 80; + const in_port_t https_port = port_in_use(443) ? 8443 : 443; + + // Create a string with the default ports + // Allocate memory for the string + char *ports = calloc(32, sizeof(char)); + if(ports == NULL) + { + log_err("Unable to allocate memory for default ports string"); + return; + } + // Create the string + snprintf(ports, 32, "%d,%ds", http_port, https_port); + + // Append IPv6 ports if IPv6 is enabled + const bool have_ipv6 = ipv6_enabled(); + if(have_ipv6) + snprintf(ports + strlen(ports), 32 - strlen(ports), + ",[::]:%d,[::]:%ds", http_port, https_port); + + // Set default values for webserver ports + if(conf->webserver.port.t == CONF_STRING_ALLOCATED) + free(conf->webserver.port.v.s); + conf->webserver.port.v.s = ports; + conf->webserver.port.t = CONF_STRING_ALLOCATED; + + log_info("Initialised webserver ports at %d (HTTP) and %d (HTTPS), IPv6 support is %s", + http_port, https_port, have_ipv6 ? "enabled" : "disabled"); } - // Create the string - snprintf(ports, 32, "%d,%ds", http_port, https_port); - - // Append IPv6 ports if IPv6 is enabled - const bool have_ipv6 = ipv6_enabled(); - if(have_ipv6) - snprintf(ports + strlen(ports), 32 - strlen(ports), - ",[::]:%d,[::]:%ds", http_port, https_port); - - // Set default values for webserver ports - if(conf->webserver.port.t == CONF_STRING_ALLOCATED) - free(conf->webserver.port.v.s); - conf->webserver.port.v.s = ports; - conf->webserver.port.t = CONF_STRING_ALLOCATED; - - log_info("Initialised webserver ports at %d (HTTP) and %d (HTTPS), IPv6 support is %s", - http_port, https_port, have_ipv6 ? "enabled" : "disabled"); // Initialize the TOML config file writeFTLtoml(true); diff --git a/src/config/config.h b/src/config/config.h index 18fa6e14d..9cee391a9 100644 --- a/src/config/config.h +++ b/src/config/config.h @@ -97,6 +97,7 @@ enum conf_type { #define FLAG_INVALIDATE_SESSIONS (1 << 3) #define FLAG_WRITE_ONLY (1 << 4) #define FLAG_ENV_VAR (1 << 5) +#define FLAG_CONF_IMPORTED (1 << 6) struct conf_item { const char *k; // item Key diff --git a/src/main.c b/src/main.c index 22e57fe5f..88d7caee6 100644 --- a/src/main.c +++ b/src/main.c @@ -185,7 +185,7 @@ int main (int argc, char *argv[]) cleanup(exit_code); if(exit_code == RESTART_FTL_CODE) - execv(argv[0], argv); + execvp(argv[0], argv); return exit_code; } diff --git a/src/setupVars.c b/src/setupVars.c index ca069d5f4..a4069c766 100644 --- a/src/setupVars.c +++ b/src/setupVars.c @@ -35,6 +35,7 @@ static void get_conf_string_from_setupVars(const char *key, struct conf_item *co free(conf_item->v.s); conf_item->v.s = strdup(setupVarsValue); conf_item->t = CONF_STRING_ALLOCATED; + conf_item->f |= FLAG_CONF_IMPORTED; // Free memory, harmless to call if read_setupVarsconf() didn't return a result clearSetupVarsArray(); @@ -380,6 +381,9 @@ void importsetupVarsConf(void) get_conf_bool_from_setupVars("DHCP_RAPID_COMMIT", &config.dhcp.rapidCommit); get_conf_bool_from_setupVars("queryLogging", &config.dns.queryLogging); + + // Ports may be temporarily stored when importing a legacy Teleporter v5 file + get_conf_string_from_setupVars("WEB_PORTS", &config.webserver.port); } char* __attribute__((pure)) find_equals(char *s) From f4e830ca17ec4cad6f1ba57c6a33be77ea71c4a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 21 Nov 2023 12:36:54 +0100 Subject: [PATCH 093/221] Only log how many sessions have been restored MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- src/database/session-table.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/database/session-table.c b/src/database/session-table.c index d185a55cd..c84eb0976 100644 --- a/src/database/session-table.c +++ b/src/database/session-table.c @@ -296,8 +296,8 @@ bool restore_db_sessions(struct session *sessions, const uint16_t max_sessions) i++; } - log_info("Restored %u/%u API session%s from the database", - i, max_sessions, max_sessions == 1 ? "" : "s"); + log_info("Restored %u API session%s from the database", + i, i == 1 ? "" : "s"); // Finalize statement if(sqlite3_finalize(stmt) != SQLITE_OK) From 1cde68bce76da3327fb08fd59dbe35d24e376273 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 21 Nov 2023 14:10:02 +0100 Subject: [PATCH 094/221] Add ability to get most recent client hostname from network table if specified by MAC address Signed-off-by: DL6ER --- src/api/list.c | 4 +- src/database/network-table.c | 105 +++++++++++++++++++++++++++++++++++ src/database/network-table.h | 8 ++- src/database/sqlite3-ext.c | 26 --------- 4 files changed, 113 insertions(+), 30 deletions(-) diff --git a/src/api/list.c b/src/api/list.c index c584a334c..f01858464 100644 --- a/src/api/list.c +++ b/src/api/list.c @@ -58,9 +58,11 @@ static int api_list_read(struct ftl_conn *api, char *name = NULL; if(table.client != NULL) { - // Try to obtain hostname if this is a valid IP address + // Try to obtain hostname if(isValidIPv4(table.client) || isValidIPv6(table.client)) name = getNameFromIP(NULL, table.client); + else if(isMAC(table.client)) + name = getNameFromMAC(table.client); } JSON_COPY_STR_TO_OBJECT(row, "client", table.client); diff --git a/src/database/network-table.c b/src/database/network-table.c index 169ba9c83..92837b31c 100644 --- a/src/database/network-table.c +++ b/src/database/network-table.c @@ -2144,6 +2144,85 @@ char *__attribute__((malloc)) getNameFromIP(sqlite3 *db, const char *ipaddr) return name; } +// Get most recently seen host name of device identified by MAC address +char *__attribute__((malloc)) getNameFromMAC(const char *client) +{ + // Return early if database is known to be broken + if(FTLDBerror()) + return NULL; + + log_info("Looking up host name for %s", client); + + // Open pihole-FTL.db database file + sqlite3 *db = NULL; + if((db = dbopen(false, false)) == NULL) + { + log_warn("getNameFromMAC(\"%s\") - Failed to open DB", client); + return NULL; + } + + // Nothing found for the same device + // Check for a host name associated with the given client as MAC address + // COLLATE NOCASE: Case-insensitive comparison + const char *querystr = "SELECT name FROM network_addresses " + "WHERE name IS NOT NULL AND " + "network_id = (SELECT id FROM network WHERE hwaddr = ? COLLATE NOCASE) " + "ORDER BY lastSeen DESC LIMIT 1"; + sqlite3_stmt *stmt = NULL; + int rc = sqlite3_prepare_v2(db, querystr, -1, &stmt, NULL); + if(rc != SQLITE_OK) + { + log_err("getNameFromMAC(\"%s\") - SQL error prepare: %s", + client, sqlite3_errstr(rc)); + dbclose(&db); + return NULL; + } + + // Bind client to prepared statement + if((rc = sqlite3_bind_text(stmt, 1, client, -1, SQLITE_STATIC)) != SQLITE_OK) + { + log_warn("getNameFromMAC(\"%s\"): Failed to bind ip: %s", + client, sqlite3_errstr(rc)); + checkFTLDBrc(rc); + sqlite3_reset(stmt); + sqlite3_finalize(stmt); + + dbclose(&db); + return NULL; + } + + char *name = NULL; + rc = sqlite3_step(stmt); + if(rc == SQLITE_ROW) + { + // Database record found (result might be empty) + name = strdup((char*)sqlite3_column_text(stmt, 0)); + + if(config.debug.resolver.v.b) + log_debug(DEBUG_RESOLVER, "Found database host name (by MAC) %s -> %s", + client, name); + } + else if(rc == SQLITE_DONE) + { + // Not found + if(config.debug.resolver.v.b) + log_debug(DEBUG_RESOLVER, " ---> not found"); + } + else + { + // Error + checkFTLDBrc(rc); + return NULL; + } + + // Finalize statement and close database handle + sqlite3_reset(stmt); + sqlite3_finalize(stmt); + + dbclose(&db); + return name; +} + // Get interface of device identified by IP address char *__attribute__((malloc)) getIfaceFromIP(sqlite3 *db, const char *ipaddr) { @@ -2425,3 +2504,29 @@ bool networkTable_deleteDevice(sqlite3 *db, const int id, const char **message) return true; } + +// Counting number of occurrences of a specific char in a string +static size_t __attribute__ ((pure)) count_char(const char *haystack, const char needle) +{ + size_t count = 0u; + while(*haystack) + if (*haystack++ == needle) + ++count; + return count; +} + +// Identify MAC addresses using a set of suitable criteria +bool __attribute__ ((pure)) isMAC(const char *input) +{ + if(input != NULL && // Valid input + strlen(input) == 17u && // MAC addresses are always 17 chars long (6 bytes + 5 colons) + count_char(input, ':') == 5u && // MAC addresses always have 5 colons + strstr(input, "::") == NULL) // No double-colons (IPv6 address abbreviation) + { + // This is a MAC address of the form AA:BB:CC:DD:EE:FF + return true; + } + + // Not a MAC address + return false; +} diff --git a/src/database/network-table.h b/src/database/network-table.h index 4b9f6952e..1cb4697d1 100644 --- a/src/database/network-table.h +++ b/src/database/network-table.h @@ -18,12 +18,14 @@ bool create_network_addresses_with_names_table(sqlite3 *db); void parse_neighbor_cache(sqlite3 *db); void updateMACVendorRecords(sqlite3 *db); bool unify_hwaddr(sqlite3 *db); -char* __attribute__((malloc)) getMACfromIP(sqlite3 *db, const char* ipaddr); +char *getMACfromIP(sqlite3 *db, const char* ipaddr) __attribute__((malloc)); int getAliasclientIDfromIP(sqlite3 *db, const char *ipaddr); -char* __attribute__((malloc)) getNameFromIP(sqlite3 *db, const char* ipaddr); -char* __attribute__((malloc)) getIfaceFromIP(sqlite3 *db, const char* ipaddr); +char *getNameFromIP(sqlite3 *db, const char* ipaddr) __attribute__((malloc)); +char *getNameFromMAC(const char *client) __attribute__((malloc)); +char *getIfaceFromIP(sqlite3 *db, const char* ipaddr) __attribute__((malloc)); void resolveNetworkTableNames(void); bool flush_network_table(void); +bool isMAC(const char *input) __attribute__ ((pure)); typedef struct { unsigned int id; diff --git a/src/database/sqlite3-ext.c b/src/database/sqlite3-ext.c index 67059ff79..f52ef029d 100644 --- a/src/database/sqlite3-ext.c +++ b/src/database/sqlite3-ext.c @@ -29,32 +29,6 @@ // isMAC() #include "network-table.h" -// Counting number of occurrences of a specific char in a string -static size_t __attribute__ ((pure)) count_char(const char *haystack, const char needle) -{ - size_t count = 0u; - while(*haystack) - if (*haystack++ == needle) - ++count; - return count; -} - -// Identify MAC addresses using a set of suitable criteria -static bool __attribute__ ((pure)) isMAC(const char *input) -{ - if(input != NULL && // Valid input - strlen(input) == 17u && // MAC addresses are always 17 chars long (6 bytes + 5 colons) - count_char(input, ':') == 5u && // MAC addresses always have 5 colons - strstr(input, "::") == NULL) // No double-colons (IPv6 address abbreviation) - { - // This is a MAC address of the form AA:BB:CC:DD:EE:FF - return true; - } - - // Not a MAC address - return false; -} - static void subnet_match_impl(sqlite3_context *context, int argc, sqlite3_value **argv) { // Exactly two arguments should be submitted to this routine From 7c53ce8d345543978681f5652b31737aab0913d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 21 Nov 2023 16:55:17 +0000 Subject: [PATCH 095/221] Log successful processing of pihole.toml on startup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- src/config/config.c | 8 +++++--- src/config/config.h | 2 +- src/main.c | 3 ++- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/config/config.c b/src/config/config.c index 7d20c57cb..d8b230971 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -1367,7 +1367,7 @@ void reset_config(struct conf_item *conf_item) } } -void readFTLconf(struct config *conf, const bool rewrite) +bool readFTLconf(struct config *conf, const bool rewrite) { // Initialize config with default values initConfig(conf); @@ -1384,7 +1384,7 @@ void readFTLconf(struct config *conf, const bool rewrite) write_dnsmasq_config(conf, false, NULL); write_custom_list(); } - return; + return true; } // On error, try to read legacy (pre-v6.0) config file. If successful, @@ -1428,7 +1428,7 @@ void readFTLconf(struct config *conf, const bool rewrite) if(ports == NULL) { log_err("Unable to allocate memory for default ports string"); - return; + return false; } // Create the string snprintf(ports, 32, "%d,%ds", http_port, https_port); @@ -1452,6 +1452,8 @@ void readFTLconf(struct config *conf, const bool rewrite) writeFTLtoml(true); write_dnsmasq_config(conf, false, NULL); write_custom_list(); + + return true; } bool getLogFilePath(void) diff --git a/src/config/config.h b/src/config/config.h index a66d8c3e4..cdacdbe38 100644 --- a/src/config/config.h +++ b/src/config/config.h @@ -324,7 +324,7 @@ void set_debug_flags(struct config *conf); void set_all_debug(struct config *conf, const bool status); void initConfig(struct config *conf); void reset_config(struct conf_item *conf_item); -void readFTLconf(struct config *conf, const bool rewrite); +bool readFTLconf(struct config *conf, const bool rewrite); bool getLogFilePath(void); struct conf_item *get_conf_item(struct config *conf, const unsigned int n); struct conf_item *get_debug_item(struct config *conf, const enum debug_flag debug); diff --git a/src/main.c b/src/main.c index 22e57fe5f..6a9898380 100644 --- a/src/main.c +++ b/src/main.c @@ -77,7 +77,8 @@ int main (int argc, char *argv[]) // Process pihole.toml configuration file // The file is rewritten after parsing to ensure that all // settings are present and have a valid value - readFTLconf(&config, true); + if(readFTLconf(&config, true)) + log_info("Parsed config file "GLOBALTOMLPATH" successfully"); // Set process priority set_nice(); From 268146d9c7701bd2b5e43870accb0cc11a07ce2e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 21 Nov 2023 18:14:10 +0100 Subject: [PATCH 096/221] Move the setupVars.conf file to setupVars.conf.old Signed-off-by: DL6ER --- src/api/teleporter.c | 6 +++--- src/config/legacy_reader.c | 5 ++++- src/setupVars.c | 17 +++++++++++++++++ 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/src/api/teleporter.c b/src/api/teleporter.c index 2350eb28e..12eb2ad1b 100644 --- a/src/api/teleporter.c +++ b/src/api/teleporter.c @@ -668,8 +668,8 @@ static int process_received_tar_gz(struct ftl_conn *api, struct upload_data *dat if(file != NULL && fileSize > 0u) { // Write file to disk - log_debug(DEBUG_API, "Writing file \"%s\" (%zu bytes) to \"%s\"", - extract_files[i].archive_name, fileSize, extract_files[i].destination); + log_info("Writing file \"%s\" (%zu bytes) to \"%s\"", + extract_files[i].archive_name, fileSize, extract_files[i].destination); FILE *fp = fopen(extract_files[i].destination, "wb"); if(fp == NULL) { @@ -693,7 +693,7 @@ static int process_received_tar_gz(struct ftl_conn *api, struct upload_data *dat log_err("Unable to open file \"%s\" for appending: %s", config.files.setupVars.v.s, strerror(errno)); else { - fprintf(fp, "WEB_PORT=%s\n", config.webserver.port.v.s); + fprintf(fp, "WEB_PORTS=%s\n", config.webserver.port.v.s); fclose(fp); } diff --git a/src/config/legacy_reader.c b/src/config/legacy_reader.c index ce09f3df0..f796131cd 100644 --- a/src/config/legacy_reader.c +++ b/src/config/legacy_reader.c @@ -113,9 +113,12 @@ const char *readFTLlegacy(struct config *conf) const char *path = NULL; FILE *fp = openFTLconf(&path); if(fp == NULL) + { + log_warn("No readable FTL config file found, using default settings"); return NULL; + } - log_notice("Reading legacy config file"); + log_info("Reading legacy config files from %s", path); // MAXDBDAYS // defaults to: 365 days diff --git a/src/setupVars.c b/src/setupVars.c index a4069c766..92fd7104b 100644 --- a/src/setupVars.c +++ b/src/setupVars.c @@ -312,6 +312,8 @@ static void get_conf_listeningMode_from_setupVars(void) void importsetupVarsConf(void) { + log_info("Migrating config from %s", config.files.setupVars.v.s); + // Try to obtain password hash from setupVars.conf get_conf_string_from_setupVars("WEBPASSWORD", &config.webserver.api.pwhash); @@ -384,6 +386,21 @@ void importsetupVarsConf(void) // Ports may be temporarily stored when importing a legacy Teleporter v5 file get_conf_string_from_setupVars("WEB_PORTS", &config.webserver.port); + + // Move the setupVars.conf file to setupVars.conf.old + char *old_setupVars = calloc(strlen(config.files.setupVars.v.s) + 5, sizeof(char)); + if(old_setupVars == NULL) + { + log_warn("Could not allocate memory for old_setupVars"); + return; + } + strcpy(old_setupVars, config.files.setupVars.v.s); + strcat(old_setupVars, ".old"); + if(rename(config.files.setupVars.v.s, old_setupVars) != 0) + log_warn("Could not move %s to %s", config.files.setupVars.v.s, old_setupVars); + else + log_info("Moved %s to %s", config.files.setupVars.v.s, old_setupVars); + free(old_setupVars); } char* __attribute__((pure)) find_equals(char *s) From a11094023135749bfe0f6834f4c585bfc8a7d839 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 21 Nov 2023 22:32:26 +0100 Subject: [PATCH 097/221] Improve config file opening logging Signed-off-by: DL6ER --- src/config/config.c | 2 ++ src/config/toml_helper.c | 8 ++++++++ src/config/toml_reader.c | 8 ++------ 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/config/config.c b/src/config/config.c index fc0ce3cec..a162f36ba 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -1361,6 +1361,8 @@ void readFTLconf(struct config *conf, const bool rewrite) } } + log_info("No config file nor backup available, using defaults"); + // If no previous config file could be read, we are likely either running // for the first time or we are upgrading from a version prior to v6.0 // In this case, we try to read the legacy config files diff --git a/src/config/toml_helper.c b/src/config/toml_helper.c index 14c46697b..d09a63f16 100644 --- a/src/config/toml_helper.c +++ b/src/config/toml_helper.c @@ -63,7 +63,11 @@ FILE * __attribute((malloc)) __attribute((nonnull(1))) openFTLtoml(const char *m // Return early if opening failed if(!fp) + { + log_info("Config %sfile %s not available: %s", + version > 0 ? "backup " : "", filename, strerror(errno)); return NULL; + } // Lock file, may block if the file is currently opened if(flock(fileno(fp), LOCK_EX) != 0) @@ -76,6 +80,10 @@ FILE * __attribute((malloc)) __attribute((nonnull(1))) openFTLtoml(const char *m return NULL; } + // Log if we are using a backup file + if(version > 0) + log_info("Using config backup %s", filename); + errno = 0; return fp; } diff --git a/src/config/toml_reader.c b/src/config/toml_reader.c index 5cef6333e..7bad6af9a 100644 --- a/src/config/toml_reader.c +++ b/src/config/toml_reader.c @@ -60,8 +60,8 @@ bool readFTLtoml(struct config *oldconf, struct config *newconf, } set_debug_flags(newconf); - log_debug(DEBUG_CONFIG, "Reading %s TOML config file: full config", - teleporter ? "teleporter" : "default"); + log_debug(DEBUG_CONFIG, "Reading %s TOML config file", + teleporter ? "teleporter" : version == 0 ? "default" : "backup"); // Read all known config items for(unsigned int i = 0; i < CONFIG_ELEMENTS; i++) @@ -140,11 +140,7 @@ static toml_table_t *parseTOML(const unsigned int version) // Try to open default config file. Use fallback if not found FILE *fp; if((fp = openFTLtoml("r", version)) == NULL) - { - log_info("No config file available (%s), using defaults", - strerror(errno)); return NULL; - } // Parse lines in the config file char errbuf[200]; From 247a3c4669f7ebbb921f94016f51f5cc80d88c45 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 21 Nov 2023 22:37:10 +0100 Subject: [PATCH 098/221] Apply suggestions from code review Co-authored-by: yubiuser Signed-off-by: DL6ER --- src/database/network-table.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/database/network-table.c b/src/database/network-table.c index 92837b31c..c5e269200 100644 --- a/src/database/network-table.c +++ b/src/database/network-table.c @@ -2151,7 +2151,7 @@ char *__attribute__((malloc)) getNameFromMAC(const char *client) if(FTLDBerror()) return NULL; - log_info("Looking up host name for %s", client); + log_debug(DEBUG_DATABASE,"Looking up host name for %s", client); // Open pihole-FTL.db database file sqlite3 *db = NULL; @@ -2161,7 +2161,6 @@ char *__attribute__((malloc)) getNameFromMAC(const char *client) return NULL; } - // Nothing found for the same device // Check for a host name associated with the given client as MAC address // COLLATE NOCASE: Case-insensitive comparison const char *querystr = "SELECT name FROM network_addresses " From c341b73d0f30623c9c34926223523266e984bcf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 22 Nov 2023 13:50:34 +0100 Subject: [PATCH 099/221] Only log number of stored sessions not total number MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- src/database/session-table.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/database/session-table.c b/src/database/session-table.c index c84eb0976..d8b7d1906 100644 --- a/src/database/session-table.c +++ b/src/database/session-table.c @@ -198,8 +198,8 @@ bool backup_db_sessions(struct session *sessions, const uint16_t max_sessions) return false; } - log_info("Stored %u/%u API session%s in the database", - api_sessions, max_sessions, max_sessions == 1 ? "" : "s"); + log_info("Stored %u API session%s in the database", + api_sessions, api_sessions == 1 ? "" : "s"); // Close database connection dbclose(&db); From b6aa531bbb6ab29843c4f6b7d35ebb22924fb436 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 22 Nov 2023 21:25:11 +0100 Subject: [PATCH 100/221] Also monitor when file was moved to overwrite our watched file Signed-off-by: DL6ER --- src/config/inotify.c | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/src/config/inotify.c b/src/config/inotify.c index e02d2f047..177673509 100644 --- a/src/config/inotify.c +++ b/src/config/inotify.c @@ -106,18 +106,38 @@ bool check_inotify_event(void) // Check if this is the event we are looking for if(event->mask & IN_CLOSE_WRITE) { + // File opened for writing was closed log_debug(DEBUG_INOTIFY, "File written: "WATCHDIR"/%s", event->name); if(strcmp(event->name, "pihole.toml") == 0) config_changed = true; } else if(event->mask & IN_CREATE) + { + // File was created log_debug(DEBUG_INOTIFY, "File created: "WATCHDIR"/%s", event->name); - else if(event->mask & IN_MOVE) - log_debug(DEBUG_INOTIFY, "File moved: "WATCHDIR"/%s", event->name); + } + else if(event->mask & IN_MOVED_FROM) + { + // File was moved (source) + log_debug(DEBUG_INOTIFY, "File moved from: "WATCHDIR"/%s", event->name); + } + else if(event->mask & IN_MOVED_TO) + { + // File was moved (target) + log_debug(DEBUG_INOTIFY, "File moved to: "WATCHDIR"/%s", event->name); + if(strcmp(event->name, "pihole.toml") == 0) + config_changed = true; + } else if(event->mask & IN_DELETE) + { + // File was deleted log_debug(DEBUG_INOTIFY, "File deleted: "WATCHDIR"/%s", event->name); + } else if(event->mask & IN_IGNORED) + { + // Watch descriptor was removed log_warn("Inotify watch descriptor for "WATCHDIR" was removed (directory deleted or unmounted?)"); + } else log_debug(DEBUG_INOTIFY, "Unknown event (%X) on watched file: "WATCHDIR"/%s", event->mask, event->name); } From 3f5fb989563ecb579a42de0b1a1b63f906a12010 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 22 Nov 2023 21:42:00 +0100 Subject: [PATCH 101/221] Update tests Signed-off-by: DL6ER --- test/test_suite.bats | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_suite.bats b/test/test_suite.bats index 0fa48c678..aca80a551 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -1575,7 +1575,7 @@ [[ ${lines[0]} == "3" ]] run bash -c 'grep -c "DEBUG_CONFIG: pihole.toml unchanged" /var/log/pihole/FTL.log' printf "%s\n" "${lines[@]}" - [[ ${lines[0]} == "3" ]] + [[ ${lines[0]} == "4" ]] run bash -c 'grep -c "DEBUG_CONFIG: Config file written to /etc/pihole/dnsmasq.conf" /var/log/pihole/FTL.log' printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "1" ]] @@ -1587,5 +1587,5 @@ [[ ${lines[0]} == "1" ]] run bash -c 'grep -c "DEBUG_CONFIG: custom.list unchanged" /var/log/pihole/FTL.log' printf "%s\n" "${lines[@]}" - [[ ${lines[0]} == "3" ]] + [[ ${lines[0]} == "4" ]] } From 3ea9ce2bbcd97fd3e5d60a2401d5b67c6632721a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 22 Nov 2023 21:47:17 +0100 Subject: [PATCH 102/221] Update embedded SQLite3 engine to version 3.44.1 Signed-off-by: DL6ER --- ...int-FTL-version-in-interactive-shell.patch | 6 +- src/database/shell.c | 3026 ++++++++++------- src/database/sqlite3.c | 255 +- src/database/sqlite3.h | 67 +- 4 files changed, 2091 insertions(+), 1263 deletions(-) diff --git a/patch/sqlite3/0001-print-FTL-version-in-interactive-shell.patch b/patch/sqlite3/0001-print-FTL-version-in-interactive-shell.patch index 5d096e216..1efac9684 100644 --- a/patch/sqlite3/0001-print-FTL-version-in-interactive-shell.patch +++ b/patch/sqlite3/0001-print-FTL-version-in-interactive-shell.patch @@ -25,6 +25,6 @@ index 6280ebf6..a5e82f70 100644 char *zHistory; int nHistory; + print_FTL_version(); - #if SHELL_WIN_UTF8_OPT - switch( console_utf8_in+2*console_utf8_out ){ - default: case 0: break; + #if CIO_WIN_WC_XLATE + # define SHELL_CIO_CHAR_SET (stdout_is_console? " (UTF-16 console I/O)" : "") + #else diff --git a/src/database/shell.c b/src/database/shell.c index 249050c28..8b63b8542 100644 --- a/src/database/shell.c +++ b/src/database/shell.c @@ -254,29 +254,1019 @@ typedef unsigned char u8; /* string conversion routines only needed on Win32 */ extern char *sqlite3_win32_unicode_to_utf8(LPCWSTR); -extern char *sqlite3_win32_mbcs_to_utf8_v2(const char *, int); -extern char *sqlite3_win32_utf8_to_mbcs_v2(const char *, int); extern LPWSTR sqlite3_win32_utf8_to_unicode(const char *zText); #endif -/* On Windows, we normally run with output mode of TEXT so that \n characters -** are automatically translated into \r\n. However, this behavior needs -** to be disabled in some cases (ex: when generating CSV output and when -** rendering quoted strings that contain \n characters). The following -** routines take care of that. +/* Use console I/O package as a direct INCLUDE. */ +#define SQLITE_INTERNAL_LINKAGE static + +#ifdef SQLITE_SHELL_FIDDLE +/* Deselect most features from the console I/O package for Fiddle. */ +# define SQLITE_CIO_NO_REDIRECT +# define SQLITE_CIO_NO_CLASSIFY +# define SQLITE_CIO_NO_TRANSLATE +# define SQLITE_CIO_NO_SETMODE +#endif +/************************* Begin ../ext/consio/console_io.h ******************/ +/* +** 2023 November 1 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +******************************************************************************** +** This file exposes various interfaces used for console and other I/O +** by the SQLite project command-line tools. These interfaces are used +** at either source conglomeration time, compilation time, or run time. +** This source provides for either inclusion into conglomerated, +** "single-source" forms or separate compilation then linking. +** +** Platform dependencies are "hidden" here by various stratagems so +** that, provided certain conditions are met, the programs using this +** source or object code compiled from it need no explicit conditional +** compilation in their source for their console and stream I/O. +** +** The symbols and functionality exposed here are not a public API. +** This code may change in tandem with other project code as needed. +** +** When this .h file and its companion .c are directly incorporated into +** a source conglomeration (such as shell.c), the preprocessor symbol +** CIO_WIN_WC_XLATE is defined as 0 or 1, reflecting whether console I/O +** translation for Windows is effected for the build. +*/ + +#ifndef SQLITE_INTERNAL_LINKAGE +# define SQLITE_INTERNAL_LINKAGE extern /* external to translation unit */ +# include +#else +# define SHELL_NO_SYSINC /* Better yet, modify mkshellc.tcl for this. */ +#endif + +#ifndef SQLITE3_H +/* # include "sqlite3.h" */ +#endif + +#ifndef SQLITE_CIO_NO_CLASSIFY + +/* Define enum for use with following function. */ +typedef enum StreamsAreConsole { + SAC_NoConsole = 0, + SAC_InConsole = 1, SAC_OutConsole = 2, SAC_ErrConsole = 4, + SAC_AnyConsole = 0x7 +} StreamsAreConsole; + +/* +** Classify the three standard I/O streams according to whether +** they are connected to a console attached to the process. +** +** Returns the bit-wise OR of SAC_{In,Out,Err}Console values, +** or SAC_NoConsole if none of the streams reaches a console. +** +** This function should be called before any I/O is done with +** the given streams. As a side-effect, the given inputs are +** recorded so that later I/O operations on them may be done +** differently than the C library FILE* I/O would be done, +** iff the stream is used for the I/O functions that follow, +** and to support the ones that use an implicit stream. +** +** On some platforms, stream or console mode alteration (aka +** "Setup") may be made which is undone by consoleRestore(). +*/ +SQLITE_INTERNAL_LINKAGE StreamsAreConsole +consoleClassifySetup( FILE *pfIn, FILE *pfOut, FILE *pfErr ); +/* A usual call for convenience: */ +#define SQLITE_STD_CONSOLE_INIT() consoleClassifySetup(stdin,stdout,stderr) + +/* +** After an initial call to consoleClassifySetup(...), renew +** the same setup it effected. (A call not after is an error.) +** This will restore state altered by consoleRestore(); +** +** Applications which run an inferior (child) process which +** inherits the same I/O streams may call this function after +** such a process exits to guard against console mode changes. +*/ +SQLITE_INTERNAL_LINKAGE void consoleRenewSetup(void); + +/* +** Undo any side-effects left by consoleClassifySetup(...). +** +** This should be called after consoleClassifySetup() and +** before the process terminates normally. It is suitable +** for use with the atexit() C library procedure. After +** this call, no console I/O should be done until one of +** console{Classify or Renew}Setup(...) is called again. +** +** Applications which run an inferior (child) process that +** inherits the same I/O streams might call this procedure +** before so that said process will have a console setup +** however users have configured it or come to expect. +*/ +SQLITE_INTERNAL_LINKAGE void SQLITE_CDECL consoleRestore( void ); + +#else /* defined(SQLITE_CIO_NO_CLASSIFY) */ +# define consoleClassifySetup(i,o,e) +# define consoleRenewSetup() +# define consoleRestore() +#endif /* defined(SQLITE_CIO_NO_CLASSIFY) */ + +#ifndef SQLITE_CIO_NO_REDIRECT +/* +** Set stream to be used for the functions below which write +** to "the designated X stream", where X is Output or Error. +** Returns the previous value. +** +** Alternatively, pass the special value, invalidFileStream, +** to get the designated stream value without setting it. +** +** Before the designated streams are set, they default to +** those passed to consoleClassifySetup(...), and before +** that is called they default to stdout and stderr. +** +** It is error to close a stream so designated, then, without +** designating another, use the corresponding {o,e}Emit(...). +*/ +SQLITE_INTERNAL_LINKAGE FILE *invalidFileStream; +SQLITE_INTERNAL_LINKAGE FILE *setOutputStream(FILE *pf); +# ifdef CONSIO_SET_ERROR_STREAM +SQLITE_INTERNAL_LINKAGE FILE *setErrorStream(FILE *pf); +# endif +#else +# define setOutputStream(pf) +# define setErrorStream(pf) +#endif /* !defined(SQLITE_CIO_NO_REDIRECT) */ + +#ifndef SQLITE_CIO_NO_TRANSLATE +/* +** Emit output like fprintf(). If the output is going to the +** console and translation from UTF-8 is necessary, perform +** the needed translation. Otherwise, write formatted output +** to the provided stream almost as-is, possibly with newline +** translation as specified by set{Binary,Text}Mode(). +*/ +SQLITE_INTERNAL_LINKAGE int fPrintfUtf8(FILE *pfO, const char *zFormat, ...); +/* Like fPrintfUtf8 except stream is always the designated output. */ +SQLITE_INTERNAL_LINKAGE int oPrintfUtf8(const char *zFormat, ...); +/* Like fPrintfUtf8 except stream is always the designated error. */ +SQLITE_INTERNAL_LINKAGE int ePrintfUtf8(const char *zFormat, ...); + +/* +** Emit output like fputs(). If the output is going to the +** console and translation from UTF-8 is necessary, perform +** the needed translation. Otherwise, write given text to the +** provided stream almost as-is, possibly with newline +** translation as specified by set{Binary,Text}Mode(). +*/ +SQLITE_INTERNAL_LINKAGE int fPutsUtf8(const char *z, FILE *pfO); +/* Like fPutsUtf8 except stream is always the designated output. */ +SQLITE_INTERNAL_LINKAGE int oPutsUtf8(const char *z); +/* Like fPutsUtf8 except stream is always the designated error. */ +SQLITE_INTERNAL_LINKAGE int ePutsUtf8(const char *z); + +/* +** Emit output like fPutsUtf8(), except that the length of the +** accepted char or character sequence is limited by nAccept. +** +** Returns the number of accepted char values. +*/ +#ifdef CONSIO_SPUTB +SQLITE_INTERNAL_LINKAGE int +fPutbUtf8(FILE *pfOut, const char *cBuf, int nAccept); +#endif +/* Like fPutbUtf8 except stream is always the designated output. */ +SQLITE_INTERNAL_LINKAGE int +oPutbUtf8(const char *cBuf, int nAccept); +/* Like fPutbUtf8 except stream is always the designated error. */ +#ifdef CONSIO_EPUTB +SQLITE_INTERNAL_LINKAGE int +ePutbUtf8(const char *cBuf, int nAccept); +#endif + +/* +** Collect input like fgets(...) with special provisions for input +** from the console on platforms that require same. Defers to the +** C library fgets() when input is not from the console. Newline +** translation may be done as set by set{Binary,Text}Mode(). As a +** convenience, pfIn==NULL is treated as stdin. +*/ +SQLITE_INTERNAL_LINKAGE char* fGetsUtf8(char *cBuf, int ncMax, FILE *pfIn); +/* Like fGetsUtf8 except stream is always the designated input. */ +/* SQLITE_INTERNAL_LINKAGE char* iGetsUtf8(char *cBuf, int ncMax); */ + +#endif /* !defined(SQLITE_CIO_NO_TRANSLATE) */ + +#ifndef SQLITE_CIO_NO_SETMODE +/* +** Set given stream for binary mode, where newline translation is +** not done, or for text mode where, for some platforms, newlines +** are translated to the platform's conventional char sequence. +** If bFlush true, flush the stream. +** +** An additional side-effect is that if the stream is one passed +** to consoleClassifySetup() as an output, it is flushed first. +** +** Note that binary/text mode has no effect on console I/O +** translation. On all platforms, newline to the console starts +** a new line and CR,LF chars from the console become a newline. +*/ +SQLITE_INTERNAL_LINKAGE void setBinaryMode(FILE *, short bFlush); +SQLITE_INTERNAL_LINKAGE void setTextMode(FILE *, short bFlush); +#endif + +#ifdef SQLITE_CIO_PROMPTED_IN +typedef struct Prompts { + int numPrompts; + const char **azPrompts; +} Prompts; + +/* +** Macros for use of a line editor. +** +** The following macros define operations involving use of a +** line-editing library or simple console interaction. +** A "T" argument is a text (char *) buffer or filename. +** A "N" argument is an integer. +** +** SHELL_ADD_HISTORY(T) // Record text as line(s) of history. +** SHELL_READ_HISTORY(T) // Read history from file named by T. +** SHELL_WRITE_HISTORY(T) // Write history to file named by T. +** SHELL_STIFLE_HISTORY(N) // Limit history to N entries. +** +** A console program which does interactive console input is +** expected to call: +** SHELL_READ_HISTORY(T) before collecting such input; +** SHELL_ADD_HISTORY(T) as record-worthy input is taken; +** SHELL_STIFLE_HISTORY(N) after console input ceases; then +** SHELL_WRITE_HISTORY(T) before the program exits. +*/ + +/* +** Retrieve a single line of input text from an input stream. +** +** If pfIn is the input stream passed to consoleClassifySetup(), +** and azPrompt is not NULL, then a prompt is issued before the +** line is collected, as selected by the isContinuation flag. +** Array azPrompt[{0,1}] holds the {main,continuation} prompt. +** +** If zBufPrior is not NULL then it is a buffer from a prior +** call to this routine that can be reused, or will be freed. +** +** The result is stored in space obtained from malloc() and +** must either be freed by the caller or else passed back to +** this function as zBufPrior for reuse. +** +** This function may call upon services of a line-editing +** library to interactively collect line edited input. +*/ +SQLITE_INTERNAL_LINKAGE char * +shellGetLine(FILE *pfIn, char *zBufPrior, int nLen, + short isContinuation, Prompts azPrompt); +#endif /* defined(SQLITE_CIO_PROMPTED_IN) */ +/* +** TBD: Define an interface for application(s) to generate +** completion candidates for use by the line-editor. +** +** This may be premature; the CLI is the only application +** that does this. Yet, getting line-editing melded into +** console I/O is desirable because a line-editing library +** may have to establish console operating mode, possibly +** in a way that interferes with the above functionality. +*/ + +#if !(defined(SQLITE_CIO_NO_UTF8SCAN)&&defined(SQLITE_CIO_NO_TRANSLATE)) +/* Skip over as much z[] input char sequence as is valid UTF-8, +** limited per nAccept char's or whole characters and containing +** no char cn such that ((1<=0 => char count, nAccept<0 => character + */ +SQLITE_INTERNAL_LINKAGE const char* +zSkipValidUtf8(const char *z, int nAccept, long ccm); + +#endif + +/************************* End ../ext/consio/console_io.h ********************/ +/************************* Begin ../ext/consio/console_io.c ******************/ +/* +** 2023 November 4 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +******************************************************************************** +** This file implements various interfaces used for console and stream I/O +** by the SQLite project command-line tools, as explained in console_io.h . +** Functions prefixed by "SQLITE_INTERNAL_LINKAGE" behave as described there. +*/ + +#ifndef SQLITE_CDECL +# define SQLITE_CDECL +#endif + +#ifndef SHELL_NO_SYSINC +# include +# include +# include +# include +# include +# include "console_io.h" +/* # include "sqlite3.h" */ +#endif + +#ifndef SQLITE_CIO_NO_TRANSLATE +# if (defined(_WIN32) || defined(WIN32)) && !SQLITE_OS_WINRT +# ifndef SHELL_NO_SYSINC +# include +# include +# undef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# include +# endif +# define CIO_WIN_WC_XLATE 1 /* Use WCHAR Windows APIs for console I/O */ +# else +# ifndef SHELL_NO_SYSINC +# include +# endif +# define CIO_WIN_WC_XLATE 0 /* Use plain C library stream I/O at console */ +# endif +#else +# define CIO_WIN_WC_XLATE 0 /* Not exposing translation routines at all */ +#endif + +#if CIO_WIN_WC_XLATE +/* Character used to represent a known-incomplete UTF-8 char group (�) */ +static WCHAR cBadGroup = 0xfffd; +#endif + +#if CIO_WIN_WC_XLATE +static HANDLE handleOfFile(FILE *pf){ + int fileDesc = _fileno(pf); + union { intptr_t osfh; HANDLE fh; } fid = { + (fileDesc>=0)? _get_osfhandle(fileDesc) : (intptr_t)INVALID_HANDLE_VALUE + }; + return fid.fh; +} +#endif + +#ifndef SQLITE_CIO_NO_TRANSLATE +typedef struct PerStreamTags { +# if CIO_WIN_WC_XLATE + HANDLE hx; + DWORD consMode; + char acIncomplete[4]; +# else + short reachesConsole; +# endif + FILE *pf; +} PerStreamTags; + +/* Define NULL-like value for things which can validly be 0. */ +# define SHELL_INVALID_FILE_PTR ((FILE *)~0) +# if CIO_WIN_WC_XLATE +# define SHELL_INVALID_CONS_MODE 0xFFFF0000 +# endif + +# if CIO_WIN_WC_XLATE +# define PST_INITIALIZER { INVALID_HANDLE_VALUE, SHELL_INVALID_CONS_MODE, \ + {0,0,0,0}, SHELL_INVALID_FILE_PTR } +# else +# define PST_INITIALIZER { 0, SHELL_INVALID_FILE_PTR } +# endif + +/* Quickly say whether a known output is going to the console. */ +# if CIO_WIN_WC_XLATE +static short pstReachesConsole(PerStreamTags *ppst){ + return (ppst->hx != INVALID_HANDLE_VALUE); +} +# else +# define pstReachesConsole(ppst) 0 +# endif + +# if CIO_WIN_WC_XLATE +static void restoreConsoleArb(PerStreamTags *ppst){ + if( pstReachesConsole(ppst) ) SetConsoleMode(ppst->hx, ppst->consMode); +} +# else +# define restoreConsoleArb(ppst) +# endif + +/* Say whether FILE* appears to be a console, collect associated info. */ +static short streamOfConsole(FILE *pf, /* out */ PerStreamTags *ppst){ +# if CIO_WIN_WC_XLATE + short rv = 0; + DWORD dwCM = SHELL_INVALID_CONS_MODE; + HANDLE fh = handleOfFile(pf); + ppst->pf = pf; + if( INVALID_HANDLE_VALUE != fh ){ + rv = (GetFileType(fh) == FILE_TYPE_CHAR && GetConsoleMode(fh,&dwCM)); + } + ppst->hx = (rv)? fh : INVALID_HANDLE_VALUE; + ppst->consMode = dwCM; + return rv; +# else + ppst->pf = pf; + ppst->reachesConsole = ( (short)isatty(fileno(pf)) ); + return ppst->reachesConsole; +# endif +} + +# if CIO_WIN_WC_XLATE +/* Define console modes for use with the Windows Console API. */ +# define SHELL_CONI_MODE \ + (ENABLE_ECHO_INPUT | ENABLE_INSERT_MODE | ENABLE_LINE_INPUT | 0x80 \ + | ENABLE_QUICK_EDIT_MODE | ENABLE_EXTENDED_FLAGS | ENABLE_PROCESSED_INPUT) +# define SHELL_CONO_MODE (ENABLE_PROCESSED_OUTPUT | ENABLE_WRAP_AT_EOL_OUTPUT \ + | ENABLE_VIRTUAL_TERMINAL_PROCESSING) +# endif + +typedef struct ConsoleInfo { + PerStreamTags pstSetup[3]; + PerStreamTags pstDesignated[3]; + StreamsAreConsole sacSetup; +} ConsoleInfo; + +static short isValidStreamInfo(PerStreamTags *ppst){ + return (ppst->pf != SHELL_INVALID_FILE_PTR); +} + +static ConsoleInfo consoleInfo = { + { /* pstSetup */ PST_INITIALIZER, PST_INITIALIZER, PST_INITIALIZER }, + { /* pstDesignated[] */ PST_INITIALIZER, PST_INITIALIZER, PST_INITIALIZER }, + SAC_NoConsole /* sacSetup */ +}; + +SQLITE_INTERNAL_LINKAGE FILE* invalidFileStream = (FILE *)~0; + +# if CIO_WIN_WC_XLATE +static void maybeSetupAsConsole(PerStreamTags *ppst, short odir){ + if( pstReachesConsole(ppst) ){ + DWORD cm = odir? SHELL_CONO_MODE : SHELL_CONI_MODE; + SetConsoleMode(ppst->hx, cm); + } +} +# else +# define maybeSetupAsConsole(ppst,odir) +# endif + +SQLITE_INTERNAL_LINKAGE void consoleRenewSetup(void){ +# if CIO_WIN_WC_XLATE + int ix = 0; + while( ix < 6 ){ + PerStreamTags *ppst = (ix<3)? + &consoleInfo.pstSetup[ix] : &consoleInfo.pstDesignated[ix-3]; + maybeSetupAsConsole(ppst, (ix % 3)>0); + ++ix; + } +# endif +} + +SQLITE_INTERNAL_LINKAGE StreamsAreConsole +consoleClassifySetup( FILE *pfIn, FILE *pfOut, FILE *pfErr ){ + StreamsAreConsole rv = SAC_NoConsole; + FILE* apf[3] = { pfIn, pfOut, pfErr }; + int ix; + for( ix = 2; ix >= 0; --ix ){ + PerStreamTags *ppst = &consoleInfo.pstSetup[ix]; + if( streamOfConsole(apf[ix], ppst) ){ + rv |= (SAC_InConsole< 0 ) fflush(apf[ix]); + } + consoleInfo.sacSetup = rv; + consoleRenewSetup(); + return rv; +} + +SQLITE_INTERNAL_LINKAGE void SQLITE_CDECL consoleRestore( void ){ +# if CIO_WIN_WC_XLATE + static ConsoleInfo *pci = &consoleInfo; + if( pci->sacSetup ){ + int ix; + for( ix=0; ix<3; ++ix ){ + if( pci->sacSetup & (SAC_InConsole<pstSetup[ix]; + SetConsoleMode(ppst->hx, ppst->consMode); + } + } + } +# endif +} +#endif /* !defined(SQLITE_CIO_NO_TRANSLATE) */ + +#ifdef SQLITE_CIO_INPUT_REDIR +/* Say whether given FILE* is among those known, via either +** consoleClassifySetup() or set{Output,Error}Stream, as +** readable, and return an associated PerStreamTags pointer +** if so. Otherwise, return 0. */ -#if (defined(_WIN32) || defined(WIN32)) && !SQLITE_OS_WINRT -static void setBinaryMode(FILE *file, int isOutput){ - if( isOutput ) fflush(file); - _setmode(_fileno(file), _O_BINARY); +static PerStreamTags * isKnownReadable(FILE *pf){ + static PerStreamTags *apst[] = { + &consoleInfo.pstDesignated[0], &consoleInfo.pstSetup[0], 0 + }; + int ix = 0; + do { + if( apst[ix]->pf == pf ) break; + } while( apst[++ix] != 0 ); + return apst[ix]; +} +#endif + +#ifndef SQLITE_CIO_NO_TRANSLATE +/* Say whether given FILE* is among those known, via either +** consoleClassifySetup() or set{Output,Error}Stream, as +** writable, and return an associated PerStreamTags pointer +** if so. Otherwise, return 0. +*/ +static PerStreamTags * isKnownWritable(FILE *pf){ + static PerStreamTags *apst[] = { + &consoleInfo.pstDesignated[1], &consoleInfo.pstDesignated[2], + &consoleInfo.pstSetup[1], &consoleInfo.pstSetup[2], 0 + }; + int ix = 0; + do { + if( apst[ix]->pf == pf ) break; + } while( apst[++ix] != 0 ); + return apst[ix]; +} + +static FILE *designateEmitStream(FILE *pf, unsigned chix){ + FILE *rv = consoleInfo.pstDesignated[chix].pf; + if( pf == invalidFileStream ) return rv; + else{ + /* Setting a possibly new output stream. */ + PerStreamTags *ppst = isKnownWritable(pf); + if( ppst != 0 ){ + PerStreamTags pst = *ppst; + consoleInfo.pstDesignated[chix] = pst; + }else streamOfConsole(pf, &consoleInfo.pstDesignated[chix]); + } + return rv; +} + +SQLITE_INTERNAL_LINKAGE FILE *setOutputStream(FILE *pf){ + return designateEmitStream(pf, 1); +} +# ifdef CONSIO_SET_ERROR_STREAM +SQLITE_INTERNAL_LINKAGE FILE *setErrorStream(FILE *pf){ + return designateEmitStream(pf, 2); +} +# endif +#endif /* !defined(SQLITE_CIO_NO_TRANSLATE) */ + +#ifndef SQLITE_CIO_NO_SETMODE +# if CIO_WIN_WC_XLATE +static void setModeFlushQ(FILE *pf, short bFlush, int mode){ + if( bFlush ) fflush(pf); + _setmode(_fileno(pf), mode); +} +# else +# define setModeFlushQ(f, b, m) if(b) fflush(f) +# endif + +SQLITE_INTERNAL_LINKAGE void setBinaryMode(FILE *pf, short bFlush){ + setModeFlushQ(pf, bFlush, _O_BINARY); +} +SQLITE_INTERNAL_LINKAGE void setTextMode(FILE *pf, short bFlush){ + setModeFlushQ(pf, bFlush, _O_TEXT); +} +# undef setModeFlushQ + +#else /* defined(SQLITE_CIO_NO_SETMODE) */ +# define setBinaryMode(f, bFlush) do{ if((bFlush)) fflush(f); }while(0) +# define setTextMode(f, bFlush) do{ if((bFlush)) fflush(f); }while(0) +#endif /* defined(SQLITE_CIO_NO_SETMODE) */ + +#ifndef SQLITE_CIO_NO_TRANSLATE +# if CIO_WIN_WC_XLATE +/* Write buffer cBuf as output to stream known to reach console, +** limited to ncTake char's. Return ncTake on success, else 0. */ +static int conZstrEmit(PerStreamTags *ppst, const char *z, int ncTake){ + int rv = 0; + if( z!=NULL ){ + int nwc = MultiByteToWideChar(CP_UTF8,0, z,ncTake, 0,0); + if( nwc > 0 ){ + WCHAR *zw = sqlite3_malloc64(nwc*sizeof(WCHAR)); + if( zw!=NULL ){ + nwc = MultiByteToWideChar(CP_UTF8,0, z,ncTake, zw,nwc); + if( nwc > 0 ){ + /* Translation from UTF-8 to UTF-16, then WCHARs out. */ + if( WriteConsoleW(ppst->hx, zw,nwc, 0, NULL) ){ + rv = ncTake; + } + } + sqlite3_free(zw); + } + } + } + return rv; +} + +/* For {f,o,e}PrintfUtf8() when stream is known to reach console. */ +static int conioVmPrintf(PerStreamTags *ppst, const char *zFormat, va_list ap){ + char *z = sqlite3_vmprintf(zFormat, ap); + if( z ){ + int rv = conZstrEmit(ppst, z, (int)strlen(z)); + sqlite3_free(z); + return rv; + }else return 0; +} +# endif /* CIO_WIN_WC_XLATE */ + +# ifdef CONSIO_GET_EMIT_STREAM +static PerStreamTags * getDesignatedEmitStream(FILE *pf, unsigned chix, + PerStreamTags *ppst){ + PerStreamTags *rv = isKnownWritable(pf); + short isValid = (rv!=0)? isValidStreamInfo(rv) : 0; + if( rv != 0 && isValid ) return rv; + streamOfConsole(pf, ppst); + return ppst; +} +# endif + +/* Get stream info, either for designated output or error stream when +** chix equals 1 or 2, or for an arbitrary stream when chix == 0. +** In either case, ppst references a caller-owned PerStreamTags +** struct which may be filled in if none of the known writable +** streams is being held by consoleInfo. The ppf parameter is an +** output when chix!=0 and an input when chix==0. + */ +static PerStreamTags * +getEmitStreamInfo(unsigned chix, PerStreamTags *ppst, + /* in/out */ FILE **ppf){ + PerStreamTags *ppstTry; + FILE *pfEmit; + if( chix > 0 ){ + ppstTry = &consoleInfo.pstDesignated[chix]; + if( !isValidStreamInfo(ppstTry) ){ + ppstTry = &consoleInfo.pstSetup[chix]; + pfEmit = ppst->pf; + }else pfEmit = ppstTry->pf; + if( !isValidStreamInfo(ppst) ){ + pfEmit = (chix > 1)? stderr : stdout; + ppstTry = ppst; + streamOfConsole(pfEmit, ppstTry); + } + *ppf = pfEmit; + }else{ + ppstTry = isKnownWritable(*ppf); + if( ppstTry != 0 ) return ppstTry; + streamOfConsole(*ppf, ppst); + return ppst; + } + return ppstTry; +} + +SQLITE_INTERNAL_LINKAGE int oPrintfUtf8(const char *zFormat, ...){ + va_list ap; + int rv; + FILE *pfOut; + PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */ +# if CIO_WIN_WC_XLATE + PerStreamTags *ppst = getEmitStreamInfo(1, &pst, &pfOut); +# else + getEmitStreamInfo(1, &pst, &pfOut); +# endif + assert(zFormat!=0); + va_start(ap, zFormat); +# if CIO_WIN_WC_XLATE + if( pstReachesConsole(ppst) ){ + rv = conioVmPrintf(ppst, zFormat, ap); + }else{ +# endif + rv = vfprintf(pfOut, zFormat, ap); +# if CIO_WIN_WC_XLATE + } +# endif + va_end(ap); + return rv; +} + +SQLITE_INTERNAL_LINKAGE int ePrintfUtf8(const char *zFormat, ...){ + va_list ap; + int rv; + FILE *pfErr; + PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */ +# if CIO_WIN_WC_XLATE + PerStreamTags *ppst = getEmitStreamInfo(2, &pst, &pfErr); +# else + getEmitStreamInfo(2, &pst, &pfErr); +# endif + assert(zFormat!=0); + va_start(ap, zFormat); +# if CIO_WIN_WC_XLATE + if( pstReachesConsole(ppst) ){ + rv = conioVmPrintf(ppst, zFormat, ap); + }else{ +# endif + rv = vfprintf(pfErr, zFormat, ap); +# if CIO_WIN_WC_XLATE + } +# endif + va_end(ap); + return rv; +} + +SQLITE_INTERNAL_LINKAGE int fPrintfUtf8(FILE *pfO, const char *zFormat, ...){ + va_list ap; + int rv; + PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */ +# if CIO_WIN_WC_XLATE + PerStreamTags *ppst = getEmitStreamInfo(0, &pst, &pfO); +# else + getEmitStreamInfo(0, &pst, &pfO); +# endif + assert(zFormat!=0); + va_start(ap, zFormat); +# if CIO_WIN_WC_XLATE + if( pstReachesConsole(ppst) ){ + maybeSetupAsConsole(ppst, 1); + rv = conioVmPrintf(ppst, zFormat, ap); + if( 0 == isKnownWritable(ppst->pf) ) restoreConsoleArb(ppst); + }else{ +# endif + rv = vfprintf(pfO, zFormat, ap); +# if CIO_WIN_WC_XLATE + } +# endif + va_end(ap); + return rv; +} + +SQLITE_INTERNAL_LINKAGE int fPutsUtf8(const char *z, FILE *pfO){ + PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */ +# if CIO_WIN_WC_XLATE + PerStreamTags *ppst = getEmitStreamInfo(0, &pst, &pfO); +# else + getEmitStreamInfo(0, &pst, &pfO); +# endif + assert(z!=0); +# if CIO_WIN_WC_XLATE + if( pstReachesConsole(ppst) ){ + int rv; + maybeSetupAsConsole(ppst, 1); + rv = conZstrEmit(ppst, z, (int)strlen(z)); + if( 0 == isKnownWritable(ppst->pf) ) restoreConsoleArb(ppst); + return rv; + }else { +# endif + return (fputs(z, pfO)<0)? 0 : (int)strlen(z); +# if CIO_WIN_WC_XLATE + } +# endif } -static void setTextMode(FILE *file, int isOutput){ - if( isOutput ) fflush(file); - _setmode(_fileno(file), _O_TEXT); + +SQLITE_INTERNAL_LINKAGE int ePutsUtf8(const char *z){ + FILE *pfErr; + PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */ +# if CIO_WIN_WC_XLATE + PerStreamTags *ppst = getEmitStreamInfo(2, &pst, &pfErr); +# else + getEmitStreamInfo(2, &pst, &pfErr); +# endif + assert(z!=0); +# if CIO_WIN_WC_XLATE + if( pstReachesConsole(ppst) ) return conZstrEmit(ppst, z, (int)strlen(z)); + else { +# endif + return (fputs(z, pfErr)<0)? 0 : (int)strlen(z); +# if CIO_WIN_WC_XLATE + } +# endif +} + +SQLITE_INTERNAL_LINKAGE int oPutsUtf8(const char *z){ + FILE *pfOut; + PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */ +# if CIO_WIN_WC_XLATE + PerStreamTags *ppst = getEmitStreamInfo(1, &pst, &pfOut); +# else + getEmitStreamInfo(1, &pst, &pfOut); +# endif + assert(z!=0); +# if CIO_WIN_WC_XLATE + if( pstReachesConsole(ppst) ) return conZstrEmit(ppst, z, (int)strlen(z)); + else { +# endif + return (fputs(z, pfOut)<0)? 0 : (int)strlen(z); +# if CIO_WIN_WC_XLATE + } +# endif } + +#endif /* !defined(SQLITE_CIO_NO_TRANSLATE) */ + +#if !(defined(SQLITE_CIO_NO_UTF8SCAN) && defined(SQLITE_CIO_NO_TRANSLATE)) +/* Skip over as much z[] input char sequence as is valid UTF-8, +** limited per nAccept char's or whole characters and containing +** no char cn such that ((1<=0 => char count, nAccept<0 => character + */ +SQLITE_INTERNAL_LINKAGE const char* +zSkipValidUtf8(const char *z, int nAccept, long ccm){ + int ng = (nAccept<0)? -nAccept : 0; + const char *pcLimit = (nAccept>=0)? z+nAccept : 0; + assert(z!=0); + while( (pcLimit)? (z= pcLimit ) return z; + else{ + char ct = *zt++; + if( ct==0 || (zt-z)>4 || (ct & 0xC0)!=0x80 ){ + /* Trailing bytes are too few, too many, or invalid. */ + return z; + } + } + } while( ((c <<= 1) & 0x40) == 0x40 ); /* Eat lead byte's count. */ + z = zt; + } + } + return z; +} +#endif /*!(defined(SQLITE_CIO_NO_UTF8SCAN)&&defined(SQLITE_CIO_NO_TRANSLATE))*/ + +#ifndef SQLITE_CIO_NO_TRANSLATE + +#ifdef CONSIO_SPUTB +SQLITE_INTERNAL_LINKAGE int +fPutbUtf8(FILE *pfO, const char *cBuf, int nAccept){ + assert(pfO!=0); +# if CIO_WIN_WC_XLATE + PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */ + PerStreamTags *ppst = getEmitStreamInfo(0, &pst, &pfO); + if( pstReachesConsole(ppst) ){ + int rv; + maybeSetupAsConsole(ppst, 1); + rv = conZstrEmit(ppst, cBuf, nAccept); + if( 0 == isKnownWritable(ppst->pf) ) restoreConsoleArb(ppst); + return rv; + }else { +# endif + return (int)fwrite(cBuf, 1, nAccept, pfO); +# if CIO_WIN_WC_XLATE + } +# endif +} +#endif /* defined(CONSIO_SPUTB) */ + +SQLITE_INTERNAL_LINKAGE int +oPutbUtf8(const char *cBuf, int nAccept){ + FILE *pfOut; + PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */ +# if CIO_WIN_WC_XLATE + PerStreamTags *ppst = getEmitStreamInfo(1, &pst, &pfOut); +# else + getEmitStreamInfo(1, &pst, &pfOut); +# endif +# if CIO_WIN_WC_XLATE + if( pstReachesConsole(ppst) ){ + return conZstrEmit(ppst, cBuf, nAccept); + }else { +# endif + return (int)fwrite(cBuf, 1, nAccept, pfOut); +# if CIO_WIN_WC_XLATE + } +# endif +} + +# ifdef CONSIO_EPUTB +SQLITE_INTERNAL_LINKAGE int +ePutbUtf8(const char *cBuf, int nAccept){ + FILE *pfErr; + PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */ + PerStreamTags *ppst = getEmitStreamInfo(2, &pst, &pfErr); +# if CIO_WIN_WC_XLATE + if( pstReachesConsole(ppst) ){ + return conZstrEmit(ppst, cBuf, nAccept); + }else { +# endif + return (int)fwrite(cBuf, 1, nAccept, pfErr); +# if CIO_WIN_WC_XLATE + } +# endif +} +# endif /* defined(CONSIO_EPUTB) */ + +SQLITE_INTERNAL_LINKAGE char* fGetsUtf8(char *cBuf, int ncMax, FILE *pfIn){ + if( pfIn==0 ) pfIn = stdin; +# if CIO_WIN_WC_XLATE + if( pfIn == consoleInfo.pstSetup[0].pf + && (consoleInfo.sacSetup & SAC_InConsole)!=0 ){ +# if CIO_WIN_WC_XLATE==1 +# define SHELL_GULP 150 /* Count of WCHARS to be gulped at a time */ + WCHAR wcBuf[SHELL_GULP+1]; + int lend = 0, noc = 0; + if( ncMax > 0 ) cBuf[0] = 0; + while( noc < ncMax-8-1 && !lend ){ + /* There is room for at least 2 more characters and a 0-terminator. */ + int na = (ncMax > SHELL_GULP*4+1 + noc)? SHELL_GULP : (ncMax-1 - noc)/4; +# undef SHELL_GULP + DWORD nbr = 0; + BOOL bRC = ReadConsoleW(consoleInfo.pstSetup[0].hx, wcBuf, na, &nbr, 0); + if( bRC && nbr>0 && (wcBuf[nbr-1]&0xF800)==0xD800 ){ + /* Last WHAR read is first of a UTF-16 surrogate pair. Grab its mate. */ + DWORD nbrx; + bRC &= ReadConsoleW(consoleInfo.pstSetup[0].hx, wcBuf+nbr, 1, &nbrx, 0); + if( bRC ) nbr += nbrx; + } + if( !bRC || (noc==0 && nbr==0) ) return 0; + if( nbr > 0 ){ + int nmb = WideCharToMultiByte(CP_UTF8, 0, wcBuf,nbr,0,0,0,0); + if( nmb != 0 && noc+nmb <= ncMax ){ + int iseg = noc; + nmb = WideCharToMultiByte(CP_UTF8, 0, wcBuf,nbr,cBuf+noc,nmb,0,0); + noc += nmb; + /* Fixup line-ends as coded by Windows for CR (or "Enter".) + ** This is done without regard for any setMode{Text,Binary}() + ** call that might have been done on the interactive input. + */ + if( noc > 0 ){ + if( cBuf[noc-1]=='\n' ){ + lend = 1; + if( noc > 1 && cBuf[noc-2]=='\r' ) cBuf[--noc-1] = '\n'; + } + } + /* Check for ^Z (anywhere in line) too, to act as EOF. */ + while( iseg < noc ){ + if( cBuf[iseg]=='\x1a' ){ + noc = iseg; /* Chop ^Z and anything following. */ + lend = 1; /* Counts as end of line too. */ + break; + } + ++iseg; + } + }else break; /* Drop apparent garbage in. (Could assert.) */ + }else break; + } + /* If got nothing, (after ^Z chop), must be at end-of-file. */ + if( noc > 0 ){ + cBuf[noc] = 0; + return cBuf; + }else return 0; +# endif + }else{ +# endif + return fgets(cBuf, ncMax, pfIn); +# if CIO_WIN_WC_XLATE + } +# endif +} +#endif /* !defined(SQLITE_CIO_NO_TRANSLATE) */ + +#undef SHELL_INVALID_FILE_PTR + +/************************* End ../ext/consio/console_io.c ********************/ + +#ifndef SQLITE_SHELL_FIDDLE +/* From here onward, fgets() is redirected to the console_io library. */ +# define fgets(b,n,f) fGetsUtf8(b,n,f) +/* + * Define macros for emitting output text in various ways: + * sputz(s, z) => emit 0-terminated string z to given stream s + * sputf(s, f, ...) => emit varargs per format f to given stream s + * oputz(z) => emit 0-terminated string z to default stream + * oputf(f, ...) => emit varargs per format f to default stream + * eputz(z) => emit 0-terminated string z to error stream + * eputf(f, ...) => emit varargs per format f to error stream + * oputb(b, n) => emit char buffer b[0..n-1] to default stream + * + * Note that the default stream is whatever has been last set via: + * setOutputStream(FILE *pf) + * This is normally the stream that CLI normal output goes to. + * For the stand-alone CLI, it is stdout with no .output redirect. + */ +# define sputz(s,z) fPutsUtf8(z,s) +# define sputf fPrintfUtf8 +# define oputz(z) oPutsUtf8(z) +# define oputf oPrintfUtf8 +# define eputz(z) ePutsUtf8(z) +# define eputf ePrintfUtf8 +# define oputb(buf,na) oPutbUtf8(buf,na) #else -# define setBinaryMode(X,Y) -# define setTextMode(X,Y) +/* For Fiddle, all console handling and emit redirection is omitted. */ +# define sputz(fp,z) fputs(z,fp) +# define sputf(fp,fmt, ...) fprintf(fp,fmt,__VA_ARGS__) +# define oputz(z) fputs(z,stdout) +# define oputf(fmt, ...) printf(fmt,__VA_ARGS__) +# define eputz(z) fputs(z,stderr) +# define eputf(fmt, ...) fprintf(stderr,fmt,__VA_ARGS__) +# define oputb(buf,na) fwrite(buf,1,na,stdout) #endif /* True if the timer is enabled */ @@ -351,10 +1341,10 @@ static void endTimer(void){ sqlite3_int64 iEnd = timeOfDay(); struct rusage sEnd; getrusage(RUSAGE_SELF, &sEnd); - printf("Run Time: real %.3f user %f sys %f\n", - (iEnd - iBegin)*0.001, - timeDiff(&sBegin.ru_utime, &sEnd.ru_utime), - timeDiff(&sBegin.ru_stime, &sEnd.ru_stime)); + oputf("Run Time: real %.3f user %f sys %f\n", + (iEnd - iBegin)*0.001, + timeDiff(&sBegin.ru_utime, &sEnd.ru_utime), + timeDiff(&sBegin.ru_stime, &sEnd.ru_stime)); } } @@ -430,10 +1420,10 @@ static void endTimer(void){ FILETIME ftCreation, ftExit, ftKernelEnd, ftUserEnd; sqlite3_int64 ftWallEnd = timeOfDay(); getProcessTimesAddr(hProcess,&ftCreation,&ftExit,&ftKernelEnd,&ftUserEnd); - printf("Run Time: real %.3f user %f sys %f\n", - (ftWallEnd - ftWallBegin)*0.001, - timeDiff(&ftUserBegin, &ftUserEnd), - timeDiff(&ftKernelBegin, &ftKernelEnd)); + oputf("Run Time: real %.3f user %f sys %f\n", + (ftWallEnd - ftWallBegin)*0.001, + timeDiff(&ftUserBegin, &ftUserEnd), + timeDiff(&ftKernelBegin, &ftKernelEnd)); } } @@ -470,28 +1460,9 @@ static int bail_on_error = 0; static int stdin_is_interactive = 1; /* -** If build is for non-RT Windows, without 3rd-party line editing, -** console input and output may be done in a UTF-8 compatible way, -** if the OS is capable of it and the --no-utf8 option is not seen. -*/ -#if (defined(_WIN32) || defined(WIN32)) && SHELL_USE_LOCAL_GETLINE \ - && !defined(SHELL_OMIT_WIN_UTF8) && !SQLITE_OS_WINRT -# define SHELL_WIN_UTF8_OPT 1 -/* Record whether to do UTF-8 console I/O translation per stream. */ - static int console_utf8_in = 0; - static int console_utf8_out = 0; -/* Record whether can do UTF-8 or --no-utf8 seen in invocation. */ - static int mbcs_opted = 1; /* Assume cannot do until shown otherwise. */ -#else -# define console_utf8_in 0 -# define console_utf8_out 0 -# define SHELL_WIN_UTF8_OPT 0 -#endif - -/* -** On Windows systems we have to know if standard output is a console -** in order to translate UTF-8 into MBCS. The following variable is -** true if translation is required. +** On Windows systems we need to know if standard output is a console +** in order to show that UTF-16 translation is done in the sign-on +** banner. The following variable is true if it is the console. */ static int stdout_is_console = 1; @@ -613,251 +1584,17 @@ static char *dynamicContinuePrompt(void){ shell_strncpy(dynPrompt.dynamicPrompt, "(x.", 4); dynPrompt.dynamicPrompt[2] = (char)('0'+dynPrompt.inParenLevel); } - shell_strncpy(dynPrompt.dynamicPrompt+3, continuePrompt+3, PROMPT_LEN_MAX-4); + shell_strncpy(dynPrompt.dynamicPrompt+3, continuePrompt+3, + PROMPT_LEN_MAX-4); } } return dynPrompt.dynamicPrompt; } #endif /* !defined(SQLITE_OMIT_DYNAPROMPT) */ -#if SHELL_WIN_UTF8_OPT -/* Following struct is used for UTF-8 console I/O. */ -static struct ConsoleState { - int stdinEof; /* EOF has been seen on console input */ - int infsMode; /* Input file stream mode upon shell start */ - UINT inCodePage; /* Input code page upon shell start */ - UINT outCodePage; /* Output code page upon shell start */ - HANDLE hConsole; /* Console input or output handle */ - DWORD consoleMode; /* Console mode upon shell start */ -} conState = { 0, 0, 0, 0, INVALID_HANDLE_VALUE, 0 }; - -#ifndef _O_U16TEXT /* For build environments lacking this constant: */ -# define _O_U16TEXT 0x20000 -#endif - -/* -** If given stream number is a console, return 1 and get some attributes, -** else return 0 and set the output attributes to invalid values. -*/ -static short console_attrs(unsigned stnum, HANDLE *pH, DWORD *pConsMode){ - static int stid[3] = { STD_INPUT_HANDLE,STD_OUTPUT_HANDLE,STD_ERROR_HANDLE }; - HANDLE h; - *pH = INVALID_HANDLE_VALUE; - *pConsMode = 0; - if( stnum > 2 ) return 0; - h = GetStdHandle(stid[stnum]); - if( h!=*pH && GetFileType(h)==FILE_TYPE_CHAR && GetConsoleMode(h,pConsMode) ){ - *pH = h; - return 1; - } - return 0; -} - -/* -** Perform a runtime test of Windows console to determine if it can -** do char-stream I/O correctly when the code page is set to CP_UTF8. -** Returns are: 1 => yes it can, 0 => no it cannot -** -** The console's output code page is momentarily set, then restored. -** So this should only be run when the process is given use of the -** console for either input or output. -*/ -static short ConsoleDoesUTF8(void){ - UINT ocp = GetConsoleOutputCP(); - const char TrialUtf8[] = { '\xC8', '\xAB' }; /* "ȫ" or 2 MBCS characters */ - WCHAR aReadBack[1] = { 0 }; /* Read back as 0x022B when decoded as UTF-8. */ - CONSOLE_SCREEN_BUFFER_INFO csbInfo = {0}; - /* Create an inactive screen buffer with which to do the experiment. */ - HANDLE hCSB = CreateConsoleScreenBuffer(GENERIC_READ|GENERIC_WRITE, 0, 0, - CONSOLE_TEXTMODE_BUFFER, NULL); - if( hCSB!=INVALID_HANDLE_VALUE ){ - COORD cpos = {0,0}; - DWORD rbc; - SetConsoleCursorPosition(hCSB, cpos); - SetConsoleOutputCP(CP_UTF8); - /* Write 2 chars which are a single character in UTF-8 but more in MBCS. */ - WriteConsoleA(hCSB, TrialUtf8, sizeof(TrialUtf8), NULL, NULL); - ReadConsoleOutputCharacterW(hCSB, &aReadBack[0], 1, cpos, &rbc); - GetConsoleScreenBufferInfo(hCSB, &csbInfo); - SetConsoleOutputCP(ocp); - CloseHandle(hCSB); - } - /* Return 1 if cursor advanced by 1 position, else 0. */ - return (short)(csbInfo.dwCursorPosition.X == 1 && aReadBack[0] == 0x022B); -} - -static short in_console = 0; -static short out_console = 0; - -/* -** Determine whether either normal I/O stream is the console, -** and whether it can do UTF-8 translation, setting globals -** in_console, out_console and mbcs_opted accordingly. -*/ -static void probe_console(void){ - HANDLE h; - DWORD cMode; - in_console = console_attrs(0, &h, &cMode); - out_console = console_attrs(1, &h, &cMode); - if( in_console || out_console ) mbcs_opted = !ConsoleDoesUTF8(); -} - -/* -** If console is used for normal I/O, absent a --no-utf8 option, -** prepare console for UTF-8 input (from either typing or suitable -** paste operations) and/or for UTF-8 output rendering. -** -** The console state upon entry is preserved, in conState, so that -** console_restore() can later restore the same console state. -** -** The globals console_utf8_in and console_utf8_out are set, for -** later use in selecting UTF-8 or MBCS console I/O translations. -** This routine depends upon globals set by probe_console(). -*/ -static void console_prepare_utf8(void){ - struct ConsoleState csWork = { 0, 0, 0, 0, INVALID_HANDLE_VALUE, 0 }; - - console_utf8_in = console_utf8_out = 0; - if( (!in_console && !out_console) || mbcs_opted ) return; - console_attrs((in_console)? 0 : 1, &conState.hConsole, &conState.consoleMode); - conState.inCodePage = GetConsoleCP(); - conState.outCodePage = GetConsoleOutputCP(); - if( in_console ){ - SetConsoleCP(CP_UTF8); - DWORD newConsoleMode = conState.consoleMode - | ENABLE_LINE_INPUT | ENABLE_PROCESSED_INPUT; - SetConsoleMode(conState.hConsole, newConsoleMode); - conState.infsMode = _setmode(_fileno(stdin), _O_U16TEXT); - console_utf8_in = 1; - } - if( out_console ){ - SetConsoleOutputCP(CP_UTF8); - console_utf8_out = 1; - } -} - -/* -** Undo the effects of console_prepare_utf8(), if any. -*/ -static void SQLITE_CDECL console_restore(void){ - if( (console_utf8_in||console_utf8_out) - && conState.hConsole!=INVALID_HANDLE_VALUE ){ - if( console_utf8_in ){ - SetConsoleCP(conState.inCodePage); - _setmode(_fileno(stdin), conState.infsMode); - } - if( console_utf8_out ) SetConsoleOutputCP(conState.outCodePage); - SetConsoleMode(conState.hConsole, conState.consoleMode); - /* Avoid multiple calls. */ - conState.hConsole = INVALID_HANDLE_VALUE; - conState.consoleMode = 0; - console_utf8_in = 0; - console_utf8_out = 0; - } -} - -/* -** Collect input like fgets(...) with special provisions for input -** from the Windows console to get around its strange coding issues. -** Defers to plain fgets() when input is not interactive or when the -** UTF-8 input is unavailable or opted out. -*/ -static char* utf8_fgets(char *buf, int ncmax, FILE *fin){ - if( fin==0 ) fin = stdin; - if( fin==stdin && stdin_is_interactive && console_utf8_in ){ -# define SQLITE_IALIM 150 - wchar_t wbuf[SQLITE_IALIM]; - int lend = 0; - int noc = 0; - if( ncmax==0 || conState.stdinEof ) return 0; - buf[0] = 0; - while( noc SQLITE_IALIM*4+1 + noc) - ? SQLITE_IALIM : (ncmax-1 - noc)/4; -# undef SQLITE_IALIM - DWORD nbr = 0; - BOOL bRC = ReadConsoleW(conState.hConsole, wbuf, na, &nbr, 0); - if( !bRC || (noc==0 && nbr==0) ) return 0; - if( nbr > 0 ){ - int nmb = WideCharToMultiByte(CP_UTF8,WC_COMPOSITECHECK|WC_DEFAULTCHAR, - wbuf,nbr,0,0,0,0); - if( nmb !=0 && noc+nmb <= ncmax ){ - int iseg = noc; - nmb = WideCharToMultiByte(CP_UTF8,WC_COMPOSITECHECK|WC_DEFAULTCHAR, - wbuf,nbr,buf+noc,nmb,0,0); - noc += nmb; - /* Fixup line-ends as coded by Windows for CR (or "Enter".)*/ - if( noc > 0 ){ - if( buf[noc-1]=='\n' ){ - lend = 1; - if( noc > 1 && buf[noc-2]=='\r' ){ - buf[noc-2] = '\n'; - --noc; - } - } - } - /* Check for ^Z (anywhere in line) too. */ - while( iseg < noc ){ - if( buf[iseg]==0x1a ){ - conState.stdinEof = 1; - noc = iseg; /* Chop ^Z and anything following. */ - break; - } - ++iseg; - } - }else break; /* Drop apparent garbage in. (Could assert.) */ - }else break; - } - /* If got nothing, (after ^Z chop), must be at end-of-file. */ - if( noc == 0 ) return 0; - buf[noc] = 0; - return buf; - }else{ - return fgets(buf, ncmax, fin); - } -} - -# define fgets(b,n,f) utf8_fgets(b,n,f) -#endif /* SHELL_WIN_UTF8_OPT */ - -/* -** Render output like fprintf(). Except, if the output is going to the -** console and if this is running on a Windows machine, and if UTF-8 -** output unavailable (or available but opted out), translate the -** output from UTF-8 into MBCS for output through 8-bit stdout stream. -** (Without -no-utf8, no translation is needed and must not be done.) -*/ -#if defined(_WIN32) || defined(WIN32) -void utf8_printf(FILE *out, const char *zFormat, ...){ - va_list ap; - va_start(ap, zFormat); - if( stdout_is_console && (out==stdout || out==stderr) && !console_utf8_out ){ - char *z1 = sqlite3_vmprintf(zFormat, ap); - char *z2 = sqlite3_win32_utf8_to_mbcs_v2(z1, 0); - sqlite3_free(z1); - fputs(z2, out); - sqlite3_free(z2); - }else{ - vfprintf(out, zFormat, ap); - } - va_end(ap); -} -#elif !defined(utf8_printf) -# define utf8_printf fprintf -#endif - -/* -** Render output like fprintf(). This should not be used on anything that -** includes string formatting (e.g. "%s"). -*/ -#if !defined(raw_printf) -# define raw_printf fprintf -#endif - /* Indicate out-of-memory and exit. */ static void shell_out_of_memory(void){ - raw_printf(stderr,"Error: out of memory\n"); + eputz("Error: out of memory\n"); exit(1); } @@ -889,18 +1626,18 @@ static void SQLITE_CDECL iotracePrintf(const char *zFormat, ...){ va_start(ap, zFormat); z = sqlite3_vmprintf(zFormat, ap); va_end(ap); - utf8_printf(iotrace, "%s", z); + sputf(iotrace, "%s", z); sqlite3_free(z); } #endif /* -** Output string zUtf to stream pOut as w characters. If w is negative, +** Output string zUtf to Out stream as w characters. If w is negative, ** then right-justify the text. W is the width in UTF-8 characters, not ** in bytes. This is different from the %*.*s specification in printf ** since with %*.*s the width is measured in bytes, not characters. */ -static void utf8_width_print(FILE *pOut, int w, const char *zUtf){ +static void utf8_width_print(int w, const char *zUtf){ int i; int n; int aw = w<0 ? -w : w; @@ -915,11 +1652,11 @@ static void utf8_width_print(FILE *pOut, int w, const char *zUtf){ } } if( n>=aw ){ - utf8_printf(pOut, "%.*s", i, zUtf); + oputf("%.*s", i, zUtf); }else if( w<0 ){ - utf8_printf(pOut, "%*s%s", aw-n, "", zUtf); + oputf("%*s%s", aw-n, "", zUtf); }else{ - utf8_printf(pOut, "%s%*s", zUtf, aw-n, ""); + oputf("%s%*s", zUtf, aw-n, ""); } } @@ -979,7 +1716,7 @@ static int strlenChar(const char *z){ ** Otherwise return 0. */ static FILE * openChrSource(const char *zFile){ -#ifdef _WIN32 +#if defined(_WIN32) || defined(WIN32) struct _stat x = {0}; # define STAT_CHR_SRC(mode) ((mode & (_S_IFCHR|_S_IFIFO|_S_IFREG))!=0) /* On Windows, open first, then check the stream nature. This order @@ -1042,23 +1779,6 @@ static char *local_getline(char *zLine, FILE *in){ break; } } -#if defined(_WIN32) || defined(WIN32) - /* For interactive input on Windows systems, with -no-utf8, - ** translate the multi-byte characterset characters into UTF-8. - ** This is the translation that predates console UTF-8 input. */ - if( stdin_is_interactive && in==stdin && !console_utf8_in ){ - char *zTrans = sqlite3_win32_mbcs_to_utf8_v2(zLine, 0); - if( zTrans ){ - i64 nTrans = strlen(zTrans)+1; - if( nTrans>nLine ){ - zLine = realloc(zLine, nTrans); - shell_check_oom(zLine); - } - memcpy(zLine, zTrans, nTrans); - sqlite3_free(zTrans); - } - } -#endif /* defined(_WIN32) || defined(WIN32) */ return zLine; } @@ -1085,7 +1805,7 @@ static char *one_input_line(FILE *in, char *zPrior, int isContinuation){ }else{ zPrompt = isContinuation ? CONTINUATION_PROMPT : mainPrompt; #if SHELL_USE_LOCAL_GETLINE - printf("%s", zPrompt); + sputz(stdout, zPrompt); fflush(stdout); do{ zResult = local_getline(zPrior, stdin); @@ -1332,7 +2052,7 @@ static void shellDtostr( char z[400]; if( n<1 ) n = 1; if( n>350 ) n = 350; - snprintf(z, sizeof(z)-1, "%#+.*e", n, r); + sqlite3_snprintf(sizeof(z), z, "%#+.*e", n, r); sqlite3_result_text(pCtx, z, -1, SQLITE_TRANSIENT); } @@ -17618,7 +18338,7 @@ static const char *modeDescr[] = { static void shellLog(void *pArg, int iErrCode, const char *zMsg){ ShellState *p = (ShellState*)pArg; if( p->pLog==0 ) return; - utf8_printf(p->pLog, "(%d) %s\n", iErrCode, zMsg); + sputf(p->pLog, "(%d) %s\n", iErrCode, zMsg); fflush(p->pLog); } @@ -17633,9 +18353,9 @@ static void shellPutsFunc( int nVal, sqlite3_value **apVal ){ - ShellState *p = (ShellState*)sqlite3_user_data(pCtx); + /* Unused: (ShellState*)sqlite3_user_data(pCtx); */ (void)nVal; - utf8_printf(p->out, "%s\n", sqlite3_value_text(apVal[0])); + oputf("%s\n", sqlite3_value_text(apVal[0])); sqlite3_result_value(pCtx, apVal[0]); } @@ -17654,8 +18374,7 @@ static void failIfSafeMode( va_start(ap, zErrMsg); zMsg = sqlite3_vmprintf(zErrMsg, ap); va_end(ap); - raw_printf(stderr, "line %d: ", p->lineno); - utf8_printf(stderr, "%s\n", zMsg); + eputf("line %d: %s\n", p->lineno, zMsg); exit(1); } } @@ -17823,7 +18542,7 @@ static void outputModePop(ShellState *p){ /* ** Output the given string as a hex-encoded blob (eg. X'1234' ) */ -static void output_hex_blob(FILE *out, const void *pBlob, int nBlob){ +static void output_hex_blob(const void *pBlob, int nBlob){ int i; unsigned char *aBlob = (unsigned char*)pBlob; @@ -17840,7 +18559,7 @@ static void output_hex_blob(FILE *out, const void *pBlob, int nBlob){ } zStr[i*2] = '\0'; - raw_printf(out,"X'%s'", zStr); + oputf("X'%s'", zStr); sqlite3_free(zStr); } @@ -17870,25 +18589,28 @@ static const char *unused_string( ** ** See also: output_quoted_escaped_string() */ -static void output_quoted_string(FILE *out, const char *z){ +static void output_quoted_string(const char *z){ int i; char c; - setBinaryMode(out, 1); +#ifndef SQLITE_SHELL_FIDDLE + FILE *pfO = setOutputStream(invalidFileStream); + setBinaryMode(pfO, 1); +#endif if( z==0 ) return; for(i=0; (c = z[i])!=0 && c!='\''; i++){} if( c==0 ){ - utf8_printf(out,"'%s'",z); + oputf("'%s'",z); }else{ - raw_printf(out, "'"); + oputz("'"); while( *z ){ for(i=0; (c = z[i])!=0 && c!='\''; i++){} if( c=='\'' ) i++; if( i ){ - utf8_printf(out, "%.*s", i, z); + oputf("%.*s", i, z); z += i; } if( c=='\'' ){ - raw_printf(out, "'"); + oputz("'"); continue; } if( c==0 ){ @@ -17896,9 +18618,13 @@ static void output_quoted_string(FILE *out, const char *z){ } z++; } - raw_printf(out, "'"); + oputz("'"); } - setTextMode(out, 1); +#ifndef SQLITE_SHELL_FIDDLE + setTextMode(pfO, 1); +#else + setTextMode(stdout, 1); +#endif } /* @@ -17910,13 +18636,16 @@ static void output_quoted_string(FILE *out, const char *z){ ** This is like output_quoted_string() but with the addition of the \r\n ** escape mechanism. */ -static void output_quoted_escaped_string(FILE *out, const char *z){ +static void output_quoted_escaped_string(const char *z){ int i; char c; - setBinaryMode(out, 1); +#ifndef SQLITE_SHELL_FIDDLE + FILE *pfO = setOutputStream(invalidFileStream); + setBinaryMode(pfO, 1); +#endif for(i=0; (c = z[i])!=0 && c!='\'' && c!='\n' && c!='\r'; i++){} if( c==0 ){ - utf8_printf(out,"'%s'",z); + oputf("'%s'",z); }else{ const char *zNL = 0; const char *zCR = 0; @@ -17928,23 +18657,23 @@ static void output_quoted_escaped_string(FILE *out, const char *z){ if( z[i]=='\r' ) nCR++; } if( nNL ){ - raw_printf(out, "replace("); + oputz("replace("); zNL = unused_string(z, "\\n", "\\012", zBuf1); } if( nCR ){ - raw_printf(out, "replace("); + oputz("replace("); zCR = unused_string(z, "\\r", "\\015", zBuf2); } - raw_printf(out, "'"); + oputz("'"); while( *z ){ for(i=0; (c = z[i])!=0 && c!='\n' && c!='\r' && c!='\''; i++){} if( c=='\'' ) i++; if( i ){ - utf8_printf(out, "%.*s", i, z); + oputf("%.*s", i, z); z += i; } if( c=='\'' ){ - raw_printf(out, "'"); + oputz("'"); continue; } if( c==0 ){ @@ -17952,93 +18681,139 @@ static void output_quoted_escaped_string(FILE *out, const char *z){ } z++; if( c=='\n' ){ - raw_printf(out, "%s", zNL); + oputz(zNL); continue; } - raw_printf(out, "%s", zCR); + oputz(zCR); } - raw_printf(out, "'"); + oputz("'"); if( nCR ){ - raw_printf(out, ",'%s',char(13))", zCR); + oputf(",'%s',char(13))", zCR); } if( nNL ){ - raw_printf(out, ",'%s',char(10))", zNL); + oputf(",'%s',char(10))", zNL); } } - setTextMode(out, 1); +#ifndef SQLITE_SHELL_FIDDLE + setTextMode(pfO, 1); +#else + setTextMode(stdout, 1); +#endif } +/* +** Find earliest of chars within s specified in zAny. +** With ns == ~0, is like strpbrk(s,zAny) and s must be 0-terminated. +*/ +static const char *anyOfInStr(const char *s, const char *zAny, size_t ns){ + const char *pcFirst = 0; + if( ns == ~(size_t)0 ) ns = strlen(s); + while(*zAny){ + const char *pc = (const char*)memchr(s, *zAny&0xff, ns); + if( pc ){ + pcFirst = pc; + ns = pcFirst - s; + } + ++zAny; + } + return pcFirst; +} /* ** Output the given string as a quoted according to C or TCL quoting rules. */ -static void output_c_string(FILE *out, const char *z){ - unsigned int c; - fputc('"', out); - while( (c = *(z++))!=0 ){ - if( c=='\\' ){ - fputc(c, out); - fputc(c, out); - }else if( c=='"' ){ - fputc('\\', out); - fputc('"', out); - }else if( c=='\t' ){ - fputc('\\', out); - fputc('t', out); - }else if( c=='\n' ){ - fputc('\\', out); - fputc('n', out); - }else if( c=='\r' ){ - fputc('\\', out); - fputc('r', out); +static void output_c_string(const char *z){ + char c; + static const char *zq = "\""; + static long ctrlMask = ~0L; + static const char *zDQBSRO = "\"\\\x7f"; /* double-quote, backslash, rubout */ + char ace[3] = "\\?"; + char cbsSay; + oputz(zq); + while( *z!=0 ){ + const char *pcDQBSRO = anyOfInStr(z, zDQBSRO, ~(size_t)0); + const char *pcPast = zSkipValidUtf8(z, INT_MAX, ctrlMask); + const char *pcEnd = (pcDQBSRO && pcDQBSRO < pcPast)? pcDQBSRO : pcPast; + if( pcEnd > z ) oputb(z, (int)(pcEnd-z)); + if( (c = *pcEnd)==0 ) break; + ++pcEnd; + switch( c ){ + case '\\': case '"': + cbsSay = (char)c; + break; + case '\t': cbsSay = 't'; break; + case '\n': cbsSay = 'n'; break; + case '\r': cbsSay = 'r'; break; + case '\f': cbsSay = 'f'; break; + default: cbsSay = 0; break; + } + if( cbsSay ){ + ace[1] = cbsSay; + oputz(ace); }else if( !isprint(c&0xff) ){ - raw_printf(out, "\\%03o", c&0xff); + oputf("\\%03o", c&0xff); }else{ - fputc(c, out); + ace[1] = (char)c; + oputz(ace+1); } + z = pcEnd; } - fputc('"', out); + oputz(zq); } /* ** Output the given string as a quoted according to JSON quoting rules. */ -static void output_json_string(FILE *out, const char *z, i64 n){ - unsigned int c; +static void output_json_string(const char *z, i64 n){ + char c; + static const char *zq = "\""; + static long ctrlMask = ~0L; + static const char *zDQBS = "\"\\"; + const char *pcLimit; + char ace[3] = "\\?"; + char cbsSay; + if( z==0 ) z = ""; - if( n<0 ) n = strlen(z); - fputc('"', out); - while( n-- ){ + pcLimit = z + ((n<0)? strlen(z) : (size_t)n); + oputz(zq); + while( z < pcLimit ){ + const char *pcDQBS = anyOfInStr(z, zDQBS, pcLimit-z); + const char *pcPast = zSkipValidUtf8(z, (int)(pcLimit-z), ctrlMask); + const char *pcEnd = (pcDQBS && pcDQBS < pcPast)? pcDQBS : pcPast; + if( pcEnd > z ){ + oputb(z, (int)(pcEnd-z)); + z = pcEnd; + } + if( z >= pcLimit ) break; c = *(z++); - if( c=='\\' || c=='"' ){ - fputc('\\', out); - fputc(c, out); + switch( c ){ + case '"': case '\\': + cbsSay = (char)c; + break; + case '\b': cbsSay = 'b'; break; + case '\f': cbsSay = 'f'; break; + case '\n': cbsSay = 'n'; break; + case '\r': cbsSay = 'r'; break; + case '\t': cbsSay = 't'; break; + default: cbsSay = 0; break; + } + if( cbsSay ){ + ace[1] = cbsSay; + oputz(ace); }else if( c<=0x1f ){ - fputc('\\', out); - if( c=='\b' ){ - fputc('b', out); - }else if( c=='\f' ){ - fputc('f', out); - }else if( c=='\n' ){ - fputc('n', out); - }else if( c=='\r' ){ - fputc('r', out); - }else if( c=='\t' ){ - fputc('t', out); - }else{ - raw_printf(out, "u%04x",c); - } + oputf("u%04x", c); }else{ - fputc(c, out); + ace[1] = (char)c; + oputz(ace+1); } } - fputc('"', out); + oputz(zq); } /* ** Output the given string with characters that are special to ** HTML escaped. */ -static void output_html_string(FILE *out, const char *z){ +static void output_html_string(const char *z){ int i; if( z==0 ) z = ""; while( *z ){ @@ -18050,18 +18825,18 @@ static void output_html_string(FILE *out, const char *z){ && z[i]!='\''; i++){} if( i>0 ){ - utf8_printf(out,"%.*s",i,z); + oputf("%.*s",i,z); } if( z[i]=='<' ){ - raw_printf(out,"<"); + oputz("<"); }else if( z[i]=='&' ){ - raw_printf(out,"&"); + oputz("&"); }else if( z[i]=='>' ){ - raw_printf(out,">"); + oputz(">"); }else if( z[i]=='\"' ){ - raw_printf(out,"""); + oputz("""); }else if( z[i]=='\'' ){ - raw_printf(out,"'"); + oputz("'"); }else{ break; } @@ -18099,9 +18874,8 @@ static const char needCsvQuote[] = { ** is only issued if bSep is true. */ static void output_csv(ShellState *p, const char *z, int bSep){ - FILE *out = p->out; if( z==0 ){ - utf8_printf(out,"%s",p->nullValue); + oputf("%s",p->nullValue); }else{ unsigned i; for(i=0; z[i]; i++){ @@ -18113,14 +18887,14 @@ static void output_csv(ShellState *p, const char *z, int bSep){ if( i==0 || strstr(z, p->colSeparator)!=0 ){ char *zQuoted = sqlite3_mprintf("\"%w\"", z); shell_check_oom(zQuoted); - utf8_printf(out, "%s", zQuoted); + oputz(zQuoted); sqlite3_free(zQuoted); }else{ - utf8_printf(out, "%s", z); + oputz(z); } } if( bSep ){ - utf8_printf(p->out, "%s", p->colSeparator); + oputz(p->colSeparator); } } @@ -18228,16 +19002,16 @@ static int shellAuth( az[1] = zA2; az[2] = zA3; az[3] = zA4; - utf8_printf(p->out, "authorizer: %s", azAction[op]); + oputf("authorizer: %s", azAction[op]); for(i=0; i<4; i++){ - raw_printf(p->out, " "); + oputz(" "); if( az[i] ){ - output_c_string(p->out, az[i]); + output_c_string(az[i]); }else{ - raw_printf(p->out, "NULL"); + oputz("NULL"); } } - raw_printf(p->out, "\n"); + oputz("\n"); if( p->bSafeMode ) (void)safeModeAuth(pClientData, op, zA1, zA2, zA3, zA4); return SQLITE_OK; } @@ -18253,7 +19027,7 @@ static int shellAuth( ** sqlite3_complete() returns false, try to terminate the comment before ** printing the result. https://sqlite.org/forum/forumpost/d7be961c5c */ -static void printSchemaLine(FILE *out, const char *z, const char *zTail){ +static void printSchemaLine(const char *z, const char *zTail){ char *zToFree = 0; if( z==0 ) return; if( zTail==0 ) return; @@ -18275,16 +19049,16 @@ static void printSchemaLine(FILE *out, const char *z, const char *zTail){ } } if( sqlite3_strglob("CREATE TABLE ['\"]*", z)==0 ){ - utf8_printf(out, "CREATE TABLE IF NOT EXISTS %s%s", z+13, zTail); + oputf("CREATE TABLE IF NOT EXISTS %s%s", z+13, zTail); }else{ - utf8_printf(out, "%s%s", z, zTail); + oputf("%s%s", z, zTail); } sqlite3_free(zToFree); } -static void printSchemaLineN(FILE *out, char *z, int n, const char *zTail){ +static void printSchemaLineN(char *z, int n, const char *zTail){ char c = z[n]; z[n] = 0; - printSchemaLine(out, z, zTail); + printSchemaLine(z, zTail); z[n] = c; } @@ -18312,7 +19086,7 @@ static void eqp_append(ShellState *p, int iEqpId, int p2, const char *zText){ if( zText==0 ) return; nText = strlen(zText); if( p->autoEQPtest ){ - utf8_printf(p->out, "%d,%d,%s\n", iEqpId, p2, zText); + oputf("%d,%d,%s\n", iEqpId, p2, zText); } pNew = sqlite3_malloc64( sizeof(*pNew) + nText ); shell_check_oom(pNew); @@ -18360,8 +19134,7 @@ static void eqp_render_level(ShellState *p, int iEqpId){ for(pRow = eqp_next_row(p, iEqpId, 0); pRow; pRow = pNext){ pNext = eqp_next_row(p, iEqpId, pRow); z = pRow->zText; - utf8_printf(p->out, "%s%s%s\n", p->sGraph.zPrefix, - pNext ? "|--" : "`--", z); + oputf("%s%s%s\n", p->sGraph.zPrefix, pNext ? "|--" : "`--", z); if( n<(i64)sizeof(p->sGraph.zPrefix)-7 ){ memcpy(&p->sGraph.zPrefix[n], pNext ? "| " : " ", 4); eqp_render_level(p, pRow->iEqpId); @@ -18381,13 +19154,13 @@ static void eqp_render(ShellState *p, i64 nCycle){ eqp_reset(p); return; } - utf8_printf(p->out, "%s\n", pRow->zText+3); + oputf("%s\n", pRow->zText+3); p->sGraph.pRow = pRow->pNext; sqlite3_free(pRow); }else if( nCycle>0 ){ - utf8_printf(p->out, "QUERY PLAN (cycles=%lld [100%%])\n", nCycle); + oputf("QUERY PLAN (cycles=%lld [100%%])\n", nCycle); }else{ - utf8_printf(p->out, "QUERY PLAN\n"); + oputz("QUERY PLAN\n"); } p->sGraph.zPrefix[0] = 0; eqp_render_level(p, 0); @@ -18403,13 +19176,13 @@ static int progress_handler(void *pClientData) { ShellState *p = (ShellState*)pClientData; p->nProgress++; if( p->nProgress>=p->mxProgress && p->mxProgress>0 ){ - raw_printf(p->out, "Progress limit reached (%u)\n", p->nProgress); + oputf("Progress limit reached (%u)\n", p->nProgress); if( p->flgProgress & SHELL_PROGRESS_RESET ) p->nProgress = 0; if( p->flgProgress & SHELL_PROGRESS_ONCE ) p->mxProgress = 0; return 1; } if( (p->flgProgress & SHELL_PROGRESS_QUIET)==0 ){ - raw_printf(p->out, "Progress %u\n", p->nProgress); + oputf("Progress %u\n", p->nProgress); } return 0; } @@ -18418,14 +19191,14 @@ static int progress_handler(void *pClientData) { /* ** Print N dashes */ -static void print_dashes(FILE *out, int N){ +static void print_dashes(int N){ const char zDash[] = "--------------------------------------------------"; const int nDash = sizeof(zDash) - 1; while( N>nDash ){ - fputs(zDash, out); + oputz(zDash); N -= nDash; } - raw_printf(out, "%.*s", N, zDash); + oputf("%.*s", N, zDash); } /* @@ -18438,15 +19211,15 @@ static void print_row_separator( ){ int i; if( nArg>0 ){ - fputs(zSep, p->out); - print_dashes(p->out, p->actualWidth[0]+2); + oputz(zSep); + print_dashes(p->actualWidth[0]+2); for(i=1; iout); - print_dashes(p->out, p->actualWidth[i]+2); + oputz(zSep); + print_dashes(p->actualWidth[i]+2); } - fputs(zSep, p->out); + oputz(zSep); } - fputs("\n", p->out); + oputz("\n"); } /* @@ -18476,10 +19249,10 @@ static int shell_callback( int len = strlen30(azCol[i] ? azCol[i] : ""); if( len>w ) w = len; } - if( p->cnt++>0 ) utf8_printf(p->out, "%s", p->rowSeparator); + if( p->cnt++>0 ) oputz(p->rowSeparator); for(i=0; iout,"%*s = %s%s", w, azCol[i], - azArg[i] ? azArg[i] : p->nullValue, p->rowSeparator); + oputf("%*s = %s%s", w, azCol[i], + azArg[i] ? azArg[i] : p->nullValue, p->rowSeparator); } break; } @@ -18506,12 +19279,12 @@ static int shell_callback( /* If this is the first row seen, print out the headers */ if( p->cnt++==0 ){ for(i=0; iout, aWidth[i], azCol[ aMap[i] ]); - fputs(i==nArg-1 ? "\n" : " ", p->out); + utf8_width_print(aWidth[i], azCol[ aMap[i] ]); + oputz(i==nArg-1 ? "\n" : " "); } for(i=0; iout, aWidth[i]); - fputs(i==nArg-1 ? "\n" : " ", p->out); + print_dashes(aWidth[i]); + oputz(i==nArg-1 ? "\n" : " "); } } @@ -18529,17 +19302,17 @@ static int shell_callback( } if( i==iIndent && p->aiIndent && p->pStmt ){ if( p->iIndentnIndent ){ - utf8_printf(p->out, "%*.s", p->aiIndent[p->iIndent], ""); + oputf("%*.s", p->aiIndent[p->iIndent], ""); } p->iIndent++; } - utf8_width_print(p->out, w, zVal ? zVal : p->nullValue); - fputs(i==nArg-1 ? "\n" : zSep, p->out); + utf8_width_print(w, zVal ? zVal : p->nullValue); + oputz(i==nArg-1 ? "\n" : zSep); } break; } case MODE_Semi: { /* .schema and .fullschema output */ - printSchemaLine(p->out, azArg[0], ";\n"); + printSchemaLine(azArg[0], ";\n"); break; } case MODE_Pretty: { /* .schema and .fullschema with --indent */ @@ -18554,7 +19327,7 @@ static int shell_callback( if( sqlite3_strlike("CREATE VIEW%", azArg[0], 0)==0 || sqlite3_strlike("CREATE TRIG%", azArg[0], 0)==0 ){ - utf8_printf(p->out, "%s;\n", azArg[0]); + oputf("%s;\n", azArg[0]); break; } z = sqlite3_mprintf("%s", azArg[0]); @@ -18587,7 +19360,7 @@ static int shell_callback( }else if( c==')' ){ nParen--; if( nLine>0 && nParen==0 && j>0 ){ - printSchemaLineN(p->out, z, j, "\n"); + printSchemaLineN(z, j, "\n"); j = 0; } } @@ -18596,7 +19369,7 @@ static int shell_callback( && (c=='(' || c=='\n' || (c==',' && !wsToEol(z+i+1))) ){ if( c=='\n' ) j--; - printSchemaLineN(p->out, z, j, "\n "); + printSchemaLineN(z, j, "\n "); j = 0; nLine++; while( IsSpace(z[i+1]) ){ i++; } @@ -18604,64 +19377,59 @@ static int shell_callback( } z[j] = 0; } - printSchemaLine(p->out, z, ";\n"); + printSchemaLine(z, ";\n"); sqlite3_free(z); break; } case MODE_List: { if( p->cnt++==0 && p->showHeader ){ for(i=0; iout,"%s%s",azCol[i], - i==nArg-1 ? p->rowSeparator : p->colSeparator); + oputf("%s%s",azCol[i], i==nArg-1 ? p->rowSeparator : p->colSeparator); } } if( azArg==0 ) break; for(i=0; inullValue; - utf8_printf(p->out, "%s", z); - if( iout, "%s", p->colSeparator); - }else{ - utf8_printf(p->out, "%s", p->rowSeparator); - } + oputz(z); + oputz((icolSeparator : p->rowSeparator); } break; } case MODE_Html: { if( p->cnt++==0 && p->showHeader ){ - raw_printf(p->out,""); + oputz(""); for(i=0; iout,"\n"); + oputz("\n"); } - raw_printf(p->out,"\n"); + oputz("\n"); } if( azArg==0 ) break; - raw_printf(p->out,""); + oputz(""); for(i=0; iout,"\n"); + oputz("\n"); } - raw_printf(p->out,"\n"); + oputz("\n"); break; } case MODE_Tcl: { if( p->cnt++==0 && p->showHeader ){ for(i=0; iout,azCol[i] ? azCol[i] : ""); - if(iout, "%s", p->colSeparator); + output_c_string(azCol[i] ? azCol[i] : ""); + if(icolSeparator); } - utf8_printf(p->out, "%s", p->rowSeparator); + oputz(p->rowSeparator); } if( azArg==0 ) break; for(i=0; iout, azArg[i] ? azArg[i] : p->nullValue); - if(iout, "%s", p->colSeparator); + output_c_string(azArg[i] ? azArg[i] : p->nullValue); + if(icolSeparator); } - utf8_printf(p->out, "%s", p->rowSeparator); + oputz(p->rowSeparator); break; } case MODE_Csv: { @@ -18670,57 +19438,57 @@ static int shell_callback( for(i=0; iout, "%s", p->rowSeparator); + oputz(p->rowSeparator); } if( nArg>0 ){ for(i=0; iout, "%s", p->rowSeparator); + oputz(p->rowSeparator); } setTextMode(p->out, 1); break; } case MODE_Insert: { if( azArg==0 ) break; - utf8_printf(p->out,"INSERT INTO %s",p->zDestTable); + oputf("INSERT INTO %s",p->zDestTable); if( p->showHeader ){ - raw_printf(p->out,"("); + oputz("("); for(i=0; i0 ) raw_printf(p->out, ","); + if( i>0 ) oputz(","); if( quoteChar(azCol[i]) ){ char *z = sqlite3_mprintf("\"%w\"", azCol[i]); shell_check_oom(z); - utf8_printf(p->out, "%s", z); + oputz(z); sqlite3_free(z); }else{ - raw_printf(p->out, "%s", azCol[i]); + oputf("%s", azCol[i]); } } - raw_printf(p->out,")"); + oputz(")"); } p->cnt++; for(i=0; iout, i>0 ? "," : " VALUES("); + oputz(i>0 ? "," : " VALUES("); if( (azArg[i]==0) || (aiType && aiType[i]==SQLITE_NULL) ){ - utf8_printf(p->out,"NULL"); + oputz("NULL"); }else if( aiType && aiType[i]==SQLITE_TEXT ){ if( ShellHasFlag(p, SHFLG_Newlines) ){ - output_quoted_string(p->out, azArg[i]); + output_quoted_string(azArg[i]); }else{ - output_quoted_escaped_string(p->out, azArg[i]); + output_quoted_escaped_string(azArg[i]); } }else if( aiType && aiType[i]==SQLITE_INTEGER ){ - utf8_printf(p->out,"%s", azArg[i]); + oputz(azArg[i]); }else if( aiType && aiType[i]==SQLITE_FLOAT ){ char z[50]; double r = sqlite3_column_double(p->pStmt, i); sqlite3_uint64 ur; memcpy(&ur,&r,sizeof(r)); if( ur==0x7ff0000000000000LL ){ - raw_printf(p->out, "9.0e+999"); + oputz("9.0e+999"); }else if( ur==0xfff0000000000000LL ){ - raw_printf(p->out, "-9.0e+999"); + oputz("-9.0e+999"); }else{ sqlite3_int64 ir = (sqlite3_int64)r; if( r==(double)ir ){ @@ -18728,21 +19496,21 @@ static int shell_callback( }else{ sqlite3_snprintf(50,z,"%!.20g", r); } - raw_printf(p->out, "%s", z); + oputz(z); } }else if( aiType && aiType[i]==SQLITE_BLOB && p->pStmt ){ const void *pBlob = sqlite3_column_blob(p->pStmt, i); int nBlob = sqlite3_column_bytes(p->pStmt, i); - output_hex_blob(p->out, pBlob, nBlob); + output_hex_blob(pBlob, nBlob); }else if( isNumber(azArg[i], 0) ){ - utf8_printf(p->out,"%s", azArg[i]); + oputz(azArg[i]); }else if( ShellHasFlag(p, SHFLG_Newlines) ){ - output_quoted_string(p->out, azArg[i]); + output_quoted_string(azArg[i]); }else{ - output_quoted_escaped_string(p->out, azArg[i]); + output_quoted_escaped_string(azArg[i]); } } - raw_printf(p->out,");\n"); + oputz(");\n"); break; } case MODE_Json: { @@ -18754,37 +19522,37 @@ static int shell_callback( } p->cnt++; for(i=0; iout, azCol[i], -1); - putc(':', p->out); + output_json_string(azCol[i], -1); + oputz(":"); if( (azArg[i]==0) || (aiType && aiType[i]==SQLITE_NULL) ){ - fputs("null",p->out); + oputz("null"); }else if( aiType && aiType[i]==SQLITE_FLOAT ){ char z[50]; double r = sqlite3_column_double(p->pStmt, i); sqlite3_uint64 ur; memcpy(&ur,&r,sizeof(r)); if( ur==0x7ff0000000000000LL ){ - raw_printf(p->out, "9.0e+999"); + oputz("9.0e+999"); }else if( ur==0xfff0000000000000LL ){ - raw_printf(p->out, "-9.0e+999"); + oputz("-9.0e+999"); }else{ sqlite3_snprintf(50,z,"%!.20g", r); - raw_printf(p->out, "%s", z); + oputz(z); } }else if( aiType && aiType[i]==SQLITE_BLOB && p->pStmt ){ const void *pBlob = sqlite3_column_blob(p->pStmt, i); int nBlob = sqlite3_column_bytes(p->pStmt, i); - output_json_string(p->out, pBlob, nBlob); + output_json_string(pBlob, nBlob); }else if( aiType && aiType[i]==SQLITE_TEXT ){ - output_json_string(p->out, azArg[i], -1); + output_json_string(azArg[i], -1); }else{ - utf8_printf(p->out,"%s", azArg[i]); + oputz(azArg[i]); } if( iout); + oputz(","); } } - putc('}', p->out); + oputz("}"); break; } case MODE_Quote: { @@ -18792,7 +19560,7 @@ static int shell_callback( if( p->cnt==0 && p->showHeader ){ for(i=0; i0 ) fputs(p->colSeparator, p->out); - output_quoted_string(p->out, azCol[i]); + output_quoted_string(azCol[i]); } fputs(p->rowSeparator, p->out); } @@ -18800,24 +19568,24 @@ static int shell_callback( for(i=0; i0 ) fputs(p->colSeparator, p->out); if( (azArg[i]==0) || (aiType && aiType[i]==SQLITE_NULL) ){ - utf8_printf(p->out,"NULL"); + oputz("NULL"); }else if( aiType && aiType[i]==SQLITE_TEXT ){ - output_quoted_string(p->out, azArg[i]); + output_quoted_string(azArg[i]); }else if( aiType && aiType[i]==SQLITE_INTEGER ){ - utf8_printf(p->out,"%s", azArg[i]); + oputz(azArg[i]); }else if( aiType && aiType[i]==SQLITE_FLOAT ){ char z[50]; double r = sqlite3_column_double(p->pStmt, i); sqlite3_snprintf(50,z,"%!.20g", r); - raw_printf(p->out, "%s", z); + oputz(z); }else if( aiType && aiType[i]==SQLITE_BLOB && p->pStmt ){ const void *pBlob = sqlite3_column_blob(p->pStmt, i); int nBlob = sqlite3_column_bytes(p->pStmt, i); - output_hex_blob(p->out, pBlob, nBlob); + output_hex_blob(pBlob, nBlob); }else if( isNumber(azArg[i], 0) ){ - utf8_printf(p->out,"%s", azArg[i]); + oputz(azArg[i]); }else{ - output_quoted_string(p->out, azArg[i]); + output_quoted_string(azArg[i]); } } fputs(p->rowSeparator, p->out); @@ -18826,17 +19594,17 @@ static int shell_callback( case MODE_Ascii: { if( p->cnt++==0 && p->showHeader ){ for(i=0; i0 ) utf8_printf(p->out, "%s", p->colSeparator); - utf8_printf(p->out,"%s",azCol[i] ? azCol[i] : ""); + if( i>0 ) oputz(p->colSeparator); + oputz(azCol[i] ? azCol[i] : ""); } - utf8_printf(p->out, "%s", p->rowSeparator); + oputz(p->rowSeparator); } if( azArg==0 ) break; for(i=0; i0 ) utf8_printf(p->out, "%s", p->colSeparator); - utf8_printf(p->out,"%s",azArg[i] ? azArg[i] : p->nullValue); + if( i>0 ) oputz(p->colSeparator); + oputz(azArg[i] ? azArg[i] : p->nullValue); } - utf8_printf(p->out, "%s", p->rowSeparator); + oputz(p->rowSeparator); break; } case MODE_EQP: { @@ -18915,7 +19683,7 @@ static void createSelftestTable(ShellState *p){ "DROP TABLE [_shell$self];" ,0,0,&zErrMsg); if( zErrMsg ){ - utf8_printf(stderr, "SELFTEST initialization failure: %s\n", zErrMsg); + eputf("SELFTEST initialization failure: %s\n", zErrMsg); sqlite3_free(zErrMsg); } sqlite3_exec(p->db, "RELEASE selftest_init",0,0,0); @@ -19018,8 +19786,8 @@ static int run_table_dump_query( rc = sqlite3_prepare_v2(p->db, zSelect, -1, &pSelect, 0); if( rc!=SQLITE_OK || !pSelect ){ char *zContext = shell_error_context(zSelect, p->db); - utf8_printf(p->out, "/**** ERROR: (%d) %s *****/\n%s", rc, - sqlite3_errmsg(p->db), zContext); + oputf("/**** ERROR: (%d) %s *****/\n%s", + rc, sqlite3_errmsg(p->db), zContext); sqlite3_free(zContext); if( (rc&0xff)!=SQLITE_CORRUPT ) p->nErr++; return rc; @@ -19028,23 +19796,22 @@ static int run_table_dump_query( nResult = sqlite3_column_count(pSelect); while( rc==SQLITE_ROW ){ z = (const char*)sqlite3_column_text(pSelect, 0); - utf8_printf(p->out, "%s", z); + oputf("%s", z); for(i=1; iout, ",%s", sqlite3_column_text(pSelect, i)); + oputf(",%s", sqlite3_column_text(pSelect, i)); } if( z==0 ) z = ""; while( z[0] && (z[0]!='-' || z[1]!='-') ) z++; if( z[0] ){ - raw_printf(p->out, "\n;\n"); + oputz("\n;\n"); }else{ - raw_printf(p->out, ";\n"); + oputz(";\n"); } rc = sqlite3_step(pSelect); } rc = sqlite3_finalize(pSelect); if( rc!=SQLITE_OK ){ - utf8_printf(p->out, "/**** ERROR: (%d) %s *****/\n", rc, - sqlite3_errmsg(p->db)); + oputf("/**** ERROR: (%d) %s *****/\n", rc, sqlite3_errmsg(p->db)); if( (rc&0xff)!=SQLITE_CORRUPT ) p->nErr++; } return rc; @@ -19080,7 +19847,7 @@ static char *save_err_msg( /* ** Attempt to display I/O stats on Linux using /proc/PID/io */ -static void displayLinuxIoStats(FILE *out){ +static void displayLinuxIoStats(void){ FILE *in; char z[200]; sqlite3_snprintf(sizeof(z), z, "/proc/%d/io", getpid()); @@ -19103,7 +19870,7 @@ static void displayLinuxIoStats(FILE *out){ for(i=0; iout, "%-36s %s\n", zLabel, zLine); + oputf("%-36s %s\n", zLabel, zLine); } /* @@ -19148,30 +19914,28 @@ static int display_stats( ){ int iCur; int iHiwtr; - FILE *out; if( pArg==0 || pArg->out==0 ) return 0; - out = pArg->out; if( pArg->pStmt && pArg->statsOn==2 ){ int nCol, i, x; sqlite3_stmt *pStmt = pArg->pStmt; char z[100]; nCol = sqlite3_column_count(pStmt); - raw_printf(out, "%-36s %d\n", "Number of output columns:", nCol); + oputf("%-36s %d\n", "Number of output columns:", nCol); for(i=0; istatsOn==3 ){ if( pArg->pStmt ){ iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_VM_STEP,bReset); - raw_printf(pArg->out, "VM-steps: %d\n", iCur); + oputf("VM-steps: %d\n", iCur); } return 0; } - displayStatLine(pArg, "Memory Used:", + displayStatLine("Memory Used:", "%lld (max %lld) bytes", SQLITE_STATUS_MEMORY_USED, bReset); - displayStatLine(pArg, "Number of Outstanding Allocations:", + displayStatLine("Number of Outstanding Allocations:", "%lld (max %lld)", SQLITE_STATUS_MALLOC_COUNT, bReset); if( pArg->shellFlgs & SHFLG_Pagecache ){ - displayStatLine(pArg, "Number of Pcache Pages Used:", + displayStatLine("Number of Pcache Pages Used:", "%lld (max %lld) pages", SQLITE_STATUS_PAGECACHE_USED, bReset); } - displayStatLine(pArg, "Number of Pcache Overflow Bytes:", + displayStatLine("Number of Pcache Overflow Bytes:", "%lld (max %lld) bytes", SQLITE_STATUS_PAGECACHE_OVERFLOW, bReset); - displayStatLine(pArg, "Largest Allocation:", + displayStatLine("Largest Allocation:", "%lld bytes", SQLITE_STATUS_MALLOC_SIZE, bReset); - displayStatLine(pArg, "Largest Pcache Allocation:", + displayStatLine("Largest Pcache Allocation:", "%lld bytes", SQLITE_STATUS_PAGECACHE_SIZE, bReset); #ifdef YYTRACKMAXSTACKDEPTH - displayStatLine(pArg, "Deepest Parser Stack:", + displayStatLine("Deepest Parser Stack:", "%lld (max %lld)", SQLITE_STATUS_PARSER_STACK, bReset); #endif @@ -19208,77 +19972,68 @@ static int display_stats( iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_USED, &iCur, &iHiwtr, bReset); - raw_printf(pArg->out, - "Lookaside Slots Used: %d (max %d)\n", - iCur, iHiwtr); + oputf("Lookaside Slots Used: %d (max %d)\n", iCur, iHiwtr); sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_HIT, &iCur, &iHiwtr, bReset); - raw_printf(pArg->out, "Successful lookaside attempts: %d\n", - iHiwtr); + oputf("Successful lookaside attempts: %d\n", iHiwtr); sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE, &iCur, &iHiwtr, bReset); - raw_printf(pArg->out, "Lookaside failures due to size: %d\n", - iHiwtr); + oputf("Lookaside failures due to size: %d\n", iHiwtr); sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL, &iCur, &iHiwtr, bReset); - raw_printf(pArg->out, "Lookaside failures due to OOM: %d\n", - iHiwtr); + oputf("Lookaside failures due to OOM: %d\n", iHiwtr); } iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_USED, &iCur, &iHiwtr, bReset); - raw_printf(pArg->out, "Pager Heap Usage: %d bytes\n", - iCur); + oputf("Pager Heap Usage: %d bytes\n", iCur); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_HIT, &iCur, &iHiwtr, 1); - raw_printf(pArg->out, "Page cache hits: %d\n", iCur); + oputf("Page cache hits: %d\n", iCur); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_MISS, &iCur, &iHiwtr, 1); - raw_printf(pArg->out, "Page cache misses: %d\n", iCur); + oputf("Page cache misses: %d\n", iCur); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_WRITE, &iCur, &iHiwtr, 1); - raw_printf(pArg->out, "Page cache writes: %d\n", iCur); + oputf("Page cache writes: %d\n", iCur); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_SPILL, &iCur, &iHiwtr, 1); - raw_printf(pArg->out, "Page cache spills: %d\n", iCur); + oputf("Page cache spills: %d\n", iCur); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_SCHEMA_USED, &iCur, &iHiwtr, bReset); - raw_printf(pArg->out, "Schema Heap Usage: %d bytes\n", - iCur); + oputf("Schema Heap Usage: %d bytes\n", iCur); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_STMT_USED, &iCur, &iHiwtr, bReset); - raw_printf(pArg->out, "Statement Heap/Lookaside Usage: %d bytes\n", - iCur); + oputf("Statement Heap/Lookaside Usage: %d bytes\n", iCur); } if( pArg->pStmt ){ int iHit, iMiss; iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_FULLSCAN_STEP, bReset); - raw_printf(pArg->out, "Fullscan Steps: %d\n", iCur); + oputf("Fullscan Steps: %d\n", iCur); iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_SORT, bReset); - raw_printf(pArg->out, "Sort Operations: %d\n", iCur); + oputf("Sort Operations: %d\n", iCur); iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_AUTOINDEX,bReset); - raw_printf(pArg->out, "Autoindex Inserts: %d\n", iCur); + oputf("Autoindex Inserts: %d\n", iCur); iHit = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_FILTER_HIT, bReset); iMiss = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_FILTER_MISS, bReset); if( iHit || iMiss ){ - raw_printf(pArg->out, "Bloom filter bypass taken: %d/%d\n", - iHit, iHit+iMiss); + oputf("Bloom filter bypass taken: %d/%d\n", iHit, iHit+iMiss); } iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_VM_STEP, bReset); - raw_printf(pArg->out, "Virtual Machine Steps: %d\n", iCur); + oputf("Virtual Machine Steps: %d\n", iCur); iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_REPREPARE,bReset); - raw_printf(pArg->out, "Reprepare operations: %d\n", iCur); + oputf("Reprepare operations: %d\n", iCur); iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_RUN, bReset); - raw_printf(pArg->out, "Number of times run: %d\n", iCur); + oputf("Number of times run: %d\n", iCur); iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_MEMUSED, bReset); - raw_printf(pArg->out, "Memory used by prepared stmt: %d\n", iCur); + oputf("Memory used by prepared stmt: %d\n", iCur); } #ifdef __linux__ - displayLinuxIoStats(pArg->out); + displayLinuxIoStats(); #endif /* Do not remove this machine readable comment: extra-stats-output-here */ @@ -19513,7 +20268,7 @@ static void display_scanstats( UNUSED_PARAMETER(pArg); #else if( pArg->scanstatsOn==3 ){ - const char *zSql = + const char *zSql = " SELECT addr, opcode, p1, p2, p3, p4, p5, comment, nexec," " round(ncycle*100.0 / (sum(ncycle) OVER ()), 2)||'%' AS cycles" " FROM bytecode(?)"; @@ -19659,17 +20414,17 @@ static void bind_prepared_stmt(ShellState *pArg, sqlite3_stmt *pStmt){ /* Draw horizontal line N characters long using unicode box ** characters */ -static void print_box_line(FILE *out, int N){ +static void print_box_line(int N){ const char zDash[] = BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24; const int nDash = sizeof(zDash) - 1; N *= 3; while( N>nDash ){ - utf8_printf(out, zDash); + oputz(zDash); N -= nDash; } - utf8_printf(out, "%.*s", N, zDash); + oputf("%.*s", N, zDash); } /* @@ -19684,15 +20439,15 @@ static void print_box_row_separator( ){ int i; if( nArg>0 ){ - utf8_printf(p->out, "%s", zSep1); - print_box_line(p->out, p->actualWidth[0]+2); + oputz(zSep1); + print_box_line(p->actualWidth[0]+2); for(i=1; iout, "%s", zSep2); - print_box_line(p->out, p->actualWidth[i]+2); + oputz(zSep2); + print_box_line(p->actualWidth[i]+2); } - utf8_printf(p->out, "%s", zSep3); + oputz(zSep3); } - fputs("\n", p->out); + oputz("\n"); } /* @@ -19955,11 +20710,11 @@ static void exec_prepared_stmt_columnar( for(i=0; iactualWidth[i]; if( p->colWidth[i]<0 ) w = -w; - utf8_width_print(p->out, w, azData[i]); + utf8_width_print(w, azData[i]); fputs(i==nColumn-1?"\n":" ", p->out); } for(i=0; iout, p->actualWidth[i]); + print_dashes(p->actualWidth[i]); fputs(i==nColumn-1?"\n":" ", p->out); } } @@ -19973,8 +20728,8 @@ static void exec_prepared_stmt_columnar( for(i=0; iactualWidth[i]; n = strlenChar(azData[i]); - utf8_printf(p->out, "%*s%s%*s", (w-n)/2, "", azData[i], (w-n+1)/2, ""); - fputs(i==nColumn-1?" |\n":" | ", p->out); + oputf("%*s%s%*s", (w-n)/2, "", azData[i], (w-n+1)/2, ""); + oputz(i==nColumn-1?" |\n":" | "); } print_row_separator(p, nColumn, "+"); break; @@ -19986,8 +20741,8 @@ static void exec_prepared_stmt_columnar( for(i=0; iactualWidth[i]; n = strlenChar(azData[i]); - utf8_printf(p->out, "%*s%s%*s", (w-n)/2, "", azData[i], (w-n+1)/2, ""); - fputs(i==nColumn-1?" |\n":" | ", p->out); + oputf("%*s%s%*s", (w-n)/2, "", azData[i], (w-n+1)/2, ""); + oputz(i==nColumn-1?" |\n":" | "); } print_row_separator(p, nColumn, "|"); break; @@ -19996,13 +20751,13 @@ static void exec_prepared_stmt_columnar( colSep = " " BOX_13 " "; rowSep = " " BOX_13 "\n"; print_box_row_separator(p, nColumn, BOX_23, BOX_234, BOX_34); - utf8_printf(p->out, BOX_13 " "); + oputz(BOX_13 " "); for(i=0; iactualWidth[i]; n = strlenChar(azData[i]); - utf8_printf(p->out, "%*s%s%*s%s", - (w-n)/2, "", azData[i], (w-n+1)/2, "", - i==nColumn-1?" "BOX_13"\n":" "BOX_13" "); + oputf("%*s%s%*s%s", + (w-n)/2, "", azData[i], (w-n+1)/2, "", + i==nColumn-1?" "BOX_13"\n":" "BOX_13" "); } print_box_row_separator(p, nColumn, BOX_123, BOX_1234, BOX_134); break; @@ -20010,28 +20765,28 @@ static void exec_prepared_stmt_columnar( } for(i=nColumn, j=0; icMode!=MODE_Column ){ - utf8_printf(p->out, "%s", p->cMode==MODE_Box?BOX_13" ":"| "); + oputz(p->cMode==MODE_Box?BOX_13" ":"| "); } z = azData[i]; if( z==0 ) z = p->nullValue; w = p->actualWidth[j]; if( p->colWidth[j]<0 ) w = -w; - utf8_width_print(p->out, w, z); + utf8_width_print(w, z); if( j==nColumn-1 ){ - utf8_printf(p->out, "%s", rowSep); + oputz(rowSep); if( bMultiLineRowExists && abRowDiv[i/nColumn-1] && i+1cMode==MODE_Table ){ print_row_separator(p, nColumn, "+"); }else if( p->cMode==MODE_Box ){ print_box_row_separator(p, nColumn, BOX_123, BOX_1234, BOX_134); }else if( p->cMode==MODE_Column ){ - raw_printf(p->out, "\n"); + oputz("\n"); } } j = -1; if( seenInterrupt ) goto columnar_end; }else{ - utf8_printf(p->out, "%s", colSep); + oputz(colSep); } } if( p->cMode==MODE_Table ){ @@ -20041,7 +20796,7 @@ static void exec_prepared_stmt_columnar( } columnar_end: if( seenInterrupt ){ - utf8_printf(p->out, "Interrupt\n"); + oputz("Interrupt\n"); } nData = (nRow+1)*nColumn; for(i=0; iout; int bVerbose = pState->expert.bVerbose; rc = sqlite3_expert_analyze(p, pzErr); @@ -20190,8 +20944,8 @@ static int expertFinish( if( bVerbose ){ const char *zCand = sqlite3_expert_report(p,0,EXPERT_REPORT_CANDIDATES); - raw_printf(out, "-- Candidates -----------------------------\n"); - raw_printf(out, "%s\n", zCand); + oputz("-- Candidates -----------------------------\n"); + oputf("%s\n", zCand); } for(i=0; i=2 && 0==cli_strncmp(z, "-sample", n) ){ if( i==(nArg-1) ){ - raw_printf(stderr, "option requires an argument: %s\n", z); + eputf("option requires an argument: %s\n", z); rc = SQLITE_ERROR; }else{ iSample = (int)integerValue(azArg[++i]); if( iSample<0 || iSample>100 ){ - raw_printf(stderr, "value out of range: %s\n", azArg[i]); + eputf("value out of range: %s\n", azArg[i]); rc = SQLITE_ERROR; } } } else{ - raw_printf(stderr, "unknown option: %s\n", z); + eputf("unknown option: %s\n", z); rc = SQLITE_ERROR; } } @@ -20257,8 +21011,7 @@ static int expertDotCommand( if( rc==SQLITE_OK ){ pState->expert.pExpert = sqlite3_expert_new(pState->db, &zErr); if( pState->expert.pExpert==0 ){ - raw_printf(stderr, "sqlite3_expert_new: %s\n", - zErr ? zErr : "out of memory"); + eputf("sqlite3_expert_new: %s\n", zErr ? zErr : "out of memory"); rc = SQLITE_ERROR; }else{ sqlite3_expert_config( @@ -20585,9 +21338,9 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){ noSys = (p->shellFlgs & SHFLG_DumpNoSys)!=0; if( cli_strcmp(zTable, "sqlite_sequence")==0 && !noSys ){ - if( !dataOnly ) raw_printf(p->out, "DELETE FROM sqlite_sequence;\n"); + if( !dataOnly ) oputz("DELETE FROM sqlite_sequence;\n"); }else if( sqlite3_strglob("sqlite_stat?", zTable)==0 && !noSys ){ - if( !dataOnly ) raw_printf(p->out, "ANALYZE sqlite_schema;\n"); + if( !dataOnly ) oputz("ANALYZE sqlite_schema;\n"); }else if( cli_strncmp(zTable, "sqlite_", 7)==0 ){ return 0; }else if( dataOnly ){ @@ -20595,7 +21348,7 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){ }else if( cli_strncmp(zSql, "CREATE VIRTUAL TABLE", 20)==0 ){ char *zIns; if( !p->writableSchema ){ - raw_printf(p->out, "PRAGMA writable_schema=ON;\n"); + oputz("PRAGMA writable_schema=ON;\n"); p->writableSchema = 1; } zIns = sqlite3_mprintf( @@ -20603,11 +21356,11 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){ "VALUES('table','%q','%q',0,'%q');", zTable, zTable, zSql); shell_check_oom(zIns); - utf8_printf(p->out, "%s\n", zIns); + oputf("%s\n", zIns); sqlite3_free(zIns); return 0; }else{ - printSchemaLine(p->out, zSql, ";\n"); + printSchemaLine(zSql, ";\n"); } if( cli_strcmp(zType, "table")==0 ){ @@ -20665,7 +21418,7 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){ p->mode = p->cMode = MODE_Insert; rc = shell_exec(p, sSelect.z, 0); if( (rc&0xff)==SQLITE_CORRUPT ){ - raw_printf(p->out, "/****** CORRUPTION ERROR *******/\n"); + oputz("/****** CORRUPTION ERROR *******/\n"); toggleSelectOrder(p->db); shell_exec(p, sSelect.z, 0); toggleSelectOrder(p->db); @@ -20696,9 +21449,9 @@ static int run_schema_dump_query( if( rc==SQLITE_CORRUPT ){ char *zQ2; int len = strlen30(zQuery); - raw_printf(p->out, "/****** CORRUPTION ERROR *******/\n"); + oputz("/****** CORRUPTION ERROR *******/\n"); if( zErr ){ - utf8_printf(p->out, "/****** %s ******/\n", zErr); + oputf("/****** %s ******/\n", zErr); sqlite3_free(zErr); zErr = 0; } @@ -20707,7 +21460,7 @@ static int run_schema_dump_query( sqlite3_snprintf(len+100, zQ2, "%s ORDER BY rowid DESC", zQuery); rc = sqlite3_exec(p->db, zQ2, dump_callback, p, &zErr); if( rc ){ - utf8_printf(p->out, "/****** ERROR: %s ******/\n", zErr); + oputf("/****** ERROR: %s ******/\n", zErr); }else{ rc = SQLITE_CORRUPT; } @@ -21063,10 +21816,10 @@ static int showHelp(FILE *out, const char *zPattern){ } if( ((hw^hh)&HH_Undoc)==0 ){ if( (hh&HH_Summary)!=0 ){ - utf8_printf(out, ".%s\n", azHelp[i]+1); + sputf(out, ".%s\n", azHelp[i]+1); ++n; }else if( (hw&HW_SummaryOnly)==0 ){ - utf8_printf(out, "%s\n", azHelp[i]); + sputf(out, "%s\n", azHelp[i]); } } } @@ -21076,7 +21829,7 @@ static int showHelp(FILE *out, const char *zPattern){ shell_check_oom(zPat); for(i=0; i65536 || (pgsz & (pgsz-1))!=0 ){ - utf8_printf(stderr, "invalid pagesize\n"); + eputz("invalid pagesize\n"); goto readHexDb_error; } for(nLine++; fgets(zLine, sizeof(zLine), in)!=0; nLine++){ @@ -21345,7 +22098,7 @@ static unsigned char *readHexDb(ShellState *p, int *pnData){ p->lineno = nLine; } sqlite3_free(a); - utf8_printf(stderr,"Error on line %d of --hexdb input\n", nLine); + eputf("Error on line %d of --hexdb input\n", nLine); return 0; } #endif /* SQLITE_OMIT_DESERIALIZE */ @@ -21421,22 +22174,19 @@ static void open_db(ShellState *p, int openFlags){ } globalDb = p->db; if( p->db==0 || SQLITE_OK!=sqlite3_errcode(p->db) ){ - utf8_printf(stderr,"Error: unable to open database \"%s\": %s\n", - zDbFilename, sqlite3_errmsg(p->db)); + eputf("Error: unable to open database \"%s\": %s\n", + zDbFilename, sqlite3_errmsg(p->db)); if( (openFlags & OPEN_DB_KEEPALIVE)==0 ){ exit(1); } sqlite3_close(p->db); sqlite3_open(":memory:", &p->db); if( p->db==0 || SQLITE_OK!=sqlite3_errcode(p->db) ){ - utf8_printf(stderr, - "Also: unable to open substitute in-memory database.\n" - ); + eputz("Also: unable to open substitute in-memory database.\n"); exit(1); }else{ - utf8_printf(stderr, - "Notice: using substitute in-memory database instead of \"%s\"\n", - zDbFilename); + eputf("Notice: using substitute in-memory database instead of \"%s\"\n", + zDbFilename); } } sqlite3_db_config(p->db, SQLITE_DBCONFIG_STMT_SCANSTATUS, (int)0, (int*)0); @@ -21543,7 +22293,7 @@ static void open_db(ShellState *p, int openFlags){ SQLITE_DESERIALIZE_RESIZEABLE | SQLITE_DESERIALIZE_FREEONCLOSE); if( rc ){ - utf8_printf(stderr, "Error: sqlite3_deserialize() returns %d\n", rc); + eputf("Error: sqlite3_deserialize() returns %d\n", rc); } if( p->szMax>0 ){ sqlite3_file_control(p->db, "main", SQLITE_FCNTL_SIZE_LIMIT, &p->szMax); @@ -21567,8 +22317,7 @@ static void open_db(ShellState *p, int openFlags){ void close_db(sqlite3 *db){ int rc = sqlite3_close(db); if( rc ){ - utf8_printf(stderr, "Error: sqlite3_close() returns %d: %s\n", - rc, sqlite3_errmsg(db)); + eputf("Error: sqlite3_close() returns %d: %s\n", rc, sqlite3_errmsg(db)); } } @@ -21729,8 +22478,7 @@ static int booleanValue(const char *zArg){ if( sqlite3_stricmp(zArg, "off")==0 || sqlite3_stricmp(zArg,"no")==0 ){ return 0; } - utf8_printf(stderr, "ERROR: Not a boolean value: \"%s\". Assuming \"no\".\n", - zArg); + eputf("ERROR: Not a boolean value: \"%s\". Assuming \"no\".\n", zArg); return 0; } @@ -21768,7 +22516,7 @@ static FILE *output_file_open(const char *zFile, int bTextMode){ }else{ f = fopen(zFile, bTextMode ? "w" : "wb"); if( f==0 ){ - utf8_printf(stderr, "Error: cannot open \"%s\"\n", zFile); + eputf("Error: cannot open \"%s\"\n", zFile); } } return f; @@ -21790,7 +22538,7 @@ static int sql_trace_callback( i64 nSql; if( p->traceOut==0 ) return 0; if( mType==SQLITE_TRACE_CLOSE ){ - utf8_printf(p->traceOut, "-- closing database connection\n"); + sputz(p->traceOut, "-- closing database connection\n"); return 0; } if( mType!=SQLITE_TRACE_ROW && pX!=0 && ((const char*)pX)[0]=='-' ){ @@ -21821,12 +22569,12 @@ static int sql_trace_callback( switch( mType ){ case SQLITE_TRACE_ROW: case SQLITE_TRACE_STMT: { - utf8_printf(p->traceOut, "%.*s;\n", (int)nSql, zSql); + sputf(p->traceOut, "%.*s;\n", (int)nSql, zSql); break; } case SQLITE_TRACE_PROFILE: { sqlite3_int64 nNanosec = pX ? *(sqlite3_int64*)pX : 0; - utf8_printf(p->traceOut, "%.*s; -- %lld ns\n", (int)nSql, zSql, nNanosec); + sputf(p->traceOut, "%.*s; -- %lld ns\n", (int)nSql, zSql, nNanosec); break; } } @@ -21933,12 +22681,11 @@ static char *SQLITE_CDECL csv_read_one_field(ImportCtx *p){ break; } if( pc==cQuote && c!='\r' ){ - utf8_printf(stderr, "%s:%d: unescaped %c character\n", - p->zFile, p->nLine, cQuote); + eputf("%s:%d: unescaped %c character\n", p->zFile, p->nLine, cQuote); } if( c==EOF ){ - utf8_printf(stderr, "%s:%d: unterminated %c-quoted field\n", - p->zFile, startLine, cQuote); + eputf("%s:%d: unterminated %c-quoted field\n", + p->zFile, startLine, cQuote); p->cTerm = c; break; } @@ -22036,9 +22783,8 @@ static void tryToCloneData( shell_check_oom(zQuery); rc = sqlite3_prepare_v2(p->db, zQuery, -1, &pQuery, 0); if( rc ){ - utf8_printf(stderr, "Error %d: %s on [%s]\n", - sqlite3_extended_errcode(p->db), sqlite3_errmsg(p->db), - zQuery); + eputf("Error %d: %s on [%s]\n", + sqlite3_extended_errcode(p->db), sqlite3_errmsg(p->db), zQuery); goto end_data_xfer; } n = sqlite3_column_count(pQuery); @@ -22054,9 +22800,8 @@ static void tryToCloneData( memcpy(zInsert+i, ");", 3); rc = sqlite3_prepare_v2(newDb, zInsert, -1, &pInsert, 0); if( rc ){ - utf8_printf(stderr, "Error %d: %s on [%s]\n", - sqlite3_extended_errcode(newDb), sqlite3_errmsg(newDb), - zInsert); + eputf("Error %d: %s on [%s]\n", + sqlite3_extended_errcode(newDb), sqlite3_errmsg(newDb), zInsert); goto end_data_xfer; } for(k=0; k<2; k++){ @@ -22091,8 +22836,8 @@ static void tryToCloneData( } /* End for */ rc = sqlite3_step(pInsert); if( rc!=SQLITE_OK && rc!=SQLITE_ROW && rc!=SQLITE_DONE ){ - utf8_printf(stderr, "Error %d: %s\n", sqlite3_extended_errcode(newDb), - sqlite3_errmsg(newDb)); + eputf("Error %d: %s\n", + sqlite3_extended_errcode(newDb), sqlite3_errmsg(newDb)); } sqlite3_reset(pInsert); cnt++; @@ -22109,7 +22854,7 @@ static void tryToCloneData( shell_check_oom(zQuery); rc = sqlite3_prepare_v2(p->db, zQuery, -1, &pQuery, 0); if( rc ){ - utf8_printf(stderr, "Warning: cannot step \"%s\" backwards", zTable); + eputf("Warning: cannot step \"%s\" backwards", zTable); break; } } /* End for(k=0...) */ @@ -22146,9 +22891,8 @@ static void tryToCloneSchema( shell_check_oom(zQuery); rc = sqlite3_prepare_v2(p->db, zQuery, -1, &pQuery, 0); if( rc ){ - utf8_printf(stderr, "Error: (%d) %s on [%s]\n", - sqlite3_extended_errcode(p->db), sqlite3_errmsg(p->db), - zQuery); + eputf("Error: (%d) %s on [%s]\n", sqlite3_extended_errcode(p->db), + sqlite3_errmsg(p->db), zQuery); goto end_schema_xfer; } while( (rc = sqlite3_step(pQuery))==SQLITE_ROW ){ @@ -22156,10 +22900,10 @@ static void tryToCloneSchema( zSql = sqlite3_column_text(pQuery, 1); if( zName==0 || zSql==0 ) continue; if( sqlite3_stricmp((char*)zName, "sqlite_sequence")!=0 ){ - printf("%s... ", zName); fflush(stdout); + sputf(stdout, "%s... ", zName); fflush(stdout); sqlite3_exec(newDb, (const char*)zSql, 0, 0, &zErrMsg); if( zErrMsg ){ - utf8_printf(stderr, "Error: %s\nSQL: [%s]\n", zErrMsg, zSql); + eputf("Error: %s\nSQL: [%s]\n", zErrMsg, zSql); sqlite3_free(zErrMsg); zErrMsg = 0; } @@ -22167,7 +22911,7 @@ static void tryToCloneSchema( if( xForEach ){ xForEach(p, newDb, (const char*)zName); } - printf("done\n"); + sputz(stdout, "done\n"); } if( rc!=SQLITE_DONE ){ sqlite3_finalize(pQuery); @@ -22177,9 +22921,8 @@ static void tryToCloneSchema( shell_check_oom(zQuery); rc = sqlite3_prepare_v2(p->db, zQuery, -1, &pQuery, 0); if( rc ){ - utf8_printf(stderr, "Error: (%d) %s on [%s]\n", - sqlite3_extended_errcode(p->db), sqlite3_errmsg(p->db), - zQuery); + eputf("Error: (%d) %s on [%s]\n", + sqlite3_extended_errcode(p->db), sqlite3_errmsg(p->db), zQuery); goto end_schema_xfer; } while( sqlite3_step(pQuery)==SQLITE_ROW ){ @@ -22187,17 +22930,17 @@ static void tryToCloneSchema( zSql = sqlite3_column_text(pQuery, 1); if( zName==0 || zSql==0 ) continue; if( sqlite3_stricmp((char*)zName, "sqlite_sequence")==0 ) continue; - printf("%s... ", zName); fflush(stdout); + sputf(stdout, "%s... ", zName); fflush(stdout); sqlite3_exec(newDb, (const char*)zSql, 0, 0, &zErrMsg); if( zErrMsg ){ - utf8_printf(stderr, "Error: %s\nSQL: [%s]\n", zErrMsg, zSql); + eputf("Error: %s\nSQL: [%s]\n", zErrMsg, zSql); sqlite3_free(zErrMsg); zErrMsg = 0; } if( xForEach ){ xForEach(p, newDb, (const char*)zName); } - printf("done\n"); + sputz(stdout, "done\n"); } } end_schema_xfer: @@ -22214,13 +22957,12 @@ static void tryToClone(ShellState *p, const char *zNewDb){ int rc; sqlite3 *newDb = 0; if( access(zNewDb,0)==0 ){ - utf8_printf(stderr, "File \"%s\" already exists.\n", zNewDb); + eputf("File \"%s\" already exists.\n", zNewDb); return; } rc = sqlite3_open(zNewDb, &newDb); if( rc ){ - utf8_printf(stderr, "Cannot create output database: %s\n", - sqlite3_errmsg(newDb)); + eputf("Cannot create output database: %s\n", sqlite3_errmsg(newDb)); }else{ sqlite3_exec(p->db, "PRAGMA writable_schema=ON;", 0, 0, 0); sqlite3_exec(newDb, "BEGIN EXCLUSIVE;", 0, 0, 0); @@ -22232,6 +22974,18 @@ static void tryToClone(ShellState *p, const char *zNewDb){ close_db(newDb); } +#ifndef SQLITE_SHELL_FIDDLE +/* +** Change the output stream (file or pipe or console) to something else. +*/ +static void output_redir(ShellState *p, FILE *pfNew){ + if( p->out != stdout ) eputz("Output already redirected.\n"); + else{ + p->out = pfNew; + setOutputStream(pfNew); + } +} + /* ** Change the output file back to stdout. ** @@ -22259,7 +23013,7 @@ static void output_reset(ShellState *p){ char *zCmd; zCmd = sqlite3_mprintf("%s %s", zXdgOpenCmd, p->zTempFile); if( system(zCmd) ){ - utf8_printf(stderr, "Failed: [%s]\n", zCmd); + eputf("Failed: [%s]\n", zCmd); }else{ /* Give the start/open/xdg-open command some time to get ** going before we continue, and potential delete the @@ -22274,7 +23028,12 @@ static void output_reset(ShellState *p){ } p->outfile[0] = 0; p->out = stdout; + setOutputStream(stdout); } +#else +# define output_redir(SS,pfO) +# define output_reset(SS) +#endif /* ** Run an SQL command and return the single integer result. @@ -22345,7 +23104,7 @@ static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){ "SELECT data FROM sqlite_dbpage(?1) WHERE pgno=1", -1, &pStmt, 0); if( rc ){ - utf8_printf(stderr, "error: %s\n", sqlite3_errmsg(p->db)); + eputf("error: %s\n", sqlite3_errmsg(p->db)); sqlite3_finalize(pStmt); return 1; } @@ -22358,28 +23117,28 @@ static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){ memcpy(aHdr, pb, 100); sqlite3_finalize(pStmt); }else{ - raw_printf(stderr, "unable to read database header\n"); + eputz("unable to read database header\n"); sqlite3_finalize(pStmt); return 1; } i = get2byteInt(aHdr+16); if( i==1 ) i = 65536; - utf8_printf(p->out, "%-20s %d\n", "database page size:", i); - utf8_printf(p->out, "%-20s %d\n", "write format:", aHdr[18]); - utf8_printf(p->out, "%-20s %d\n", "read format:", aHdr[19]); - utf8_printf(p->out, "%-20s %d\n", "reserved bytes:", aHdr[20]); + oputf("%-20s %d\n", "database page size:", i); + oputf("%-20s %d\n", "write format:", aHdr[18]); + oputf("%-20s %d\n", "read format:", aHdr[19]); + oputf("%-20s %d\n", "reserved bytes:", aHdr[20]); for(i=0; iout, "%-20s %u", aField[i].zName, val); + oputf("%-20s %u", aField[i].zName, val); switch( ofst ){ case 56: { - if( val==1 ) raw_printf(p->out, " (utf8)"); - if( val==2 ) raw_printf(p->out, " (utf16le)"); - if( val==3 ) raw_printf(p->out, " (utf16be)"); + if( val==1 ) oputz(" (utf8)"); + if( val==2 ) oputz(" (utf16le)"); + if( val==3 ) oputz(" (utf16be)"); } } - raw_printf(p->out, "\n"); + oputz("\n"); } if( zDb==0 ){ zSchemaTab = sqlite3_mprintf("main.sqlite_schema"); @@ -22392,11 +23151,11 @@ static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){ char *zSql = sqlite3_mprintf(aQuery[i].zSql, zSchemaTab); int val = db_int(p->db, zSql); sqlite3_free(zSql); - utf8_printf(p->out, "%-20s %d\n", aQuery[i].zName, val); + oputf("%-20s %d\n", aQuery[i].zName, val); } sqlite3_free(zSchemaTab); sqlite3_file_control(p->db, zDb, SQLITE_FCNTL_DATA_VERSION, &iDataVersion); - utf8_printf(p->out, "%-20s %u\n", "data version", iDataVersion); + oputf("%-20s %u\n", "data version", iDataVersion); return 0; } #endif /* SQLITE_SHELL_HAVE_RECOVER */ @@ -22406,7 +23165,7 @@ static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){ */ static int shellDatabaseError(sqlite3 *db){ const char *zErr = sqlite3_errmsg(db); - utf8_printf(stderr, "Error: %s\n", zErr); + eputf("Error: %s\n", zErr); return 1; } @@ -22641,7 +23400,6 @@ static int lintFkeyIndexes( int nArg /* Number of entries in azArg[] */ ){ sqlite3 *db = pState->db; /* Database handle to query "main" db of */ - FILE *out = pState->out; /* Stream to write non-error output to */ int bVerbose = 0; /* If -verbose is present */ int bGroupByParent = 0; /* If -groupbyparent is present */ int i; /* To iterate through azArg[] */ @@ -22723,9 +23481,7 @@ static int lintFkeyIndexes( zIndent = " "; } else{ - raw_printf(stderr, "Usage: %s %s ?-verbose? ?-groupbyparent?\n", - azArg[0], azArg[1] - ); + eputf("Usage: %s %s ?-verbose? ?-groupbyparent?\n", azArg[0], azArg[1]); return SQLITE_ERROR; } } @@ -22769,23 +23525,23 @@ static int lintFkeyIndexes( if( rc!=SQLITE_OK ) break; if( res<0 ){ - raw_printf(stderr, "Error: internal error"); + eputz("Error: internal error"); break; }else{ if( bGroupByParent && (bVerbose || res==0) && (zPrev==0 || sqlite3_stricmp(zParent, zPrev)) ){ - raw_printf(out, "-- Parent table %s\n", zParent); + oputf("-- Parent table %s\n", zParent); sqlite3_free(zPrev); zPrev = sqlite3_mprintf("%s", zParent); } if( res==0 ){ - raw_printf(out, "%s%s --> %s\n", zIndent, zCI, zTarget); + oputf("%s%s --> %s\n", zIndent, zCI, zTarget); }else if( bVerbose ){ - raw_printf(out, "%s/* no extra indexes required for %s -> %s */\n", - zIndent, zFrom, zTarget + oputf("%s/* no extra indexes required for %s -> %s */\n", + zIndent, zFrom, zTarget ); } } @@ -22793,16 +23549,16 @@ static int lintFkeyIndexes( sqlite3_free(zPrev); if( rc!=SQLITE_OK ){ - raw_printf(stderr, "%s\n", sqlite3_errmsg(db)); + eputf("%s\n", sqlite3_errmsg(db)); } rc2 = sqlite3_finalize(pSql); if( rc==SQLITE_OK && rc2!=SQLITE_OK ){ rc = rc2; - raw_printf(stderr, "%s\n", sqlite3_errmsg(db)); + eputf("%s\n", sqlite3_errmsg(db)); } }else{ - raw_printf(stderr, "%s\n", sqlite3_errmsg(db)); + eputf("%s\n", sqlite3_errmsg(db)); } return rc; @@ -22822,9 +23578,9 @@ static int lintDotCommand( return lintFkeyIndexes(pState, azArg, nArg); usage: - raw_printf(stderr, "Usage %s sub-command ?switches...?\n", azArg[0]); - raw_printf(stderr, "Where sub-commands are:\n"); - raw_printf(stderr, " fkey-indexes\n"); + eputf("Usage %s sub-command ?switches...?\n", azArg[0]); + eputz("Where sub-commands are:\n"); + eputz(" fkey-indexes\n"); return SQLITE_ERROR; } @@ -22839,9 +23595,7 @@ static void shellPrepare( if( *pRc==SQLITE_OK ){ int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0); if( rc!=SQLITE_OK ){ - raw_printf(stderr, "sql error: %s (%d)\n", - sqlite3_errmsg(db), sqlite3_errcode(db) - ); + eputf("sql error: %s (%d)\n", sqlite3_errmsg(db), sqlite3_errcode(db)); *pRc = rc; } } @@ -22892,7 +23646,7 @@ void shellFinalize( int rc = sqlite3_finalize(pStmt); if( *pRc==SQLITE_OK ){ if( rc!=SQLITE_OK ){ - raw_printf(stderr, "SQL error: %s\n", sqlite3_errmsg(db)); + eputf("SQL error: %s\n", sqlite3_errmsg(db)); } *pRc = rc; } @@ -22913,7 +23667,7 @@ void shellReset( if( *pRc==SQLITE_OK ){ if( rc!=SQLITE_OK ){ sqlite3 *db = sqlite3_db_handle(pStmt); - raw_printf(stderr, "SQL error: %s\n", sqlite3_errmsg(db)); + eputf("SQL error: %s\n", sqlite3_errmsg(db)); } *pRc = rc; } @@ -22963,11 +23717,11 @@ static int arErrorMsg(ArCommand *pAr, const char *zFmt, ...){ va_start(ap, zFmt); z = sqlite3_vmprintf(zFmt, ap); va_end(ap); - utf8_printf(stderr, "Error: %s\n", z); + eputf("Error: %s\n", z); if( pAr->fromCmdLine ){ - utf8_printf(stderr, "Use \"-A\" for more help\n"); + eputz("Use \"-A\" for more help\n"); }else{ - utf8_printf(stderr, "Use \".archive --help\" for more help\n"); + eputz("Use \".archive --help\" for more help\n"); } sqlite3_free(z); return SQLITE_ERROR; @@ -23067,7 +23821,7 @@ static int arParseCommand( struct ArSwitch *pEnd = &aSwitch[nSwitch]; if( nArg<=1 ){ - utf8_printf(stderr, "Wrong number of arguments. Usage:\n"); + eputz("Wrong number of arguments. Usage:\n"); return arUsage(stderr); }else{ char *z = azArg[1]; @@ -23173,7 +23927,7 @@ static int arParseCommand( } } if( pAr->eCmd==0 ){ - utf8_printf(stderr, "Required argument missing. Usage:\n"); + eputz("Required argument missing. Usage:\n"); return arUsage(stderr); } return SQLITE_OK; @@ -23216,7 +23970,7 @@ static int arCheckEntries(ArCommand *pAr){ } shellReset(&rc, pTest); if( rc==SQLITE_OK && bOk==0 ){ - utf8_printf(stderr, "not found in archive: %s\n", z); + eputf("not found in archive: %s\n", z); rc = SQLITE_ERROR; } } @@ -23283,18 +24037,15 @@ static int arListCommand(ArCommand *pAr){ shellPreparePrintf(pAr->db, &rc, &pSql, zSql, azCols[pAr->bVerbose], pAr->zSrcTable, zWhere); if( pAr->bDryRun ){ - utf8_printf(pAr->p->out, "%s\n", sqlite3_sql(pSql)); + oputf("%s\n", sqlite3_sql(pSql)); }else{ while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){ if( pAr->bVerbose ){ - utf8_printf(pAr->p->out, "%s % 10d %s %s\n", - sqlite3_column_text(pSql, 0), - sqlite3_column_int(pSql, 1), - sqlite3_column_text(pSql, 2), - sqlite3_column_text(pSql, 3) - ); + oputf("%s % 10d %s %s\n", + sqlite3_column_text(pSql, 0), sqlite3_column_int(pSql, 1), + sqlite3_column_text(pSql, 2),sqlite3_column_text(pSql, 3)); }else{ - utf8_printf(pAr->p->out, "%s\n", sqlite3_column_text(pSql, 0)); + oputf("%s\n", sqlite3_column_text(pSql, 0)); } } } @@ -23303,7 +24054,6 @@ static int arListCommand(ArCommand *pAr){ return rc; } - /* ** Implementation of .ar "Remove" command. */ @@ -23322,7 +24072,7 @@ static int arRemoveCommand(ArCommand *pAr){ zSql = sqlite3_mprintf("DELETE FROM %s WHERE %s;", pAr->zSrcTable, zWhere); if( pAr->bDryRun ){ - utf8_printf(pAr->p->out, "%s\n", zSql); + oputf("%s\n", zSql); }else{ char *zErr = 0; rc = sqlite3_exec(pAr->db, "SAVEPOINT ar;", 0, 0, 0); @@ -23335,7 +24085,7 @@ static int arRemoveCommand(ArCommand *pAr){ } } if( zErr ){ - utf8_printf(stdout, "ERROR: %s\n", zErr); + sputf(stdout, "ERROR: %s\n", zErr); /* stdout? */ sqlite3_free(zErr); } } @@ -23399,11 +24149,11 @@ static int arExtractCommand(ArCommand *pAr){ j = sqlite3_bind_parameter_index(pSql, "$dirOnly"); sqlite3_bind_int(pSql, j, i); if( pAr->bDryRun ){ - utf8_printf(pAr->p->out, "%s\n", sqlite3_sql(pSql)); + oputf("%s\n", sqlite3_sql(pSql)); }else{ while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){ if( i==0 && pAr->bVerbose ){ - utf8_printf(pAr->p->out, "%s\n", sqlite3_column_text(pSql, 0)); + oputf("%s\n", sqlite3_column_text(pSql, 0)); } } } @@ -23423,13 +24173,13 @@ static int arExtractCommand(ArCommand *pAr){ static int arExecSql(ArCommand *pAr, const char *zSql){ int rc; if( pAr->bDryRun ){ - utf8_printf(pAr->p->out, "%s\n", zSql); + oputf("%s\n", zSql); rc = SQLITE_OK; }else{ char *zErr = 0; rc = sqlite3_exec(pAr->db, zSql, 0, 0, &zErr); if( zErr ){ - utf8_printf(stdout, "ERROR: %s\n", zErr); + sputf(stdout, "ERROR: %s\n", zErr); sqlite3_free(zErr); } } @@ -23604,15 +24354,13 @@ static int arDotCommand( } cmd.db = 0; if( cmd.bDryRun ){ - utf8_printf(pState->out, "-- open database '%s'%s\n", cmd.zFile, - eDbType==SHELL_OPEN_APPENDVFS ? " using 'apndvfs'" : ""); + oputf("-- open database '%s'%s\n", cmd.zFile, + eDbType==SHELL_OPEN_APPENDVFS ? " using 'apndvfs'" : ""); } rc = sqlite3_open_v2(cmd.zFile, &cmd.db, flags, eDbType==SHELL_OPEN_APPENDVFS ? "apndvfs" : 0); if( rc!=SQLITE_OK ){ - utf8_printf(stderr, "cannot open file: %s (%s)\n", - cmd.zFile, sqlite3_errmsg(cmd.db) - ); + eputf("cannot open file: %s (%s)\n", cmd.zFile, sqlite3_errmsg(cmd.db)); goto end_ar_command; } sqlite3_fileio_init(cmd.db, 0, 0); @@ -23625,7 +24373,7 @@ static int arDotCommand( if( cmd.eCmd!=AR_CMD_CREATE && sqlite3_table_column_metadata(cmd.db,0,"sqlar","name",0,0,0,0,0) ){ - utf8_printf(stderr, "database does not contain an 'sqlar' table\n"); + eputz("database does not contain an 'sqlar' table\n"); rc = SQLITE_ERROR; goto end_ar_command; } @@ -23683,7 +24431,7 @@ static int arDotCommand( */ static int recoverSqlCb(void *pCtx, const char *zSql){ ShellState *pState = (ShellState*)pCtx; - utf8_printf(pState->out, "%s;\n", zSql); + sputf(pState->out, "%s;\n", zSql); return SQLITE_OK; } @@ -23726,7 +24474,7 @@ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ bRowids = 0; } else{ - utf8_printf(stderr, "unexpected option: %s\n", azArg[i]); + eputf("unexpected option: %s\n", azArg[i]); showHelp(pState->out, azArg[0]); return 1; } @@ -23745,7 +24493,7 @@ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ if( sqlite3_recover_errcode(p)!=SQLITE_OK ){ const char *zErr = sqlite3_recover_errmsg(p); int errCode = sqlite3_recover_errcode(p); - raw_printf(stderr, "sql error: %s (%d)\n", zErr, errCode); + eputf("sql error: %s (%d)\n", zErr, errCode); } rc = sqlite3_recover_finish(p); return rc; @@ -23770,7 +24518,7 @@ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ #define rc_err_oom_die(rc) \ if( rc==SQLITE_NOMEM ) shell_check_oom(0); \ else if(!(rc==SQLITE_OK||rc==SQLITE_DONE)) \ - fprintf(stderr,"E:%d\n",rc), assert(0) + eputf("E:%d\n",rc), assert(0) #else static void rc_err_oom_die(int rc){ if( rc==SQLITE_NOMEM ) shell_check_oom(0); @@ -23910,6 +24658,7 @@ FROM (\ sqlite3_exec(*pDb,"drop table if exists ColNames;" "drop view if exists RepeatedNames;",0,0,0); #endif +#undef SHELL_COLFIX_DB rc = sqlite3_exec(*pDb, zTabMake, 0, 0, 0); rc_err_oom_die(rc); } @@ -24023,7 +24772,7 @@ static int do_meta_command(char *zLine, ShellState *p){ #ifndef SQLITE_OMIT_AUTHORIZATION if( c=='a' && cli_strncmp(azArg[0], "auth", n)==0 ){ if( nArg!=2 ){ - raw_printf(stderr, "Usage: .auth ON|OFF\n"); + eputz("Usage: .auth ON|OFF\n"); rc = 1; goto meta_command_exit; } @@ -24070,7 +24819,7 @@ static int do_meta_command(char *zLine, ShellState *p){ bAsync = 1; }else { - utf8_printf(stderr, "unknown option: %s\n", azArg[j]); + eputf("unknown option: %s\n", azArg[j]); return 1; } }else if( zDestFile==0 ){ @@ -24079,19 +24828,19 @@ static int do_meta_command(char *zLine, ShellState *p){ zDb = zDestFile; zDestFile = azArg[j]; }else{ - raw_printf(stderr, "Usage: .backup ?DB? ?OPTIONS? FILENAME\n"); + eputz("Usage: .backup ?DB? ?OPTIONS? FILENAME\n"); return 1; } } if( zDestFile==0 ){ - raw_printf(stderr, "missing FILENAME argument on .backup\n"); + eputz("missing FILENAME argument on .backup\n"); return 1; } if( zDb==0 ) zDb = "main"; rc = sqlite3_open_v2(zDestFile, &pDest, SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE, zVfs); if( rc!=SQLITE_OK ){ - utf8_printf(stderr, "Error: cannot open \"%s\"\n", zDestFile); + eputf("Error: cannot open \"%s\"\n", zDestFile); close_db(pDest); return 1; } @@ -24102,7 +24851,7 @@ static int do_meta_command(char *zLine, ShellState *p){ open_db(p, 0); pBackup = sqlite3_backup_init(pDest, "main", p->db, zDb); if( pBackup==0 ){ - utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(pDest)); + eputf("Error: %s\n", sqlite3_errmsg(pDest)); close_db(pDest); return 1; } @@ -24111,7 +24860,7 @@ static int do_meta_command(char *zLine, ShellState *p){ if( rc==SQLITE_DONE ){ rc = 0; }else{ - utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(pDest)); + eputf("Error: %s\n", sqlite3_errmsg(pDest)); rc = 1; } close_db(pDest); @@ -24122,7 +24871,7 @@ static int do_meta_command(char *zLine, ShellState *p){ if( nArg==2 ){ bail_on_error = booleanValue(azArg[1]); }else{ - raw_printf(stderr, "Usage: .bail on|off\n"); + eputz("Usage: .bail on|off\n"); rc = 1; } }else @@ -24136,9 +24885,8 @@ static int do_meta_command(char *zLine, ShellState *p){ setTextMode(p->out, 1); } }else{ - raw_printf(stderr, "The \".binary\" command is deprecated." - " Use \".crnl\" instead.\n"); - raw_printf(stderr, "Usage: .binary on|off\n"); + eputz("The \".binary\" command is deprecated. Use \".crnl\" instead.\n" + "Usage: .binary on|off\n"); rc = 1; } }else @@ -24162,11 +24910,11 @@ static int do_meta_command(char *zLine, ShellState *p){ rc = chdir(azArg[1]); #endif if( rc ){ - utf8_printf(stderr, "Cannot change to directory \"%s\"\n", azArg[1]); + eputf("Cannot change to directory \"%s\"\n", azArg[1]); rc = 1; } }else{ - raw_printf(stderr, "Usage: .cd DIRECTORY\n"); + eputz("Usage: .cd DIRECTORY\n"); rc = 1; } }else @@ -24176,7 +24924,7 @@ static int do_meta_command(char *zLine, ShellState *p){ if( nArg==2 ){ setOrClearFlag(p, SHFLG_CountChanges, azArg[1]); }else{ - raw_printf(stderr, "Usage: .changes on|off\n"); + eputz("Usage: .changes on|off\n"); rc = 1; } }else @@ -24190,17 +24938,16 @@ static int do_meta_command(char *zLine, ShellState *p){ char *zRes = 0; output_reset(p); if( nArg!=2 ){ - raw_printf(stderr, "Usage: .check GLOB-PATTERN\n"); + eputz("Usage: .check GLOB-PATTERN\n"); rc = 2; }else if( (zRes = readFile("testcase-out.txt", 0))==0 ){ rc = 2; }else if( testcase_glob(azArg[1],zRes)==0 ){ - utf8_printf(stderr, - "testcase-%s FAILED\n Expected: [%s]\n Got: [%s]\n", - p->zTestcase, azArg[1], zRes); + eputf("testcase-%s FAILED\n Expected: [%s]\n Got: [%s]\n", + p->zTestcase, azArg[1], zRes); rc = 1; }else{ - utf8_printf(stdout, "testcase-%s ok\n", p->zTestcase); + oputf("testcase-%s ok\n", p->zTestcase); p->nCheck++; } sqlite3_free(zRes); @@ -24213,7 +24960,7 @@ static int do_meta_command(char *zLine, ShellState *p){ if( nArg==2 ){ tryToClone(p, azArg[1]); }else{ - raw_printf(stderr, "Usage: .clone FILENAME\n"); + eputz("Usage: .clone FILENAME\n"); rc = 1; } }else @@ -24233,9 +24980,9 @@ static int do_meta_command(char *zLine, ShellState *p){ zFile = "(temporary-file)"; } if( p->pAuxDb == &p->aAuxDb[i] ){ - utf8_printf(stdout, "ACTIVE %d: %s\n", i, zFile); + sputf(stdout, "ACTIVE %d: %s\n", i, zFile); }else if( p->aAuxDb[i].db!=0 ){ - utf8_printf(stdout, " %d: %s\n", i, zFile); + sputf(stdout, " %d: %s\n", i, zFile); } } }else if( nArg==2 && IsDigit(azArg[1][0]) && azArg[1][1]==0 ){ @@ -24252,7 +24999,7 @@ static int do_meta_command(char *zLine, ShellState *p){ if( i<0 || i>=ArraySize(p->aAuxDb) ){ /* No-op */ }else if( p->pAuxDb == &p->aAuxDb[i] ){ - raw_printf(stderr, "cannot close the active database connection\n"); + eputz("cannot close the active database connection\n"); rc = 1; }else if( p->aAuxDb[i].db ){ session_close_all(p, i); @@ -24260,7 +25007,7 @@ static int do_meta_command(char *zLine, ShellState *p){ p->aAuxDb[i].db = 0; } }else{ - raw_printf(stderr, "Usage: .connection [close] [CONNECTION-NUMBER]\n"); + eputz("Usage: .connection [close] [CONNECTION-NUMBER]\n"); rc = 1; } }else @@ -24274,9 +25021,9 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else{ #if !defined(_WIN32) && !defined(WIN32) - raw_printf(stderr, "The \".crnl\" is a no-op on non-Windows machines.\n"); + eputz("The \".crnl\" is a no-op on non-Windows machines.\n"); #endif - raw_printf(stderr, "Usage: .crnl on|off\n"); + eputz("Usage: .crnl on|off\n"); rc = 1; } }else @@ -24289,7 +25036,7 @@ static int do_meta_command(char *zLine, ShellState *p){ open_db(p, 0); rc = sqlite3_prepare_v2(p->db, "PRAGMA database_list", -1, &pStmt, 0); if( rc ){ - utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(p->db)); + eputf("Error: %s\n", sqlite3_errmsg(p->db)); rc = 1; }else{ while( sqlite3_step(pStmt)==SQLITE_ROW ){ @@ -24308,11 +25055,9 @@ static int do_meta_command(char *zLine, ShellState *p){ int eTxn = sqlite3_txn_state(p->db, azName[i*2]); int bRdonly = sqlite3_db_readonly(p->db, azName[i*2]); const char *z = azName[i*2+1]; - utf8_printf(p->out, "%s: %s %s%s\n", - azName[i*2], - z && z[0] ? z : "\"\"", - bRdonly ? "r/o" : "r/w", - eTxn==SQLITE_TXN_NONE ? "" : + oputf("%s: %s %s%s\n", + azName[i*2], z && z[0] ? z : "\"\"", bRdonly ? "r/o" : "r/w", + eTxn==SQLITE_TXN_NONE ? "" : eTxn==SQLITE_TXN_READ ? " read-txn" : " write-txn"); free(azName[i*2]); free(azName[i*2+1]); @@ -24352,12 +25097,12 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_db_config(p->db, aDbConfig[ii].op, booleanValue(azArg[2]), 0); } sqlite3_db_config(p->db, aDbConfig[ii].op, -1, &v); - utf8_printf(p->out, "%19s %s\n", aDbConfig[ii].zName, v ? "on" : "off"); + oputf("%19s %s\n", aDbConfig[ii].zName, v ? "on" : "off"); if( nArg>1 ) break; } if( nArg>1 && ii==ArraySize(aDbConfig) ){ - utf8_printf(stderr, "Error: unknown dbconfig \"%s\"\n", azArg[1]); - utf8_printf(stderr, "Enter \".dbconfig\" with no arguments for a list\n"); + eputf("Error: unknown dbconfig \"%s\"\n", azArg[1]); + eputz("Enter \".dbconfig\" with no arguments for a list\n"); } }else @@ -24387,8 +25132,8 @@ static int do_meta_command(char *zLine, ShellState *p){ if( z[0]=='-' ) z++; if( cli_strcmp(z,"preserve-rowids")==0 ){ #ifdef SQLITE_OMIT_VIRTUALTABLE - raw_printf(stderr, "The --preserve-rowids option is not compatible" - " with SQLITE_OMIT_VIRTUALTABLE\n"); + eputz("The --preserve-rowids option is not compatible" + " with SQLITE_OMIT_VIRTUALTABLE\n"); rc = 1; sqlite3_free(zLike); goto meta_command_exit; @@ -24406,7 +25151,7 @@ static int do_meta_command(char *zLine, ShellState *p){ ShellSetFlag(p, SHFLG_DumpNoSys); }else { - raw_printf(stderr, "Unknown option \"%s\" on \".dump\"\n", azArg[i]); + eputf("Unknown option \"%s\" on \".dump\"\n", azArg[i]); rc = 1; sqlite3_free(zLike); goto meta_command_exit; @@ -24440,8 +25185,8 @@ static int do_meta_command(char *zLine, ShellState *p){ /* When playing back a "dump", the content might appear in an order ** which causes immediate foreign key constraints to be violated. ** So disable foreign-key constraint enforcement to prevent problems. */ - raw_printf(p->out, "PRAGMA foreign_keys=OFF;\n"); - raw_printf(p->out, "BEGIN TRANSACTION;\n"); + oputz("PRAGMA foreign_keys=OFF;\n"); + oputz("BEGIN TRANSACTION;\n"); } p->writableSchema = 0; p->showHeader = 0; @@ -24472,13 +25217,13 @@ static int do_meta_command(char *zLine, ShellState *p){ } sqlite3_free(zLike); if( p->writableSchema ){ - raw_printf(p->out, "PRAGMA writable_schema=OFF;\n"); + oputz("PRAGMA writable_schema=OFF;\n"); p->writableSchema = 0; } sqlite3_exec(p->db, "PRAGMA writable_schema=OFF;", 0, 0, 0); sqlite3_exec(p->db, "RELEASE dump;", 0, 0, 0); if( (p->shellFlgs & SHFLG_DumpDataOnly)==0 ){ - raw_printf(p->out, p->nErr?"ROLLBACK; -- due to errors\n":"COMMIT;\n"); + oputz(p->nErr?"ROLLBACK; -- due to errors\n":"COMMIT;\n"); } p->showHeader = savedShowHeader; p->shellFlgs = savedShellFlags; @@ -24488,7 +25233,7 @@ static int do_meta_command(char *zLine, ShellState *p){ if( nArg==2 ){ setOrClearFlag(p, SHFLG_Echo, azArg[1]); }else{ - raw_printf(stderr, "Usage: .echo on|off\n"); + eputz("Usage: .echo on|off\n"); rc = 1; } }else @@ -24519,7 +25264,7 @@ static int do_meta_command(char *zLine, ShellState *p){ p->autoEQP = (u8)booleanValue(azArg[1]); } }else{ - raw_printf(stderr, "Usage: .eqp off|on|trace|trigger|full\n"); + eputz("Usage: .eqp off|on|trace|trigger|full\n"); rc = 1; } }else @@ -24558,9 +25303,8 @@ static int do_meta_command(char *zLine, ShellState *p){ #ifndef SQLITE_OMIT_VIRTUALTABLE if( c=='e' && cli_strncmp(azArg[0], "expert", n)==0 ){ if( p->bSafeMode ){ - raw_printf(stderr, - "Cannot run experimental commands such as \"%s\" in safe mode\n", - azArg[0]); + eputf("Cannot run experimental commands such as \"%s\" in safe mode\n", + azArg[0]); rc = 1; }else{ open_db(p, 0); @@ -24616,10 +25360,9 @@ static int do_meta_command(char *zLine, ShellState *p){ /* --help lists all file-controls */ if( cli_strcmp(zCmd,"help")==0 ){ - utf8_printf(p->out, "Available file-controls:\n"); + oputz("Available file-controls:\n"); for(i=0; iout, " .filectrl %s %s\n", - aCtrl[i].zCtrlName, aCtrl[i].zUsage); + oputf(" .filectrl %s %s\n", aCtrl[i].zCtrlName, aCtrl[i].zUsage); } rc = 1; goto meta_command_exit; @@ -24634,16 +25377,16 @@ static int do_meta_command(char *zLine, ShellState *p){ filectrl = aCtrl[i].ctrlCode; iCtrl = i; }else{ - utf8_printf(stderr, "Error: ambiguous file-control: \"%s\"\n" - "Use \".filectrl --help\" for help\n", zCmd); + eputf("Error: ambiguous file-control: \"%s\"\n" + "Use \".filectrl --help\" for help\n", zCmd); rc = 1; goto meta_command_exit; } } } if( filectrl<0 ){ - utf8_printf(stderr,"Error: unknown file-control: %s\n" - "Use \".filectrl --help\" for help\n", zCmd); + eputf("Error: unknown file-control: %s\n" + "Use \".filectrl --help\" for help\n", zCmd); }else{ switch(filectrl){ case SQLITE_FCNTL_SIZE_LIMIT: { @@ -24686,7 +25429,7 @@ static int do_meta_command(char *zLine, ShellState *p){ if( nArg!=2 ) break; sqlite3_file_control(p->db, zSchema, filectrl, &z); if( z ){ - utf8_printf(p->out, "%s\n", z); + oputf("%s\n", z); sqlite3_free(z); } isOk = 2; @@ -24700,19 +25443,19 @@ static int do_meta_command(char *zLine, ShellState *p){ } x = -1; sqlite3_file_control(p->db, zSchema, filectrl, &x); - utf8_printf(p->out,"%d\n", x); + oputf("%d\n", x); isOk = 2; break; } } } if( isOk==0 && iCtrl>=0 ){ - utf8_printf(p->out, "Usage: .filectrl %s %s\n", zCmd,aCtrl[iCtrl].zUsage); + oputf("Usage: .filectrl %s %s\n", zCmd,aCtrl[iCtrl].zUsage); rc = 1; }else if( isOk==1 ){ char zBuf[100]; sqlite3_snprintf(sizeof(zBuf), zBuf, "%lld", iRes); - raw_printf(p->out, "%s\n", zBuf); + oputf("%s\n", zBuf); } }else @@ -24727,7 +25470,7 @@ static int do_meta_command(char *zLine, ShellState *p){ nArg = 1; } if( nArg!=1 ){ - raw_printf(stderr, "Usage: .fullschema ?--indent?\n"); + eputz("Usage: .fullschema ?--indent?\n"); rc = 1; goto meta_command_exit; } @@ -24753,15 +25496,15 @@ static int do_meta_command(char *zLine, ShellState *p){ } } if( doStats==0 ){ - raw_printf(p->out, "/* No STAT tables available */\n"); + oputz("/* No STAT tables available */\n"); }else{ - raw_printf(p->out, "ANALYZE sqlite_schema;\n"); + oputz("ANALYZE sqlite_schema;\n"); data.cMode = data.mode = MODE_Insert; data.zDestTable = "sqlite_stat1"; shell_exec(&data, "SELECT * FROM sqlite_stat1", 0); data.zDestTable = "sqlite_stat4"; shell_exec(&data, "SELECT * FROM sqlite_stat4", 0); - raw_printf(p->out, "ANALYZE sqlite_schema;\n"); + oputz("ANALYZE sqlite_schema;\n"); } }else @@ -24770,7 +25513,7 @@ static int do_meta_command(char *zLine, ShellState *p){ p->showHeader = booleanValue(azArg[1]); p->shellFlgs |= SHFLG_HeaderSet; }else{ - raw_printf(stderr, "Usage: .headers on|off\n"); + eputz("Usage: .headers on|off\n"); rc = 1; } }else @@ -24779,7 +25522,7 @@ static int do_meta_command(char *zLine, ShellState *p){ if( nArg>=2 ){ n = showHelp(p->out, azArg[1]); if( n==0 ){ - utf8_printf(p->out, "Nothing matches '%s'\n", azArg[1]); + oputf("Nothing matches '%s'\n", azArg[1]); } }else{ showHelp(p->out, 0); @@ -24823,7 +25566,7 @@ static int do_meta_command(char *zLine, ShellState *p){ }else if( zTable==0 ){ zTable = z; }else{ - utf8_printf(p->out, "ERROR: extra argument: \"%s\". Usage:\n", z); + oputf("ERROR: extra argument: \"%s\". Usage:\n", z); showHelp(p->out, "import"); goto meta_command_exit; } @@ -24844,14 +25587,14 @@ static int do_meta_command(char *zLine, ShellState *p){ xRead = csv_read_one_field; useOutputMode = 0; }else{ - utf8_printf(p->out, "ERROR: unknown option: \"%s\". Usage:\n", z); + oputf("ERROR: unknown option: \"%s\". Usage:\n", z); showHelp(p->out, "import"); goto meta_command_exit; } } if( zTable==0 ){ - utf8_printf(p->out, "ERROR: missing %s argument. Usage:\n", - zFile==0 ? "FILE" : "TABLE"); + oputf("ERROR: missing %s argument. Usage:\n", + zFile==0 ? "FILE" : "TABLE"); showHelp(p->out, "import"); goto meta_command_exit; } @@ -24862,20 +25605,17 @@ static int do_meta_command(char *zLine, ShellState *p){ ** the column and row separator characters from the output mode. */ nSep = strlen30(p->colSeparator); if( nSep==0 ){ - raw_printf(stderr, - "Error: non-null column separator required for import\n"); + eputz("Error: non-null column separator required for import\n"); goto meta_command_exit; } if( nSep>1 ){ - raw_printf(stderr, - "Error: multi-character column separators not allowed" + eputz("Error: multi-character column separators not allowed" " for import\n"); goto meta_command_exit; } nSep = strlen30(p->rowSeparator); if( nSep==0 ){ - raw_printf(stderr, - "Error: non-null row separator required for import\n"); + eputz("Error: non-null row separator required for import\n"); goto meta_command_exit; } if( nSep==2 && p->mode==MODE_Csv @@ -24889,8 +25629,8 @@ static int do_meta_command(char *zLine, ShellState *p){ nSep = strlen30(p->rowSeparator); } if( nSep>1 ){ - raw_printf(stderr, "Error: multi-character row separators not allowed" - " for import\n"); + eputz("Error: multi-character row separators not allowed" + " for import\n"); goto meta_command_exit; } sCtx.cColSep = (u8)p->colSeparator[0]; @@ -24900,7 +25640,7 @@ static int do_meta_command(char *zLine, ShellState *p){ sCtx.nLine = 1; if( sCtx.zFile[0]=='|' ){ #ifdef SQLITE_OMIT_POPEN - raw_printf(stderr, "Error: pipes are not supported in this OS\n"); + eputz("Error: pipes are not supported in this OS\n"); goto meta_command_exit; #else sCtx.in = popen(sCtx.zFile+1, "r"); @@ -24912,19 +25652,19 @@ static int do_meta_command(char *zLine, ShellState *p){ sCtx.xCloser = fclose; } if( sCtx.in==0 ){ - utf8_printf(stderr, "Error: cannot open \"%s\"\n", zFile); + eputf("Error: cannot open \"%s\"\n", zFile); goto meta_command_exit; } if( eVerbose>=2 || (eVerbose>=1 && useOutputMode) ){ char zSep[2]; zSep[1] = 0; zSep[0] = sCtx.cColSep; - utf8_printf(p->out, "Column separator "); - output_c_string(p->out, zSep); - utf8_printf(p->out, ", row separator "); + oputz("Column separator "); + output_c_string(zSep); + oputz(", row separator "); zSep[0] = sCtx.cRowSep; - output_c_string(p->out, zSep); - utf8_printf(p->out, "\n"); + output_c_string(zSep); + oputz("\n"); } sCtx.z = sqlite3_malloc64(120); if( sCtx.z==0 ){ @@ -24959,14 +25699,14 @@ static int do_meta_command(char *zLine, ShellState *p){ } zColDefs = zAutoColumn(0, &dbCols, &zRenames); if( zRenames!=0 ){ - utf8_printf((stdin_is_interactive && p->in==stdin)? p->out : stderr, - "Columns renamed during .import %s due to duplicates:\n" - "%s\n", sCtx.zFile, zRenames); + sputf((stdin_is_interactive && p->in==stdin)? p->out : stderr, + "Columns renamed during .import %s due to duplicates:\n" + "%s\n", sCtx.zFile, zRenames); sqlite3_free(zRenames); } assert(dbCols==0); if( zColDefs==0 ){ - utf8_printf(stderr,"%s: empty file\n", sCtx.zFile); + eputf("%s: empty file\n", sCtx.zFile); import_fail: sqlite3_free(zCreate); sqlite3_free(zSql); @@ -24977,11 +25717,11 @@ static int do_meta_command(char *zLine, ShellState *p){ } zCreate = sqlite3_mprintf("%z%z\n", zCreate, zColDefs); if( eVerbose>=1 ){ - utf8_printf(p->out, "%s\n", zCreate); + oputf("%s\n", zCreate); } rc = sqlite3_exec(p->db, zCreate, 0, 0, 0); if( rc ){ - utf8_printf(stderr, "%s failed:\n%s\n", zCreate, sqlite3_errmsg(p->db)); + eputf("%s failed:\n%s\n", zCreate, sqlite3_errmsg(p->db)); goto import_fail; } sqlite3_free(zCreate); @@ -24990,7 +25730,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } if( rc ){ if (pStmt) sqlite3_finalize(pStmt); - utf8_printf(stderr,"Error: %s\n", sqlite3_errmsg(p->db)); + eputf("Error: %s\n", sqlite3_errmsg(p->db)); goto import_fail; } sqlite3_free(zSql); @@ -25012,11 +25752,11 @@ static int do_meta_command(char *zLine, ShellState *p){ zSql[j++] = ')'; zSql[j] = 0; if( eVerbose>=2 ){ - utf8_printf(p->out, "Insert using: %s\n", zSql); + oputf("Insert using: %s\n", zSql); } rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); if( rc ){ - utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(p->db)); + eputf("Error: %s\n", sqlite3_errmsg(p->db)); if (pStmt) sqlite3_finalize(pStmt); goto import_fail; } @@ -25049,9 +25789,9 @@ static int do_meta_command(char *zLine, ShellState *p){ } sqlite3_bind_text(pStmt, i+1, z, -1, SQLITE_TRANSIENT); if( i=nCol ){ sqlite3_step(pStmt); rc = sqlite3_reset(pStmt); if( rc!=SQLITE_OK ){ - utf8_printf(stderr, "%s:%d: INSERT failed: %s\n", sCtx.zFile, - startLine, sqlite3_errmsg(p->db)); + eputf("%s:%d: INSERT failed: %s\n", + sCtx.zFile, startLine, sqlite3_errmsg(p->db)); sCtx.nErr++; }else{ sCtx.nRow++; @@ -25082,9 +25821,8 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_finalize(pStmt); if( needCommit ) sqlite3_exec(p->db, "COMMIT", 0, 0, 0); if( eVerbose>0 ){ - utf8_printf(p->out, - "Added %d rows with %d errors using %d lines of input\n", - sCtx.nRow, sCtx.nErr, sCtx.nLine-1); + oputf("Added %d rows with %d errors using %d lines of input\n", + sCtx.nRow, sCtx.nErr, sCtx.nLine-1); } }else #endif /* !defined(SQLITE_SHELL_FIDDLE) */ @@ -25099,14 +25837,14 @@ static int do_meta_command(char *zLine, ShellState *p){ int lenPK = 0; /* Length of the PRIMARY KEY string for isWO tables */ int i; if( !ShellHasFlag(p,SHFLG_TestingMode) ){ - utf8_printf(stderr, ".%s unavailable without --unsafe-testing\n", - "imposter"); + eputf(".%s unavailable without --unsafe-testing\n", + "imposter"); rc = 1; goto meta_command_exit; } if( !(nArg==3 || (nArg==2 && sqlite3_stricmp(azArg[1],"off")==0)) ){ - utf8_printf(stderr, "Usage: .imposter INDEX IMPOSTER\n" - " .imposter off\n"); + eputz("Usage: .imposter INDEX IMPOSTER\n" + " .imposter off\n"); /* Also allowed, but not documented: ** ** .imposter TABLE IMPOSTER @@ -25165,7 +25903,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } sqlite3_finalize(pStmt); if( i==0 || tnum==0 ){ - utf8_printf(stderr, "no such index: \"%s\"\n", azArg[1]); + eputf("no such index: \"%s\"\n", azArg[1]); rc = 1; sqlite3_free(zCollist); goto meta_command_exit; @@ -25180,16 +25918,14 @@ static int do_meta_command(char *zLine, ShellState *p){ rc = sqlite3_exec(p->db, zSql, 0, 0, 0); sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->db, "main", 0, 0); if( rc ){ - utf8_printf(stderr, "Error in [%s]: %s\n", zSql, sqlite3_errmsg(p->db)); + eputf("Error in [%s]: %s\n", zSql, sqlite3_errmsg(p->db)); }else{ - utf8_printf(stdout, "%s;\n", zSql); - raw_printf(stdout, - "WARNING: writing to an imposter table will corrupt the \"%s\" %s!\n", - azArg[1], isWO ? "table" : "index" - ); + sputf(stdout, "%s;\n", zSql); + sputf(stdout, "WARNING: writing to an imposter table will corrupt" + " the \"%s\" %s!\n", azArg[1], isWO ? "table" : "index"); } }else{ - raw_printf(stderr, "SQLITE_TESTCTRL_IMPOSTER returns %d\n", rc); + eputf("SQLITE_TESTCTRL_IMPOSTER returns %d\n", rc); rc = 1; } sqlite3_free(zSql); @@ -25209,7 +25945,7 @@ static int do_meta_command(char *zLine, ShellState *p){ }else{ iotrace = fopen(azArg[1], "w"); if( iotrace==0 ){ - utf8_printf(stderr, "Error: cannot open \"%s\"\n", azArg[1]); + eputf("Error: cannot open \"%s\"\n", azArg[1]); sqlite3IoTrace = 0; rc = 1; }else{ @@ -25241,11 +25977,11 @@ static int do_meta_command(char *zLine, ShellState *p){ open_db(p, 0); if( nArg==1 ){ for(i=0; idb, aLimit[i].limitCode, -1)); + sputf(stdout, "%20s %d\n", aLimit[i].zLimitName, + sqlite3_limit(p->db, aLimit[i].limitCode, -1)); } }else if( nArg>3 ){ - raw_printf(stderr, "Usage: .limit NAME ?NEW-VALUE?\n"); + eputz("Usage: .limit NAME ?NEW-VALUE?\n"); rc = 1; goto meta_command_exit; }else{ @@ -25256,16 +25992,16 @@ static int do_meta_command(char *zLine, ShellState *p){ if( iLimit<0 ){ iLimit = i; }else{ - utf8_printf(stderr, "ambiguous limit: \"%s\"\n", azArg[1]); + eputf("ambiguous limit: \"%s\"\n", azArg[1]); rc = 1; goto meta_command_exit; } } } if( iLimit<0 ){ - utf8_printf(stderr, "unknown limit: \"%s\"\n" - "enter \".limits\" with no arguments for a list.\n", - azArg[1]); + eputf("unknown limit: \"%s\"\n" + "enter \".limits\" with no arguments for a list.\n", + azArg[1]); rc = 1; goto meta_command_exit; } @@ -25273,8 +26009,8 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_limit(p->db, aLimit[iLimit].limitCode, (int)integerValue(azArg[2])); } - printf("%20s %d\n", aLimit[iLimit].zLimitName, - sqlite3_limit(p->db, aLimit[iLimit].limitCode, -1)); + sputf(stdout, "%20s %d\n", aLimit[iLimit].zLimitName, + sqlite3_limit(p->db, aLimit[iLimit].limitCode, -1)); } }else @@ -25290,7 +26026,7 @@ static int do_meta_command(char *zLine, ShellState *p){ failIfSafeMode(p, "cannot run .load in safe mode"); if( nArg<2 || azArg[1][0]==0 ){ /* Must have a non-empty FILE. (Will not load self.) */ - raw_printf(stderr, "Usage: .load FILE ?ENTRYPOINT?\n"); + eputz("Usage: .load FILE ?ENTRYPOINT?\n"); rc = 1; goto meta_command_exit; } @@ -25299,7 +26035,7 @@ static int do_meta_command(char *zLine, ShellState *p){ open_db(p, 0); rc = sqlite3_load_extension(p->db, zFile, zProc, &zErrMsg); if( rc!=SQLITE_OK ){ - utf8_printf(stderr, "Error: %s\n", zErrMsg); + eputf("Error: %s\n", zErrMsg); sqlite3_free(zErrMsg); rc = 1; } @@ -25308,7 +26044,7 @@ static int do_meta_command(char *zLine, ShellState *p){ if( c=='l' && cli_strncmp(azArg[0], "log", n)==0 ){ if( nArg!=2 ){ - raw_printf(stderr, "Usage: .log FILENAME\n"); + eputz("Usage: .log FILENAME\n"); rc = 1; }else{ const char *zFile = azArg[1]; @@ -25316,8 +26052,8 @@ static int do_meta_command(char *zLine, ShellState *p){ && cli_strcmp(zFile,"on")!=0 && cli_strcmp(zFile,"off")!=0 ){ - raw_printf(stdout, "cannot set .log to anything other " - "than \"on\" or \"off\"\n"); + sputz(stdout, "cannot set .log to anything other" + " than \"on\" or \"off\"\n"); zFile = "off"; } output_file_close(p->pLog); @@ -25356,17 +26092,17 @@ static int do_meta_command(char *zLine, ShellState *p){ }else if( zTabname==0 ){ zTabname = z; }else if( z[0]=='-' ){ - utf8_printf(stderr, "unknown option: %s\n", z); - utf8_printf(stderr, "options:\n" - " --noquote\n" - " --quote\n" - " --wordwrap on/off\n" - " --wrap N\n" - " --ww\n"); + eputf("unknown option: %s\n", z); + eputz("options:\n" + " --noquote\n" + " --quote\n" + " --wordwrap on/off\n" + " --wrap N\n" + " --ww\n"); rc = 1; goto meta_command_exit; }else{ - utf8_printf(stderr, "extra argument: \"%s\"\n", z); + eputf("extra argument: \"%s\"\n", z); rc = 1; goto meta_command_exit; } @@ -25375,14 +26111,12 @@ static int do_meta_command(char *zLine, ShellState *p){ if( p->mode==MODE_Column || (p->mode>=MODE_Markdown && p->mode<=MODE_Box) ){ - raw_printf - (p->out, - "current output mode: %s --wrap %d --wordwrap %s --%squote\n", - modeDescr[p->mode], p->cmOpts.iWrap, - p->cmOpts.bWordWrap ? "on" : "off", - p->cmOpts.bQuote ? "" : "no"); + oputf("current output mode: %s --wrap %d --wordwrap %s --%squote\n", + modeDescr[p->mode], p->cmOpts.iWrap, + p->cmOpts.bWordWrap ? "on" : "off", + p->cmOpts.bQuote ? "" : "no"); }else{ - raw_printf(p->out, "current output mode: %s\n", modeDescr[p->mode]); + oputf("current output mode: %s\n", modeDescr[p->mode]); } zMode = modeDescr[p->mode]; } @@ -25441,9 +26175,9 @@ static int do_meta_command(char *zLine, ShellState *p){ }else if( cli_strncmp(zMode,"json",n2)==0 ){ p->mode = MODE_Json; }else{ - raw_printf(stderr, "Error: mode should be one of: " - "ascii box column csv html insert json line list markdown " - "qbox quote table tabs tcl\n"); + eputz("Error: mode should be one of: " + "ascii box column csv html insert json line list markdown " + "qbox quote table tabs tcl\n"); rc = 1; } p->cMode = p->mode; @@ -25452,11 +26186,11 @@ static int do_meta_command(char *zLine, ShellState *p){ #ifndef SQLITE_SHELL_FIDDLE if( c=='n' && cli_strcmp(azArg[0], "nonce")==0 ){ if( nArg!=2 ){ - raw_printf(stderr, "Usage: .nonce NONCE\n"); + eputz("Usage: .nonce NONCE\n"); rc = 1; }else if( p->zNonce==0 || cli_strcmp(azArg[1],p->zNonce)!=0 ){ - raw_printf(stderr, "line %d: incorrect nonce: \"%s\"\n", - p->lineno, azArg[1]); + eputf("line %d: incorrect nonce: \"%s\"\n", + p->lineno, azArg[1]); exit(1); }else{ p->bSafeMode = 0; @@ -25471,7 +26205,7 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_snprintf(sizeof(p->nullValue), p->nullValue, "%.*s", (int)ArraySize(p->nullValue)-1, azArg[1]); }else{ - raw_printf(stderr, "Usage: .nullvalue STRING\n"); + eputz("Usage: .nullvalue STRING\n"); rc = 1; } }else @@ -25510,11 +26244,11 @@ static int do_meta_command(char *zLine, ShellState *p){ }else #endif /* !SQLITE_SHELL_FIDDLE */ if( z[0]=='-' ){ - utf8_printf(stderr, "unknown option: %s\n", z); + eputf("unknown option: %s\n", z); rc = 1; goto meta_command_exit; }else if( zFN ){ - utf8_printf(stderr, "extra argument: \"%s\"\n", z); + eputf("extra argument: \"%s\"\n", z); rc = 1; goto meta_command_exit; }else{ @@ -25556,7 +26290,7 @@ static int do_meta_command(char *zLine, ShellState *p){ p->pAuxDb->zDbFilename = zNewFilename; open_db(p, OPEN_DB_KEEPALIVE); if( p->db==0 ){ - utf8_printf(stderr, "Error: cannot open '%s'\n", zNewFilename); + eputf("Error: cannot open '%s'\n", zNewFilename); sqlite3_free(zNewFilename); }else{ p->pAuxDb->zFreeOnClose = zNewFilename; @@ -25580,9 +26314,9 @@ static int do_meta_command(char *zLine, ShellState *p){ int i; int eMode = 0; int bOnce = 0; /* 0: .output, 1: .once, 2: .excel */ - unsigned char zBOM[4]; /* Byte-order mark to using if --bom is present */ + static const char *zBomUtf8 = "\xef\xbb\xbf"; + const char *zBom = 0; - zBOM[0] = 0; failIfSafeMode(p, "cannot run .%s in safe mode", azArg[0]); if( c=='e' ){ eMode = 'x'; @@ -25595,17 +26329,13 @@ static int do_meta_command(char *zLine, ShellState *p){ if( z[0]=='-' ){ if( z[1]=='-' ) z++; if( cli_strcmp(z,"-bom")==0 ){ - zBOM[0] = 0xef; - zBOM[1] = 0xbb; - zBOM[2] = 0xbf; - zBOM[3] = 0; + zBom = zBomUtf8; }else if( c!='e' && cli_strcmp(z,"-x")==0 ){ eMode = 'x'; /* spreadsheet */ }else if( c!='e' && cli_strcmp(z,"-e")==0 ){ eMode = 'e'; /* text editor */ }else{ - utf8_printf(p->out, "ERROR: unknown option: \"%s\". Usage:\n", - azArg[i]); + oputf("ERROR: unknown option: \"%s\". Usage:\n", azArg[i]); showHelp(p->out, azArg[0]); rc = 1; goto meta_command_exit; @@ -25617,8 +26347,7 @@ static int do_meta_command(char *zLine, ShellState *p){ break; } }else{ - utf8_printf(p->out,"ERROR: extra parameter: \"%s\". Usage:\n", - azArg[i]); + oputf("ERROR: extra parameter: \"%s\". Usage:\n", azArg[i]); showHelp(p->out, azArg[0]); rc = 1; sqlite3_free(zFile); @@ -25657,30 +26386,30 @@ static int do_meta_command(char *zLine, ShellState *p){ shell_check_oom(zFile); if( zFile[0]=='|' ){ #ifdef SQLITE_OMIT_POPEN - raw_printf(stderr, "Error: pipes are not supported in this OS\n"); + eputz("Error: pipes are not supported in this OS\n"); rc = 1; - p->out = stdout; + output_redir(p, stdout); #else - p->out = popen(zFile + 1, "w"); - if( p->out==0 ){ - utf8_printf(stderr,"Error: cannot open pipe \"%s\"\n", zFile + 1); - p->out = stdout; + FILE *pfPipe = popen(zFile + 1, "w"); + if( pfPipe==0 ){ + eputf("Error: cannot open pipe \"%s\"\n", zFile + 1); rc = 1; }else{ - if( zBOM[0] ) fwrite(zBOM, 1, 3, p->out); + output_redir(p, pfPipe); + if( zBom ) oputz(zBom); sqlite3_snprintf(sizeof(p->outfile), p->outfile, "%s", zFile); } #endif }else{ - p->out = output_file_open(zFile, bTxtMode); - if( p->out==0 ){ + FILE *pfFile = output_file_open(zFile, bTxtMode); + if( pfFile==0 ){ if( cli_strcmp(zFile,"off")!=0 ){ - utf8_printf(stderr,"Error: cannot write to \"%s\"\n", zFile); + eputf("Error: cannot write to \"%s\"\n", zFile); } - p->out = stdout; rc = 1; } else { - if( zBOM[0] ) fwrite(zBOM, 1, 3, p->out); + output_redir(p, pfFile); + if( zBom ) oputz(zBom); sqlite3_snprintf(sizeof(p->outfile), p->outfile, "%s", zFile); } } @@ -25721,8 +26450,8 @@ static int do_meta_command(char *zLine, ShellState *p){ "SELECT key, quote(value) " "FROM temp.sqlite_parameters;", -1, &pStmt, 0); while( rx==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){ - utf8_printf(p->out, "%-*s %s\n", len, sqlite3_column_text(pStmt,0), - sqlite3_column_text(pStmt,1)); + oputf("%-*s %s\n", len, sqlite3_column_text(pStmt,0), + sqlite3_column_text(pStmt,1)); } sqlite3_finalize(pStmt); } @@ -25766,7 +26495,7 @@ static int do_meta_command(char *zLine, ShellState *p){ rx = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); sqlite3_free(zSql); if( rx!=SQLITE_OK ){ - utf8_printf(p->out, "Error: %s\n", sqlite3_errmsg(p->db)); + oputf("Error: %s\n", sqlite3_errmsg(p->db)); sqlite3_finalize(pStmt); pStmt = 0; rc = 1; @@ -25795,10 +26524,10 @@ static int do_meta_command(char *zLine, ShellState *p){ if( c=='p' && n>=3 && cli_strncmp(azArg[0], "print", n)==0 ){ int i; for(i=1; i1 ) raw_printf(p->out, " "); - utf8_printf(p->out, "%s", azArg[i]); + if( i>1 ) oputz(" "); + oputz(azArg[i]); } - raw_printf(p->out, "\n"); + oputz("\n"); }else #ifndef SQLITE_OMIT_PROGRESS_CALLBACK @@ -25827,7 +26556,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } if( cli_strcmp(z,"limit")==0 ){ if( i+1>=nArg ){ - utf8_printf(stderr, "Error: missing argument on --limit\n"); + eputz("Error: missing argument on --limit\n"); rc = 1; goto meta_command_exit; }else{ @@ -25835,7 +26564,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } continue; } - utf8_printf(stderr, "Error: unknown option: \"%s\"\n", azArg[i]); + eputf("Error: unknown option: \"%s\"\n", azArg[i]); rc = 1; goto meta_command_exit; }else{ @@ -25868,19 +26597,19 @@ static int do_meta_command(char *zLine, ShellState *p){ int savedLineno = p->lineno; failIfSafeMode(p, "cannot run .read in safe mode"); if( nArg!=2 ){ - raw_printf(stderr, "Usage: .read FILE\n"); + eputz("Usage: .read FILE\n"); rc = 1; goto meta_command_exit; } if( azArg[1][0]=='|' ){ #ifdef SQLITE_OMIT_POPEN - raw_printf(stderr, "Error: pipes are not supported in this OS\n"); + eputz("Error: pipes are not supported in this OS\n"); rc = 1; p->out = stdout; #else p->in = popen(azArg[1]+1, "r"); if( p->in==0 ){ - utf8_printf(stderr, "Error: cannot open \"%s\"\n", azArg[1]); + eputf("Error: cannot open \"%s\"\n", azArg[1]); rc = 1; }else{ rc = process_input(p); @@ -25888,7 +26617,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } #endif }else if( (p->in = openChrSource(azArg[1]))==0 ){ - utf8_printf(stderr,"Error: cannot open \"%s\"\n", azArg[1]); + eputf("Error: cannot open \"%s\"\n", azArg[1]); rc = 1; }else{ rc = process_input(p); @@ -25915,20 +26644,20 @@ static int do_meta_command(char *zLine, ShellState *p){ zSrcFile = azArg[2]; zDb = azArg[1]; }else{ - raw_printf(stderr, "Usage: .restore ?DB? FILE\n"); + eputz("Usage: .restore ?DB? FILE\n"); rc = 1; goto meta_command_exit; } rc = sqlite3_open(zSrcFile, &pSrc); if( rc!=SQLITE_OK ){ - utf8_printf(stderr, "Error: cannot open \"%s\"\n", zSrcFile); + eputf("Error: cannot open \"%s\"\n", zSrcFile); close_db(pSrc); return 1; } open_db(p, 0); pBackup = sqlite3_backup_init(p->db, zDb, pSrc, "main"); if( pBackup==0 ){ - utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(p->db)); + eputf("Error: %s\n", sqlite3_errmsg(p->db)); close_db(pSrc); return 1; } @@ -25943,10 +26672,10 @@ static int do_meta_command(char *zLine, ShellState *p){ if( rc==SQLITE_DONE ){ rc = 0; }else if( rc==SQLITE_BUSY || rc==SQLITE_LOCKED ){ - raw_printf(stderr, "Error: source database is busy\n"); + eputz("Error: source database is busy\n"); rc = 1; }else{ - utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(p->db)); + eputf("Error: %s\n", sqlite3_errmsg(p->db)); rc = 1; } close_db(pSrc); @@ -25967,11 +26696,15 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_db_config( p->db, SQLITE_DBCONFIG_STMT_SCANSTATUS, p->scanstatsOn, (int*)0 ); -#ifndef SQLITE_ENABLE_STMT_SCANSTATUS - raw_printf(stderr, "Warning: .scanstats not available in this build.\n"); +#if !defined(SQLITE_ENABLE_STMT_SCANSTATUS) + eputz("Warning: .scanstats not available in this build.\n"); +#elif !defined(SQLITE_ENABLE_BYTECODE_VTAB) + if( p->scanstatsOn==3 ){ + eputz("Warning: \".scanstats vm\" not available in this build.\n"); + } #endif }else{ - raw_printf(stderr, "Usage: .scanstats on|off|est\n"); + eputz("Usage: .scanstats on|off|est\n"); rc = 1; } }else @@ -26000,14 +26733,13 @@ static int do_meta_command(char *zLine, ShellState *p){ }else if( optionMatch(azArg[ii],"nosys") ){ bNoSystemTabs = 1; }else if( azArg[ii][0]=='-' ){ - utf8_printf(stderr, "Unknown option: \"%s\"\n", azArg[ii]); + eputf("Unknown option: \"%s\"\n", azArg[ii]); rc = 1; goto meta_command_exit; }else if( zName==0 ){ zName = azArg[ii]; }else{ - raw_printf(stderr, - "Usage: .schema ?--indent? ?--nosys? ?LIKE-PATTERN?\n"); + eputz("Usage: .schema ?--indent? ?--nosys? ?LIKE-PATTERN?\n"); rc = 1; goto meta_command_exit; } @@ -26040,7 +26772,7 @@ static int do_meta_command(char *zLine, ShellState *p){ rc = sqlite3_prepare_v2(p->db, "SELECT name FROM pragma_database_list", -1, &pStmt, 0); if( rc ){ - utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(p->db)); + eputf("Error: %s\n", sqlite3_errmsg(p->db)); sqlite3_finalize(pStmt); rc = 1; goto meta_command_exit; @@ -26102,18 +26834,18 @@ static int do_meta_command(char *zLine, ShellState *p){ appendText(&sSelect, "sql IS NOT NULL" " ORDER BY snum, rowid", 0); if( bDebug ){ - utf8_printf(p->out, "SQL: %s;\n", sSelect.z); + oputf("SQL: %s;\n", sSelect.z); }else{ rc = sqlite3_exec(p->db, sSelect.z, callback, &data, &zErrMsg); } freeText(&sSelect); } if( zErrMsg ){ - utf8_printf(stderr,"Error: %s\n", zErrMsg); + eputf("Error: %s\n", zErrMsg); sqlite3_free(zErrMsg); rc = 1; }else if( rc != SQLITE_OK ){ - raw_printf(stderr,"Error: querying schema information\n"); + eputz("Error: querying schema information\n"); rc = 1; }else{ rc = 0; @@ -26159,11 +26891,11 @@ static int do_meta_command(char *zLine, ShellState *p){ if( nCmd!=2 ) goto session_syntax_error; if( pSession->p==0 ){ session_not_open: - raw_printf(stderr, "ERROR: No sessions are open\n"); + eputz("ERROR: No sessions are open\n"); }else{ rc = sqlite3session_attach(pSession->p, azCmd[1]); if( rc ){ - raw_printf(stderr, "ERROR: sqlite3session_attach() returns %d\n", rc); + eputf("ERROR: sqlite3session_attach() returns %d\n",rc); rc = 0; } } @@ -26182,8 +26914,8 @@ static int do_meta_command(char *zLine, ShellState *p){ if( pSession->p==0 ) goto session_not_open; out = fopen(azCmd[1], "wb"); if( out==0 ){ - utf8_printf(stderr, "ERROR: cannot open \"%s\" for writing\n", - azCmd[1]); + eputf("ERROR: cannot open \"%s\" for writing\n", + azCmd[1]); }else{ int szChng; void *pChng; @@ -26193,13 +26925,12 @@ static int do_meta_command(char *zLine, ShellState *p){ rc = sqlite3session_patchset(pSession->p, &szChng, &pChng); } if( rc ){ - printf("Error: error code %d\n", rc); + sputf(stdout, "Error: error code %d\n", rc); rc = 0; } if( pChng && fwrite(pChng, szChng, 1, out)!=1 ){ - raw_printf(stderr, "ERROR: Failed to write entire %d-byte output\n", - szChng); + eputf("ERROR: Failed to write entire %d-byte output\n", szChng); } sqlite3_free(pChng); fclose(out); @@ -26226,8 +26957,7 @@ static int do_meta_command(char *zLine, ShellState *p){ ii = nCmd==1 ? -1 : booleanValue(azCmd[1]); if( pAuxDb->nSession ){ ii = sqlite3session_enable(pSession->p, ii); - utf8_printf(p->out, "session %s enable flag = %d\n", - pSession->zName, ii); + oputf("session %s enable flag = %d\n", pSession->zName, ii); } }else @@ -26244,10 +26974,7 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_free(pSession->azFilter); nByte = sizeof(pSession->azFilter[0])*(nCmd-1); pSession->azFilter = sqlite3_malloc( nByte ); - if( pSession->azFilter==0 ){ - raw_printf(stderr, "Error: out or memory\n"); - exit(1); - } + shell_check_oom( pSession->azFilter ); for(ii=1; iiazFilter[ii-1] = sqlite3_mprintf("%s", azCmd[ii]); shell_check_oom(x); @@ -26265,8 +26992,7 @@ static int do_meta_command(char *zLine, ShellState *p){ ii = nCmd==1 ? -1 : booleanValue(azCmd[1]); if( pAuxDb->nSession ){ ii = sqlite3session_indirect(pSession->p, ii); - utf8_printf(p->out, "session %s indirect flag = %d\n", - pSession->zName, ii); + oputf("session %s indirect flag = %d\n", pSession->zName, ii); } }else @@ -26278,8 +27004,7 @@ static int do_meta_command(char *zLine, ShellState *p){ if( nCmd!=1 ) goto session_syntax_error; if( pAuxDb->nSession ){ ii = sqlite3session_isempty(pSession->p); - utf8_printf(p->out, "session %s isempty flag = %d\n", - pSession->zName, ii); + oputf("session %s isempty flag = %d\n", pSession->zName, ii); } }else @@ -26288,7 +27013,7 @@ static int do_meta_command(char *zLine, ShellState *p){ */ if( cli_strcmp(azCmd[0],"list")==0 ){ for(i=0; inSession; i++){ - utf8_printf(p->out, "%d %s\n", i, pAuxDb->aSession[i].zName); + oputf("%d %s\n", i, pAuxDb->aSession[i].zName); } }else @@ -26303,19 +27028,18 @@ static int do_meta_command(char *zLine, ShellState *p){ if( zName[0]==0 ) goto session_syntax_error; for(i=0; inSession; i++){ if( cli_strcmp(pAuxDb->aSession[i].zName,zName)==0 ){ - utf8_printf(stderr, "Session \"%s\" already exists\n", zName); + eputf("Session \"%s\" already exists\n", zName); goto meta_command_exit; } } if( pAuxDb->nSession>=ArraySize(pAuxDb->aSession) ){ - raw_printf(stderr, - "Maximum of %d sessions\n", ArraySize(pAuxDb->aSession)); + eputf("Maximum of %d sessions\n", ArraySize(pAuxDb->aSession)); goto meta_command_exit; } pSession = &pAuxDb->aSession[pAuxDb->nSession]; rc = sqlite3session_create(p->db, azCmd[1], &pSession->p); if( rc ){ - raw_printf(stderr, "Cannot open session: error code=%d\n", rc); + eputf("Cannot open session: error code=%d\n", rc); rc = 0; goto meta_command_exit; } @@ -26339,7 +27063,7 @@ static int do_meta_command(char *zLine, ShellState *p){ int i, v; for(i=1; iout, "%s: %d 0x%x\n", azArg[i], v, v); + oputf("%s: %d 0x%x\n", azArg[i], v, v); } } if( cli_strncmp(azArg[0]+9, "integer", n-9)==0 ){ @@ -26348,7 +27072,7 @@ static int do_meta_command(char *zLine, ShellState *p){ char zBuf[200]; v = integerValue(azArg[i]); sqlite3_snprintf(sizeof(zBuf),zBuf,"%s: %lld 0x%llx\n", azArg[i],v,v); - utf8_printf(p->out, "%s", zBuf); + oputz(zBuf); } } }else @@ -26375,9 +27099,8 @@ static int do_meta_command(char *zLine, ShellState *p){ bVerbose++; }else { - utf8_printf(stderr, "Unknown option \"%s\" on \"%s\"\n", - azArg[i], azArg[0]); - raw_printf(stderr, "Should be one of: --init -v\n"); + eputf("Unknown option \"%s\" on \"%s\"\n", azArg[i], azArg[0]); + eputz("Should be one of: --init -v\n"); rc = 1; goto meta_command_exit; } @@ -26406,7 +27129,7 @@ static int do_meta_command(char *zLine, ShellState *p){ -1, &pStmt, 0); } if( rc ){ - raw_printf(stderr, "Error querying the selftest table\n"); + eputz("Error querying the selftest table\n"); rc = 1; sqlite3_finalize(pStmt); goto meta_command_exit; @@ -26422,10 +27145,10 @@ static int do_meta_command(char *zLine, ShellState *p){ if( zAns==0 ) continue; k = 0; if( bVerbose>0 ){ - printf("%d: %s %s\n", tno, zOp, zSql); + sputf(stdout, "%d: %s %s\n", tno, zOp, zSql); } if( cli_strcmp(zOp,"memo")==0 ){ - utf8_printf(p->out, "%s\n", zSql); + oputf("%s\n", zSql); }else if( cli_strcmp(zOp,"run")==0 ){ char *zErrMsg = 0; @@ -26434,23 +27157,22 @@ static int do_meta_command(char *zLine, ShellState *p){ rc = sqlite3_exec(p->db, zSql, captureOutputCallback, &str, &zErrMsg); nTest++; if( bVerbose ){ - utf8_printf(p->out, "Result: %s\n", str.z); + oputf("Result: %s\n", str.z); } if( rc || zErrMsg ){ nErr++; rc = 1; - utf8_printf(p->out, "%d: error-code-%d: %s\n", tno, rc, zErrMsg); + oputf("%d: error-code-%d: %s\n", tno, rc, zErrMsg); sqlite3_free(zErrMsg); }else if( cli_strcmp(zAns,str.z)!=0 ){ nErr++; rc = 1; - utf8_printf(p->out, "%d: Expected: [%s]\n", tno, zAns); - utf8_printf(p->out, "%d: Got: [%s]\n", tno, str.z); + oputf("%d: Expected: [%s]\n", tno, zAns); + oputf("%d: Got: [%s]\n", tno, str.z); } - }else - { - utf8_printf(stderr, - "Unknown operation \"%s\" on selftest line %d\n", zOp, tno); + } + else{ + eputf("Unknown operation \"%s\" on selftest line %d\n", zOp, tno); rc = 1; break; } @@ -26458,12 +27180,12 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_finalize(pStmt); } /* End loop over k */ freeText(&str); - utf8_printf(p->out, "%d errors out of %d tests\n", nErr, nTest); + oputf("%d errors out of %d tests\n", nErr, nTest); }else if( c=='s' && cli_strncmp(azArg[0], "separator", n)==0 ){ if( nArg<2 || nArg>3 ){ - raw_printf(stderr, "Usage: .separator COL ?ROW?\n"); + eputz("Usage: .separator COL ?ROW?\n"); rc = 1; } if( nArg>=2 ){ @@ -26506,14 +27228,13 @@ static int do_meta_command(char *zLine, ShellState *p){ bDebug = 1; }else { - utf8_printf(stderr, "Unknown option \"%s\" on \"%s\"\n", - azArg[i], azArg[0]); + eputf("Unknown option \"%s\" on \"%s\"\n", azArg[i], azArg[0]); showHelp(p->out, azArg[0]); rc = 1; goto meta_command_exit; } }else if( zLike ){ - raw_printf(stderr, "Usage: .sha3sum ?OPTIONS? ?LIKE-PATTERN?\n"); + eputz("Usage: .sha3sum ?OPTIONS? ?LIKE-PATTERN?\n"); rc = 1; goto meta_command_exit; }else{ @@ -26585,7 +27306,7 @@ static int do_meta_command(char *zLine, ShellState *p){ freeText(&sQuery); freeText(&sSql); if( bDebug ){ - utf8_printf(p->out, "%s\n", zSql); + oputf("%s\n", zSql); }else{ shell_exec(p, zSql, 0); } @@ -26615,7 +27336,7 @@ static int do_meta_command(char *zLine, ShellState *p){ "' OR ') as query, tname from tabcols group by tname)" , zRevText); shell_check_oom(zRevText); - if( bDebug ) utf8_printf(p->out, "%s\n", zRevText); + if( bDebug ) oputf("%s\n", zRevText); lrc = sqlite3_prepare_v2(p->db, zRevText, -1, &pStmt, 0); if( lrc!=SQLITE_OK ){ /* assert(lrc==SQLITE_NOMEM); // might also be SQLITE_ERROR if the @@ -26628,7 +27349,7 @@ static int do_meta_command(char *zLine, ShellState *p){ const char *zGenQuery = (char*)sqlite3_column_text(pStmt,0); sqlite3_stmt *pCheckStmt; lrc = sqlite3_prepare_v2(p->db, zGenQuery, -1, &pCheckStmt, 0); - if( bDebug ) utf8_printf(p->out, "%s\n", zGenQuery); + if( bDebug ) oputf("%s\n", zGenQuery); if( lrc!=SQLITE_OK ){ rc = 1; }else{ @@ -26636,9 +27357,8 @@ static int do_meta_command(char *zLine, ShellState *p){ double countIrreversible = sqlite3_column_double(pCheckStmt, 0); if( countIrreversible>0 ){ int sz = (int)(countIrreversible + 0.5); - utf8_printf(stderr, - "Digest includes %d invalidly encoded text field%s.\n", - sz, (sz>1)? "s": ""); + eputf("Digest includes %d invalidly encoded text field%s.\n", + sz, (sz>1)? "s": ""); } } sqlite3_finalize(pCheckStmt); @@ -26646,7 +27366,7 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_finalize(pStmt); } } - if( rc ) utf8_printf(stderr, ".sha3sum failed.\n"); + if( rc ) eputz(".sha3sum failed.\n"); sqlite3_free(zRevText); } #endif /* !defined(*_OMIT_SCHEMA_PRAGMAS) && !defined(*_OMIT_VIRTUALTABLE) */ @@ -26662,7 +27382,7 @@ static int do_meta_command(char *zLine, ShellState *p){ int i, x; failIfSafeMode(p, "cannot run .%s in safe mode", azArg[0]); if( nArg<2 ){ - raw_printf(stderr, "Usage: .system COMMAND\n"); + eputz("Usage: .system COMMAND\n"); rc = 1; goto meta_command_exit; } @@ -26671,9 +27391,11 @@ static int do_meta_command(char *zLine, ShellState *p){ zCmd = sqlite3_mprintf(strchr(azArg[i],' ')==0?"%z %s":"%z \"%s\"", zCmd, azArg[i]); } + consoleRestore(); x = zCmd!=0 ? system(zCmd) : 1; + consoleRenewSetup(); sqlite3_free(zCmd); - if( x ) raw_printf(stderr, "System command returns %d\n", x); + if( x ) eputf("System command returns %d\n", x); }else #endif /* !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_FIDDLE) */ @@ -26682,52 +27404,51 @@ static int do_meta_command(char *zLine, ShellState *p){ const char *zOut; int i; if( nArg!=1 ){ - raw_printf(stderr, "Usage: .show\n"); + eputz("Usage: .show\n"); rc = 1; goto meta_command_exit; } - utf8_printf(p->out, "%12.12s: %s\n","echo", - azBool[ShellHasFlag(p, SHFLG_Echo)]); - utf8_printf(p->out, "%12.12s: %s\n","eqp", azBool[p->autoEQP&3]); - utf8_printf(p->out, "%12.12s: %s\n","explain", - p->mode==MODE_Explain ? "on" : p->autoExplain ? "auto" : "off"); - utf8_printf(p->out,"%12.12s: %s\n","headers", azBool[p->showHeader!=0]); + oputf("%12.12s: %s\n","echo", + azBool[ShellHasFlag(p, SHFLG_Echo)]); + oputf("%12.12s: %s\n","eqp", azBool[p->autoEQP&3]); + oputf("%12.12s: %s\n","explain", + p->mode==MODE_Explain ? "on" : p->autoExplain ? "auto" : "off"); + oputf("%12.12s: %s\n","headers", azBool[p->showHeader!=0]); if( p->mode==MODE_Column || (p->mode>=MODE_Markdown && p->mode<=MODE_Box) ){ - utf8_printf - (p->out, "%12.12s: %s --wrap %d --wordwrap %s --%squote\n", "mode", - modeDescr[p->mode], p->cmOpts.iWrap, - p->cmOpts.bWordWrap ? "on" : "off", - p->cmOpts.bQuote ? "" : "no"); + oputf("%12.12s: %s --wrap %d --wordwrap %s --%squote\n", "mode", + modeDescr[p->mode], p->cmOpts.iWrap, + p->cmOpts.bWordWrap ? "on" : "off", + p->cmOpts.bQuote ? "" : "no"); }else{ - utf8_printf(p->out, "%12.12s: %s\n","mode", modeDescr[p->mode]); - } - utf8_printf(p->out, "%12.12s: ", "nullvalue"); - output_c_string(p->out, p->nullValue); - raw_printf(p->out, "\n"); - utf8_printf(p->out,"%12.12s: %s\n","output", - strlen30(p->outfile) ? p->outfile : "stdout"); - utf8_printf(p->out,"%12.12s: ", "colseparator"); - output_c_string(p->out, p->colSeparator); - raw_printf(p->out, "\n"); - utf8_printf(p->out,"%12.12s: ", "rowseparator"); - output_c_string(p->out, p->rowSeparator); - raw_printf(p->out, "\n"); + oputf("%12.12s: %s\n","mode", modeDescr[p->mode]); + } + oputf("%12.12s: ", "nullvalue"); + output_c_string(p->nullValue); + oputz("\n"); + oputf("%12.12s: %s\n","output", + strlen30(p->outfile) ? p->outfile : "stdout"); + oputf("%12.12s: ", "colseparator"); + output_c_string(p->colSeparator); + oputz("\n"); + oputf("%12.12s: ", "rowseparator"); + output_c_string(p->rowSeparator); + oputz("\n"); switch( p->statsOn ){ case 0: zOut = "off"; break; default: zOut = "on"; break; case 2: zOut = "stmt"; break; case 3: zOut = "vmstep"; break; } - utf8_printf(p->out, "%12.12s: %s\n","stats", zOut); - utf8_printf(p->out, "%12.12s: ", "width"); + oputf("%12.12s: %s\n","stats", zOut); + oputf("%12.12s: ", "width"); for (i=0;inWidth;i++) { - raw_printf(p->out, "%d ", p->colWidth[i]); + oputf("%d ", p->colWidth[i]); } - raw_printf(p->out, "\n"); - utf8_printf(p->out, "%12.12s: %s\n", "filename", - p->pAuxDb->zDbFilename ? p->pAuxDb->zDbFilename : ""); + oputz("\n"); + oputf("%12.12s: %s\n", "filename", + p->pAuxDb->zDbFilename ? p->pAuxDb->zDbFilename : ""); }else if( c=='s' && cli_strncmp(azArg[0], "stats", n)==0 ){ @@ -26742,7 +27463,7 @@ static int do_meta_command(char *zLine, ShellState *p){ }else if( nArg==1 ){ display_stats(p->db, p, 0); }else{ - raw_printf(stderr, "Usage: .stats ?on|off|stmt|vmstep?\n"); + eputz("Usage: .stats ?on|off|stmt|vmstep?\n"); rc = 1; } }else @@ -26768,7 +27489,7 @@ static int do_meta_command(char *zLine, ShellState *p){ /* It is an historical accident that the .indexes command shows an error ** when called with the wrong number of arguments whereas the .tables ** command does not. */ - raw_printf(stderr, "Usage: .indexes ?LIKE-PATTERN?\n"); + eputz("Usage: .indexes ?LIKE-PATTERN?\n"); rc = 1; sqlite3_finalize(pStmt); goto meta_command_exit; @@ -26844,10 +27565,9 @@ static int do_meta_command(char *zLine, ShellState *p){ for(i=0; iout, "%s%-*s", zSp, maxlen, - azResult[j] ? azResult[j]:""); + oputf("%s%-*s", zSp, maxlen, azResult[j] ? azResult[j]:""); } - raw_printf(p->out, "\n"); + oputz("\n"); } } @@ -26861,7 +27581,7 @@ static int do_meta_command(char *zLine, ShellState *p){ output_reset(p); p->out = output_file_open("testcase-out.txt", 0); if( p->out==0 ){ - raw_printf(stderr, "Error: cannot open 'testcase-out.txt'\n"); + eputz("Error: cannot open 'testcase-out.txt'\n"); } if( nArg>=2 ){ sqlite3_snprintf(sizeof(p->zTestcase), p->zTestcase, "%s", azArg[1]); @@ -26902,7 +27622,7 @@ static int do_meta_command(char *zLine, ShellState *p){ {"seek_count", SQLITE_TESTCTRL_SEEK_COUNT, 0, "" }, {"sorter_mmap", SQLITE_TESTCTRL_SORTER_MMAP, 0, "NMAX" }, {"tune", SQLITE_TESTCTRL_TUNE, 1, "ID VALUE" }, - {"uselongdouble", SQLITE_TESTCTRL_USELONGDOUBLE,0,"?BOOLEAN|\"default\"?"}, + {"uselongdouble", SQLITE_TESTCTRL_USELONGDOUBLE,0,"?BOOLEAN|\"default\"?"}, }; int testctrl = -1; int iCtrl = -1; @@ -26922,11 +27642,11 @@ static int do_meta_command(char *zLine, ShellState *p){ /* --help lists all test-controls */ if( cli_strcmp(zCmd,"help")==0 ){ - utf8_printf(p->out, "Available test-controls:\n"); + oputz("Available test-controls:\n"); for(i=0; iout, " .testctrl %s %s\n", - aCtrl[i].zCtrlName, aCtrl[i].zUsage); + oputf(" .testctrl %s %s\n", + aCtrl[i].zCtrlName, aCtrl[i].zUsage); } rc = 1; goto meta_command_exit; @@ -26942,16 +27662,16 @@ static int do_meta_command(char *zLine, ShellState *p){ testctrl = aCtrl[i].ctrlCode; iCtrl = i; }else{ - utf8_printf(stderr, "Error: ambiguous test-control: \"%s\"\n" - "Use \".testctrl --help\" for help\n", zCmd); + eputf("Error: ambiguous test-control: \"%s\"\n" + "Use \".testctrl --help\" for help\n", zCmd); rc = 1; goto meta_command_exit; } } } if( testctrl<0 ){ - utf8_printf(stderr,"Error: unknown test-control: %s\n" - "Use \".testctrl --help\" for help\n", zCmd); + eputf("Error: unknown test-control: %s\n" + "Use \".testctrl --help\" for help\n", zCmd); }else{ switch(testctrl){ @@ -26991,7 +27711,7 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3 *db; if( ii==0 && cli_strcmp(azArg[2],"random")==0 ){ sqlite3_randomness(sizeof(ii),&ii); - printf("-- random seed: %d\n", ii); + sputf(stdout, "-- random seed: %d\n", ii); } if( nArg==3 ){ db = 0; @@ -27059,7 +27779,7 @@ static int do_meta_command(char *zLine, ShellState *p){ case SQLITE_TESTCTRL_SEEK_COUNT: { u64 x = 0; rc2 = sqlite3_test_control(testctrl, p->db, &x); - utf8_printf(p->out, "%llu\n", x); + oputf("%llu\n", x); isOk = 3; break; } @@ -27090,11 +27810,11 @@ static int do_meta_command(char *zLine, ShellState *p){ int val = 0; rc2 = sqlite3_test_control(testctrl, -id, &val); if( rc2!=SQLITE_OK ) break; - if( id>1 ) utf8_printf(p->out, " "); - utf8_printf(p->out, "%d: %d", id, val); + if( id>1 ) oputz(" "); + oputf("%d: %d", id, val); id++; } - if( id>1 ) utf8_printf(p->out, "\n"); + if( id>1 ) oputz("\n"); isOk = 3; } break; @@ -27110,12 +27830,12 @@ static int do_meta_command(char *zLine, ShellState *p){ } } if( isOk==0 && iCtrl>=0 ){ - utf8_printf(p->out, "Usage: .testctrl %s %s\n", zCmd,aCtrl[iCtrl].zUsage); + oputf("Usage: .testctrl %s %s\n", zCmd,aCtrl[iCtrl].zUsage); rc = 1; }else if( isOk==1 ){ - raw_printf(p->out, "%d\n", rc2); + oputf("%d\n", rc2); }else if( isOk==2 ){ - raw_printf(p->out, "0x%08x\n", rc2); + oputf("0x%08x\n", rc2); } }else #endif /* !defined(SQLITE_UNTESTABLE) */ @@ -27129,11 +27849,11 @@ static int do_meta_command(char *zLine, ShellState *p){ if( nArg==2 ){ enableTimer = booleanValue(azArg[1]); if( enableTimer && !HAS_TIMER ){ - raw_printf(stderr, "Error: timer not available on this system.\n"); + eputz("Error: timer not available on this system.\n"); enableTimer = 0; } }else{ - raw_printf(stderr, "Usage: .timer on|off\n"); + eputz("Usage: .timer on|off\n"); rc = 1; } }else @@ -27170,7 +27890,7 @@ static int do_meta_command(char *zLine, ShellState *p){ mType |= SQLITE_TRACE_CLOSE; } else { - raw_printf(stderr, "Unknown option \"%s\" on \".trace\"\n", z); + eputf("Unknown option \"%s\" on \".trace\"\n", z); rc = 1; goto meta_command_exit; } @@ -27194,7 +27914,7 @@ static int do_meta_command(char *zLine, ShellState *p){ int lenOpt; char *zOpt; if( nArg<2 ){ - raw_printf(stderr, "Usage: .unmodule [--allexcept] NAME ...\n"); + eputz("Usage: .unmodule [--allexcept] NAME ...\n"); rc = 1; goto meta_command_exit; } @@ -27216,60 +27936,60 @@ static int do_meta_command(char *zLine, ShellState *p){ #if SQLITE_USER_AUTHENTICATION if( c=='u' && cli_strncmp(azArg[0], "user", n)==0 ){ if( nArg<2 ){ - raw_printf(stderr, "Usage: .user SUBCOMMAND ...\n"); + eputz("Usage: .user SUBCOMMAND ...\n"); rc = 1; goto meta_command_exit; } open_db(p, 0); if( cli_strcmp(azArg[1],"login")==0 ){ if( nArg!=4 ){ - raw_printf(stderr, "Usage: .user login USER PASSWORD\n"); + eputz("Usage: .user login USER PASSWORD\n"); rc = 1; goto meta_command_exit; } rc = sqlite3_user_authenticate(p->db, azArg[2], azArg[3], strlen30(azArg[3])); if( rc ){ - utf8_printf(stderr, "Authentication failed for user %s\n", azArg[2]); + eputf("Authentication failed for user %s\n", azArg[2]); rc = 1; } }else if( cli_strcmp(azArg[1],"add")==0 ){ if( nArg!=5 ){ - raw_printf(stderr, "Usage: .user add USER PASSWORD ISADMIN\n"); + eputz("Usage: .user add USER PASSWORD ISADMIN\n"); rc = 1; goto meta_command_exit; } rc = sqlite3_user_add(p->db, azArg[2], azArg[3], strlen30(azArg[3]), booleanValue(azArg[4])); if( rc ){ - raw_printf(stderr, "User-Add failed: %d\n", rc); + eputf("User-Add failed: %d\n", rc); rc = 1; } }else if( cli_strcmp(azArg[1],"edit")==0 ){ if( nArg!=5 ){ - raw_printf(stderr, "Usage: .user edit USER PASSWORD ISADMIN\n"); + eputz("Usage: .user edit USER PASSWORD ISADMIN\n"); rc = 1; goto meta_command_exit; } rc = sqlite3_user_change(p->db, azArg[2], azArg[3], strlen30(azArg[3]), booleanValue(azArg[4])); if( rc ){ - raw_printf(stderr, "User-Edit failed: %d\n", rc); + eputf("User-Edit failed: %d\n", rc); rc = 1; } }else if( cli_strcmp(azArg[1],"delete")==0 ){ if( nArg!=3 ){ - raw_printf(stderr, "Usage: .user delete USER\n"); + eputz("Usage: .user delete USER\n"); rc = 1; goto meta_command_exit; } rc = sqlite3_user_delete(p->db, azArg[2]); if( rc ){ - raw_printf(stderr, "User-Delete failed: %d\n", rc); + eputf("User-Delete failed: %d\n", rc); rc = 1; } }else{ - raw_printf(stderr, "Usage: .user login|add|edit|delete ...\n"); + eputz("Usage: .user login|add|edit|delete ...\n"); rc = 1; goto meta_command_exit; } @@ -27278,21 +27998,21 @@ static int do_meta_command(char *zLine, ShellState *p){ if( c=='v' && cli_strncmp(azArg[0], "version", n)==0 ){ char *zPtrSz = sizeof(void*)==8 ? "64-bit" : "32-bit"; - utf8_printf(p->out, "SQLite %s %s\n" /*extra-version-info*/, - sqlite3_libversion(), sqlite3_sourceid()); + oputf("SQLite %s %s\n" /*extra-version-info*/, + sqlite3_libversion(), sqlite3_sourceid()); #if SQLITE_HAVE_ZLIB - utf8_printf(p->out, "zlib version %s\n", zlibVersion()); + oputf("zlib version %s\n", zlibVersion()); #endif #define CTIMEOPT_VAL_(opt) #opt #define CTIMEOPT_VAL(opt) CTIMEOPT_VAL_(opt) #if defined(__clang__) && defined(__clang_major__) - utf8_printf(p->out, "clang-" CTIMEOPT_VAL(__clang_major__) "." - CTIMEOPT_VAL(__clang_minor__) "." - CTIMEOPT_VAL(__clang_patchlevel__) " (%s)\n", zPtrSz); + oputf("clang-" CTIMEOPT_VAL(__clang_major__) "." + CTIMEOPT_VAL(__clang_minor__) "." + CTIMEOPT_VAL(__clang_patchlevel__) " (%s)\n", zPtrSz); #elif defined(_MSC_VER) - utf8_printf(p->out, "msvc-" CTIMEOPT_VAL(_MSC_VER) " (%s)\n", zPtrSz); + oputf("msvc-" CTIMEOPT_VAL(_MSC_VER) " (%s)\n", zPtrSz); #elif defined(__GNUC__) && defined(__VERSION__) - utf8_printf(p->out, "gcc-" __VERSION__ " (%s)\n", zPtrSz); + oputf("gcc-" __VERSION__ " (%s)\n", zPtrSz); #endif }else @@ -27302,10 +28022,10 @@ static int do_meta_command(char *zLine, ShellState *p){ if( p->db ){ sqlite3_file_control(p->db, zDbName, SQLITE_FCNTL_VFS_POINTER, &pVfs); if( pVfs ){ - utf8_printf(p->out, "vfs.zName = \"%s\"\n", pVfs->zName); - raw_printf(p->out, "vfs.iVersion = %d\n", pVfs->iVersion); - raw_printf(p->out, "vfs.szOsFile = %d\n", pVfs->szOsFile); - raw_printf(p->out, "vfs.mxPathname = %d\n", pVfs->mxPathname); + oputf("vfs.zName = \"%s\"\n", pVfs->zName); + oputf("vfs.iVersion = %d\n", pVfs->iVersion); + oputf("vfs.szOsFile = %d\n", pVfs->szOsFile); + oputf("vfs.mxPathname = %d\n", pVfs->mxPathname); } } }else @@ -27317,13 +28037,13 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_file_control(p->db, "main", SQLITE_FCNTL_VFS_POINTER, &pCurrent); } for(pVfs=sqlite3_vfs_find(0); pVfs; pVfs=pVfs->pNext){ - utf8_printf(p->out, "vfs.zName = \"%s\"%s\n", pVfs->zName, - pVfs==pCurrent ? " <--- CURRENT" : ""); - raw_printf(p->out, "vfs.iVersion = %d\n", pVfs->iVersion); - raw_printf(p->out, "vfs.szOsFile = %d\n", pVfs->szOsFile); - raw_printf(p->out, "vfs.mxPathname = %d\n", pVfs->mxPathname); + oputf("vfs.zName = \"%s\"%s\n", pVfs->zName, + pVfs==pCurrent ? " <--- CURRENT" : ""); + oputf("vfs.iVersion = %d\n", pVfs->iVersion); + oputf("vfs.szOsFile = %d\n", pVfs->szOsFile); + oputf("vfs.mxPathname = %d\n", pVfs->mxPathname); if( pVfs->pNext ){ - raw_printf(p->out, "-----------------------------------\n"); + oputz("-----------------------------------\n"); } } }else @@ -27334,7 +28054,7 @@ static int do_meta_command(char *zLine, ShellState *p){ if( p->db ){ sqlite3_file_control(p->db, zDbName, SQLITE_FCNTL_VFSNAME, &zVfsName); if( zVfsName ){ - utf8_printf(p->out, "%s\n", zVfsName); + oputf("%s\n", zVfsName); sqlite3_free(zVfsName); } } @@ -27358,8 +28078,8 @@ static int do_meta_command(char *zLine, ShellState *p){ }else { - utf8_printf(stderr, "Error: unknown command or invalid arguments: " - " \"%s\". Enter \".help\" for help\n", azArg[0]); + eputf("Error: unknown command or invalid arguments: " + " \"%s\". Enter \".help\" for help\n", azArg[0]); rc = 1; } @@ -27549,7 +28269,7 @@ static int runOneSqlLine(ShellState *p, char *zSql, FILE *in, int startline){ }else{ sqlite3_snprintf(sizeof(zPrefix), zPrefix, "%s:", zErrorType); } - utf8_printf(stderr, "%s %s\n", zPrefix, zErrorTail); + eputf("%s %s\n", zPrefix, zErrorTail); sqlite3_free(zErrMsg); zErrMsg = 0; return 1; @@ -27558,13 +28278,13 @@ static int runOneSqlLine(ShellState *p, char *zSql, FILE *in, int startline){ sqlite3_snprintf(sizeof(zLineBuf), zLineBuf, "changes: %lld total_changes: %lld", sqlite3_changes64(p->db), sqlite3_total_changes64(p->db)); - raw_printf(p->out, "%s\n", zLineBuf); + oputf("%s\n", zLineBuf); } return 0; } static void echo_group_input(ShellState *p, const char *zDo){ - if( ShellHasFlag(p, SHFLG_Echo) ) utf8_printf(p->out, "%s\n", zDo); + if( ShellHasFlag(p, SHFLG_Echo) ) oputf("%s\n", zDo); } #ifdef SQLITE_SHELL_FIDDLE @@ -27622,8 +28342,8 @@ static int process_input(ShellState *p){ if( p->inputNesting==MAX_INPUT_NESTING ){ /* This will be more informative in a later version. */ - utf8_printf(stderr,"Input nesting limit (%d) reached at line %d." - " Check recursion.\n", MAX_INPUT_NESTING, p->lineno); + eputf("Input nesting limit (%d) reached at line %d." + " Check recursion.\n", MAX_INPUT_NESTING, p->lineno); return 1; } ++p->inputNesting; @@ -27634,7 +28354,7 @@ static int process_input(ShellState *p){ zLine = one_input_line(p->in, zLine, nSql>0); if( zLine==0 ){ /* End of input */ - if( p->in==0 && stdin_is_interactive ) printf("\n"); + if( p->in==0 && stdin_is_interactive ) oputz("\n"); break; } if( seenInterrupt ){ @@ -27844,8 +28564,8 @@ static void process_sqliterc( if( sqliterc == NULL ){ home_dir = find_home_dir(0); if( home_dir==0 ){ - raw_printf(stderr, "-- warning: cannot find home directory;" - " cannot read ~/.sqliterc\n"); + eputz("-- warning: cannot find home directory;" + " cannot read ~/.sqliterc\n"); return; } zBuf = sqlite3_mprintf("%s/.sqliterc",home_dir); @@ -27855,12 +28575,12 @@ static void process_sqliterc( p->in = fopen(sqliterc,"rb"); if( p->in ){ if( stdin_is_interactive ){ - utf8_printf(stderr,"-- Loading resources from %s\n",sqliterc); + eputf("-- Loading resources from %s\n", sqliterc); } if( process_input(p) && bail_on_error ) exit(1); fclose(p->in); }else if( sqliterc_override!=0 ){ - utf8_printf(stderr,"cannot open: \"%s\"\n", sqliterc); + eputf("cannot open: \"%s\"\n", sqliterc); if( bail_on_error ) exit(1); } p->in = inSaved; @@ -27910,9 +28630,6 @@ static const char zOptions[] = " -multiplex enable the multiplexor VFS\n" #endif " -newline SEP set output row separator. Default: '\\n'\n" -#if SHELL_WIN_UTF8_OPT - " -no-utf8 do not try to set up UTF-8 output (for legacy)\n" -#endif " -nofollow refuse to open symbolic links to database files\n" " -nonce STRING set the safe-mode escape nonce\n" " -nullvalue TEXT set text string for NULL values. Default ''\n" @@ -27929,9 +28646,6 @@ static const char zOptions[] = " -table set output mode to 'table'\n" " -tabs set output mode to 'tabs'\n" " -unsafe-testing allow unsafe commands and modes for testing\n" -#if SHELL_WIN_UTF8_OPT && 0 /* Option is accepted, but is now the default. */ - " -utf8 setup interactive console code page for UTF-8\n" -#endif " -version show SQLite version\n" " -vfs NAME use NAME as the default VFS\n" #ifdef SQLITE_ENABLE_VFSTRACE @@ -27942,14 +28656,13 @@ static const char zOptions[] = #endif ; static void usage(int showDetail){ - utf8_printf(stderr, - "Usage: %s [OPTIONS] [FILENAME [SQL]]\n" - "FILENAME is the name of an SQLite database. A new database is created\n" - "if the file does not previously exist. Defaults to :memory:.\n", Argv0); + eputf("Usage: %s [OPTIONS] [FILENAME [SQL]]\n" + "FILENAME is the name of an SQLite database. A new database is created\n" + "if the file does not previously exist. Defaults to :memory:.\n", Argv0); if( showDetail ){ - utf8_printf(stderr, "OPTIONS include:\n%s", zOptions); + eputf("OPTIONS include:\n%s", zOptions); }else{ - raw_printf(stderr, "Use the -help option for additional information\n"); + eputz("Use the -help option for additional information\n"); } exit(1); } @@ -27960,8 +28673,8 @@ static void usage(int showDetail){ */ static void verify_uninitialized(void){ if( sqlite3_config(-1)==SQLITE_MISUSE ){ - utf8_printf(stdout, "WARNING: attempt to configure SQLite after" - " initialization.\n"); + sputz(stdout, "WARNING: attempt to configure SQLite after" + " initialization.\n"); } } @@ -27990,7 +28703,7 @@ static void main_init(ShellState *data) { /* ** Output text to the console in a font that attracts extra attention. */ -#ifdef _WIN32 +#if defined(_WIN32) || defined(WIN32) static void printBold(const char *zText){ #if !SQLITE_OS_WINRT HANDLE out = GetStdHandle(STD_OUTPUT_HANDLE); @@ -28000,14 +28713,14 @@ static void printBold(const char *zText){ FOREGROUND_RED|FOREGROUND_INTENSITY ); #endif - printf("%s", zText); + oputz(zText); #if !SQLITE_OS_WINRT SetConsoleTextAttribute(out, defaultScreenInfo.wAttributes); #endif } #else static void printBold(const char *zText){ - printf("\033[1m%s\033[0m", zText); + oputf("\033[1m%s\033[0m", zText); } #endif @@ -28017,15 +28730,14 @@ static void printBold(const char *zText){ */ static char *cmdline_option_value(int argc, char **argv, int i){ if( i==argc ){ - utf8_printf(stderr, "%s: Error: missing argument to %s\n", - argv[0], argv[argc-1]); + eputf("%s: Error: missing argument to %s\n", argv[0], argv[argc-1]); exit(1); } return argv[i]; } static void sayAbnormalExit(void){ - if( seenInterrupt ) fprintf(stderr, "Program interrupted.\n"); + if( seenInterrupt ) eputz("Program interrupted.\n"); } #ifndef SQLITE_SHELL_IS_UTF8 @@ -28055,6 +28767,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ # define data shellState #else ShellState data; + StreamsAreConsole consStreams = SAC_NoConsole; #endif const char *zInitFile = 0; int i; @@ -28076,12 +28789,10 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ stdout_is_console = 1; data.wasm.zDefaultDbName = "/fiddle.sqlite3"; #else - stdin_is_interactive = isatty(0); - stdout_is_console = isatty(1); -#endif -#if SHELL_WIN_UTF8_OPT - probe_console(); /* Check for console I/O and UTF-8 capability. */ - if( !mbcs_opted ) atexit(console_restore); + consStreams = consoleClassifySetup(stdin, stdout, stderr); + stdin_is_interactive = (consStreams & SAC_InConsole)!=0; + stdout_is_console = (consStreams & SAC_OutConsole)!=0; + atexit(consoleRestore); #endif atexit(sayAbnormalExit); #ifdef SQLITE_DEBUG @@ -28090,9 +28801,8 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ #if !defined(_WIN32_WCE) if( getenv("SQLITE_DEBUG_BREAK") ){ if( isatty(0) && isatty(2) ){ - fprintf(stderr, - "attach debugger to process %d and press any key to continue.\n", - GETPID()); + eputf("attach debugger to process %d and press any key to continue.\n", + GETPID()); fgetc(stdin); }else{ #if defined(_WIN32) || defined(WIN32) @@ -28112,14 +28822,14 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ signal(SIGINT, interrupt_handler); #elif (defined(_WIN32) || defined(WIN32)) && !defined(_WIN32_WCE) if( !SetConsoleCtrlHandler(ConsoleCtrlHandler, TRUE) ){ - fprintf(stderr, "No ^C handler.\n"); + eputz("No ^C handler.\n"); } #endif #if USE_SYSTEM_SQLITE+0!=1 if( cli_strncmp(sqlite3_sourceid(),SQLITE_SOURCE_ID,60)!=0 ){ - utf8_printf(stderr, "SQLite header and source version mismatch\n%s\n%s\n", - sqlite3_sourceid(), SQLITE_SOURCE_ID); + eputf("SQLite header and source version mismatch\n%s\n%s\n", + sqlite3_sourceid(), SQLITE_SOURCE_ID); exit(1); } #endif @@ -28215,14 +28925,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ */ stdin_is_interactive = 0; }else if( cli_strcmp(z,"-utf8")==0 ){ -#if SHELL_WIN_UTF8_OPT - /* Option accepted, but is ignored except for this diagnostic. */ - if( mbcs_opted ) fprintf(stderr, "Cannot do UTF-8 at this console.\n"); -#endif /* SHELL_WIN_UTF8_OPT */ }else if( cli_strcmp(z,"-no-utf8")==0 ){ -#if SHELL_WIN_UTF8_OPT - mbcs_opted = 1; -#endif /* SHELL_WIN_UTF8_OPT */ }else if( cli_strcmp(z,"-heap")==0 ){ #if defined(SQLITE_ENABLE_MEMSYS3) || defined(SQLITE_ENABLE_MEMSYS5) const char *zSize; @@ -28357,26 +29060,17 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ if( pVfs ){ sqlite3_vfs_register(pVfs, 1); }else{ - utf8_printf(stderr, "no such VFS: \"%s\"\n", zVfs); + eputf("no such VFS: \"%s\"\n", zVfs); exit(1); } } -#if SHELL_WIN_UTF8_OPT - /* Get indicated Windows console setup done before running invocation commands. */ - if( in_console || out_console ){ - console_prepare_utf8(); - } - if( !in_console ){ - setBinaryMode(stdin, 0); - } -#endif if( data.pAuxDb->zDbFilename==0 ){ #ifndef SQLITE_OMIT_MEMORYDB data.pAuxDb->zDbFilename = ":memory:"; warnInmemoryDb = argc==1; #else - utf8_printf(stderr,"%s: Error: no database filename specified\n", Argv0); + eputf("%s: Error: no database filename specified\n", Argv0); return 1; #endif } @@ -28493,8 +29187,8 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ }else if( cli_strcmp(z,"-bail")==0 ){ /* No-op. The bail_on_error flag should already be set. */ }else if( cli_strcmp(z,"-version")==0 ){ - printf("%s %s (%d-bit)\n", sqlite3_libversion(), sqlite3_sourceid(), - 8*(int)sizeof(char*)); + oputf("%s %s (%d-bit)\n", sqlite3_libversion(), sqlite3_sourceid(), + 8*(int)sizeof(char*)); return 0; }else if( cli_strcmp(z,"-interactive")==0 ){ /* already handled */ @@ -28550,18 +29244,18 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ open_db(&data, 0); rc = shell_exec(&data, z, &zErrMsg); if( zErrMsg!=0 ){ - utf8_printf(stderr,"Error: %s\n", zErrMsg); + eputf("Error: %s\n", zErrMsg); if( bail_on_error ) return rc!=0 ? rc : 1; }else if( rc!=0 ){ - utf8_printf(stderr,"Error: unable to process SQL \"%s\"\n", z); + eputf("Error: unable to process SQL \"%s\"\n", z); if( bail_on_error ) return rc; } } #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) }else if( cli_strncmp(z, "-A", 2)==0 ){ if( nCmd>0 ){ - utf8_printf(stderr, "Error: cannot mix regular SQL or dot-commands" - " with \"%s\"\n", z); + eputf("Error: cannot mix regular SQL or dot-commands" + " with \"%s\"\n", z); return 1; } open_db(&data, OPEN_DB_ZIPFILE); @@ -28579,8 +29273,8 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ }else if( cli_strcmp(z,"-unsafe-testing")==0 ){ /* Acted upon in first pass. */ }else{ - utf8_printf(stderr,"%s: Error: unknown option: %s\n", Argv0, z); - raw_printf(stderr,"Use -help for a list of options.\n"); + eputf("%s: Error: unknown option: %s\n", Argv0, z); + eputz("Use -help for a list of options.\n"); return 1; } data.cMode = data.mode; @@ -28604,9 +29298,9 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ rc = shell_exec(&data, azCmd[i], &zErrMsg); if( zErrMsg || rc ){ if( zErrMsg!=0 ){ - utf8_printf(stderr,"Error: %s\n", zErrMsg); + eputf("Error: %s\n", zErrMsg); }else{ - utf8_printf(stderr,"Error: unable to process SQL: %s\n", azCmd[i]); + eputf("Error: unable to process SQL: %s\n", azCmd[i]); } sqlite3_free(zErrMsg); free(azCmd); @@ -28620,27 +29314,21 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ if( stdin_is_interactive ){ char *zHome; char *zHistory; - const char *zCharset = ""; int nHistory; print_FTL_version(); -#if SHELL_WIN_UTF8_OPT - switch( console_utf8_in+2*console_utf8_out ){ - default: case 0: break; - case 1: zCharset = " (utf8 in)"; break; - case 2: zCharset = " (utf8 out)"; break; - case 3: zCharset = " (utf8 I/O)"; break; - } -#endif - printf( - "SQLite version %s %.19s%s\n" /*extra-version-info*/ - "Enter \".help\" for usage hints.\n", - sqlite3_libversion(), sqlite3_sourceid(), zCharset - ); +#if CIO_WIN_WC_XLATE +# define SHELL_CIO_CHAR_SET (stdout_is_console? " (UTF-16 console I/O)" : "") +#else +# define SHELL_CIO_CHAR_SET "" +#endif + oputf("SQLite version %s %.19s%s\n" /*extra-version-info*/ + "Enter \".help\" for usage hints.\n", + sqlite3_libversion(), sqlite3_sourceid(), SHELL_CIO_CHAR_SET); if( warnInmemoryDb ){ - printf("Connected to a "); + oputz("Connected to a "); printBold("transient in-memory database"); - printf(".\nUse \".open FILENAME\" to reopen on a " - "persistent database.\n"); + oputz(".\nUse \".open FILENAME\" to reopen on a" + " persistent database.\n"); } zHistory = getenv("SQLITE_HISTORY"); if( zHistory ){ @@ -28700,8 +29388,8 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ memset(&data, 0, sizeof(data)); #ifdef SQLITE_DEBUG if( sqlite3_memory_used()>mem_main_enter ){ - utf8_printf(stderr, "Memory leaked: %u bytes\n", - (unsigned int)(sqlite3_memory_used()-mem_main_enter)); + eputf("Memory leaked: %u bytes\n", + (unsigned int)(sqlite3_memory_used()-mem_main_enter)); } #endif #endif /* !SQLITE_SHELL_FIDDLE */ diff --git a/src/database/sqlite3.c b/src/database/sqlite3.c index aacd507cb..592320ff6 100644 --- a/src/database/sqlite3.c +++ b/src/database/sqlite3.c @@ -1,6 +1,6 @@ /****************************************************************************** ** This file is an amalgamation of many separate C source files from SQLite -** version 3.44.0. By combining all the individual C code files into this +** version 3.44.1. By combining all the individual C code files into this ** single large file, the entire code can be compiled as a single translation ** unit. This allows many compilers to do optimizations that would not be ** possible if the files were compiled separately. Performance improvements @@ -18,7 +18,7 @@ ** separate file. This file contains only code for the core SQLite library. ** ** The content in this amalgamation comes from Fossil check-in -** 17129ba1ff7f0daf37100ee82d507aef7827. +** d295f48e8f367b066b881780c98bdf980a1d. */ #define SQLITE_CORE 1 #define SQLITE_AMALGAMATION 1 @@ -459,9 +459,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.44.0" -#define SQLITE_VERSION_NUMBER 3044000 -#define SQLITE_SOURCE_ID "2023-11-01 11:23:50 17129ba1ff7f0daf37100ee82d507aef7827cf38de1866e2633096ae6ad81301" +#define SQLITE_VERSION "3.44.1" +#define SQLITE_VERSION_NUMBER 3044001 +#define SQLITE_SOURCE_ID "2023-11-22 14:18:12 d295f48e8f367b066b881780c98bdf980a1d550397d5ba0b0e49842c95b3e8b4" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -5886,13 +5886,27 @@ SQLITE_API int sqlite3_create_window_function( ** ** ** [[SQLITE_SUBTYPE]]
SQLITE_SUBTYPE
-** The SQLITE_SUBTYPE flag indicates to SQLite that a function may call +** The SQLITE_SUBTYPE flag indicates to SQLite that a function might call ** [sqlite3_value_subtype()] to inspect the sub-types of its arguments. -** Specifying this flag makes no difference for scalar or aggregate user -** functions. However, if it is not specified for a user-defined window -** function, then any sub-types belonging to arguments passed to the window -** function may be discarded before the window function is called (i.e. -** sqlite3_value_subtype() will always return 0). +** This flag instructs SQLite to omit some corner-case optimizations that +** might disrupt the operation of the [sqlite3_value_subtype()] function, +** causing it to return zero rather than the correct subtype(). +** SQL functions that invokes [sqlite3_value_subtype()] should have this +** property. If the SQLITE_SUBTYPE property is omitted, then the return +** value from [sqlite3_value_subtype()] might sometimes be zero even though +** a non-zero subtype was specified by the function argument expression. +** +** [[SQLITE_RESULT_SUBTYPE]]
SQLITE_RESULT_SUBTYPE
+** The SQLITE_RESULT_SUBTYPE flag indicates to SQLite that a function might call +** [sqlite3_result_subtype()] to cause a sub-type to be associated with its +** result. +** Every function that invokes [sqlite3_result_subtype()] should have this +** property. If it does not, then the call to [sqlite3_result_subtype()] +** might become a no-op if the function is used as term in an +** [expression index]. On the other hand, SQL functions that never invoke +** [sqlite3_result_subtype()] should avoid setting this property, as the +** purpose of this property is to disable certain optimizations that are +** incompatible with subtypes. **
** */ @@ -5900,6 +5914,7 @@ SQLITE_API int sqlite3_create_window_function( #define SQLITE_DIRECTONLY 0x000080000 #define SQLITE_SUBTYPE 0x000100000 #define SQLITE_INNOCUOUS 0x000200000 +#define SQLITE_RESULT_SUBTYPE 0x001000000 /* ** CAPI3REF: Deprecated Functions @@ -6096,6 +6111,12 @@ SQLITE_API int sqlite3_value_encoding(sqlite3_value*); ** information can be used to pass a limited amount of context from ** one SQL function to another. Use the [sqlite3_result_subtype()] ** routine to set the subtype for the return value of an SQL function. +** +** Every [application-defined SQL function] that invoke this interface +** should include the [SQLITE_SUBTYPE] property in the text +** encoding argument when the function is [sqlite3_create_function|registered]. +** If the [SQLITE_SUBTYPE] property is omitted, then sqlite3_value_subtype() +** might return zero instead of the upstream subtype in some corner cases. */ SQLITE_API unsigned int sqlite3_value_subtype(sqlite3_value*); @@ -6226,14 +6247,22 @@ SQLITE_API sqlite3 *sqlite3_context_db_handle(sqlite3_context*); **
  • ^(when sqlite3_set_auxdata() is invoked again on the same ** parameter)^, or **
  • ^(during the original sqlite3_set_auxdata() call when a memory -** allocation error occurs.)^ +** allocation error occurs.)^ +**
  • ^(during the original sqlite3_set_auxdata() call if the function +** is evaluated during query planning instead of during query execution, +** as sometimes happens with [SQLITE_ENABLE_STAT4].)^ ** -** Note the last bullet in particular. The destructor X in +** Note the last two bullets in particular. The destructor X in ** sqlite3_set_auxdata(C,N,P,X) might be called immediately, before the ** sqlite3_set_auxdata() interface even returns. Hence sqlite3_set_auxdata() ** should be called near the end of the function implementation and the ** function implementation should not make any use of P after -** sqlite3_set_auxdata() has been called. +** sqlite3_set_auxdata() has been called. Furthermore, a call to +** sqlite3_get_auxdata() that occurs immediately after a corresponding call +** to sqlite3_set_auxdata() might still return NULL if an out-of-memory +** condition occurred during the sqlite3_set_auxdata() call or if the +** function is being evaluated during query planning rather than during +** query execution. ** ** ^(In practice, auxiliary data is preserved between function calls for ** function parameters that are compile-time constants, including literal @@ -6507,6 +6536,20 @@ SQLITE_API int sqlite3_result_zeroblob64(sqlite3_context*, sqlite3_uint64 n); ** higher order bits are discarded. ** The number of subtype bytes preserved by SQLite might increase ** in future releases of SQLite. +** +** Every [application-defined SQL function] that invokes this interface +** should include the [SQLITE_RESULT_SUBTYPE] property in its +** text encoding argument when the SQL function is +** [sqlite3_create_function|registered]. If the [SQLITE_RESULT_SUBTYPE] +** property is omitted from the function that invokes sqlite3_result_subtype(), +** then in some cases the sqlite3_result_subtype() might fail to set +** the result subtype. +** +** If SQLite is compiled with -DSQLITE_STRICT_SUBTYPE=1, then any +** SQL function that invokes the sqlite3_result_subtype() interface +** and that does not have the SQLITE_RESULT_SUBTYPE property will raise +** an error. Future versions of SQLite might enable -DSQLITE_STRICT_SUBTYPE=1 +** by default. */ SQLITE_API void sqlite3_result_subtype(sqlite3_context*,unsigned int); @@ -17811,14 +17854,15 @@ struct FuncDestructor { #define SQLITE_FUNC_SLOCHNG 0x2000 /* "Slow Change". Value constant during a ** single query - might change over time */ #define SQLITE_FUNC_TEST 0x4000 /* Built-in testing functions */ -/* 0x8000 -- available for reuse */ +#define SQLITE_FUNC_RUNONLY 0x8000 /* Cannot be used by valueFromFunction */ #define SQLITE_FUNC_WINDOW 0x00010000 /* Built-in window-only function */ #define SQLITE_FUNC_INTERNAL 0x00040000 /* For use by NestedParse() only */ #define SQLITE_FUNC_DIRECT 0x00080000 /* Not for use in TRIGGERs or VIEWs */ -#define SQLITE_FUNC_SUBTYPE 0x00100000 /* Result likely to have sub-type */ +/* SQLITE_SUBTYPE 0x00100000 // Consumer of subtypes */ #define SQLITE_FUNC_UNSAFE 0x00200000 /* Function has side effects */ #define SQLITE_FUNC_INLINE 0x00400000 /* Functions implemented in-line */ #define SQLITE_FUNC_BUILTIN 0x00800000 /* This is a built-in function */ +/* SQLITE_RESULT_SUBTYPE 0x01000000 // Generator of subtypes */ #define SQLITE_FUNC_ANYORDER 0x08000000 /* count/min/max aggregate */ /* Identifier numbers for each in-line function */ @@ -17910,9 +17954,10 @@ struct FuncDestructor { #define MFUNCTION(zName, nArg, xPtr, xFunc) \ {nArg, SQLITE_FUNC_BUILTIN|SQLITE_FUNC_CONSTANT|SQLITE_UTF8, \ xPtr, 0, xFunc, 0, 0, 0, #zName, {0} } -#define JFUNCTION(zName, nArg, iArg, xFunc) \ - {nArg, SQLITE_FUNC_BUILTIN|SQLITE_DETERMINISTIC|\ - SQLITE_FUNC_CONSTANT|SQLITE_UTF8, \ +#define JFUNCTION(zName, nArg, bUseCache, bWS, bRS, iArg, xFunc) \ + {nArg, SQLITE_FUNC_BUILTIN|SQLITE_DETERMINISTIC|SQLITE_FUNC_CONSTANT|\ + SQLITE_UTF8|((bUseCache)*SQLITE_FUNC_RUNONLY)|\ + ((bRS)*SQLITE_SUBTYPE)|((bWS)*SQLITE_RESULT_SUBTYPE), \ SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, 0, #zName, {0} } #define INLINE_FUNC(zName, nArg, iArg, mFlags) \ {nArg, SQLITE_FUNC_BUILTIN|\ @@ -29453,7 +29498,7 @@ SQLITE_PRIVATE void sqlite3MemoryBarrier(void){ SQLITE_MEMORY_BARRIER; #elif defined(__GNUC__) __sync_synchronize(); -#elif MSVC_VERSION>=1300 +#elif MSVC_VERSION>=1400 _ReadWriteBarrier(); #elif defined(MemoryBarrier) MemoryBarrier(); @@ -61447,10 +61492,13 @@ SQLITE_PRIVATE int sqlite3PagerOpen( */ SQLITE_API sqlite3_file *sqlite3_database_file_object(const char *zName){ Pager *pPager; + const char *p; while( zName[-1]!=0 || zName[-2]!=0 || zName[-3]!=0 || zName[-4]!=0 ){ zName--; } - pPager = *(Pager**)(zName - 4 - sizeof(Pager*)); + p = zName - 4 - sizeof(Pager*); + assert( EIGHT_BYTE_ALIGNMENT(p) ); + pPager = *(Pager**)p; return pPager->fd; } @@ -83411,7 +83459,7 @@ static int valueFromFunction( #endif assert( pFunc ); if( (pFunc->funcFlags & (SQLITE_FUNC_CONSTANT|SQLITE_FUNC_SLOCHNG))==0 - || (pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL) + || (pFunc->funcFlags & (SQLITE_FUNC_NEEDCOLL|SQLITE_FUNC_RUNONLY))!=0 ){ return SQLITE_OK; } @@ -89952,6 +90000,18 @@ SQLITE_API void sqlite3_result_subtype(sqlite3_context *pCtx, unsigned int eSubt #ifdef SQLITE_ENABLE_API_ARMOR if( pCtx==0 ) return; #endif +#if defined(SQLITE_STRICT_SUBTYPE) && SQLITE_STRICT_SUBTYPE+0!=0 + if( pCtx->pFunc!=0 + && (pCtx->pFunc->funcFlags & SQLITE_RESULT_SUBTYPE)==0 + ){ + char zErr[200]; + sqlite3_snprintf(sizeof(zErr), zErr, + "misuse of sqlite3_result_subtype() by %s()", + pCtx->pFunc->zName); + sqlite3_result_error(pCtx, zErr, -1); + return; + } +#endif /* SQLITE_STRICT_SUBTYPE */ pOut = pCtx->pOut; assert( sqlite3_mutex_held(pOut->db->mutex) ); pOut->eSubtype = eSubtype & 0xff; @@ -100321,7 +100381,7 @@ case OP_VCheck: { /* out2 */ pTab = pOp->p4.pTab; assert( pTab!=0 ); assert( IsVirtual(pTab) ); - assert( pTab->u.vtab.p!=0 ); + if( pTab->u.vtab.p==0 ) break; pVtab = pTab->u.vtab.p->pVtab; assert( pVtab!=0 ); pModule = pVtab->pModule; @@ -113917,8 +113977,8 @@ SQLITE_PRIVATE int sqlite3ExprListCompare(const ExprList *pA, const ExprList *pB */ SQLITE_PRIVATE int sqlite3ExprCompareSkip(Expr *pA,Expr *pB, int iTab){ return sqlite3ExprCompare(0, - sqlite3ExprSkipCollateAndLikely(pA), - sqlite3ExprSkipCollateAndLikely(pB), + sqlite3ExprSkipCollate(pA), + sqlite3ExprSkipCollate(pB), iTab); } @@ -147605,10 +147665,11 @@ static void selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){ SrcList *pTabList; SrcItem *pFrom; - assert( p->selFlags & SF_Resolved ); if( p->selFlags & SF_HasTypeInfo ) return; p->selFlags |= SF_HasTypeInfo; pParse = pWalker->pParse; + testcase( (p->selFlags & SF_Resolved)==0 ); + assert( (p->selFlags & SF_Resolved) || IN_RENAME_OBJECT ); pTabList = p->pSrc; for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ Table *pTab = pFrom->pTab; @@ -148630,6 +148691,7 @@ SQLITE_PRIVATE int sqlite3Select( TREETRACE(0x1000,pParse,p, ("LEFT-JOIN simplifies to JOIN on term %d\n",i)); pItem->fg.jointype &= ~(JT_LEFT|JT_OUTER); + unsetJoinExpr(p->pWhere, pItem->iCursor, 0); } } if( pItem->fg.jointype & JT_LTORJ ){ @@ -148644,17 +148706,15 @@ SQLITE_PRIVATE int sqlite3Select( TREETRACE(0x1000,pParse,p, ("RIGHT-JOIN simplifies to JOIN on term %d\n",j)); pI2->fg.jointype &= ~(JT_RIGHT|JT_OUTER); + unsetJoinExpr(p->pWhere, pI2->iCursor, 1); } } } - for(j=pTabList->nSrc-1; j>=i; j--){ + for(j=pTabList->nSrc-1; j>=0; j--){ pTabList->a[j].fg.jointype &= ~JT_LTORJ; if( pTabList->a[j].fg.jointype & JT_RIGHT ) break; } } - assert( pItem->iCursor>=0 ); - unsetJoinExpr(p->pWhere, pItem->iCursor, - pTabList->a[0].fg.jointype & JT_LTORJ); } /* No further action if this term of the FROM clause is not a subquery */ @@ -166058,6 +166118,20 @@ static SQLITE_NOINLINE void whereAddIndexedExpr( continue; } if( sqlite3ExprIsConstant(pExpr) ) continue; + if( pExpr->op==TK_FUNCTION ){ + /* Functions that might set a subtype should not be replaced by the + ** value taken from an expression index since the index omits the + ** subtype. https://sqlite.org/forum/forumpost/68d284c86b082c3e */ + int n; + FuncDef *pDef; + sqlite3 *db = pParse->db; + assert( ExprUseXList(pExpr) ); + n = pExpr->x.pList ? pExpr->x.pList->nExpr : 0; + pDef = sqlite3FindFunction(db, pExpr->u.zToken, n, ENC(db), 0); + if( pDef==0 || (pDef->funcFlags & SQLITE_RESULT_SUBTYPE)!=0 ){ + continue; + } + } p = sqlite3DbMallocRaw(pParse->db, sizeof(IndexedExpr)); if( p==0 ) break; p->pIENext = pParse->pIdxEpr; @@ -168240,7 +168314,7 @@ SQLITE_PRIVATE int sqlite3WindowRewrite(Parse *pParse, Select *p){ assert( ExprUseXList(pWin->pOwner) ); assert( pWin->pWFunc!=0 ); pArgs = pWin->pOwner->x.pList; - if( pWin->pWFunc->funcFlags & SQLITE_FUNC_SUBTYPE ){ + if( pWin->pWFunc->funcFlags & SQLITE_SUBTYPE ){ selectWindowRewriteEList(pParse, pMWin, pSrc, pArgs, pTab, &pSublist); pWin->iArgCol = (pSublist ? pSublist->nExpr : 0); pWin->bExprArgs = 1; @@ -179412,7 +179486,7 @@ SQLITE_PRIVATE int sqlite3CreateFunc( assert( SQLITE_FUNC_CONSTANT==SQLITE_DETERMINISTIC ); assert( SQLITE_FUNC_DIRECT==SQLITE_DIRECTONLY ); extraFlags = enc & (SQLITE_DETERMINISTIC|SQLITE_DIRECTONLY| - SQLITE_SUBTYPE|SQLITE_INNOCUOUS); + SQLITE_SUBTYPE|SQLITE_INNOCUOUS|SQLITE_RESULT_SUBTYPE); enc &= (SQLITE_FUNC_ENCMASK|SQLITE_ANY); /* The SQLITE_INNOCUOUS flag is the same bit as SQLITE_FUNC_UNSAFE. But @@ -202993,13 +203067,19 @@ static void jsonAppendNormalizedString(JsonString *p, const char *zIn, u32 N){ zIn++; N -= 2; while( N>0 ){ - for(i=0; i0 ){ jsonAppendRawNZ(p, zIn, i); zIn += i; N -= i; if( N==0 ) break; } + if( zIn[0]=='"' ){ + jsonAppendRawNZ(p, "\\\"", 2); + zIn++; + N--; + continue; + } assert( zIn[0]=='\\' ); switch( (u8)zIn[1] ){ case '\'': @@ -203394,7 +203474,8 @@ static void jsonReturnJson( JsonParse *pParse, /* The complete JSON */ JsonNode *pNode, /* Node to return */ sqlite3_context *pCtx, /* Return value for this function */ - int bGenerateAlt /* Also store the rendered text in zAlt */ + int bGenerateAlt, /* Also store the rendered text in zAlt */ + int omitSubtype /* Do not call sqlite3_result_subtype() */ ){ JsonString s; if( pParse->oom ){ @@ -203409,7 +203490,7 @@ static void jsonReturnJson( pParse->nAlt = s.nUsed; } jsonResult(&s); - sqlite3_result_subtype(pCtx, JSON_SUBTYPE); + if( !omitSubtype ) sqlite3_result_subtype(pCtx, JSON_SUBTYPE); } } @@ -203450,7 +203531,8 @@ static u32 jsonHexToInt4(const char *z){ static void jsonReturn( JsonParse *pParse, /* Complete JSON parse tree */ JsonNode *pNode, /* Node to return */ - sqlite3_context *pCtx /* Return value for this function */ + sqlite3_context *pCtx, /* Return value for this function */ + int omitSubtype /* Do not call sqlite3_result_subtype() */ ){ switch( pNode->eType ){ default: { @@ -203596,7 +203678,7 @@ static void jsonReturn( } case JSON_ARRAY: case JSON_OBJECT: { - jsonReturnJson(pParse, pNode, pCtx, 0); + jsonReturnJson(pParse, pNode, pCtx, 0, omitSubtype); break; } } @@ -204948,7 +205030,7 @@ static void jsonParseFunc( printf("iSubst = %u\n", p->iSubst); printf("iHold = %u\n", p->iHold); jsonDebugPrintNodeEntries(p->aNode, p->nNode); - jsonReturnJson(p, p->aNode, ctx, 1); + jsonReturnJson(p, p->aNode, ctx, 1, 0); } /* @@ -205134,15 +205216,14 @@ static void jsonExtractFunc( } if( pNode ){ if( flags & JSON_JSON ){ - jsonReturnJson(p, pNode, ctx, 0); + jsonReturnJson(p, pNode, ctx, 0, 0); }else{ - jsonReturn(p, pNode, ctx); - sqlite3_result_subtype(ctx, 0); + jsonReturn(p, pNode, ctx, 1); } } }else{ pNode = jsonLookup(p, zPath, 0, ctx); - if( p->nErr==0 && pNode ) jsonReturn(p, pNode, ctx); + if( p->nErr==0 && pNode ) jsonReturn(p, pNode, ctx, 0); } }else{ /* Two or more PATH arguments results in a JSON array with each @@ -205268,7 +205349,7 @@ static void jsonPatchFunc( if( pResult && pX->oom==0 ){ jsonDebugPrintParse(pX); jsonDebugPrintNode(pResult); - jsonReturnJson(pX, pResult, ctx, 0); + jsonReturnJson(pX, pResult, ctx, 0, 0); }else{ sqlite3_result_error_nomem(ctx); } @@ -205347,7 +205428,7 @@ static void jsonRemoveFunc( } } if( (pParse->aNode[0].jnFlags & JNODE_REMOVE)==0 ){ - jsonReturnJson(pParse, pParse->aNode, ctx, 1); + jsonReturnJson(pParse, pParse->aNode, ctx, 1, 0); } remove_done: jsonDebugPrintParse(p); @@ -205476,7 +205557,7 @@ static void jsonReplaceFunc( jsonReplaceNode(ctx, pParse, (u32)(pNode - pParse->aNode), argv[i+1]); } } - jsonReturnJson(pParse, pParse->aNode, ctx, 1); + jsonReturnJson(pParse, pParse->aNode, ctx, 1, 0); replace_err: jsonDebugPrintParse(pParse); jsonParseFree(pParse); @@ -205530,7 +205611,7 @@ static void jsonSetFunc( } } jsonDebugPrintParse(pParse); - jsonReturnJson(pParse, pParse->aNode, ctx, 1); + jsonReturnJson(pParse, pParse->aNode, ctx, 1, 0); jsonSetDone: jsonParseFree(pParse); } @@ -206045,7 +206126,7 @@ static int jsonEachColumn( case JEACH_KEY: { if( p->i==0 ) break; if( p->eType==JSON_OBJECT ){ - jsonReturn(&p->sParse, pThis, ctx); + jsonReturn(&p->sParse, pThis, ctx, 0); }else if( p->eType==JSON_ARRAY ){ u32 iKey; if( p->bRecursive ){ @@ -206061,7 +206142,7 @@ static int jsonEachColumn( } case JEACH_VALUE: { if( pThis->jnFlags & JNODE_LABEL ) pThis++; - jsonReturn(&p->sParse, pThis, ctx); + jsonReturn(&p->sParse, pThis, ctx, 0); break; } case JEACH_TYPE: { @@ -206072,7 +206153,7 @@ static int jsonEachColumn( case JEACH_ATOM: { if( pThis->jnFlags & JNODE_LABEL ) pThis++; if( pThis->eType>=JSON_ARRAY ) break; - jsonReturn(&p->sParse, pThis, ctx); + jsonReturn(&p->sParse, pThis, ctx, 0); break; } case JEACH_ID: { @@ -206365,34 +206446,43 @@ static sqlite3_module jsonTreeModule = { SQLITE_PRIVATE void sqlite3RegisterJsonFunctions(void){ #ifndef SQLITE_OMIT_JSON static FuncDef aJsonFunc[] = { - JFUNCTION(json, 1, 0, jsonRemoveFunc), - JFUNCTION(json_array, -1, 0, jsonArrayFunc), - JFUNCTION(json_array_length, 1, 0, jsonArrayLengthFunc), - JFUNCTION(json_array_length, 2, 0, jsonArrayLengthFunc), - JFUNCTION(json_error_position,1, 0, jsonErrorFunc), - JFUNCTION(json_extract, -1, 0, jsonExtractFunc), - JFUNCTION(->, 2, JSON_JSON, jsonExtractFunc), - JFUNCTION(->>, 2, JSON_SQL, jsonExtractFunc), - JFUNCTION(json_insert, -1, 0, jsonSetFunc), - JFUNCTION(json_object, -1, 0, jsonObjectFunc), - JFUNCTION(json_patch, 2, 0, jsonPatchFunc), - JFUNCTION(json_quote, 1, 0, jsonQuoteFunc), - JFUNCTION(json_remove, -1, 0, jsonRemoveFunc), - JFUNCTION(json_replace, -1, 0, jsonReplaceFunc), - JFUNCTION(json_set, -1, JSON_ISSET, jsonSetFunc), - JFUNCTION(json_type, 1, 0, jsonTypeFunc), - JFUNCTION(json_type, 2, 0, jsonTypeFunc), - JFUNCTION(json_valid, 1, 0, jsonValidFunc), -#if SQLITE_DEBUG - JFUNCTION(json_parse, 1, 0, jsonParseFunc), - JFUNCTION(json_test1, 1, 0, jsonTest1Func), + /* calls sqlite3_result_subtype() */ + /* | */ + /* Uses cache ______ | __ calls sqlite3_value_subtype() */ + /* | | | */ + /* Num args _________ | | | ___ Flags */ + /* | | | | | */ + /* | | | | | */ + JFUNCTION(json, 1, 1, 1, 0, 0, jsonRemoveFunc), + JFUNCTION(json_array, -1, 0, 1, 1, 0, jsonArrayFunc), + JFUNCTION(json_array_length, 1, 1, 0, 0, 0, jsonArrayLengthFunc), + JFUNCTION(json_array_length, 2, 1, 0, 0, 0, jsonArrayLengthFunc), + JFUNCTION(json_error_position,1, 1, 0, 0, 0, jsonErrorFunc), + JFUNCTION(json_extract, -1, 1, 1, 0, 0, jsonExtractFunc), + JFUNCTION(->, 2, 1, 1, 0, JSON_JSON, jsonExtractFunc), + JFUNCTION(->>, 2, 1, 0, 0, JSON_SQL, jsonExtractFunc), + JFUNCTION(json_insert, -1, 1, 1, 1, 0, jsonSetFunc), + JFUNCTION(json_object, -1, 0, 1, 1, 0, jsonObjectFunc), + JFUNCTION(json_patch, 2, 1, 1, 0, 0, jsonPatchFunc), + JFUNCTION(json_quote, 1, 0, 1, 1, 0, jsonQuoteFunc), + JFUNCTION(json_remove, -1, 1, 1, 0, 0, jsonRemoveFunc), + JFUNCTION(json_replace, -1, 1, 1, 1, 0, jsonReplaceFunc), + JFUNCTION(json_set, -1, 1, 1, 1, JSON_ISSET, jsonSetFunc), + JFUNCTION(json_type, 1, 1, 0, 0, 0, jsonTypeFunc), + JFUNCTION(json_type, 2, 1, 0, 0, 0, jsonTypeFunc), + JFUNCTION(json_valid, 1, 1, 0, 0, 0, jsonValidFunc), +#ifdef SQLITE_DEBUG + JFUNCTION(json_parse, 1, 1, 1, 0, 0, jsonParseFunc), + JFUNCTION(json_test1, 1, 1, 0, 1, 0, jsonTest1Func), #endif WAGGREGATE(json_group_array, 1, 0, 0, jsonArrayStep, jsonArrayFinal, jsonArrayValue, jsonGroupInverse, - SQLITE_SUBTYPE|SQLITE_UTF8|SQLITE_DETERMINISTIC), + SQLITE_SUBTYPE|SQLITE_RESULT_SUBTYPE|SQLITE_UTF8| + SQLITE_DETERMINISTIC), WAGGREGATE(json_group_object, 2, 0, 0, jsonObjectStep, jsonObjectFinal, jsonObjectValue, jsonGroupInverse, - SQLITE_SUBTYPE|SQLITE_UTF8|SQLITE_DETERMINISTIC) + SQLITE_SUBTYPE|SQLITE_RESULT_SUBTYPE|SQLITE_UTF8| + SQLITE_DETERMINISTIC) }; sqlite3InsertBuiltinFuncs(aJsonFunc, ArraySize(aJsonFunc)); #endif @@ -236129,10 +236219,8 @@ static Fts5HashEntry *fts5HashEntryMerge( } /* -** Extract all tokens from hash table iHash and link them into a list -** in sorted order. The hash table is cleared before returning. It is -** the responsibility of the caller to free the elements of the returned -** list. +** Link all tokens from hash table iHash into a list in sorted order. The +** tokens are not removed from the hash table. */ static int fts5HashEntrySort( Fts5Hash *pHash, @@ -238998,6 +239086,14 @@ static void fts5SegIterHashInit( pLeaf->p = (u8*)pList; } } + + /* The call to sqlite3Fts5HashScanInit() causes the hash table to + ** fill the size field of all existing position lists. This means they + ** can no longer be appended to. Since the only scenario in which they + ** can be appended to is if the previous operation on this table was + ** a DELETE, by clearing the Fts5Index.bDelete flag we can avoid this + ** possibility altogether. */ + p->bDelete = 0; }else{ p->rc = sqlite3Fts5HashQuery(p->pHash, sizeof(Fts5Data), (const char*)pTerm, nTerm, (void**)&pLeaf, &nList @@ -240675,7 +240771,7 @@ static void fts5WriteAppendPoslistData( const u8 *a = aData; int n = nData; - assert( p->pConfig->pgsz>0 ); + assert( p->pConfig->pgsz>0 || p->rc!=SQLITE_OK ); while( p->rc==SQLITE_OK && (pPage->buf.n + pPage->pgidx.n + n)>=p->pConfig->pgsz ){ @@ -241935,8 +242031,9 @@ static int sqlite3Fts5IndexOptimize(Fts5Index *p){ assert( p->rc==SQLITE_OK ); fts5IndexFlush(p); - assert( p->nContentlessDelete==0 ); + assert( p->rc!=SQLITE_OK || p->nContentlessDelete==0 ); pStruct = fts5StructureRead(p); + assert( p->rc!=SQLITE_OK || pStruct!=0 ); fts5StructureInvalidate(p); if( pStruct ){ @@ -247513,7 +247610,7 @@ static void fts5SourceIdFunc( ){ assert( nArg==0 ); UNUSED_PARAM2(nArg, apUnused); - sqlite3_result_text(pCtx, "fts5: 2023-11-01 11:23:50 17129ba1ff7f0daf37100ee82d507aef7827cf38de1866e2633096ae6ad81301", -1, SQLITE_TRANSIENT); + sqlite3_result_text(pCtx, "fts5: 2023-11-22 14:18:12 d295f48e8f367b066b881780c98bdf980a1d550397d5ba0b0e49842c95b3e8b4", -1, SQLITE_TRANSIENT); } /* diff --git a/src/database/sqlite3.h b/src/database/sqlite3.h index d4f1c810c..fce162d69 100644 --- a/src/database/sqlite3.h +++ b/src/database/sqlite3.h @@ -146,9 +146,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.44.0" -#define SQLITE_VERSION_NUMBER 3044000 -#define SQLITE_SOURCE_ID "2023-11-01 11:23:50 17129ba1ff7f0daf37100ee82d507aef7827cf38de1866e2633096ae6ad81301" +#define SQLITE_VERSION "3.44.1" +#define SQLITE_VERSION_NUMBER 3044001 +#define SQLITE_SOURCE_ID "2023-11-22 14:18:12 d295f48e8f367b066b881780c98bdf980a1d550397d5ba0b0e49842c95b3e8b4" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -5573,13 +5573,27 @@ SQLITE_API int sqlite3_create_window_function( ** ** ** [[SQLITE_SUBTYPE]]
    SQLITE_SUBTYPE
    -** The SQLITE_SUBTYPE flag indicates to SQLite that a function may call +** The SQLITE_SUBTYPE flag indicates to SQLite that a function might call ** [sqlite3_value_subtype()] to inspect the sub-types of its arguments. -** Specifying this flag makes no difference for scalar or aggregate user -** functions. However, if it is not specified for a user-defined window -** function, then any sub-types belonging to arguments passed to the window -** function may be discarded before the window function is called (i.e. -** sqlite3_value_subtype() will always return 0). +** This flag instructs SQLite to omit some corner-case optimizations that +** might disrupt the operation of the [sqlite3_value_subtype()] function, +** causing it to return zero rather than the correct subtype(). +** SQL functions that invokes [sqlite3_value_subtype()] should have this +** property. If the SQLITE_SUBTYPE property is omitted, then the return +** value from [sqlite3_value_subtype()] might sometimes be zero even though +** a non-zero subtype was specified by the function argument expression. +** +** [[SQLITE_RESULT_SUBTYPE]]
    SQLITE_RESULT_SUBTYPE
    +** The SQLITE_RESULT_SUBTYPE flag indicates to SQLite that a function might call +** [sqlite3_result_subtype()] to cause a sub-type to be associated with its +** result. +** Every function that invokes [sqlite3_result_subtype()] should have this +** property. If it does not, then the call to [sqlite3_result_subtype()] +** might become a no-op if the function is used as term in an +** [expression index]. On the other hand, SQL functions that never invoke +** [sqlite3_result_subtype()] should avoid setting this property, as the +** purpose of this property is to disable certain optimizations that are +** incompatible with subtypes. **
    ** */ @@ -5587,6 +5601,7 @@ SQLITE_API int sqlite3_create_window_function( #define SQLITE_DIRECTONLY 0x000080000 #define SQLITE_SUBTYPE 0x000100000 #define SQLITE_INNOCUOUS 0x000200000 +#define SQLITE_RESULT_SUBTYPE 0x001000000 /* ** CAPI3REF: Deprecated Functions @@ -5783,6 +5798,12 @@ SQLITE_API int sqlite3_value_encoding(sqlite3_value*); ** information can be used to pass a limited amount of context from ** one SQL function to another. Use the [sqlite3_result_subtype()] ** routine to set the subtype for the return value of an SQL function. +** +** Every [application-defined SQL function] that invoke this interface +** should include the [SQLITE_SUBTYPE] property in the text +** encoding argument when the function is [sqlite3_create_function|registered]. +** If the [SQLITE_SUBTYPE] property is omitted, then sqlite3_value_subtype() +** might return zero instead of the upstream subtype in some corner cases. */ SQLITE_API unsigned int sqlite3_value_subtype(sqlite3_value*); @@ -5913,14 +5934,22 @@ SQLITE_API sqlite3 *sqlite3_context_db_handle(sqlite3_context*); **
  • ^(when sqlite3_set_auxdata() is invoked again on the same ** parameter)^, or **
  • ^(during the original sqlite3_set_auxdata() call when a memory -** allocation error occurs.)^ +** allocation error occurs.)^ +**
  • ^(during the original sqlite3_set_auxdata() call if the function +** is evaluated during query planning instead of during query execution, +** as sometimes happens with [SQLITE_ENABLE_STAT4].)^ ** -** Note the last bullet in particular. The destructor X in +** Note the last two bullets in particular. The destructor X in ** sqlite3_set_auxdata(C,N,P,X) might be called immediately, before the ** sqlite3_set_auxdata() interface even returns. Hence sqlite3_set_auxdata() ** should be called near the end of the function implementation and the ** function implementation should not make any use of P after -** sqlite3_set_auxdata() has been called. +** sqlite3_set_auxdata() has been called. Furthermore, a call to +** sqlite3_get_auxdata() that occurs immediately after a corresponding call +** to sqlite3_set_auxdata() might still return NULL if an out-of-memory +** condition occurred during the sqlite3_set_auxdata() call or if the +** function is being evaluated during query planning rather than during +** query execution. ** ** ^(In practice, auxiliary data is preserved between function calls for ** function parameters that are compile-time constants, including literal @@ -6194,6 +6223,20 @@ SQLITE_API int sqlite3_result_zeroblob64(sqlite3_context*, sqlite3_uint64 n); ** higher order bits are discarded. ** The number of subtype bytes preserved by SQLite might increase ** in future releases of SQLite. +** +** Every [application-defined SQL function] that invokes this interface +** should include the [SQLITE_RESULT_SUBTYPE] property in its +** text encoding argument when the SQL function is +** [sqlite3_create_function|registered]. If the [SQLITE_RESULT_SUBTYPE] +** property is omitted from the function that invokes sqlite3_result_subtype(), +** then in some cases the sqlite3_result_subtype() might fail to set +** the result subtype. +** +** If SQLite is compiled with -DSQLITE_STRICT_SUBTYPE=1, then any +** SQL function that invokes the sqlite3_result_subtype() interface +** and that does not have the SQLITE_RESULT_SUBTYPE property will raise +** an error. Future versions of SQLite might enable -DSQLITE_STRICT_SUBTYPE=1 +** by default. */ SQLITE_API void sqlite3_result_subtype(sqlite3_context*,unsigned int); From ca617b69462d6b513b84881f62468a9294042e06 Mon Sep 17 00:00:00 2001 From: yubiuser Date: Wed, 22 Nov 2023 21:54:46 +0100 Subject: [PATCH 103/221] Address reviewer's comment Co-authored-by: DL6ER Signed-off-by: yubiuser --- src/config/config.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config/config.c b/src/config/config.c index d8b230971..b8eac3d3a 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -1453,7 +1453,7 @@ bool readFTLconf(struct config *conf, const bool rewrite) write_dnsmasq_config(conf, false, NULL); write_custom_list(); - return true; + return false; } bool getLogFilePath(void) From 29ce1ee8e81972c10a64f69ddddb16552a0d4e85 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 22 Nov 2023 22:40:50 +0100 Subject: [PATCH 104/221] Do not rotate pihole.toml when opening pihole.toml.tmp for writing Signed-off-by: DL6ER --- src/config/toml_helper.c | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/config/toml_helper.c b/src/config/toml_helper.c index d09a63f16..49c1da520 100644 --- a/src/config/toml_helper.c +++ b/src/config/toml_helper.c @@ -26,12 +26,6 @@ // Open the TOML file for reading or writing FILE * __attribute((malloc)) __attribute((nonnull(1))) openFTLtoml(const char *mode, const unsigned int version) { - FILE *fp; - // Rotate config file, no rotation is done when the file is opened for - // reading (mode == "r") - if(mode[0] != 'r') - rotate_files(GLOBALTOMLPATH, NULL); - // This should not happen, install a safeguard anyway to unveil // possible future coding issues early on if(mode[0] == 'w' && version != 0) @@ -59,7 +53,7 @@ FILE * __attribute((malloc)) __attribute((nonnull(1))) openFTLtoml(const char *m } // Try to open config file - fp = fopen(filename, mode); + FILE *fp = fopen(filename, mode); // Return early if opening failed if(!fp) From f4798929e1332ddbd692e7d46f3ddf412e9870c1 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 15 Nov 2023 12:35:13 +0100 Subject: [PATCH 105/221] Only reload pihole.toml if the content changed Signed-off-by: DL6ER --- src/config/config.c | 20 ++++++++++++++++++++ src/config/toml_writer.c | 6 ++++++ src/files.c | 37 +++++++++++++++++++++++++++++++++++++ src/files.h | 3 +++ src/gc.c | 1 - 5 files changed, 66 insertions(+), 1 deletion(-) diff --git a/src/config/config.c b/src/config/config.c index b8eac3d3a..74c19608f 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -29,9 +29,12 @@ #include "api/api.h" // exit_code #include "signals.h" +// sha256sum() +#include "files.h" struct config config = { 0 }; static bool config_initialized = false; +uint8_t last_checksum[SHA256_DIGEST_SIZE] = { 0 }; // Private prototypes static bool port_in_use(const in_port_t port); @@ -1562,6 +1565,23 @@ void replace_config(struct config *newconf) void reread_config(void) { + + // Create checksum of config file + uint8_t checksum[SHA256_DIGEST_SIZE]; + if(!sha256sum(GLOBALTOMLPATH, checksum)) + { + log_err("Unable to create checksum of %s, not re-reading config file", GLOBALTOMLPATH); + return; + } + + // Compare checksums + if(memcmp(checksum, last_checksum, SHA256_DIGEST_SIZE) == 0) + { + log_debug(DEBUG_CONFIG, "Checksum of %s has not changed, not re-reading config file", GLOBALTOMLPATH); + return; + } + + log_info("Reloading config due to pihole.toml change"); struct config conf_copy; duplicate_config(&conf_copy, &config); diff --git a/src/config/toml_writer.c b/src/config/toml_writer.c index c756bc12b..f53959e50 100644 --- a/src/config/toml_writer.c +++ b/src/config/toml_writer.c @@ -22,6 +22,9 @@ // files_different() #include "files.h" +// defined in config/config.c +extern uint8_t last_checksum[SHA256_DIGEST_SIZE]; + static void migrate_config(void) { // Migrating dhcp.domain -> dns.domain @@ -176,5 +179,8 @@ bool writeFTLtoml(const bool verbose) log_debug(DEBUG_CONFIG, "pihole.toml unchanged"); } + if(!sha256sum(GLOBALTOMLPATH, last_checksum)) + log_err("Unable to create checksum of %s", GLOBALTOMLPATH); + return true; } diff --git a/src/files.c b/src/files.c index 525f57e3a..36455f40f 100644 --- a/src/files.c +++ b/src/files.c @@ -688,3 +688,40 @@ bool files_different(const char *pathA, const char* pathB, unsigned int from) return different; } + +// Create SHA256 checksum of a file +bool sha256sum(const char *path, uint8_t checksum[SHA256_DIGEST_SIZE]) +{ + // Open file + FILE *fp = fopen(path, "rb"); + if(fp == NULL) + { + log_warn("sha256_file(): Failed to open \"%s\" for reading: %s", path, strerror(errno)); + return false; + } + + // Initialize SHA2-256 context + struct sha256_ctx ctx; + sha256_init(&ctx); + + // Read file in chunks of bytes + const size_t pagesize = getpagesize(); + unsigned char *buf = calloc(pagesize, sizeof(char)); + size_t len; + while((len = fread(buf, sizeof(char), pagesize, fp)) > 0) + { + // Update SHA256 context + sha256_update(&ctx, len, buf); + } + + // Finalize SHA256 context + sha256_digest(&ctx, SHA256_DIGEST_SIZE, checksum); + + // Close file + fclose(fp); + + // Free memory + free(buf); + + return true; +} diff --git a/src/files.h b/src/files.h index 329ce876e..51770ef60 100644 --- a/src/files.h +++ b/src/files.h @@ -14,6 +14,8 @@ #include // setmntent() #include +// SHA256_DIGEST_SIZE +#include #define ZIP_ROTATIONS 3 #define MAX_ROTATIONS 15 @@ -31,6 +33,7 @@ struct mntent *get_filesystem_details(const char *path); bool directory_exists(const char *path); void rotate_files(const char *path, char **first_file); bool files_different(const char *pathA, const char* pathB, unsigned int from); +bool sha256sum(const char *path, uint8_t checksum[SHA256_DIGEST_SIZE]); int parse_line(char *line, char **key, char **value); diff --git a/src/gc.c b/src/gc.c index 322a92ab0..ad793413d 100644 --- a/src/gc.c +++ b/src/gc.c @@ -374,7 +374,6 @@ void *GC_thread(void *val) if(check_inotify_event()) { // Reload config - log_info("Reloading config due to pihole.toml change"); reread_config(); } From adf8a13e9d8302db23998d4970c60d47aa67d7bb Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 15 Nov 2023 14:20:59 +0100 Subject: [PATCH 106/221] Add pihole-FTL sha256sum Signed-off-by: DL6ER --- src/args.c | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/args.c b/src/args.c index 57550e52c..381b7f3bc 100644 --- a/src/args.c +++ b/src/args.c @@ -62,6 +62,8 @@ #include "config/password.h" // idn2_to_ascii_lz() #include +// sha256sum() +#include "files.h" // defined in dnsmasq.c extern void print_dnsmasq_version(const char *yellow, const char *green, const char *bold, const char *normal); @@ -476,6 +478,24 @@ void parse_args(int argc, char* argv[]) } } + // sha256sum mode + if(argc == 3 && strcmp(argv[1], "sha256sum") == 0) + { + // Enable stdout printing + cli_mode = true; + uint8_t checksum[SHA256_DIGEST_SIZE]; + if(!sha256sum(argv[2], checksum)) + exit(EXIT_FAILURE); + + // Convert checksum to hex string + char hex[SHA256_DIGEST_SIZE*2+1]; + sha256_raw_to_hex(checksum, hex); + + // Print result + printf("%s %s\n", hex, argv[2]); + exit(EXIT_SUCCESS); + } + // start from 1, as argv[0] is the executable name for(int i = 1; i < argc; i++) { @@ -934,6 +954,7 @@ void parse_args(int argc, char* argv[]) printf(" Decoding: %spihole-FTL idn2 -d %spunycode%s\n\n", green, cyan, normal); printf("%sOther:%s\n", yellow, normal); + printf("\t%ssha256sum %sfile%s Calculate SHA256 checksum of a file\n", green, cyan, normal); printf("\t%sdhcp-discover%s Discover DHCP servers in the local\n", green, normal); printf("\t network\n"); printf("\t%sarp-scan %s[-a/-x]%s Use ARP to scan local network for\n", green, cyan, normal); From ea7753d3ec60905921ec12a20f14fac673a3303f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 15 Nov 2023 14:27:53 +0100 Subject: [PATCH 107/221] Add SHA256 CI test Signed-off-by: DL6ER --- test/test_suite.bats | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/test/test_suite.bats b/test/test_suite.bats index aca80a551..d8d5cbaca 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -1524,6 +1524,12 @@ [[ $status == 0 ]] } +@test "SHA256 checksum working" { + run bash -c './pihole-FTL sha256sum test/test.pem' + printf "%s\n" "${lines[@]}" + [[ ${lines[0]} == "eae293f0c30369935a7457a789658bedebf92d544e7526bc43aa07883a597fa9 test/test.pem" ]] +} + @test "API validation" { run python3 test/api/checkAPI.py printf "%s\n" "${lines[@]}" @@ -1575,7 +1581,7 @@ [[ ${lines[0]} == "3" ]] run bash -c 'grep -c "DEBUG_CONFIG: pihole.toml unchanged" /var/log/pihole/FTL.log' printf "%s\n" "${lines[@]}" - [[ ${lines[0]} == "4" ]] + [[ ${lines[0]} == "3" ]] run bash -c 'grep -c "DEBUG_CONFIG: Config file written to /etc/pihole/dnsmasq.conf" /var/log/pihole/FTL.log' printf "%s\n" "${lines[@]}" [[ ${lines[0]} == "1" ]] @@ -1587,5 +1593,5 @@ [[ ${lines[0]} == "1" ]] run bash -c 'grep -c "DEBUG_CONFIG: custom.list unchanged" /var/log/pihole/FTL.log' printf "%s\n" "${lines[@]}" - [[ ${lines[0]} == "4" ]] + [[ ${lines[0]} == "3" ]] } From 0734a2769a1f7592ce34d5f76e810319fb9da1b6 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 23 Nov 2023 09:36:35 +0100 Subject: [PATCH 108/221] Include PID of the currently running FTL instance in the logs response. This allows clients to easily detect when FTL is restarted to reset their nextID to zero. Signed-off-by: DL6ER --- src/api/docs/content/specs/logs.yaml | 4 ++++ src/api/logs.c | 3 +++ 2 files changed, 7 insertions(+) diff --git a/src/api/docs/content/specs/logs.yaml b/src/api/docs/content/specs/logs.yaml index 1aa799e65..e65a8c589 100644 --- a/src/api/docs/content/specs/logs.yaml +++ b/src/api/docs/content/specs/logs.yaml @@ -132,6 +132,10 @@ components: type: integer description: Next ID to query if checking for new log lines example: 229 + pid: + type: integer + description: Process ID of FTL. When this changes, FTL was restarted and nextID should be reset to 0. + example: 2258 file: type: string description: Path to respective log file on disk diff --git a/src/api/logs.c b/src/api/logs.c index c078327b3..e6678b8ac 100644 --- a/src/api/logs.c +++ b/src/api/logs.c @@ -15,6 +15,8 @@ // struct fifologData #include "log.h" #include "config/config.h" +// main_pid() +#include "signals.h" // fifologData is allocated in shared memory for cross-fork compatibility int api_logs(struct ftl_conn *api) @@ -70,6 +72,7 @@ int api_logs(struct ftl_conn *api) } JSON_ADD_ITEM_TO_OBJECT(json, "log", log); JSON_ADD_NUMBER_TO_OBJECT(json, "nextID", fifo_log->logs[api->opts.which].next_id); + JSON_ADD_NUMBER_TO_OBJECT(json, "pid", main_pid()); // Add file name const char *logfile = NULL; From ff37972f8182e25db0c547e12f5c337b5e41e3fb Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 23 Nov 2023 10:27:58 +0100 Subject: [PATCH 109/221] Clients with zweo queries should not be returned unless ?withzero=true is set Signed-off-by: DL6ER --- src/api/stats.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/api/stats.c b/src/api/stats.c index 0c06b1f78..eaf8b93be 100644 --- a/src/api/stats.c +++ b/src/api/stats.c @@ -366,18 +366,18 @@ int api_stats_top_clients(struct ftl_conn *api) continue; // Skip this client if there is a filter on it - bool skip_domain = false; + bool skip_client = false; for(unsigned int j = 0; j < excludeClients; j++) { cJSON *item = cJSON_GetArrayItem(config.webserver.api.excludeClients.v.json, j); if(strcmp(getstr(client->ippos), item->valuestring) == 0 || strcmp(getstr(client->namepos), item->valuestring) == 0) { - skip_domain = true; + skip_client = true; break; } } - if(skip_domain) + if(skip_client) continue; // Hidden client, probably due to privacy level. Skip this in the top lists @@ -391,7 +391,7 @@ int api_stats_top_clients(struct ftl_conn *api) // Return this client if either // - "withzero" option is set, and/or // - the client made at least one query within the most recent 24 hours - if(includezeroclients || count > 0) + if(includezeroclients || client_count > 0) { cJSON *client_item = JSON_NEW_OBJECT(); JSON_REF_STR_IN_OBJECT(client_item, "name", client_name); From d5eb664e1f950ec22eeff20086b554105dc141be Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 23 Nov 2023 22:17:10 +0100 Subject: [PATCH 110/221] Remove undocumented withzero parameter Signed-off-by: DL6ER --- src/api/stats.c | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/api/stats.c b/src/api/stats.c index eaf8b93be..7453b5109 100644 --- a/src/api/stats.c +++ b/src/api/stats.c @@ -293,7 +293,6 @@ int api_stats_top_domains(struct ftl_conn *api) int api_stats_top_clients(struct ftl_conn *api) { int count = 10; - bool includezeroclients = false; int *temparray = calloc(2*counters->clients, sizeof(int*)); if(temparray == NULL) { @@ -325,9 +324,6 @@ int api_stats_top_clients(struct ftl_conn *api) // Does the user request a non-default number of replies? // Note: We do not accept zero query requests here get_int_var(api->request->query_string, "count", &count); - - // Show also clients which have not been active recently? - get_bool_var(api->request->query_string, "withzero", &includezeroclients); } // Lock shared memory @@ -388,10 +384,9 @@ int api_stats_top_clients(struct ftl_conn *api) const char *client_ip = getstr(client->ippos); const char *client_name = getstr(client->namepos); - // Return this client if either - // - "withzero" option is set, and/or - // - the client made at least one query within the most recent 24 hours - if(includezeroclients || client_count > 0) + // Return this client if the client made at least one query + // within the most recent 24 hours + if(client_count > 0) { cJSON *client_item = JSON_NEW_OBJECT(); JSON_REF_STR_IN_OBJECT(client_item, "name", client_name); From 409b6a40a38ca4d2228a61b8e097bd1b954f525e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 26 Nov 2023 10:41:30 +0100 Subject: [PATCH 111/221] Do not allocate too much memory for /api/stats/upstreams Signed-off-by: DL6ER --- src/api/stats.c | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/src/api/stats.c b/src/api/stats.c index a9874750f..d80314658 100644 --- a/src/api/stats.c +++ b/src/api/stats.c @@ -141,7 +141,8 @@ int api_stats_top_domains(struct ftl_conn *api) { int count = 10; bool audit = false; - int *temparray = calloc(2*counters->domains, sizeof(int*)); + const int domains = counters->domains; + int *temparray = calloc(2*domains, sizeof(int*)); if(temparray == NULL) { log_err("Memory allocation failed in %s()", __FUNCTION__); @@ -181,7 +182,7 @@ int api_stats_top_domains(struct ftl_conn *api) // Lock shared memory lock_shm(); - for(int domainID=0; domainID < counters->domains; domainID++) + for(int domainID=0; domainID < domains; domainID++) { // Get domain pointer const domainsData* domain = getDomain(domainID, true); @@ -197,7 +198,7 @@ int api_stats_top_domains(struct ftl_conn *api) } // Sort temporary array - qsort(temparray, counters->domains, sizeof(int[2]), cmpdesc); + qsort(temparray, domains, sizeof(int[2]), cmpdesc); // Get filter const char* filter = read_setupVarsconf("API_QUERY_LOG_SHOW"); @@ -221,7 +222,7 @@ int api_stats_top_domains(struct ftl_conn *api) int n = 0; cJSON *top_domains = JSON_NEW_ARRAY(); - for(int i = 0; i < counters->domains; i++) + for(int i = 0; i < domains; i++) { // Get sorted index const int domainID = temparray[2*i + 0]; @@ -298,7 +299,8 @@ int api_stats_top_clients(struct ftl_conn *api) { int count = 10; bool includezeroclients = false; - int *temparray = calloc(2*counters->clients, sizeof(int*)); + const int clients = counters->clients; + int *temparray = calloc(2*clients, sizeof(int*)); if(temparray == NULL) { log_err("Memory allocation failed in api_stats_top_clients()"); @@ -337,7 +339,7 @@ int api_stats_top_clients(struct ftl_conn *api) // Lock shared memory lock_shm(); - for(int clientID = 0; clientID < counters->clients; clientID++) + for(int clientID = 0; clientID < clients; clientID++) { // Get client pointer const clientsData* client = getClient(clientID, true); @@ -352,14 +354,14 @@ int api_stats_top_clients(struct ftl_conn *api) } // Sort temporary array - qsort(temparray, counters->clients, sizeof(int[2]), cmpdesc); + qsort(temparray, clients, sizeof(int[2]), cmpdesc); // Get clients which the user doesn't want to see unsigned int excludeClients = cJSON_GetArraySize(config.webserver.api.excludeClients.v.json); int n = 0; cJSON *top_clients = JSON_NEW_ARRAY(); - for(int i=0; i < counters->clients; i++) + for(int i=0; i < clients; i++) { // Get sorted indices and counter values (may be either total or blocked count) const int clientID = temparray[2*i + 0]; @@ -423,9 +425,9 @@ int api_stats_top_clients(struct ftl_conn *api) int api_stats_upstreams(struct ftl_conn *api) { - const int forwarded = get_forwarded_count(); unsigned int totalcount = 0; - int *temparray = calloc(2*forwarded, sizeof(int*)); + const int upstreams = counters->upstreams; + int *temparray = calloc(2*upstreams, sizeof(int*)); if(temparray == NULL) { log_err("Memory allocation failed in api_stats_upstreams()"); @@ -435,7 +437,7 @@ int api_stats_upstreams(struct ftl_conn *api) // Lock shared memory lock_shm(); - for(int upstreamID = 0; upstreamID < counters->upstreams; upstreamID++) + for(int upstreamID = 0; upstreamID < upstreams; upstreamID++) { // Get upstream pointer const upstreamsData* upstream = getUpstream(upstreamID, true); @@ -452,11 +454,11 @@ int api_stats_upstreams(struct ftl_conn *api) } // Sort temporary array in descending order - qsort(temparray, counters->upstreams, sizeof(int[2]), cmpdesc); + qsort(temparray, upstreams, sizeof(int[2]), cmpdesc); // Loop over available forward destinations - cJSON *upstreams = JSON_NEW_ARRAY(); - for(int i = -2; i < min(counters->upstreams, 8); i++) + cJSON *top_upstreams = JSON_NEW_ARRAY(); + for(int i = -2; i < upstreams; i++) { int count = 0; const char* ip, *name; @@ -524,7 +526,7 @@ int api_stats_upstreams(struct ftl_conn *api) JSON_ADD_NUMBER_TO_OBJECT(statistics, "response", responsetime); JSON_ADD_NUMBER_TO_OBJECT(statistics, "variance", uncertainty); JSON_ADD_ITEM_TO_OBJECT(upstream, "statistics", statistics); - JSON_ADD_ITEM_TO_ARRAY(upstreams, upstream); + JSON_ADD_ITEM_TO_ARRAY(top_upstreams, upstream); } } @@ -532,7 +534,7 @@ int api_stats_upstreams(struct ftl_conn *api) free(temparray); cJSON *json = JSON_NEW_OBJECT(); - JSON_ADD_ITEM_TO_OBJECT(json, "upstreams", upstreams); + JSON_ADD_ITEM_TO_OBJECT(json, "upstreams", top_upstreams); const int forwarded_queries = get_forwarded_count(); JSON_ADD_NUMBER_TO_OBJECT(json, "forwarded_queries", forwarded_queries); JSON_ADD_NUMBER_TO_OBJECT(json, "total_queries", counters->queries); From 164aeef0360ed5542a299e901a636f68bd55ca51 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 26 Nov 2023 10:45:15 +0100 Subject: [PATCH 112/221] Simplify /api/stats/upstreams to use global upstream counter instead of looping of over the respective overTime structure Signed-off-by: DL6ER --- src/api/stats.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/api/stats.c b/src/api/stats.c index d80314658..44cfa151a 100644 --- a/src/api/stats.c +++ b/src/api/stats.c @@ -445,12 +445,8 @@ int api_stats_upstreams(struct ftl_conn *api) continue; temparray[2*upstreamID + 0] = upstreamID; - - unsigned int count = 0; - for(unsigned i = 0; i < ArraySize(upstream->overTime); i++) - count += upstream->overTime[i]; - temparray[2*upstreamID + 1] = count; - totalcount += count; + temparray[2*upstreamID + 1] = upstream->count; + totalcount += upstream->count; } // Sort temporary array in descending order From d3826c97431d79731560fb36e7f86a4704041716 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 26 Nov 2023 10:48:59 +0100 Subject: [PATCH 113/221] Remove obsolete overTime structure stored for each upstream. This reduces memory consumption per upstream destination from 640 to 56 bytes each Signed-off-by: DL6ER --- src/database/query-table.c | 1 - src/datastructure.h | 1 - src/dnsmasq_interface.c | 10 +--------- src/overTime.c | 28 ---------------------------- 4 files changed, 1 insertion(+), 39 deletions(-) diff --git a/src/database/query-table.c b/src/database/query-table.c index 8e8fb9db8..0e2344f6b 100644 --- a/src/database/query-table.c +++ b/src/database/query-table.c @@ -1168,7 +1168,6 @@ void DB_read_queries(void) upstreamsData *upstream = getUpstream(upstreamID, true); if(upstream != NULL) { - upstream->overTime[timeidx]++; upstream->lastQuery = queryTimeStamp; upstream->count++; } diff --git a/src/datastructure.h b/src/datastructure.h index 5ab8faf97..b94358aad 100644 --- a/src/datastructure.h +++ b/src/datastructure.h @@ -63,7 +63,6 @@ typedef struct { int count; int failed; unsigned int responses; - int overTime[OVERTIME_SLOTS]; size_t ippos; size_t namepos; double rtime; diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index f450b600b..bb6bb0f88 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -1693,11 +1693,7 @@ static void FTL_forwarded(const unsigned int flags, const char *name, const unio upstreamsData *upstream = getUpstream(upstreamID, true); if(upstream != NULL) - { - // Update overTime counts - const int timeidx = getOverTimeID(query->timestamp); - upstream->overTime[timeidx]++; - } + upstream->count++; // Proceed only if // - current query has not been marked as replied to so far @@ -2322,11 +2318,7 @@ static void query_blocked(queriesData* query, domainsData* domain, clientsData* // Get forward pointer upstreamsData* upstream = getUpstream(query->upstreamID, true); if(upstream != NULL) - { - const int timeidx = getOverTimeID(query->timestamp); - upstream->overTime[timeidx]--; upstream->count--; - } } else if(is_blocked(query->status)) { diff --git a/src/overTime.c b/src/overTime.c index 5148b908f..39681311e 100644 --- a/src/overTime.c +++ b/src/overTime.c @@ -53,18 +53,6 @@ static void initSlot(const unsigned int index, const time_t timestamp) client->overTime[index] = 0; } } - - // Zero overTime counter for all known upstream destinations - for(int upstreamID = 0; upstreamID < counters->upstreams; upstreamID++) - { - // Get client pointer - upstreamsData* upstream = getUpstream(upstreamID, true); - if(upstream != NULL) - { - // Set overTime data to zero - upstream->overTime[index] = 0; - } - } } void initOverTime(void) @@ -202,22 +190,6 @@ void moveOverTimeMemory(const time_t mintime) remainingSlots*sizeof(*client->overTime)); } - // Process upstream data - for(int upstreamID = 0; upstreamID < counters->upstreams; upstreamID++) - { - upstreamsData *upstream = getUpstream(upstreamID, true); - if(!upstream) - continue; - - // Adjust upstream's queries counter - upstream->count -= upstream->overTime[0]; - - // Move upstream-specific overTime memory - memmove(&(upstream->overTime[0]), - &(upstream->overTime[moveOverTime]), - remainingSlots*sizeof(*upstream->overTime)); - } - // Iterate over new overTime region and initialize it for(unsigned int timeidx = remainingSlots; timeidx < OVERTIME_SLOTS ; timeidx++) { From 399ca110385d7bd9c3fff2481c52c1f0fc3e7e36 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 26 Nov 2023 21:44:55 +0100 Subject: [PATCH 114/221] Add extra debug statements Signed-off-by: DL6ER --- src/database/query-table.c | 2 ++ src/datastructure.c | 4 ++++ src/dnsmasq_interface.c | 6 ++++++ src/gc.c | 29 ++++++++++++++++++++--------- 4 files changed, 32 insertions(+), 9 deletions(-) diff --git a/src/database/query-table.c b/src/database/query-table.c index 0e2344f6b..b8b2766fe 100644 --- a/src/database/query-table.c +++ b/src/database/query-table.c @@ -1065,6 +1065,7 @@ void DB_read_queries(void) query->qtype = type - 100; } counters->querytype[query->type]++; + log_debug(DEBUG_GC, "GC: query type %d set (database), new count = %d", query->type, counters->querytype[query->type]); // Status is set below query->domainID = domainID; @@ -1076,6 +1077,7 @@ void DB_read_queries(void) query->dnssec = dnssec; query->reply = reply; counters->reply[query->reply]++; + log_debug(DEBUG_GC, "GC: reply type %d set (database), new count = %d", query->reply, counters->reply[query->reply]); query->response = reply_time; query->CNAME_domainID = -1; // Initialize flags diff --git a/src/datastructure.c b/src/datastructure.c index b62cf30a1..fd502ab15 100644 --- a/src/datastructure.c +++ b/src/datastructure.c @@ -1037,8 +1037,12 @@ void _query_set_status(queriesData *query, const enum query_status new_status, c // else: update global counters, ... if(!init) + { counters->status[old_status]--; + log_debug(DEBUG_GC, "GC: status %d removed (!init), new count = %d", QUERY_UNKNOWN, counters->status[QUERY_UNKNOWN]); + } counters->status[new_status]++; + log_debug(DEBUG_GC, "GC: status %d set, new count = %d", new_status, counters->status[new_status]); // ... update overTime counters, ... const int timeidx = getOverTimeID(query->timestamp); diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index bb6bb0f88..530d16dd8 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -747,6 +747,7 @@ bool _FTL_new_query(const unsigned int flags, const char *name, query->timestamp = querytimestamp; query->type = querytype; counters->querytype[querytype]++; + log_debug(DEBUG_GC, "GC: query type %d set (new query), new count = %d", query->type, counters->querytype[query->type]); query->qtype = qtype; query->id = id; // Has to be set before calling query_set_status() @@ -763,6 +764,7 @@ bool _FTL_new_query(const unsigned int flags, const char *name, // Initialize reply type query->reply = REPLY_UNKNOWN; counters->reply[REPLY_UNKNOWN]++; + log_debug(DEBUG_GC, "GC: reply type %d set (new query), new count = %d", query->reply, counters->reply[query->reply]); // Store DNSSEC result for this domain query->dnssec = DNSSEC_UNKNOWN; query->CNAME_domainID = -1; @@ -2729,10 +2731,12 @@ static void _query_set_reply(const unsigned int flags, const enum reply_type rep // Subtract from old reply counter counters->reply[query->reply]--; + log_debug(DEBUG_GC, "GC: reply type %d removed (set_reply), new count = %d", query->reply, counters->reply[query->reply]); // Add to new reply counter counters->reply[new_reply]++; // Store reply type query->reply = new_reply; + log_debug(DEBUG_GC, "GC: reply type %d added (set_reply), new count = %d", query->reply, counters->reply[query->reply]); // Save response time // Skipped internally if already computed @@ -3349,8 +3353,10 @@ void FTL_multiple_replies(const int id, int *firstID) // Copy relevant information over counters->reply[duplicated_query->reply]--; + log_debug(DEBUG_GC, "GC: duplicated_query reply type %d removed, new count = %d", duplicated_query->reply, counters->reply[duplicated_query->reply]); duplicated_query->reply = source_query->reply; counters->reply[duplicated_query->reply]++; + log_debug(DEBUG_GC, "GC: duplicated_query reply type %d set, new count = %d", duplicated_query->reply, counters->reply[duplicated_query->reply]); duplicated_query->dnssec = source_query->dnssec; duplicated_query->flags.complete = true; diff --git a/src/gc.c b/src/gc.c index 3305aa36c..410cc073f 100644 --- a/src/gc.c +++ b/src/gc.c @@ -297,8 +297,8 @@ void runGC(const time_t now, time_t *lastGCrun, const bool flush) } // Process all queries - int removed = 0; - for(long int i=0; i < counters->queries; i++) + unsigned int removed = 0; + for(long int i = 0; i < counters->queries; i++) { queriesData* query = getQuery(i, true); if(query == NULL) @@ -308,19 +308,25 @@ void runGC(const time_t now, time_t *lastGCrun, const bool flush) if(query->timestamp > mintime) break; - // Adjust client counter (total and overTime) - clientsData* client = getClient(query->clientID, true); + // Adjust overTime counter const int timeidx = getOverTimeID(query->timestamp); overTime[timeidx].total--; + + // Adjust client counter (total and overTime) + clientsData* client = getClient(query->clientID, true); if(client != NULL) change_clientcount(client, -1, 0, timeidx, -1); // Adjust domain counter (no overTime information) - domainsData* domain = getDomain(query->domainID, true); + domainsData *domain = getDomain(query->domainID, true); if(domain != NULL) domain->count--; - // Get upstream pointer + // Adjust upstream counter (no overTime information) + upstreamsData *upstream = getUpstream(query->upstreamID, true); + if(upstream != NULL) + // Adjust upstream counter + upstream->count--; // Change other counters according to status of this query switch(query->status) @@ -354,24 +360,29 @@ void runGC(const time_t now, time_t *lastGCrun, const bool flush) if(client != NULL) change_clientcount(client, 0, -1, -1, 0); break; - case QUERY_IN_PROGRESS: // Don't have to do anything here + case QUERY_IN_PROGRESS: // fall through case QUERY_STATUS_MAX: // fall through default: - /* That cannot happen */ + // Don't have to do anything here break; } // Update reply counters counters->reply[query->reply]--; + log_debug(DEBUG_GC, "GC: reply type %d removed (GC), new count = %d", query->reply, counters->reply[query->reply]); // Update type counters counters->querytype[query->type]--; + log_debug(DEBUG_GC, "GC: query type %d removed (GC), new count = %d", query->type, counters->querytype[query->type]); // Subtract UNKNOWN from the counters before // setting the status if different. // Minus one here and plus one below = net zero if(query->status != QUERY_UNKNOWN) + { counters->status[QUERY_UNKNOWN]--; + log_debug(DEBUG_GC, "GC: status %d removed (GC), new count = %d", QUERY_UNKNOWN, counters->status[QUERY_UNKNOWN]); + } // Set query again to UNKNOWN to reset the counters query_set_status(query, QUERY_UNKNOWN); @@ -417,7 +428,7 @@ void runGC(const time_t now, time_t *lastGCrun, const bool flush) // Determine if overTime memory needs to get moved moveOverTimeMemory(mintime); - log_debug(DEBUG_GC, "GC removed %i queries (took %.2f ms)", removed, timer_elapsed_msec(GC_TIMER)); + log_debug(DEBUG_GC, "GC removed %u queries (took %.2f ms)", removed, timer_elapsed_msec(GC_TIMER)); // Release thread lock if(!flush) From 0f5d3970bbbe6e59b0ca3cbf015d1acd2a76bbfe Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 26 Nov 2023 22:24:23 +0100 Subject: [PATCH 115/221] Implement wildcard X.509 SAN/CN (subject) domain checking Signed-off-by: DL6ER --- src/webserver/x509.c | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/webserver/x509.c b/src/webserver/x509.c index e4d224fec..0679e49ed 100644 --- a/src/webserver/x509.c +++ b/src/webserver/x509.c @@ -294,6 +294,23 @@ bool generate_certificate(const char* certfile, bool rsa, const char *domain) return true; } +static bool check_wildcard_domain(char *san, const size_t san_len, const char *domain) +{ + // Also check if the SAN is a wildcard domain and if the domain + // matches the wildcard (e.g. "*.pi-hole.net" and "abc.pi-hole.net") + const bool is_wild = san_len > 2 && san[0] == '*' && san[1] == '.'; + if(!is_wild) + return false; + + // The domain must be at least as long as the wildcard domain + if(strlen(domain) < san_len - 1) + return false; + + // Check if the domain ends with the wildcard domain + const char *wild_domain = domain + strlen(domain) - san_len + 2; + return strcasecmp(wild_domain, san + 2) == 0; +} + // This function reads a X.509 certificate from a file and prints a // human-readable representation of the certificate to stdout. If a domain is // specified, we only check if this domain is present in the certificate. @@ -363,6 +380,14 @@ enum cert_check read_certificate(const char* certfile, const char *domain, const found = true; break; } + + // Also check if the SAN is a wildcard domain and if the domain + // matches the wildcard + if(check_wildcard_domain((char*)san.san.unstructured_name.p, san.san.unstructured_name.len, domain)) + { + found = true; + break; + } next_san: // Go to next SAN sans = sans->next; @@ -378,6 +403,10 @@ enum cert_check read_certificate(const char* certfile, const char *domain, const // Check subject == "" else if(strcasecmp(domain, subject) == 0) found = true; + // Also check if the subject is a wildcard domain and if the domain + // matches the wildcard + else if(check_wildcard_domain(subject, strlen(subject), domain)) + found = true; } From 8b8218b2a308ba81e0a3e1e101e388ca7b1e7291 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 26 Nov 2023 22:31:51 +0100 Subject: [PATCH 116/221] Fix comment Signed-off-by: DL6ER --- src/api/stats.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/api/stats.c b/src/api/stats.c index 7453b5109..c1106cc54 100644 --- a/src/api/stats.c +++ b/src/api/stats.c @@ -163,7 +163,7 @@ int api_stats_top_domains(struct ftl_conn *api) // /api/stats/top_domains?blocked=true if(api->request->query_string != NULL) { - // Should blocked clients be shown? + // Should blocked domains be shown? get_bool_var(api->request->query_string, "blocked", &blocked); // Does the user request a non-default number of replies? @@ -229,7 +229,7 @@ int api_stats_top_domains(struct ftl_conn *api) // Skip this domain if there is a filter on it (but only if not in audit mode) if(!audit) { - // Check if this client should be skipped + // Check if this domain should be skipped bool skip_domain = false; for(unsigned int j = 0; j < excludeDomains; j++) { From 7727c04425f404c5f75bc134e3f0ae613e183430 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 27 Nov 2023 09:33:47 +0100 Subject: [PATCH 117/221] Escape invalid domains possibly containing control characters Signed-off-by: DL6ER --- src/tools/gravity-parseList.c | 43 ++++++++++++++++++++++++++++++----- 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/src/tools/gravity-parseList.c b/src/tools/gravity-parseList.c index 912ad358f..12072ea79 100644 --- a/src/tools/gravity-parseList.c +++ b/src/tools/gravity-parseList.c @@ -104,9 +104,8 @@ inline bool __attribute__((pure)) valid_domain(const char *domain, const size_t } // Validate ABP domain name -static inline bool __attribute__((pure)) valid_abp_domain(const char *line, const bool antigravity) +static inline bool __attribute__((pure)) valid_abp_domain(const char *line, const size_t len, const bool antigravity) { - const size_t len = strlen(line); if(antigravity) { @@ -263,6 +262,7 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis const size_t print_step = fsize / 20; // Print progress every 100/20 = 5% int last_progress = 0; char *invalid_domains_list[MAX_INVALID_DOMAINS] = { NULL }; + ssize_t invalid_domains_list_lengths[MAX_INVALID_DOMAINS] = { -1 }; unsigned int invalid_domains_list_len = 0; unsigned int exact_domains = 0, abp_domains = 0, invalid_domains = 0; while((read = getline(&line, &len, fpin)) != -1) @@ -313,7 +313,7 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis exact_domains++; } else if(line[0] == (antigravity ? '@' : '|') && // <- ABP-style match - valid_abp_domain(line, antigravity)) // <- Valid ABP domain + valid_abp_domain(line, read, antigravity)) // <- Valid ABP domain { // ABP-style match (see comments above) if(checkOnly) @@ -364,7 +364,12 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis bool found = false; for(unsigned int i = 0; i < invalid_domains_list_len; i++) { - if(strcmp(invalid_domains_list[i], line) == 0) + // Do not compare against unset entries + if(invalid_domains_list[i] == NULL || invalid_domains_list_lengths[i] == -1) + break; + + // Compare against the current domain + if(memcmp(invalid_domains_list[i], line, min(read, invalid_domains_list_lengths[i])) == 0) { found = true; break; @@ -373,7 +378,20 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis // If not found, add it to the list if(!found) - invalid_domains_list[invalid_domains_list_len++] = strdup(line); + { + invalid_domains_list[invalid_domains_list_len] = calloc(read + 1, sizeof(char)); + if(invalid_domains_list[invalid_domains_list_len] == NULL) + { + printf("%s %s Unable to allocate memory for invalid domains list\n", over, cross); + fclose(fpin); + sqlite3_close(db); + return EXIT_FAILURE; + } + memcpy(invalid_domains_list[invalid_domains_list_len], line, read); + invalid_domains_list[invalid_domains_list_len][read] = '\0'; + invalid_domains_list_lengths[invalid_domains_list_len] = read; + invalid_domains_list_len++; + } } invalid_domains++; @@ -504,7 +522,20 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis { puts(" Sample of non-domain entries:"); for(unsigned int i = 0; i < invalid_domains_list_len; i++) - printf(" - \"%s\"\n", invalid_domains_list[i]); + { + // Print indentation + printf(" - "); + // Print domain (escape non-printable characters) + for(ssize_t j = 0; j < invalid_domains_list_lengths[i]; j++) + if(isgraph(invalid_domains_list[i][j])) + putchar(invalid_domains_list[i][j]); + else + // Escape non-printable characters + printf("\\x%02x", (unsigned char)invalid_domains_list[i][j]); + // Print newline + puts(""); + } + // Print final newline puts(""); } From 9ef42fb37525f0279894f6e4d652e3748fd4d443 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 27 Nov 2023 10:21:49 +0100 Subject: [PATCH 118/221] Fix comparison of the final line and add more debug logging during file comparison (debug.config) Signed-off-by: DL6ER --- src/files.c | 52 ++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 40 insertions(+), 12 deletions(-) diff --git a/src/files.c b/src/files.c index 0f36def4f..1f12e8c94 100644 --- a/src/files.c +++ b/src/files.c @@ -607,30 +607,53 @@ bool files_different(const char *pathA, const char* pathB, unsigned int from) } // Compare both files line by line - char *lineA = NULL; - size_t lenA = 0; - ssize_t readA; - char *lineB = NULL; - size_t lenB = 0; - ssize_t readB; + char *lineA = NULL, *lineB = NULL; + size_t lenA = 0, lenB = 0; + ssize_t readA = 0, readB = 0; bool different = false; - while((readA = getline(&lineA, &lenA, fpA)) != -1 && - (readB = getline(&lineB, &lenB, fpB)) != -1) + unsigned int lineno = 0; + while(true) { + // Read lines from both files + readA = getline(&lineA, &lenA, fpA); + readB = getline(&lineB, &lenB, fpB); + + // Check if we reached the end of any of the files + if(readA < 0 || readB < 0) + break; + // Skip lines until we reach the requested line number - if(from > 0) - { - from--; + if(from > ++lineno) continue; - } + + // Remove possible trailing newline characters + if(lineA[readA - 1] == '\n') + lineA[readA - 1] = '\0'; + if(lineB[readB - 1] == '\n') + lineB[readB - 1] = '\0'; + // Compare lines if(strcmp(lineA, lineB) != 0) { different = true; + log_debug(DEBUG_CONFIG, "Files %s and %s differ at line %u", + pathA, pathB, lineno); + log_debug(DEBUG_CONFIG, "-> %s:%u = '%s'", pathA, lineno, readA < 0 ? "" : lineA); + log_debug(DEBUG_CONFIG, "-> %s:%u = '%s'", pathB, lineno, readB < 0 ? "" : lineB); break; } } + // Check if one file has more lines than the other + if(!different && readA != readB) + { + different = true; + log_debug(DEBUG_CONFIG, "Files %s and %s differ at the final line %u", + pathA, pathB, lineno); + log_debug(DEBUG_CONFIG, "-> %s:%u = '%s'", pathA, lineno, readA < 0 ? "" : lineA); + log_debug(DEBUG_CONFIG, "-> %s:%u = '%s'", pathB, lineno, readB < 0 ? "" : lineB); + } + // Free memory free(lineA); free(lineB); @@ -639,6 +662,11 @@ bool files_different(const char *pathA, const char* pathB, unsigned int from) fclose(fpA); fclose(fpB); + // Log result (if not already done above) + if(!different) + log_debug(DEBUG_CONFIG, "Files %s and %s are identical (skipped the first %u line%s)", + pathA, pathB, from, from == 1 ? "" : "s"); + return different; } From b017c1c20c1166f593de49c064e60649452e673f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 27 Nov 2023 13:27:04 +0100 Subject: [PATCH 119/221] The SAN is not NUL-terminated, we need to use the specified length explicitly Signed-off-by: DL6ER --- src/webserver/x509.c | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/src/webserver/x509.c b/src/webserver/x509.c index 0679e49ed..f44075964 100644 --- a/src/webserver/x509.c +++ b/src/webserver/x509.c @@ -294,21 +294,24 @@ bool generate_certificate(const char* certfile, bool rsa, const char *domain) return true; } -static bool check_wildcard_domain(char *san, const size_t san_len, const char *domain) +static bool check_wildcard_domain(const char *domain, char *san, const size_t san_len) { // Also check if the SAN is a wildcard domain and if the domain // matches the wildcard (e.g. "*.pi-hole.net" and "abc.pi-hole.net") - const bool is_wild = san_len > 2 && san[0] == '*' && san[1] == '.'; + const bool is_wild = san_len > 1 && san[0] == '*'; if(!is_wild) return false; // The domain must be at least as long as the wildcard domain - if(strlen(domain) < san_len - 1) + const size_t domain_len = strlen(domain); + if(domain_len < san_len - 1) return false; // Check if the domain ends with the wildcard domain - const char *wild_domain = domain + strlen(domain) - san_len + 2; - return strcasecmp(wild_domain, san + 2) == 0; + // Attention: The SAN is not NUL-terminated, so we need to + // use the length field + const char *wild_domain = domain + domain_len - san_len + 1; + return strncasecmp(wild_domain, san + 1, san_len) == 0; } // This function reads a X.509 certificate from a file and prints a @@ -375,20 +378,27 @@ enum cert_check read_certificate(const char* certfile, const char *domain, const goto next_san; // Check if the SAN matches the domain + // Attention: The SAN is not NUL-terminated, so we need to + // use the length field if(strncasecmp(domain, (char*)san.san.unstructured_name.p, san.san.unstructured_name.len) == 0) { found = true; + mbedtls_x509_free_subject_alt_name(&san); break; } // Also check if the SAN is a wildcard domain and if the domain // matches the wildcard - if(check_wildcard_domain((char*)san.san.unstructured_name.p, san.san.unstructured_name.len, domain)) + if(check_wildcard_domain(domain, (char*)san.san.unstructured_name.p, san.san.unstructured_name.len)) { found = true; + mbedtls_x509_free_subject_alt_name(&san); break; } next_san: + // Free resources + mbedtls_x509_free_subject_alt_name(&san); + // Go to next SAN sans = sans->next; } @@ -405,7 +415,7 @@ enum cert_check read_certificate(const char* certfile, const char *domain, const found = true; // Also check if the subject is a wildcard domain and if the domain // matches the wildcard - else if(check_wildcard_domain(subject, strlen(subject), domain)) + else if(check_wildcard_domain(domain, subject, strlen(subject))) found = true; } From bd126166107a54b6312a7fd7615843694c40ce1b Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 27 Nov 2023 13:34:00 +0100 Subject: [PATCH 120/221] Do not pin SID to one specific IP address Signed-off-by: DL6ER --- src/api/auth.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/api/auth.c b/src/api/auth.c index d0f89128f..7d3bd7be5 100644 --- a/src/api/auth.c +++ b/src/api/auth.c @@ -202,7 +202,6 @@ int check_client_auth(struct ftl_conn *api, const bool is_api) { if(auth_data[i].used && auth_data[i].valid_until >= now && - strcmp(auth_data[i].remote_addr, api->request->remote_addr) == 0 && strcmp(auth_data[i].sid, sid) == 0) { if(need_csrf && strcmp(auth_data[i].csrf, csrf) != 0) @@ -217,10 +216,7 @@ int check_client_auth(struct ftl_conn *api, const bool is_api) } if(user_id > API_AUTH_UNAUTHORIZED) { - // Authentication successful: - // - We know this client - // - The session is (still) valid - // - The IP matches the one we know for this SID + // Authentication successful: valid session // Update timestamp of this client to extend // the validity of their API authentication @@ -245,8 +241,8 @@ int check_client_auth(struct ftl_conn *api, const bool is_api) { char timestr[128]; get_timestr(timestr, auth_data[user_id].valid_until, false, false); - log_debug(DEBUG_API, "Recognized known user: user_id %i, valid_until: %s, remote_addr %s", - user_id, timestr, auth_data[user_id].remote_addr); + log_debug(DEBUG_API, "Recognized known user: user_id %i, valid_until: %s, remote_addr %s (%s at login)", + user_id, timestr, api->request->remote_addr, auth_data[user_id].remote_addr); } } else From 9cfb107cc73315d9edb87a1e79a780596743fa0f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 27 Nov 2023 13:41:04 +0100 Subject: [PATCH 121/221] Improve API POST /auth documentation Signed-off-by: DL6ER --- src/api/docs/content/specs/auth.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/api/docs/content/specs/auth.yaml b/src/api/docs/content/specs/auth.yaml index 9ba516941..febfd0591 100644 --- a/src/api/docs/content/specs/auth.yaml +++ b/src/api/docs/content/specs/auth.yaml @@ -44,7 +44,11 @@ components: operationId: "add_auth" security: [] description: | - Login with a password. The password is not stored in the session, and neither when to generating the session token. + Authenticate using a password. The password isn't stored in the session or used to create the session token. Instead, the session token is produced using a cryptographically secure random number generator. A CSRF token is utilized to guard against CSRF attacks and is necessary when using Cookie-based authentication. However, it's not needed with other authentication methods. + + Both the Session ID (SID) and CSRF token remain valid for the session's duration. The session can be extended before its expiration by performing any authenticated action. By default, the session lasts for 5 minutes. It can be invalidated by either logging out or deleting the session. Additionally, the session becomes invalid when the password is altered or a new application password is created. + + If two-factor authentication (2FA) is activated, the Time-based One-Time Password (TOTP) token must be included in the request body. Be aware that the TOTP token, generated by your authenticator app, is only valid for 30 seconds. If the TOTP token is missing, invalid, or has been used previously, the login attempt will be unsuccessful. requestBody: description: Callback payload content: From 4f0800b85464d93764ee6d212dccbc2970a88b12 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 27 Nov 2023 14:07:26 +0100 Subject: [PATCH 122/221] Update embedded SQLite3 engine to version 3.44.2 Signed-off-by: DL6ER --- src/database/shell.c | 6 +++--- src/database/sqlite3.c | 43 +++++++++++++++++++++++++----------------- src/database/sqlite3.h | 6 +++--- 3 files changed, 32 insertions(+), 23 deletions(-) diff --git a/src/database/shell.c b/src/database/shell.c index 8b63b8542..6704b7b11 100644 --- a/src/database/shell.c +++ b/src/database/shell.c @@ -896,8 +896,8 @@ static PerStreamTags * getDesignatedEmitStream(FILE *pf, unsigned chix, ** chix equals 1 or 2, or for an arbitrary stream when chix == 0. ** In either case, ppst references a caller-owned PerStreamTags ** struct which may be filled in if none of the known writable -** streams is being held by consoleInfo. The ppf parameter is an -** output when chix!=0 and an input when chix==0. +** streams is being held by consoleInfo. The ppf parameter is a +** byref output when chix!=0 and a byref input when chix==0. */ static PerStreamTags * getEmitStreamInfo(unsigned chix, PerStreamTags *ppst, @@ -910,7 +910,7 @@ getEmitStreamInfo(unsigned chix, PerStreamTags *ppst, ppstTry = &consoleInfo.pstSetup[chix]; pfEmit = ppst->pf; }else pfEmit = ppstTry->pf; - if( !isValidStreamInfo(ppst) ){ + if( !isValidStreamInfo(ppstTry) ){ pfEmit = (chix > 1)? stderr : stdout; ppstTry = ppst; streamOfConsole(pfEmit, ppstTry); diff --git a/src/database/sqlite3.c b/src/database/sqlite3.c index 592320ff6..a28b3a7bf 100644 --- a/src/database/sqlite3.c +++ b/src/database/sqlite3.c @@ -1,6 +1,6 @@ /****************************************************************************** ** This file is an amalgamation of many separate C source files from SQLite -** version 3.44.1. By combining all the individual C code files into this +** version 3.44.2. By combining all the individual C code files into this ** single large file, the entire code can be compiled as a single translation ** unit. This allows many compilers to do optimizations that would not be ** possible if the files were compiled separately. Performance improvements @@ -18,7 +18,7 @@ ** separate file. This file contains only code for the core SQLite library. ** ** The content in this amalgamation comes from Fossil check-in -** d295f48e8f367b066b881780c98bdf980a1d. +** ebead0e7230cd33bcec9f95d2183069565b9. */ #define SQLITE_CORE 1 #define SQLITE_AMALGAMATION 1 @@ -459,9 +459,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.44.1" -#define SQLITE_VERSION_NUMBER 3044001 -#define SQLITE_SOURCE_ID "2023-11-22 14:18:12 d295f48e8f367b066b881780c98bdf980a1d550397d5ba0b0e49842c95b3e8b4" +#define SQLITE_VERSION "3.44.2" +#define SQLITE_VERSION_NUMBER 3044002 +#define SQLITE_SOURCE_ID "2023-11-24 11:41:44 ebead0e7230cd33bcec9f95d2183069565b9e709bf745c9b5db65cc0cbf92c0f" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -84183,10 +84183,11 @@ static int growOpArray(Vdbe *v, int nOp){ ** sqlite3CantopenError(lineno) */ static void test_addop_breakpoint(int pc, Op *pOp){ - static int n = 0; + static u64 n = 0; (void)pc; (void)pOp; n++; + if( n==LARGEST_UINT64 ) abort(); /* so that n is used, preventing a warning */ } #endif @@ -92330,11 +92331,12 @@ SQLITE_API int sqlite3_found_count = 0; ** sqlite3CantopenError(lineno) */ static void test_trace_breakpoint(int pc, Op *pOp, Vdbe *v){ - static int n = 0; + static u64 n = 0; (void)pc; (void)pOp; (void)v; n++; + if( n==LARGEST_UINT64 ) abort(); /* So that n is used, preventing a warning */ } #endif @@ -143612,7 +143614,8 @@ SQLITE_PRIVATE void sqlite3SubqueryColumnTypes( NameContext sNC; assert( pSelect!=0 ); - assert( (pSelect->selFlags & SF_Resolved)!=0 ); + testcase( (pSelect->selFlags & SF_Resolved)==0 ); + assert( (pSelect->selFlags & SF_Resolved)!=0 || IN_RENAME_OBJECT ); assert( pTab->nCol==pSelect->pEList->nExpr || pParse->nErr>0 ); assert( aff==SQLITE_AFF_NONE || aff==SQLITE_AFF_BLOB ); if( db->mallocFailed || IN_RENAME_OBJECT ) return; @@ -241504,18 +241507,24 @@ static void fts5DoSecureDelete( iOff = iStart; - /* Set variable bLastInDoclist to true if this entry happens to be - ** the last rowid in the doclist for its term. */ + /* If the position-list for the entry being removed flows over past + ** the end of this page, delete the portion of the position-list on the + ** next page and beyond. + ** + ** Set variable bLastInDoclist to true if this entry happens + ** to be the last rowid in the doclist for its term. */ + if( iNextOff>=iPgIdx ){ + int pgno = pSeg->iLeafPgno+1; + fts5SecureDeleteOverflow(p, pSeg->pSeg, pgno, &bLastInDoclist); + iNextOff = iPgIdx; + } + if( pSeg->bDel==0 ){ - if( iNextOff>=iPgIdx ){ - int pgno = pSeg->iLeafPgno+1; - fts5SecureDeleteOverflow(p, pSeg->pSeg, pgno, &bLastInDoclist); - iNextOff = iPgIdx; - }else{ + if( iNextOff!=iPgIdx ){ /* Loop through the page-footer. If iNextOff (offset of the ** entry following the one we are removing) is equal to the ** offset of a key on this page, then the entry is the last - ** in its doclist. */ + ** in its doclist. */ int iKeyOff = 0; for(iIdx=0; iIdx Date: Mon, 27 Nov 2023 14:25:25 +0100 Subject: [PATCH 123/221] Also check wildcards prefixed by "CN=" in the subject name of the certificate Signed-off-by: DL6ER --- src/webserver/x509.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/webserver/x509.c b/src/webserver/x509.c index f44075964..4dbd714c1 100644 --- a/src/webserver/x509.c +++ b/src/webserver/x509.c @@ -405,17 +405,24 @@ enum cert_check read_certificate(const char* certfile, const char *domain, const // Also check against the common name (CN) field char subject[MBEDTLS_X509_MAX_DN_NAME_SIZE]; - if(mbedtls_x509_dn_gets(subject, sizeof(subject), &crt.subject) > 0) + const size_t subject_len = mbedtls_x509_dn_gets(subject, sizeof(subject), &crt.subject); + if(subject_len > 0) { - // Check subject == "CN=" - if(strlen(subject) > 3 && strncasecmp(subject, "CN=", 3) == 0 && strcasecmp(domain, subject + 3) == 0) - found = true; + if(subject_len > 3 && strncasecmp(subject, "CN=", 3) == 0) + { + // Check subject + 3 == "CN=" to skip the "CN=" prefix + if(strncasecmp(domain, subject + 3, subject_len) == 0) + found = true; + // Also check if the subject is a wildcard domain + else if(check_wildcard_domain(domain, subject + 3, subject_len - 3)) + found = true; + } // Check subject == "" else if(strcasecmp(domain, subject) == 0) found = true; // Also check if the subject is a wildcard domain and if the domain // matches the wildcard - else if(check_wildcard_domain(domain, subject, strlen(subject))) + else if(check_wildcard_domain(domain, subject, subject_len)) found = true; } From 3f4502c01b1aae4f12420492ab8f6e0d8b0f5a80 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 27 Nov 2023 14:41:38 +0100 Subject: [PATCH 124/221] Add comments Signed-off-by: DL6ER --- src/webserver/x509.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/webserver/x509.c b/src/webserver/x509.c index 4dbd714c1..432c17b5a 100644 --- a/src/webserver/x509.c +++ b/src/webserver/x509.c @@ -383,6 +383,7 @@ enum cert_check read_certificate(const char* certfile, const char *domain, const if(strncasecmp(domain, (char*)san.san.unstructured_name.p, san.san.unstructured_name.len) == 0) { found = true; + // Free resources mbedtls_x509_free_subject_alt_name(&san); break; } @@ -392,6 +393,7 @@ enum cert_check read_certificate(const char* certfile, const char *domain, const if(check_wildcard_domain(domain, (char*)san.san.unstructured_name.p, san.san.unstructured_name.len)) { found = true; + // Free resources mbedtls_x509_free_subject_alt_name(&san); break; } @@ -408,10 +410,11 @@ enum cert_check read_certificate(const char* certfile, const char *domain, const const size_t subject_len = mbedtls_x509_dn_gets(subject, sizeof(subject), &crt.subject); if(subject_len > 0) { + // Check subjects prefixed with "CN=" if(subject_len > 3 && strncasecmp(subject, "CN=", 3) == 0) { - // Check subject + 3 == "CN=" to skip the "CN=" prefix - if(strncasecmp(domain, subject + 3, subject_len) == 0) + // Check subject + 3 to skip the prefix + if(strncasecmp(domain, subject + 3, subject_len - 3) == 0) found = true; // Also check if the subject is a wildcard domain else if(check_wildcard_domain(domain, subject + 3, subject_len - 3)) From e4417692b867f558fac4fd66219b4abcb79b3575 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 27 Nov 2023 23:06:42 +0100 Subject: [PATCH 125/221] Extend escaping to the checkList function Signed-off-by: DL6ER --- src/tools/gravity-parseList.c | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/src/tools/gravity-parseList.c b/src/tools/gravity-parseList.c index 12072ea79..61856134a 100644 --- a/src/tools/gravity-parseList.c +++ b/src/tools/gravity-parseList.c @@ -152,6 +152,17 @@ static inline bool is_false_positive(const char *line) return false; } +// Print domain (escape non-printable characters) +static void print_escaped(const char *str, const ssize_t len) +{ + for(ssize_t j = 0; j < len; j++) + if(isgraph(str[j])) + putchar(str[j]); + else + // Escape non-printable characters + printf("\\x%02x", (unsigned char)str[j]); +} + int gravity_parseList(const char *infile, const char *outfile, const char *adlistIDstr, const bool checkOnly, const bool antigravity) { @@ -353,7 +364,9 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis { // Increment counter invalid_domains++; - printf("%s %s Invalid domain on line %zu: %s\n", over, cross, lineno, line); + printf("%s %s Invalid domain on line %zu: ", over, cross, lineno); + print_escaped(line, read); + puts(""); continue; } // Add the domain to invalid_domains_list only @@ -525,13 +538,7 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis { // Print indentation printf(" - "); - // Print domain (escape non-printable characters) - for(ssize_t j = 0; j < invalid_domains_list_lengths[i]; j++) - if(isgraph(invalid_domains_list[i][j])) - putchar(invalid_domains_list[i][j]); - else - // Escape non-printable characters - printf("\\x%02x", (unsigned char)invalid_domains_list[i][j]); + print_escaped(invalid_domains_list[i], invalid_domains_list_lengths[i]); // Print newline puts(""); } From 7edec2e2d42802ea2023277d698707d1275b5553 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 27 Nov 2023 23:08:31 +0100 Subject: [PATCH 126/221] Improve language Co-authored-by: yubiuser Signed-off-by: DL6ER --- src/api/docs/content/specs/auth.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/docs/content/specs/auth.yaml b/src/api/docs/content/specs/auth.yaml index febfd0591..1a665f437 100644 --- a/src/api/docs/content/specs/auth.yaml +++ b/src/api/docs/content/specs/auth.yaml @@ -44,7 +44,7 @@ components: operationId: "add_auth" security: [] description: | - Authenticate using a password. The password isn't stored in the session or used to create the session token. Instead, the session token is produced using a cryptographically secure random number generator. A CSRF token is utilized to guard against CSRF attacks and is necessary when using Cookie-based authentication. However, it's not needed with other authentication methods. + Authenticate using a password. The password isn't stored in the session nor used to create the session token. Instead, the session token is produced using a cryptographically secure random number generator. A CSRF token is utilized to guard against CSRF attacks and is necessary when using Cookie-based authentication. However, it's not needed with other authentication methods. Both the Session ID (SID) and CSRF token remain valid for the session's duration. The session can be extended before its expiration by performing any authenticated action. By default, the session lasts for 5 minutes. It can be invalidated by either logging out or deleting the session. Additionally, the session becomes invalid when the password is altered or a new application password is created. From 6a3502a0ec1423d8f59973d8a47d46417a55fe7d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 27 Nov 2023 23:18:56 +0100 Subject: [PATCH 127/221] Disable cache optimizer for any negative TTL Signed-off-by: DL6ER --- src/config/config.c | 6 +++--- src/config/dnsmasq_config.c | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/config/config.c b/src/config/config.c index 4e756198c..5f3e85798 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -557,10 +557,10 @@ void initConfig(struct config *conf) conf->dns.cache.size.d.ui = 10000u; conf->dns.cache.optimizer.k = "dns.cache.optimizer"; - conf->dns.cache.optimizer.h = "Query cache optimizer: If a DNS name exists in the cache, but its time-to-live has expired only recently, the data will be used anyway (a refreshing from upstream is triggered). This can improve DNS query delays especially over unreliable Internet connections. This feature comes at the expense of possibly sometimes returning out-of-date data and less efficient cache utilisation, since old data cannot be flushed when its TTL expires, so the cache becomes mostly least-recently-used. To mitigate issues caused by massively outdated DNS replies, the maximum overaging of cached records is limited. We strongly recommend staying below 86400 (1 day) with this option."; - conf->dns.cache.optimizer.t = CONF_UINT; + conf->dns.cache.optimizer.h = "Query cache optimizer: If a DNS name exists in the cache, but its time-to-live has expired only recently, the data will be used anyway (a refreshing from upstream is triggered). This can improve DNS query delays especially over unreliable Internet connections. This feature comes at the expense of possibly sometimes returning out-of-date data and less efficient cache utilization, since old data cannot be flushed when its TTL expires, so the cache becomes mostly least-recently-used. To mitigate issues caused by massively outdated DNS replies, the maximum overaging of cached records is limited. We strongly recommend staying below 86400 (1 day) with this option.\n Setting the TTL excess time to zero will serve stale cache data regardless how long it has expired. This is not recommended as it may lead to stale data being served for a long time. Setting this option to any negative value will disable this feature altogether."; + conf->dns.cache.optimizer.t = CONF_INT; conf->dns.cache.optimizer.f = FLAG_RESTART_FTL | FLAG_ADVANCED_SETTING; - conf->dns.cache.optimizer.d.ui = 3600u; + conf->dns.cache.optimizer.d.i = 3600u; // sub-struct dns.blocking conf->dns.blocking.active.k = "dns.blocking.active"; diff --git a/src/config/dnsmasq_config.c b/src/config/dnsmasq_config.c index 92bda8b71..35e1cba9d 100644 --- a/src/config/dnsmasq_config.c +++ b/src/config/dnsmasq_config.c @@ -408,10 +408,10 @@ bool __attribute__((const)) write_dnsmasq_config(struct config *conf, bool test_ fputs("\n", pihole_conf); } - if(conf->dns.cache.optimizer.v.ui > 0u) + if(conf->dns.cache.optimizer.v.i > -1) { fputs("# Use stale cache entries for a given number of seconds to optimize cache utilization\n", pihole_conf); - fprintf(pihole_conf, "use-stale-cache=%u\n", conf->dns.cache.optimizer.v.ui); + fprintf(pihole_conf, "use-stale-cache=%i\n", conf->dns.cache.optimizer.v.i); fputs("\n", pihole_conf); } From b9b373504c795188cd9e1dfebad8c427a9df25ec Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 28 Nov 2023 00:09:33 +0100 Subject: [PATCH 128/221] Increase default value of webserver.session.timeout Signed-off-by: DL6ER --- src/config/config.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config/config.c b/src/config/config.c index 4e756198c..345bee06e 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -869,7 +869,7 @@ void initConfig(struct config *conf) conf->webserver.session.timeout.k = "webserver.session.timeout"; conf->webserver.session.timeout.h = "Session timeout in seconds. If a session is inactive for more than this time, it will be terminated. Sessions are continuously refreshed by the web interface, preventing sessions from timing out while the web interface is open.\n This option may also be used to make logins persistent for long times, e.g. 86400 seconds (24 hours), 604800 seconds (7 days) or 2592000 seconds (30 days). Note that the total number of concurrent sessions is limited so setting this value too high may result in users being rejected and unable to log in if there are already too many sessions active."; conf->webserver.session.timeout.t = CONF_UINT; - conf->webserver.session.timeout.d.ui = 300u; + conf->webserver.session.timeout.d.ui = 1800u; conf->webserver.session.restore.k = "webserver.session.restore"; conf->webserver.session.restore.h = "Should Pi-hole backup and restore sessions from the database? This is useful if you want to keep your sessions after a restart of the web interface."; From 71d726c1745275290765f78628583ccfe2999419 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 28 Nov 2023 09:31:48 +0100 Subject: [PATCH 129/221] Remove deprecated dhcp.domain setting Signed-off-by: DL6ER --- src/api/docs/content/specs/config.yaml | 5 ----- src/config/config.c | 11 ++--------- src/config/config.h | 1 - src/config/toml_writer.c | 22 ---------------------- test/pihole.toml | 11 +---------- 5 files changed, 3 insertions(+), 47 deletions(-) diff --git a/src/api/docs/content/specs/config.yaml b/src/api/docs/content/specs/config.yaml index 66a48f5b8..46577c5f5 100644 --- a/src/api/docs/content/specs/config.yaml +++ b/src/api/docs/content/specs/config.yaml @@ -311,10 +311,6 @@ components: netmask: type: string x-format: ipv4 - domain: - type: string - description: | - *Note:* This setting is deprecated and will be removed in a future release. Use dns.domain instead. leaseTime: type: string ipv6: @@ -645,7 +641,6 @@ components: start: "192.168.0.10" end: "192.168.0.250" router: "192.168.0.1" - domain: "lan" netmask: "0.0.0.0" leaseTime: "24h" ipv6: true diff --git a/src/config/config.c b/src/config/config.c index 4e756198c..888034c00 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -681,8 +681,8 @@ void initConfig(struct config *conf) conf->dns.revServer.target.f = FLAG_RESTART_FTL; conf->dns.revServer.domain.k = "dns.revServer.domain"; - conf->dns.revServer.domain.h = "Domain used for the reverse server feature"; - conf->dns.revServer.domain.a = cJSON_CreateStringReference(", typically set to the same value as dhcp.domain"); + conf->dns.revServer.domain.h = "Domain used for the reverse server feature (e.g., \"fritz.box\")"; + conf->dns.revServer.domain.a = cJSON_CreateStringReference(""); conf->dns.revServer.domain.t = CONF_STRING; conf->dns.revServer.domain.d.s = (char*)""; conf->dns.revServer.domain.f = FLAG_RESTART_FTL; @@ -715,13 +715,6 @@ void initConfig(struct config *conf) conf->dhcp.router.f = FLAG_RESTART_FTL; memset(&conf->dhcp.router.d.in_addr, 0, sizeof(struct in_addr)); - conf->dhcp.domain.k = "dhcp.domain"; - conf->dhcp.domain.h = "The DNS domain used by your Pi-hole (*** DEPRECATED ***)\n This setting is deprecated and will be removed in a future version. Please use dns.domain instead. Setting it to any non-default value will overwrite the value of dns.domain if it is still set to its default value."; - conf->dhcp.domain.a = cJSON_CreateStringReference(""); - conf->dhcp.domain.t = CONF_STRING; - conf->dhcp.domain.f = FLAG_RESTART_FTL | FLAG_ADVANCED_SETTING; - conf->dhcp.domain.d.s = (char*)"lan"; - conf->dhcp.netmask.k = "dhcp.netmask"; conf->dhcp.netmask.h = "The netmask used by your Pi-hole. For directly connected networks (i.e., networks on which the machine running Pi-hole has an interface) the netmask is optional and may be set to an empty string (\"\"): it will then be determined from the interface configuration itself. For networks which receive DHCP service via a relay agent, we cannot determine the netmask itself, so it should explicitly be specified, otherwise Pi-hole guesses based on the class (A, B or C) of the network address."; conf->dhcp.netmask.a = cJSON_CreateStringReference(" (e.g., \"255.255.255.0\") or empty string (\"\") for auto-discovery"); diff --git a/src/config/config.h b/src/config/config.h index cdacdbe38..12dd51944 100644 --- a/src/config/config.h +++ b/src/config/config.h @@ -183,7 +183,6 @@ struct config { struct conf_item start; struct conf_item end; struct conf_item router; - struct conf_item domain; struct conf_item netmask; struct conf_item leaseTime; struct conf_item ipv6; diff --git a/src/config/toml_writer.c b/src/config/toml_writer.c index 5f892daf4..340900963 100644 --- a/src/config/toml_writer.c +++ b/src/config/toml_writer.c @@ -25,25 +25,6 @@ // defined in config/config.c extern uint8_t last_checksum[SHA256_DIGEST_SIZE]; -static void migrate_config(void) -{ - // Migrating dhcp.domain -> dns.domain - if(strcmp(config.dns.domain.v.s, config.dns.domain.d.s) == 0) - { - // If the domain is the same as the default, check if the dhcp domain - // is different from the default. If so, migrate it - if(strcmp(config.dhcp.domain.v.s, config.dhcp.domain.d.s) != 0) - { - // Migrate dhcp.domain -> dns.domain - log_info("Migrating dhcp.domain = \"%s\" -> dns.domain", config.dhcp.domain.v.s); - if(config.dns.domain.t == CONF_STRING_ALLOCATED) - free(config.dns.domain.v.s); - config.dns.domain.v.s = strdup(config.dhcp.domain.v.s); - config.dns.domain.t = CONF_STRING_ALLOCATED; - } - } -} - bool writeFTLtoml(const bool verbose) { // Try to open a temporary config file for writing @@ -68,9 +49,6 @@ bool writeFTLtoml(const bool verbose) fputs(timestring, fp); fputs("\n\n", fp); - // Perform possible config migration - migrate_config(); - // Iterate over configuration and store it into the file char *last_path = (char*)""; for(unsigned int i = 0; i < CONFIG_ELEMENTS; i++) diff --git a/test/pihole.toml b/test/pihole.toml index 3aaa217d9..33b189745 100644 --- a/test/pihole.toml +++ b/test/pihole.toml @@ -370,7 +370,7 @@ # Domain used for the reverse server feature # # Possible values are: - # , typically set to the same value as dhcp.domain + # (e.g., "fritz.box") domain = "" [dhcp] @@ -396,15 +396,6 @@ # , e.g., "192.168.0.1" router = "" - # The DNS domain used by your Pi-hole (*** DEPRECATED ***) - # This setting is deprecated and will be removed in a future version. Please use - # dns.domain instead. Setting it to any non-default value will overwrite the value of - # dns.domain if it is still set to its default value. - # - # Possible values are: - # - domain = "lan" - # The netmask used by your Pi-hole. For directly connected networks (i.e., networks on # which the machine running Pi-hole has an interface) the netmask is optional and may # be set to "0.0.0.0": it will then be determined from the interface configuration From 34e44f2453be0a67a47d543b68eae06a436b3f65 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 28 Nov 2023 15:44:58 +0100 Subject: [PATCH 130/221] Set imported database query IDs to counter value and fix GC pointer magic (update comment to explain better what happens here) Signed-off-by: DL6ER --- src/database/query-table.c | 6 +++--- src/datastructure.c | 4 ++-- src/dnsmasq_interface.c | 12 +++++------ src/gc.c | 43 +++++++++++++++++++++++++++++--------- 4 files changed, 44 insertions(+), 21 deletions(-) diff --git a/src/database/query-table.c b/src/database/query-table.c index b8b2766fe..8a4fa6544 100644 --- a/src/database/query-table.c +++ b/src/database/query-table.c @@ -1065,19 +1065,19 @@ void DB_read_queries(void) query->qtype = type - 100; } counters->querytype[query->type]++; - log_debug(DEBUG_GC, "GC: query type %d set (database), new count = %d", query->type, counters->querytype[query->type]); + log_debug(DEBUG_GC, "query type %d set (database), ID = %d, new count = %d", query->type, counters->queries, counters->querytype[query->type]); // Status is set below query->domainID = domainID; query->clientID = clientID; query->upstreamID = upstreamID; - query->id = 0; + query->id = counters->queries; query->response = 0; query->flags.response_calculated = reply_time_avail; query->dnssec = dnssec; query->reply = reply; counters->reply[query->reply]++; - log_debug(DEBUG_GC, "GC: reply type %d set (database), new count = %d", query->reply, counters->reply[query->reply]); + log_debug(DEBUG_GC, "reply type %d set (database), ID = %d, new count = %d", query->reply, counters->queries, counters->reply[query->reply]); query->response = reply_time; query->CNAME_domainID = -1; // Initialize flags diff --git a/src/datastructure.c b/src/datastructure.c index fd502ab15..8cf2e5aef 100644 --- a/src/datastructure.c +++ b/src/datastructure.c @@ -1039,10 +1039,10 @@ void _query_set_status(queriesData *query, const enum query_status new_status, c if(!init) { counters->status[old_status]--; - log_debug(DEBUG_GC, "GC: status %d removed (!init), new count = %d", QUERY_UNKNOWN, counters->status[QUERY_UNKNOWN]); + log_debug(DEBUG_GC, "status %d removed (!init), ID = %d, new count = %d", QUERY_UNKNOWN, query->id, counters->status[QUERY_UNKNOWN]); } counters->status[new_status]++; - log_debug(DEBUG_GC, "GC: status %d set, new count = %d", new_status, counters->status[new_status]); + log_debug(DEBUG_GC, "status %d set, ID = %d, new count = %d", new_status, query->id, counters->status[new_status]); // ... update overTime counters, ... const int timeidx = getOverTimeID(query->timestamp); diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index 530d16dd8..90aaf7c5a 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -747,7 +747,7 @@ bool _FTL_new_query(const unsigned int flags, const char *name, query->timestamp = querytimestamp; query->type = querytype; counters->querytype[querytype]++; - log_debug(DEBUG_GC, "GC: query type %d set (new query), new count = %d", query->type, counters->querytype[query->type]); + log_debug(DEBUG_GC, "query type %d set (new query), ID = %d, new count = %d", query->type, id, counters->querytype[query->type]); query->qtype = qtype; query->id = id; // Has to be set before calling query_set_status() @@ -764,7 +764,7 @@ bool _FTL_new_query(const unsigned int flags, const char *name, // Initialize reply type query->reply = REPLY_UNKNOWN; counters->reply[REPLY_UNKNOWN]++; - log_debug(DEBUG_GC, "GC: reply type %d set (new query), new count = %d", query->reply, counters->reply[query->reply]); + log_debug(DEBUG_GC, "reply type %d set (new query), ID = %d, new count = %d", query->reply, query->id, counters->reply[query->reply]); // Store DNSSEC result for this domain query->dnssec = DNSSEC_UNKNOWN; query->CNAME_domainID = -1; @@ -2731,12 +2731,12 @@ static void _query_set_reply(const unsigned int flags, const enum reply_type rep // Subtract from old reply counter counters->reply[query->reply]--; - log_debug(DEBUG_GC, "GC: reply type %d removed (set_reply), new count = %d", query->reply, counters->reply[query->reply]); + log_debug(DEBUG_GC, "reply type %d removed (set_reply), ID = %d, new count = %d", query->reply, query->id, counters->reply[query->reply]); // Add to new reply counter counters->reply[new_reply]++; // Store reply type query->reply = new_reply; - log_debug(DEBUG_GC, "GC: reply type %d added (set_reply), new count = %d", query->reply, counters->reply[query->reply]); + log_debug(DEBUG_GC, "reply type %d added (set_reply), ID = %d, new count = %d", query->reply, query->id, counters->reply[query->reply]); // Save response time // Skipped internally if already computed @@ -3353,10 +3353,10 @@ void FTL_multiple_replies(const int id, int *firstID) // Copy relevant information over counters->reply[duplicated_query->reply]--; - log_debug(DEBUG_GC, "GC: duplicated_query reply type %d removed, new count = %d", duplicated_query->reply, counters->reply[duplicated_query->reply]); + log_debug(DEBUG_GC, "duplicated_query reply type %d removed, ID = %d, new count = %d", duplicated_query->reply, duplicated_query->id, counters->reply[duplicated_query->reply]); duplicated_query->reply = source_query->reply; counters->reply[duplicated_query->reply]++; - log_debug(DEBUG_GC, "GC: duplicated_query reply type %d set, new count = %d", duplicated_query->reply, counters->reply[duplicated_query->reply]); + log_debug(DEBUG_GC, "duplicated_query reply type %d set, ID = %d, new count = %d", duplicated_query->reply, duplicated_query->id, counters->reply[duplicated_query->reply]); duplicated_query->dnssec = source_query->dnssec; duplicated_query->flags.complete = true; diff --git a/src/gc.c b/src/gc.c index 410cc073f..e4b484186 100644 --- a/src/gc.c +++ b/src/gc.c @@ -369,11 +369,11 @@ void runGC(const time_t now, time_t *lastGCrun, const bool flush) // Update reply counters counters->reply[query->reply]--; - log_debug(DEBUG_GC, "GC: reply type %d removed (GC), new count = %d", query->reply, counters->reply[query->reply]); + log_debug(DEBUG_GC, "reply type %d removed (GC), ID = %d, new count = %d", query->reply, query->id, counters->reply[query->reply]); // Update type counters counters->querytype[query->type]--; - log_debug(DEBUG_GC, "GC: query type %d removed (GC), new count = %d", query->type, counters->querytype[query->type]); + log_debug(DEBUG_GC, "query type %d removed (GC), ID = %d, new count = %d", query->type, query->id, counters->querytype[query->type]); // Subtract UNKNOWN from the counters before // setting the status if different. @@ -381,7 +381,7 @@ void runGC(const time_t now, time_t *lastGCrun, const bool flush) if(query->status != QUERY_UNKNOWN) { counters->status[QUERY_UNKNOWN]--; - log_debug(DEBUG_GC, "GC: status %d removed (GC), new count = %d", QUERY_UNKNOWN, counters->status[QUERY_UNKNOWN]); + log_debug(DEBUG_GC, "status %d removed (GC), ID = %d, new count = %d", QUERY_UNKNOWN, query->id, counters->status[QUERY_UNKNOWN]); } // Set query again to UNKNOWN to reset the counters @@ -404,19 +404,42 @@ void runGC(const time_t now, time_t *lastGCrun, const bool flush) { // Move memory forward to keep only what we want // Note: for overlapping memory blocks, memmove() is a safer approach than memcpy() - // Example: (I = now invalid, X = still valid queries, F = free space) - // Before: IIIIIIXXXXFF - // After: XXXXFFFFFFFF + // + // ┌──────────────────────┐ + // │ Example: removed = 5 │▒ + // │ │▒ + // │ query with ID = 6 │▒ + // │ is moved to ID = 0, │▒ + // │ 7 ─> 1, 8 ─> 2, etc. │▒ + // │ │▒ + // │ ID: 111111 │▒ + // │ 0123456789012345 │▒ + // │ │▒ + // │ ......QQQQ------ │▒ + // │ vvvv │▒ + // │ ┌─────┘│││ │▒ + // │ │┌─────┘││ │▒ + // │ ││┌─────┘│ │▒ + // │ │││┌─────┘ │▒ + // │ vvvv │▒ + // │ QQQQ------------ │▒ + // └──────────────────────┘▒ + // ▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒ + // + // Legend: . = removed queries, Q = valid queries, - = free space + // + // We move the memory block starting at the first valid query (index 5) to the + // beginning of the memory block, overwriting the invalid queries (index 0-4). + // The remaining memory (index 5-15) is then zeroed out by memset() below. queriesData *dest = getQuery(0, true); - // Note: we use "removed - 1" here because the ID of the last query is "counters->queries - 1" - queriesData *src = getQuery(removed - 1, true); - if(dest && src) + queriesData *src = getQuery(removed, true); + if(dest != NULL && src != NULL) memmove(dest, src, (counters->queries - removed)*sizeof(queriesData)); // Update queries counter counters->queries -= removed; - // ensure remaining memory is zeroed out (marked as "F" in the above example) + // Ensure remaining memory is zeroed out (marked as "F" in the above example) queriesData *tail = getQuery(counters->queries, true); if(tail) memset(tail, 0, (counters->queries_MAX - counters->queries)*sizeof(queriesData)); From 4422c3930790cc5a562ff58c0a9f20efef44a771 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 28 Nov 2023 19:22:20 +0100 Subject: [PATCH 131/221] Minor config comment fix Signed-off-by: DL6ER --- src/config/config.c | 2 +- test/pihole.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/config/config.c b/src/config/config.c index 345bee06e..862b51fc8 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -970,7 +970,7 @@ void initConfig(struct config *conf) conf->webserver.api.excludeDomains.k = "webserver.api.excludeDomains"; conf->webserver.api.excludeDomains.h = "Array of domains to be excluded from certain API responses\n Example: [ \"google.de\", \"pi-hole.net\" ]"; - conf->webserver.api.excludeDomains.a = cJSON_CreateStringReference("array of IP addresses and/or hostnames"); + conf->webserver.api.excludeDomains.a = cJSON_CreateStringReference("array of domains"); conf->webserver.api.excludeDomains.t = CONF_JSON_STRING_ARRAY; conf->webserver.api.excludeDomains.d.json = cJSON_CreateArray(); diff --git a/test/pihole.toml b/test/pihole.toml index 3aaa217d9..dfd974f33 100644 --- a/test/pihole.toml +++ b/test/pihole.toml @@ -694,7 +694,7 @@ # Example: [ "google.de", "pi-hole.net" ] # # Possible values are: - # array of IP addresses and/or hostnames + # array of domains excludeDomains = [] # How much history should be imported from the database [seconds]? (max 24*60*60 = From d982b46052f3caddad95ff3e0f4d5cd5ec4dc757 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 28 Nov 2023 21:05:54 +0100 Subject: [PATCH 132/221] Fix link from message table to list table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- src/database/message-table.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/database/message-table.c b/src/database/message-table.c index ba1b75919..6696fcd74 100644 --- a/src/database/message-table.c +++ b/src/database/message-table.c @@ -659,7 +659,7 @@ static void format_inaccessible_adlist_message(char *plain, const int sizeof_pla char *escaped_address = escape_html(address); - if(snprintf(html, sizeof_html, "List with ID %d (%s) was inaccessible during last gravity run", + if(snprintf(html, sizeof_html, "List with ID %d (%s) was inaccessible during last gravity run", dbindex, dbindex, escaped_address) > sizeof_html) log_warn("format_inaccessible_adlist_message(): Buffer too small to hold HTML message, warning truncated"); From 01af195c776367a5931debb103a07b2b7baf1628 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 28 Nov 2023 22:16:38 +0100 Subject: [PATCH 133/221] Update src/config/dnsmasq_config.c Co-authored-by: yubiuser Signed-off-by: DL6ER --- src/config/dnsmasq_config.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/config/dnsmasq_config.c b/src/config/dnsmasq_config.c index 35e1cba9d..cc262bc77 100644 --- a/src/config/dnsmasq_config.c +++ b/src/config/dnsmasq_config.c @@ -411,6 +411,7 @@ bool __attribute__((const)) write_dnsmasq_config(struct config *conf, bool test_ if(conf->dns.cache.optimizer.v.i > -1) { fputs("# Use stale cache entries for a given number of seconds to optimize cache utilization\n", pihole_conf); + fputs("# Setting the time to zero will serve stale cache data regardless how long it has expired.\n", pihole_conf); fprintf(pihole_conf, "use-stale-cache=%i\n", conf->dns.cache.optimizer.v.i); fputs("\n", pihole_conf); } From 3eb0e5183321328068b1554ffdc04861170b43c3 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 28 Nov 2023 22:44:32 +0100 Subject: [PATCH 134/221] Use case-insensitive matching for the gravity subcommands Signed-off-by: DL6ER --- src/args.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/args.c b/src/args.c index 381b7f3bc..7a7895bd7 100644 --- a/src/args.c +++ b/src/args.c @@ -388,14 +388,14 @@ void parse_args(int argc, char* argv[]) const bool antigravity = strcmp(argv[1], "antigravity") == 0; // pihole-FTL gravity parseList - if(argc == 6 && strcmp(argv[2], "parseList") == 0) + if(argc == 6 && strcasecmp(argv[2], "parseList") == 0) { // Parse the given list and write the result to the given file exit(gravity_parseList(argv[3], argv[4], argv[5], false, antigravity)); } // pihole-FTL gravity checkList - if(argc == 4 && strcmp(argv[2], "checkList") == 0) + if(argc == 4 && strcasecmp(argv[2], "checkList") == 0) { // Parse the given list and write the result to the given file exit(gravity_parseList(argv[3], "", "-1", true, antigravity)); From 392b39e0d0f798c86537c90258ee0c393d3cffa7 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 29 Nov 2023 21:25:44 +0100 Subject: [PATCH 135/221] Minor tweak ensuring we also decommission UNKNOWN queries properly when they expire Signed-off-by: DL6ER --- src/gc.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/gc.c b/src/gc.c index 5c2677de9..5d9aa144e 100644 --- a/src/gc.c +++ b/src/gc.c @@ -378,11 +378,8 @@ void runGC(const time_t now, time_t *lastGCrun, const bool flush) // Subtract UNKNOWN from the counters before // setting the status if different. // Minus one here and plus one below = net zero - if(query->status != QUERY_UNKNOWN) - { - counters->status[QUERY_UNKNOWN]--; - log_debug(DEBUG_GC, "status %d removed (GC), ID = %d, new count = %d", QUERY_UNKNOWN, query->id, counters->status[QUERY_UNKNOWN]); - } + counters->status[QUERY_UNKNOWN]--; + log_debug(DEBUG_GC, "status %d removed (GC), ID = %d, new count = %d", QUERY_UNKNOWN, query->id, counters->status[QUERY_UNKNOWN]); // Set query again to UNKNOWN to reset the counters query_set_status(query, QUERY_UNKNOWN); From a52cae48f46f1418871334fdff2e1bf390fafa36 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 29 Nov 2023 22:05:10 +0100 Subject: [PATCH 136/221] Remove obsolete string escaping routines. They have been necessary for the Telnet API, however, this is gone and our JSON functions know how to deal with spaces Signed-off-by: DL6ER --- src/dnsmasq_interface.c | 2 +- src/shmem.c | 90 +++-------------------------------------- src/shmem.h | 11 ----- 3 files changed, 7 insertions(+), 96 deletions(-) diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index 90aaf7c5a..d77feba59 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -2084,7 +2084,7 @@ static void FTL_reply(const unsigned int flags, const char *name, const union al // else: This is a reply from upstream // Check if this domain matches exactly - const bool isExactMatch = strcmp_escaped(name, getstr(domain->domainpos)); + const bool isExactMatch = strcmp(name, getstr(domain->domainpos)); if((flags & F_CONFIG) && isExactMatch && !query->flags.complete) { diff --git a/src/shmem.c b/src/shmem.c index 415305f69..1ee6b1b42 100644 --- a/src/shmem.c +++ b/src/shmem.c @@ -206,73 +206,6 @@ static bool chown_shmem(SharedMemory *sharedMemory, struct passwd *ent_pw) return true; } -// A function that duplicates a string and replaces all characters "s" by "r" -static char *__attribute__ ((malloc)) str_replace(const char *input, - const char s, - const char r, - unsigned int *N) -{ - // Duplicate string - char *copy = strdup(input); - if(!copy) - return NULL; - - // Woring pointer - char *ix = copy; - // Loop over string until there are no further "s" chars in the string - while((ix = strchr(ix, s)) != NULL) - { - *ix++ = r; - (*N)++; - } - - return copy; -} - -char *__attribute__ ((malloc)) str_escape(const char *input, unsigned int *N) -{ - // If no escaping is done, this routine returns the original pointer - // and N stays 0 - *N = 0; - if(strchr(input, ' ') != NULL) - { - // Replace any spaces by ~ if we find them in the domain name - // This is necessary as our telnet API uses space delimiters - return str_replace(input, ' ', '~', N); - } - - return strdup(input); -} - -bool strcmp_escaped(const char *a, const char *b) -{ - unsigned int Na, Nb; - - // Input check - if(a == NULL || b == NULL) - return false; - - // Escape both inputs - char *aa = str_escape(a, &Na); - char *bb = str_escape(b, &Nb); - - // Check for memory errors - if(!aa || !bb) - { - if(aa) free(aa); - if(bb) free(bb); - return false; - } - - const char result = strcasecmp(aa, bb) == 0; - - free(aa); - free(bb); - - return result; -} - - size_t _addstr(const char *input, const char *func, const int line, const char *file) { if(input == NULL) @@ -304,22 +237,12 @@ size_t _addstr(const char *input, const char *func, const int line, const char * len = avail_mem; } - unsigned int N = 0; - char *str = str_escape(input, &N); - - if(N > 0) - log_info("FTL replaced %u invalid characters with ~ in the query \"%s\"", N, str); - // Search buffer for existence of exact same string - char *str_pos = memmem(shm_strings.ptr, shmSettings->next_str_pos, str, len); + char *str_pos = memmem(shm_strings.ptr, shmSettings->next_str_pos, input, len); if(str_pos != NULL) { - log_debug(DEBUG_SHMEM, "Reusing existing string \"%s\" at position %zd in %s() (%s:%i)", - str, str_pos - (char*)shm_strings.ptr, func, short_path(file), line); - - // If the string already exists, we can free the memory allocated - // for the escaped string - free(str); + log_debug(DEBUG_SHMEM, "Reusing existing string \"%s\" at %zd in %s() (%s:%i)", + input, str_pos - (char*)shm_strings.ptr, func, short_path(file), line); // Return position of existing string return (str_pos - (char*)shm_strings.ptr); @@ -327,11 +250,10 @@ size_t _addstr(const char *input, const char *func, const int line, const char * // Debugging output log_debug(DEBUG_SHMEM, "Adding \"%s\" (len %zu) to buffer in %s() (%s:%i), next_str_pos is %u", - str, len, func, short_path(file), line, shmSettings->next_str_pos); + input, len, func, short_path(file), line, shmSettings->next_str_pos); - // Copy the C string pointed by str into the shared string buffer - strncpy(&((char*)shm_strings.ptr)[shmSettings->next_str_pos], str, len); - free(str); + // Copy the C string pointed by input into the shared string buffer + strncpy(&((char*)shm_strings.ptr)[shmSettings->next_str_pos], input, len); // Increment string length counter shmSettings->next_str_pos += len; diff --git a/src/shmem.h b/src/shmem.h index f92de6545..8a6ed6275 100644 --- a/src/shmem.h +++ b/src/shmem.h @@ -116,17 +116,6 @@ size_t _addstr(const char *str, const char *func, const int line, const char *fi #define getstr(pos) _getstr(pos, __FUNCTION__, __LINE__, __FILE__) const char *_getstr(const size_t pos, const char *func, const int line, const char *file); -/** - * Escapes a string by replacing special characters, such as spaces - * The input string is always duplicated, ensure to free it after use - */ -char *str_escape(const char *input, unsigned int *N) __attribute__ ((malloc)); - -/** - * Compare two strings. Escape them if needed - */ -bool strcmp_escaped(const char *a, const char *b); - /** * Create a new overTime client shared memory block. * This also updates `overTimeClientData`. From 9d0cc59cbad074aa328f7a902bb744d668b3f6c9 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 29 Nov 2023 22:09:38 +0100 Subject: [PATCH 137/221] Improve shmem comments Signed-off-by: DL6ER --- src/shmem.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/shmem.c b/src/shmem.c index 1ee6b1b42..1d267146e 100644 --- a/src/shmem.c +++ b/src/shmem.c @@ -127,6 +127,10 @@ static size_t get_optimal_object_size(const size_t objsize, const size_t minsize // Private prototypes static void *enlarge_shmem_struct(const char type); +// Calculate and format the memory usage of the shared memory segment used by +// FTL +// The function returns the percentage of used memory. A human-readable string +// is stored in the buffer passed to this function. static int get_dev_shm_usage(char buffer[64]) { char buffer2[64] = { 0 }; @@ -206,6 +210,10 @@ static bool chown_shmem(SharedMemory *sharedMemory, struct passwd *ent_pw) return true; } +// Add string to our shared memory buffer +// This function checks if the string already exists in the buffer and returns +// the position of the existing string if it does. Otherwise, it adds the +// string to the buffer and returns the position of the newly added string. size_t _addstr(const char *input, const char *func, const int line, const char *file) { if(input == NULL) @@ -262,6 +270,7 @@ size_t _addstr(const char *input, const char *func, const int line, const char * return (shmSettings->next_str_pos - len); } +// Get string from shared memory buffer const char *_getstr(const size_t pos, const char *func, const int line, const char *file) { // Only access the string memory if this memory region has already been set @@ -274,7 +283,7 @@ const char *_getstr(const size_t pos, const char *func, const int line, const ch } } -/// Create a mutex for shared memory +// Create a mutex for shared memory static void create_mutex(pthread_mutex_t *lock) { log_debug(DEBUG_SHMEM, "Creating SHM mutex lock"); pthread_mutexattr_t lock_attr = {}; @@ -312,9 +321,9 @@ static void create_mutex(pthread_mutex_t *lock) { pthread_mutexattr_destroy(&lock_attr); } +// Remap shared object pointers which might have changed static void remap_shm(void) { - // Remap shared object pointers which might have changed realloc_shm(&shm_queries, counters->queries_MAX, sizeof(queriesData), false); queries = (queriesData*)shm_queries.ptr; From d212f2663469b02e95374b0f6b0be74c1f4316a2 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 30 Nov 2023 12:52:58 +0100 Subject: [PATCH 138/221] Add CHAOS TXT records to dynamically get the available API ports: - domain.api.ftl will use the configured web domain, e.g. "https://pi.hole:443/api" - local.api.ftl will use the hard-coded string "localhost" instead, e.g., "https://localhost:443/api" - api.ftl is an alias for domain.api.ftl Signed-off-by: DL6ER --- src/dnsmasq/cache.c | 15 +++++++ src/dnsmasq/dnsmasq.h | 2 + src/dnsmasq/option.c | 5 ++- src/webserver/webserver.c | 93 +++++++++++++++++++++++++++++++++++++++ src/webserver/webserver.h | 1 + 5 files changed, 115 insertions(+), 1 deletion(-) diff --git a/src/dnsmasq/cache.c b/src/dnsmasq/cache.c index 09dc998d1..01f5a3061 100644 --- a/src/dnsmasq/cache.c +++ b/src/dnsmasq/cache.c @@ -16,6 +16,7 @@ #include "dnsmasq.h" #include "../dnsmasq_interface.h" +#include "webserver/webserver.h" static struct crec *cache_head = NULL, *cache_tail = NULL, **hash_table = NULL; #ifdef HAVE_DHCP @@ -1745,6 +1746,20 @@ int cache_make_stat(struct txt_record *t) case TXT_PRIVACYLEVEL: sprintf(buff+1, "%d", *pihole_privacylevel); break; + case TXT_API_DOMAIN: + { + t->len = get_api_string(&buff, true); + t->txt = (unsigned char *)buff; + + return 1; + } + case TXT_API_LOCAL: + { + t->len = get_api_string(&buff, false); + t->txt = (unsigned char *)buff; + + return 1; + } /* -------------------- */ case TXT_STAT_SERVERS: diff --git a/src/dnsmasq/dnsmasq.h b/src/dnsmasq/dnsmasq.h index b8a83aaa5..cd52530eb 100644 --- a/src/dnsmasq/dnsmasq.h +++ b/src/dnsmasq/dnsmasq.h @@ -382,6 +382,8 @@ struct naptr { #define TXT_STAT_SERVERS 7 /* Pi-hole modification */ #define TXT_PRIVACYLEVEL 123 +#define TXT_API_DOMAIN 124 +#define TXT_API_LOCAL 125 /************************/ #endif diff --git a/src/dnsmasq/option.c b/src/dnsmasq/option.c index e6ef7e3a0..2e3c3a000 100644 --- a/src/dnsmasq/option.c +++ b/src/dnsmasq/option.c @@ -5961,7 +5961,10 @@ void read_opts(int argc, char **argv, char *compile_opts) add_txt("servers.bind", NULL, TXT_STAT_SERVERS); /* Pi-hole modification */ add_txt("privacylevel.pihole", NULL, TXT_PRIVACYLEVEL); - add_txt("version.FTL", (char*)get_FTL_version(), 0 ); + add_txt("version.ftl", (char*)get_FTL_version(), 0 ); + add_txt("api.ftl", NULL, TXT_API_DOMAIN); + add_txt("domain.api.ftl", NULL, TXT_API_DOMAIN); + add_txt("local.api.ftl", NULL, TXT_API_LOCAL); /************************/ } #endif diff --git a/src/webserver/webserver.c b/src/webserver/webserver.c index b5c3c6bf7..9159dc752 100644 --- a/src/webserver/webserver.c +++ b/src/webserver/webserver.c @@ -204,6 +204,7 @@ void FTL_mbed_debug(void *user_param, int level, const char *file, int line, con static struct serverports { bool is_secure; + bool is_redirect; unsigned char protocol; // 1 = IPv4, 2 = IPv4+IPv6, 3 = IPv6 in_port_t port; } server_ports[MAXPORTS] = { 0 }; @@ -227,6 +228,7 @@ static void get_server_ports(void) // Store port information server_ports[i].port = mgports[i].port; server_ports[i].is_secure = mgports[i].is_ssl; + server_ports[i].is_redirect = mgports[i].is_redirect; server_ports[i].protocol = mgports[i].protocol; // Store HTTPS port if not already set @@ -246,6 +248,97 @@ in_port_t __attribute__((pure)) get_https_port(void) return https_port; } +unsigned short get_api_string(char **buf, const bool domain) +{ + // Initialize buffer to empty string + size_t len = 0; + // First byte has the length of the first string + **buf = 0; + const char *domain_str = domain ? config.webserver.domain.v.s : "localhost"; + size_t api_str_size = strlen(domain_str) + 20; + + // Check if the string is too long for the TXT record + if(api_str_size > 255) + { + log_err("API URL too long for TXT record!"); + return 0; + } + + // TXT record format: + // + // 0 length of first string (unsigned char n) + // 1 to (n+1) first string + // (n+2) length of second string (unsigned char m) + // (n+3) to (n+m+3) second string + // ... + // This is repeated for every port, so the total length is + // (n+1) + (n+m+3) + (n+m+3) + ... + // + // This is implemented in the loop below + + // Loop over all ports + for(unsigned int i = 0; i < MAXPORTS; i++) + { + // Skip ports that are not configured or redirected + if(server_ports[i].port == 0 || server_ports[i].is_redirect) + continue; + + // Reallocate additional memory for every port + if((*buf = realloc(*buf, (i+1)*api_str_size)) == NULL) + { + log_err("Failed to reallocate API URL buffer!"); + return 0; + } + const size_t bufsz = (i+1)*api_str_size; + + // Append API URL to buffer + // We add this at buffer + 1 because the first byte is the + // length of the string, which we don't know yet + char *api_str = calloc(api_str_size, sizeof(char)); + const ssize_t this_len = snprintf(api_str, bufsz - len - 1, "http%s://%s:%d/api/", + server_ports[i].is_secure ? "s" : "", + domain_str, server_ports[i].port); + // Check if snprintf() failed + if(this_len < 0) + { + log_err("Failed to append API URL to buffer: %s", strerror(errno)); + return 0; + } + + // Check if snprintf() truncated the string (this should never + // happen as we allocate enough memory for the domain to fit) + if((size_t)this_len >= bufsz - len - 1) + { + log_err("API URL buffer too small!"); + return 0; + } + + // Check if this string is already present in the buffer + if(memmem(*buf, len, api_str, this_len) != NULL) + { + // This string is already present, so skip it + free(api_str); + log_debug(DEBUG_API, "Skipping duplicate API URL: %s", api_str); + continue; + } + + // Append string to buffer (one byte after the current end of + // the buffer to leave space for the length byte) + strcpy(*buf + len + 1, api_str); + free(api_str); + + // Set first byte to the length of the string (see breakdown + // above) + (*buf)[len] = (unsigned char)this_len; + + // Increase total length + len += this_len + 1; + } + + // Return total length + return (unsigned short)len; +} + void http_init(void) { log_web("Initializing HTTP server on port %s", config.webserver.port.v.s); diff --git a/src/webserver/webserver.h b/src/webserver/webserver.h index d90b44941..941a60e47 100644 --- a/src/webserver/webserver.h +++ b/src/webserver/webserver.h @@ -14,5 +14,6 @@ void http_init(void); void http_terminate(void); in_port_t get_https_port(void) __attribute__((pure)); +unsigned short get_api_string(char **buf, const bool domain); #endif // WEBSERVER_H \ No newline at end of file From fde6c29fecb562526d3123b840a92cb3a076b130 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 30 Nov 2023 13:10:50 +0100 Subject: [PATCH 139/221] Add CI tests for CHAOS TXT *.api.ftl Signed-off-by: DL6ER --- test/test_suite.bats | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/test/test_suite.bats b/test/test_suite.bats index dfc1bff1b..e74d845f5 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -904,6 +904,26 @@ [[ "${lines[@]}" == *"status: NOERROR"* ]] } +@test "API addresses reported correctly by CHAOS TXT domain.api.ftl" { + run bash -c 'dig CHAOS TXT domain.api.ftl +short @127.0.0.1' + printf "dig (full): %s\n" "${lines[@]}" + [[ ${lines[0]} == '"http://pi.hole:80/api/" "https://pi.hole:443/api/"' ]] +} + +@test "API addresses reported correctly by CHAOS TXT local.api.ftl" { + run bash -c 'dig CHAOS TXT local.api.ftl +short @127.0.0.1' + printf "dig (full): %s\n" "${lines[@]}" + [[ ${lines[0]} == '"http://localhost:80/api/" "https://localhost:443/api/"' ]] +} + +@test "API addresses reported by CHAOS TXT api.ftl identical to domain.api.ftl" { + run bash -c 'dig CHAOS TXT api.ftl +short @127.0.0.1' + api="${lines[0]}" + run bash -c 'dig CHAOS TXT domain.api.ftl +short @127.0.0.1' + domain_api="${lines[0]}" + [[ "${api}" == "${domain_api}" ]] +} + # x86_64-musl is built on busybox which has a slightly different # variant of ls displaying three, instead of one, spaces between the # user and group names. From 7f293224399cb0b3b89b3787c9d539f9d6e42c06 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 30 Nov 2023 20:05:24 +0100 Subject: [PATCH 140/221] Domain comparision should be case-insensitive Signed-off-by: DL6ER --- src/dnsmasq_interface.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index d77feba59..d880db64a 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -2084,7 +2084,7 @@ static void FTL_reply(const unsigned int flags, const char *name, const union al // else: This is a reply from upstream // Check if this domain matches exactly - const bool isExactMatch = strcmp(name, getstr(domain->domainpos)); + const bool isExactMatch = name != NULL && strcasecmp(name, getstr(domain->domainpos)) == 0; if((flags & F_CONFIG) && isExactMatch && !query->flags.complete) { From c36dbca9e9a8bbad50d4a7b4ce327c54bf79db5f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 30 Nov 2023 20:42:59 +0100 Subject: [PATCH 141/221] Add missing bits to the .gitignore file Signed-off-by: DL6ER --- .gitignore | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.gitignore b/.gitignore index 111747d87..075dad8fc 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ version~ # CMake files generated during compilation /cmake/ +/cmake_ci/ /cmake-build-debug/ /cmake-build-release/ @@ -18,6 +19,15 @@ version~ /.vscode/ /build/ +# __pycache__ files (API tests) +__pycache__/ + +# When patch fails to apply a patch segment to the original file, it saves the +# temporary original file copy out durably as *.orig, dumps the rejected segment +# to *.rej, and continues trying to apply patch segments. +*.orig +*.rej + # MAC->Vendor database files tools/manuf.data tools/macvendor.db From 242b35eba83970e496b3a7984d10501551d5e0db Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 30 Nov 2023 21:59:39 +0100 Subject: [PATCH 142/221] Remove CH TXT privacylevel.pihole entry Signed-off-by: DL6ER --- src/dnsmasq/cache.c | 4 ---- src/dnsmasq/dnsmasq.h | 1 - src/dnsmasq/option.c | 1 - src/webserver/webserver.h | 2 ++ 4 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/dnsmasq/cache.c b/src/dnsmasq/cache.c index 01f5a3061..133737da2 100644 --- a/src/dnsmasq/cache.c +++ b/src/dnsmasq/cache.c @@ -15,7 +15,6 @@ */ #include "dnsmasq.h" -#include "../dnsmasq_interface.h" #include "webserver/webserver.h" static struct crec *cache_head = NULL, *cache_tail = NULL, **hash_table = NULL; @@ -1743,9 +1742,6 @@ int cache_make_stat(struct txt_record *t) #endif /* Pi-hole modification */ - case TXT_PRIVACYLEVEL: - sprintf(buff+1, "%d", *pihole_privacylevel); - break; case TXT_API_DOMAIN: { t->len = get_api_string(&buff, true); diff --git a/src/dnsmasq/dnsmasq.h b/src/dnsmasq/dnsmasq.h index cd52530eb..3f1d47516 100644 --- a/src/dnsmasq/dnsmasq.h +++ b/src/dnsmasq/dnsmasq.h @@ -381,7 +381,6 @@ struct naptr { #define TXT_STAT_AUTH 6 #define TXT_STAT_SERVERS 7 /* Pi-hole modification */ -#define TXT_PRIVACYLEVEL 123 #define TXT_API_DOMAIN 124 #define TXT_API_LOCAL 125 /************************/ diff --git a/src/dnsmasq/option.c b/src/dnsmasq/option.c index 2e3c3a000..647525aff 100644 --- a/src/dnsmasq/option.c +++ b/src/dnsmasq/option.c @@ -5960,7 +5960,6 @@ void read_opts(int argc, char **argv, char *compile_opts) #endif add_txt("servers.bind", NULL, TXT_STAT_SERVERS); /* Pi-hole modification */ - add_txt("privacylevel.pihole", NULL, TXT_PRIVACYLEVEL); add_txt("version.ftl", (char*)get_FTL_version(), 0 ); add_txt("api.ftl", NULL, TXT_API_DOMAIN); add_txt("domain.api.ftl", NULL, TXT_API_DOMAIN); diff --git a/src/webserver/webserver.h b/src/webserver/webserver.h index 941a60e47..d87001f42 100644 --- a/src/webserver/webserver.h +++ b/src/webserver/webserver.h @@ -10,6 +10,8 @@ #ifndef WEBSERVER_H #define WEBSERVER_H +#include + void http_init(void); void http_terminate(void); From 00d6ae6bd66ad2a141dcf35389803fbf078379a1 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 1 Dec 2023 10:26:18 +0100 Subject: [PATCH 143/221] Add config option ftl.gravity_tmp Signed-off-by: DL6ER --- src/api/docs/content/specs/config.yaml | 3 +++ src/config/config.c | 7 +++++++ src/config/config.h | 1 + test/pihole.toml | 7 +++++++ 4 files changed, 18 insertions(+) diff --git a/src/api/docs/content/specs/config.yaml b/src/api/docs/content/specs/config.yaml index 46577c5f5..b688a77d8 100644 --- a/src/api/docs/content/specs/config.yaml +++ b/src/api/docs/content/specs/config.yaml @@ -438,6 +438,8 @@ components: type: string gravity: type: string + gravity_tmp: + type: string macvendor: type: string setupVars: @@ -698,6 +700,7 @@ components: pid: "/run/pihole-FTL.pid" database: "/etc/pihole/pihole-FTL.db" gravity: "/etc/pihole/gravity.db" + gravity_tmp: "/tmp" macvendor: "/etc/pihole/macvendor.db" setupVars: "/etc/pihole/setupVars.conf" pcap: "" diff --git a/src/config/config.c b/src/config/config.c index 5d6eaf1d8..b8360efe4 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -1020,6 +1020,13 @@ void initConfig(struct config *conf) conf->files.gravity.f = FLAG_ADVANCED_SETTING | FLAG_RESTART_FTL; conf->files.gravity.d.s = (char*)"/etc/pihole/gravity.db"; + conf->files.gravity_tmp.k = "files.gravity_tmp"; + conf->files.gravity_tmp.h = "A temporary directory where Pi-hole can store files during gravity updates. This directory must be writable by the user running gravity (typically pihole)."; + conf->files.gravity_tmp.a = cJSON_CreateStringReference(""); + conf->files.gravity_tmp.t = CONF_STRING; + conf->files.gravity_tmp.f = FLAG_ADVANCED_SETTING | FLAG_RESTART_FTL; + conf->files.gravity_tmp.d.s = (char*)"/tmp"; + conf->files.macvendor.k = "files.macvendor"; conf->files.macvendor.h = "The database containing MAC -> Vendor information for the network table"; conf->files.macvendor.a = cJSON_CreateStringReference(""); diff --git a/src/config/config.h b/src/config/config.h index 12dd51944..800520cfb 100644 --- a/src/config/config.h +++ b/src/config/config.h @@ -253,6 +253,7 @@ struct config { struct conf_item pid; struct conf_item database; struct conf_item gravity; + struct conf_item gravity_tmp; struct conf_item macvendor; struct conf_item setupVars; struct conf_item pcap; diff --git a/test/pihole.toml b/test/pihole.toml index 46909b67d..0915daa59 100644 --- a/test/pihole.toml +++ b/test/pihole.toml @@ -730,6 +730,13 @@ # gravity = "/etc/pihole/gravity.db" + # A temporary directory where Pi-hole can store files during gravity updates. This + # directory must be writable by the user running gravity (typically pihole). + # + # Possible values are: + # + gravity_tmp = "/tmp" + # The database containing MAC -> Vendor information for the network table # # Possible values are: From 3a59b49df01005a047abbca207860f38f69550e6 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 1 Dec 2023 12:20:29 +0100 Subject: [PATCH 144/221] Add misc.extraLogging to enable log-queries=extra defaulting to false Signed-off-by: DL6ER --- src/api/docs/content/specs/config.yaml | 3 +++ src/config/config.c | 6 ++++++ src/config/config.h | 1 + src/config/dnsmasq_config.c | 5 ++++- test/pihole.toml | 8 ++++++++ 5 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/api/docs/content/specs/config.yaml b/src/api/docs/content/specs/config.yaml index 46577c5f5..23f19b4a3 100644 --- a/src/api/docs/content/specs/config.yaml +++ b/src/api/docs/content/specs/config.yaml @@ -470,6 +470,8 @@ components: type: array items: type: string + extraLogging: + type: boolean check: type: object properties: @@ -712,6 +714,7 @@ components: privacylevel: 0 etc_dnsmasq_d: false dnsmasq_lines: [ ] + extraLogging: false check: load: true shmem: 90 diff --git a/src/config/config.c b/src/config/config.c index 5d6eaf1d8..68b4348a5 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -1105,6 +1105,12 @@ void initConfig(struct config *conf) conf->misc.dnsmasq_lines.f = FLAG_ADVANCED_SETTING | FLAG_RESTART_FTL; conf->misc.dnsmasq_lines.d.json = cJSON_CreateArray(); + conf->misc.extraLogging.k = "misc.extraLogging"; + conf->misc.extraLogging.h = "Log additional information about queries and replies to pihole.log\n When this setting is enabled, the log has extra information at the start of each line. This consists of a serial number which ties together the log lines associated with an individual query, and the IP address of the requestor. This setting is only effective if dns.queryLogging is enabled, too. This option is only useful for debugging and is not recommended for normal use."; + conf->misc.extraLogging.t = CONF_BOOL; + conf->misc.extraLogging.f = FLAG_RESTART_FTL; + conf->misc.extraLogging.d.b = false; + // sub-struct misc.check conf->misc.check.load.k = "misc.check.load"; conf->misc.check.load.h = "Pi-hole is very lightweight on resources. Nevertheless, this does not mean that you should run Pi-hole on a server that is otherwise extremely busy as queuing on the system can lead to unnecessary delays in DNS operation as the system becomes less and less usable as the system load increases because all resources are permanently in use. To account for this, FTL regularly checks the system load. To bring this to your attention, FTL warns about excessive load when the 15 minute system load average exceeds the number of cores.\n This check can be disabled with this setting."; diff --git a/src/config/config.h b/src/config/config.h index 12dd51944..6fc03c794 100644 --- a/src/config/config.h +++ b/src/config/config.h @@ -270,6 +270,7 @@ struct config { struct conf_item addr2line; struct conf_item etc_dnsmasq_d; struct conf_item dnsmasq_lines; + struct conf_item extraLogging; struct { struct conf_item load; struct conf_item shmem; diff --git a/src/config/dnsmasq_config.c b/src/config/dnsmasq_config.c index cc262bc77..eb2230512 100644 --- a/src/config/dnsmasq_config.c +++ b/src/config/dnsmasq_config.c @@ -345,7 +345,10 @@ bool __attribute__((const)) write_dnsmasq_config(struct config *conf, bool test_ if(conf->dns.queryLogging.v.b) { fputs("# Enable query logging\n", pihole_conf); - fputs("log-queries\n", pihole_conf); + if(conf->misc.extraLogging.v.b) + fputs("#log-queries=extra\n", pihole_conf); + else + fputs("log-queries\n", pihole_conf); fputs("log-async\n", pihole_conf); fputs("\n", pihole_conf); } diff --git a/test/pihole.toml b/test/pihole.toml index 46909b67d..57b0c38de 100644 --- a/test/pihole.toml +++ b/test/pihole.toml @@ -822,6 +822,14 @@ # Use this option with extra care. dnsmasq_lines = [] + # Log additional information about queries and replies to pihole.log + # When this setting is enabled, the log has extra information at the start of each + # line. This consists of a serial number which ties together the log lines associated + # with an individual query, and the IP address of the requestor. This setting is only + # effective if dns.queryLogging is enabled, too. This option is only useful for + # debugging and is not recommended for normal use. + extraLogging = false + [misc.check] # Pi-hole is very lightweight on resources. Nevertheless, this does not mean that you # should run Pi-hole on a server that is otherwise extremely busy as queuing on the From f89f207246888943c5d8176658337af4c487f979 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 1 Dec 2023 12:30:22 +0100 Subject: [PATCH 145/221] Never recycle alias-clients Signed-off-by: DL6ER --- src/gc.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/gc.c b/src/gc.c index 5d9aa144e..df3c6b770 100644 --- a/src/gc.c +++ b/src/gc.c @@ -86,6 +86,11 @@ static void recycle(void) if(client == NULL) continue; + // Never recycle aliasclients (they are not counted above but + // are only indirectly referenced by other clients) + if(client->flags.aliasclient) + continue; + log_debug(DEBUG_GC, "Recycling client %s (ID %d, lastQuery at %.3f)", getstr(client->ippos), clientID, client->lastQuery); From d01a62f0539d72d43b3dbb1d9ff03cbb1bfbbe1e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 1 Dec 2023 14:59:57 +0100 Subject: [PATCH 146/221] Add priority string in logs (if applicable) Signed-off-by: DL6ER --- src/api/docs/content/specs/logs.yaml | 4 ++++ src/api/logs.c | 1 + src/dnsmasq_interface.c | 2 +- src/log.c | 16 +++++++++++----- src/log.h | 3 ++- 5 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/api/docs/content/specs/logs.yaml b/src/api/docs/content/specs/logs.yaml index e65a8c589..b816f20cc 100644 --- a/src/api/docs/content/specs/logs.yaml +++ b/src/api/docs/content/specs/logs.yaml @@ -119,6 +119,10 @@ components: message: type: string description: Log line content + prio: + type: string + nullable: true + description: Log line priority (if available) example: - timestamp: 1611729969.0 message: "started, version pi-hole-2.84 cachesize 10000" diff --git a/src/api/logs.c b/src/api/logs.c index e6678b8ac..35f3bb3e5 100644 --- a/src/api/logs.c +++ b/src/api/logs.c @@ -68,6 +68,7 @@ int api_logs(struct ftl_conn *api) cJSON *entry = JSON_NEW_OBJECT(); JSON_ADD_NUMBER_TO_OBJECT(entry, "timestamp", fifo_log->logs[api->opts.which].timestamp[i]); JSON_REF_STR_IN_OBJECT(entry, "message", fifo_log->logs[api->opts.which].message[i]); + JSON_REF_STR_IN_OBJECT(entry, "prio", fifo_log->logs[api->opts.which].prio[i]); JSON_ADD_ITEM_TO_ARRAY(log, entry); } JSON_ADD_ITEM_TO_OBJECT(json, "log", log); diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index d880db64a..071e73d2f 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -3403,7 +3403,7 @@ void FTL_dnsmasq_log(const char *payload, const int length) lock_shm(); // Add to FIFO buffer - add_to_fifo_buffer(FIFO_DNSMASQ, payload, length); + add_to_fifo_buffer(FIFO_DNSMASQ, payload, NULL, length); // Unlock SHM unlock_shm(); diff --git a/src/log.c b/src/log.c index 3b5902d10..94cd61f0e 100644 --- a/src/log.c +++ b/src/log.c @@ -274,6 +274,8 @@ void __attribute__ ((format (gnu_printf, 3, 4))) _FTL_log(const int priority, co const int mpid = main_pid(); // Get the process ID of the main FTL process const int tid = gettid(); // Get the thread ID of the calling process + const char *prio = priostr(priority, flag); + // There are four cases we have to differentiate here: if(pid == tid) if(is_fork(mpid, pid)) @@ -295,7 +297,7 @@ void __attribute__ ((format (gnu_printf, 3, 4))) _FTL_log(const int priority, co { // Only print time/ID string when not in direct user interaction (CLI mode) if(!cli_mode) - printf("%s [%s] %s: ", timestring, idstr, priostr(priority, flag)); + printf("%s [%s] %s: ", timestring, idstr, prio); va_start(args, format); vprintf(format, args); va_end(args); @@ -310,7 +312,7 @@ void __attribute__ ((format (gnu_printf, 3, 4))) _FTL_log(const int priority, co va_start(args, format); const size_t len = vsnprintf(buffer, MAX_MSG_FIFO, format, args) + 1u; /* include zero-terminator */ va_end(args); - add_to_fifo_buffer(FIFO_FTL, buffer, len > MAX_MSG_FIFO ? MAX_MSG_FIFO : len); + add_to_fifo_buffer(FIFO_FTL, buffer, prio, len > MAX_MSG_FIFO ? MAX_MSG_FIFO : len); if(config.files.log.ftl.v.s != NULL) { @@ -321,7 +323,7 @@ void __attribute__ ((format (gnu_printf, 3, 4))) _FTL_log(const int priority, co if(logfile != NULL) { // Prepend message with identification string and priority - fprintf(logfile, "%s [%s] %s: ", timestring, idstr, priostr(priority, flag)); + fprintf(logfile, "%s [%s] %s: ", timestring, idstr, prio); // Log message va_start(args, format); @@ -361,7 +363,7 @@ void __attribute__ ((format (gnu_printf, 1, 2))) log_web(const char *format, ... va_start(args, format); const size_t len = vsnprintf(buffer, MAX_MSG_FIFO, format, args) + 1u; /* include zero-terminator */ va_end(args); - add_to_fifo_buffer(FIFO_WEBSERVER, buffer, len > MAX_MSG_FIFO ? MAX_MSG_FIFO : len); + add_to_fifo_buffer(FIFO_WEBSERVER, buffer, NULL, len > MAX_MSG_FIFO ? MAX_MSG_FIFO : len); // Get human-readable time get_timestr(timestring, now, true, false); @@ -721,7 +723,7 @@ void dnsmasq_diagnosis_warning(char *message) logg_warn_dnsmasq_message(skipStr("warning: ", message)); } -void add_to_fifo_buffer(const enum fifo_logs which, const char *payload, const size_t length) +void add_to_fifo_buffer(const enum fifo_logs which, const char *payload, const char *prio, const size_t length) { const double now = double_time(); @@ -735,6 +737,7 @@ void add_to_fifo_buffer(const enum fifo_logs which, const char *payload, const s // Log is full, move everything one slot forward to make space for a new record at the end // This pruges the oldest message from the list (it is overwritten by the second message) memmove(&fifo_log->logs[which].message[0][0], &fifo_log->logs[which].message[1][0], (LOG_SIZE - 1u) * MAX_MSG_FIFO); + memmove(&fifo_log->logs[which].prio[0], &fifo_log->logs[which].prio[1], (LOG_SIZE - 1u) * sizeof(fifo_log->logs[which].prio[0])); memmove(&fifo_log->logs[which].timestamp[0], &fifo_log->logs[which].timestamp[1], (LOG_SIZE - 1u) * sizeof(fifo_log->logs[which].timestamp[0])); idx = LOG_SIZE - 1u; } @@ -758,6 +761,9 @@ void add_to_fifo_buffer(const enum fifo_logs which, const char *payload, const s // Set timestamp fifo_log->logs[which].timestamp[idx] = now; + + // Set prio (if available) + fifo_log->logs[which].prio[idx] = prio; } bool flush_dnsmasq_log(void) diff --git a/src/log.h b/src/log.h index 5c5bf8881..fd5f58f0e 100644 --- a/src/log.h +++ b/src/log.h @@ -90,7 +90,7 @@ const char *short_path(const char *full_path) __attribute__ ((pure)); // Defaults to 512 [512 * 256 above = use 128 KB of memory for the log] #define LOG_SIZE 515u -void add_to_fifo_buffer(const enum fifo_logs which, const char *payload, const size_t length); +void add_to_fifo_buffer(const enum fifo_logs which, const char *payload, const char *prio, const size_t length); bool flush_dnsmasq_log(void); @@ -99,6 +99,7 @@ typedef struct { unsigned int next_id; double timestamp[LOG_SIZE]; char message[LOG_SIZE][MAX_MSG_FIFO]; + const char *prio[LOG_SIZE]; } logs[FIFO_MAX]; } fifologData; From 849c48dc6bc2e2220e2b9ff18ef85dcf839c708e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 1 Dec 2023 15:13:51 +0100 Subject: [PATCH 147/221] Simplify debugstr() function Signed-off-by: DL6ER --- src/config/legacy_reader.c | 7 +-- src/config/toml_reader.c | 3 +- src/log.c | 95 +++++++++++++------------------------- src/log.h | 2 +- 4 files changed, 35 insertions(+), 72 deletions(-) diff --git a/src/config/legacy_reader.c b/src/config/legacy_reader.c index 9ae6e664b..d10ef9138 100644 --- a/src/config/legacy_reader.c +++ b/src/config/legacy_reader.c @@ -824,12 +824,7 @@ static void readDebugingSettingsLegacy(FILE *fp) setDebugOption(fp, "DEBUG_ALL", ~(enum debug_flag)0); for(enum debug_flag flag = DEBUG_DATABASE; flag < DEBUG_EXTRA; flag <<= 1) - { - // DEBUG_DATABASE - const char *name; - debugstr(flag, &name); - setDebugOption(fp, name, flag); - } + setDebugOption(fp, debugstr(flag), flag); // Parse debug options set_debug_flags(&config); diff --git a/src/config/toml_reader.c b/src/config/toml_reader.c index 7bad6af9a..b2bf26e3f 100644 --- a/src/config/toml_reader.c +++ b/src/config/toml_reader.c @@ -206,11 +206,10 @@ static void reportDebugFlags(void) // Read all known debug config items for(unsigned int debug_flag = 1; debug_flag < DEBUG_ELEMENTS; debug_flag++) { - const char *name; // Get name of debug flag // We do not need to add an offset as this loop starts counting // at 1 - debugstr(debug_flag, &name); + const char *name = debugstr(debug_flag); // Calculate number of spaces to nicely align output int spaces = 20 - strlen(name); // Print debug flag diff --git a/src/log.c b/src/log.c index 94cd61f0e..4eff29106 100644 --- a/src/log.c +++ b/src/log.c @@ -125,9 +125,8 @@ unsigned int get_year(const time_t timein) return tm.tm_year + 1900; } -static const char *priostr(const int priority, const enum debug_flag flag) +static const char * __attribute__((const)) priostr(const int priority, const enum debug_flag flag) { - const char *name; switch (priority) { // system is unusable @@ -153,105 +152,75 @@ static const char *priostr(const int priority, const enum debug_flag flag) return "INFO"; // debug-level messages case LOG_DEBUG: - debugstr(flag, &name); - return name; + return debugstr(flag); // invalid option default: return "UNKNOWN"; } } -void debugstr(const enum debug_flag flag, const char **name) +const char *debugstr(const enum debug_flag flag) { switch (flag) { case DEBUG_DATABASE: - *name = "DEBUG_DATABASE"; - return; + return "DEBUG_DATABASE"; case DEBUG_NETWORKING: - *name = "DEBUG_NETWORKING"; - return; + return "DEBUG_NETWORKING"; case DEBUG_LOCKS: - *name = "DEBUG_LOCKS"; - return; + return "DEBUG_LOCKS"; case DEBUG_QUERIES: - *name = "DEBUG_QUERIES"; - return; + return "DEBUG_QUERIES"; case DEBUG_FLAGS: - *name = "DEBUG_FLAGS"; - return; + return "DEBUG_FLAGS"; case DEBUG_SHMEM: - *name = "DEBUG_SHMEM"; - return; + return "DEBUG_SHMEM"; case DEBUG_GC: - *name = "DEBUG_GC"; - return; + return "DEBUG_GC"; case DEBUG_ARP: - *name = "DEBUG_ARP"; - return; + return "DEBUG_ARP"; case DEBUG_REGEX: - *name = "DEBUG_REGEX"; - return; + return "DEBUG_REGEX"; case DEBUG_API: - *name = "DEBUG_API"; - return; + return "DEBUG_API"; case DEBUG_TLS: - *name = "DEBUG_TLS"; - return; + return "DEBUG_TLS"; case DEBUG_OVERTIME: - *name = "DEBUG_OVERTIME"; - return; + return "DEBUG_OVERTIME"; case DEBUG_STATUS: - *name = "DEBUG_STATUS"; - return; + return "DEBUG_STATUS"; case DEBUG_CAPS: - *name = "DEBUG_CAPS"; - return; + return "DEBUG_CAPS"; case DEBUG_DNSSEC: - *name = "DEBUG_DNSSEC"; - return; + return "DEBUG_DNSSEC"; case DEBUG_VECTORS: - *name = "DEBUG_VECTORS"; - return; + return "DEBUG_VECTORS"; case DEBUG_RESOLVER: - *name = "DEBUG_RESOLVER"; - return; + return "DEBUG_RESOLVER"; case DEBUG_EDNS0: - *name = "DEBUG_EDNS0"; - return; + return "DEBUG_EDNS0"; case DEBUG_CLIENTS: - *name = "DEBUG_CLIENTS"; - return; + return "DEBUG_CLIENTS"; case DEBUG_ALIASCLIENTS: - *name = "DEBUG_ALIASCLIENTS"; - return; + return "DEBUG_ALIASCLIENTS"; case DEBUG_EVENTS: - *name = "DEBUG_EVENTS"; - return; + return "DEBUG_EVENTS"; case DEBUG_HELPER: - *name = "DEBUG_HELPER"; - return; + return "DEBUG_HELPER"; case DEBUG_EXTRA: - *name = "DEBUG_EXTRA"; - return; + return "DEBUG_EXTRA"; case DEBUG_CONFIG: - *name = "DEBUG_CONFIG"; - return; + return "DEBUG_CONFIG"; case DEBUG_INOTIFY: - *name = "DEBUG_INOTIFY"; - return; + return "DEBUG_INOTIFY"; case DEBUG_WEBSERVER: - *name = "DEBUG_WEBSERVER"; - return; + return "DEBUG_WEBSERVER"; case DEBUG_RESERVED: - *name = "DEBUG_RESERVED"; - return; + return "DEBUG_RESERVED"; case DEBUG_MAX: - *name = "DEBUG_MAX"; - return; + return "DEBUG_MAX"; default: - *name = "DEBUG_ANY"; - return; + return "DEBUG_ANY"; } } diff --git a/src/log.h b/src/log.h index fd5f58f0e..594959213 100644 --- a/src/log.h +++ b/src/log.h @@ -51,7 +51,7 @@ const char *get_FTL_version(void); void log_FTL_version(bool crashreport); double double_time(void); void get_timestr(char timestring[TIMESTR_SIZE], const time_t timein, const bool millis, const bool uri_compatible); -void debugstr(const enum debug_flag flag, const char **name); +const char *debugstr(const enum debug_flag flag) __attribute__((const)); void log_web(const char *format, ...) __attribute__ ((format (gnu_printf, 1, 2))); const char *get_ordinal_suffix(unsigned int number) __attribute__ ((const)); void print_FTL_version(void); From 0165d7beb7e94241327b5af6fcec47eec7ab73e7 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 2 Dec 2023 18:09:01 +0100 Subject: [PATCH 148/221] Fix extra logging feature Signed-off-by: DL6ER --- src/config/dnsmasq_config.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config/dnsmasq_config.c b/src/config/dnsmasq_config.c index eb2230512..c0865a1cd 100644 --- a/src/config/dnsmasq_config.c +++ b/src/config/dnsmasq_config.c @@ -346,7 +346,7 @@ bool __attribute__((const)) write_dnsmasq_config(struct config *conf, bool test_ { fputs("# Enable query logging\n", pihole_conf); if(conf->misc.extraLogging.v.b) - fputs("#log-queries=extra\n", pihole_conf); + fputs("log-queries=extra\n", pihole_conf); else fputs("log-queries\n", pihole_conf); fputs("log-async\n", pihole_conf); From 56fd63b9d807c457ea64807e30a9f79773d6bdc0 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 3 Dec 2023 21:08:27 +0100 Subject: [PATCH 149/221] Use resource details obtained from kernel to compute a ten minute average of FTL's CPU utilization Signed-off-by: DL6ER --- src/api/docs/content/specs/info.yaml | 2 +- src/daemon.c | 35 ++++++++++++++++------------ src/daemon.h | 2 +- src/gc.c | 9 +++++-- 4 files changed, 29 insertions(+), 19 deletions(-) diff --git a/src/api/docs/content/specs/info.yaml b/src/api/docs/content/specs/info.yaml index a3ed864f5..65f118ea0 100644 --- a/src/api/docs/content/specs/info.yaml +++ b/src/api/docs/content/specs/info.yaml @@ -732,7 +732,7 @@ components: example: 0.1 "%cpu": type: number - description: Percentage of total CPU used by FTL + description: Percentage of total CPU used by FTL (ten seconds average) example: 1.2 allow_destructive: type: boolean diff --git a/src/daemon.c b/src/daemon.c index 60b659265..b4d07d23a 100644 --- a/src/daemon.c +++ b/src/daemon.c @@ -356,26 +356,31 @@ void cleanup(const int ret) log_info("########## FTL terminated after%s (code %i)! ##########", buffer, ret); } -static clock_t last_clock = -1; +static float last_clock = 0.0f; static float cpu_usage = 0.0f; -void calc_cpu_usage(void) +void calc_cpu_usage(const unsigned int interval) { - // Get the current CPU usage - const clock_t clk = clock(); - if(clk == (clock_t)-1) + // Get the current resource usage + // RUSAGE_SELF means here "the calling process" which is the sum of all + // resources used by all threads in the process + struct rusage usage = { 0 }; + if(getrusage(RUSAGE_SELF, &usage) != 0) { - log_warn("calc_cpu_usage() failed: %s", strerror(errno)); + log_err("Unable to obtain CPU usage: %s (%i)", strerror(errno), errno); return; } - if(last_clock == -1) - { - // Initialize the value and return - last_clock = clk; - return; - } - // Percentage of CPU time spent executing instructions - cpu_usage = 100.0f * ((float)clk - (float)last_clock) / CLOCKS_PER_SEC; - last_clock = clk; + + // Calculate the CPU usage: it is the total time spent in user mode and + // kernel mode by this process since the total time since the last call + // to this function. 100% means one core is fully used, 200% means two + // cores are fully used, etc. + const float this_clock = usage.ru_utime.tv_sec + usage.ru_stime.tv_sec + 1e-6 * (usage.ru_utime.tv_usec + usage.ru_stime.tv_usec); + + // Calculate the CPU usage in this interval + cpu_usage = 100.0 * (this_clock - last_clock) / interval; + + // Store the current time for the next call to this function + last_clock = this_clock; } float __attribute__((pure)) get_cpu_percentage(void) diff --git a/src/daemon.h b/src/daemon.h index fb96c63e1..bd86cb147 100644 --- a/src/daemon.h +++ b/src/daemon.h @@ -21,7 +21,7 @@ void delay_startup(void); bool is_fork(const pid_t mpid, const pid_t pid) __attribute__ ((const)); void cleanup(const int ret); void set_nice(void); -void calc_cpu_usage(void); +void calc_cpu_usage(const unsigned int interval); float get_cpu_percentage(void) __attribute__((pure)); bool ipv6_enabled(void); void init_locale(void); diff --git a/src/gc.c b/src/gc.c index df3c6b770..98b5b9da3 100644 --- a/src/gc.c +++ b/src/gc.c @@ -37,6 +37,10 @@ // default: 300 seconds #define RCinterval 300 +// CPU usage calculation interval +// default: 10 seconds +#define CPU_AVERAGE_INTERVAL 10 + bool doGC = false; // Recycle old clients and domains in our internal data structure @@ -518,8 +522,9 @@ void *GC_thread(void *val) break; // Calculate average CPU usage - // This is done every second to get averaged values - calc_cpu_usage(); + // This is done once every ten seconds to get averaged values + if(now - lastResourceCheck >= CPU_AVERAGE_INTERVAL) + calc_cpu_usage(CPU_AVERAGE_INTERVAL); // Check available resources if(now - lastResourceCheck >= RCinterval) From 066a38f466b512d7c04ee411dd068f060941e008 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 3 Dec 2023 22:56:41 +0100 Subject: [PATCH 150/221] Don't even try to start webserver if webserver.port is an empty string Signed-off-by: DL6ER --- src/config/config.c | 2 +- src/webserver/webserver.c | 9 ++++++++- test/pihole.toml | 13 +++++++------ 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/src/config/config.c b/src/config/config.c index 68b4348a5..e7389ba66 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -840,7 +840,7 @@ void initConfig(struct config *conf) conf->webserver.acl.d.s = (char*)""; conf->webserver.port.k = "webserver.port"; - conf->webserver.port.h = "Ports to be used by the webserver.\n Comma-separated list of ports to listen on. It is possible to specify an IP address to bind to. In this case, an IP address and a colon must be prepended to the port number. For example, to bind to the loopback interface on port 80 (IPv4) and to all interfaces port 8080 (IPv4), use \"127.0.0.1:80,8080\". \"[::]:80\" can be used to listen to IPv6 connections to port 80. IPv6 addresses of network interfaces can be specified as well, e.g. \"[::1]:80\" for the IPv6 loopback interface. [::]:80 will bind to port 80 IPv6 only.\n In order to use port 80 for all interfaces, both IPv4 and IPv6, use either the configuration \"80,[::]:80\" (create one socket for IPv4 and one for IPv6 only), or \"+80\" (create one socket for both, IPv4 and IPv6). The + notation to use IPv4 and IPv6 will only work if no network interface is specified. Depending on your operating system version and IPv6 network environment, some configurations might not work as expected, so you have to test to find the configuration most suitable for your needs. In case \"+80\" does not work for your environment, you need to use \"80,[::]:80\".\n If the port is TLS/SSL, a letter 's' must be appended, for example, \"80,443s\" will open port 80 and port 443, and connections on port 443 will be encrypted. For non-encrypted ports, it is allowed to append letter 'r' (as in redirect). Redirected ports will redirect all their traffic to the first configured SSL port. For example, if webserver.port is \"80r,443s\", then all HTTP traffic coming at port 80 will be redirected to HTTPS port 443."; + conf->webserver.port.h = "Ports to be used by the webserver.\n Comma-separated list of ports to listen on. It is possible to specify an IP address to bind to. In this case, an IP address and a colon must be prepended to the port number. For example, to bind to the loopback interface on port 80 (IPv4) and to all interfaces port 8080 (IPv4), use \"127.0.0.1:80,8080\". \"[::]:80\" can be used to listen to IPv6 connections to port 80. IPv6 addresses of network interfaces can be specified as well, e.g. \"[::1]:80\" for the IPv6 loopback interface. [::]:80 will bind to port 80 IPv6 only.\n In order to use port 80 for all interfaces, both IPv4 and IPv6, use either the configuration \"80,[::]:80\" (create one socket for IPv4 and one for IPv6 only), or \"+80\" (create one socket for both, IPv4 and IPv6). The + notation to use IPv4 and IPv6 will only work if no network interface is specified. Depending on your operating system version and IPv6 network environment, some configurations might not work as expected, so you have to test to find the configuration most suitable for your needs. In case \"+80\" does not work for your environment, you need to use \"80,[::]:80\".\n If the port is TLS/SSL, a letter 's' must be appended, for example, \"80,443s\" will open port 80 and port 443, and connections on port 443 will be encrypted. For non-encrypted ports, it is allowed to append letter 'r' (as in redirect). Redirected ports will redirect all their traffic to the first configured SSL port. For example, if webserver.port is \"80r,443s\", then all HTTP traffic coming at port 80 will be redirected to HTTPS port 443. If this value is not set (empty string), the web server will not be started and, hence, the API will not be available."; conf->webserver.port.a = cJSON_CreateStringReference("comma-separated list of <[ip_address:]port>"); conf->webserver.port.f = FLAG_ADVANCED_SETTING | FLAG_RESTART_FTL; conf->webserver.port.t = CONF_STRING; diff --git a/src/webserver/webserver.c b/src/webserver/webserver.c index 9159dc752..215843646 100644 --- a/src/webserver/webserver.c +++ b/src/webserver/webserver.c @@ -341,9 +341,15 @@ unsigned short get_api_string(char **buf, const bool domain) void http_init(void) { - log_web("Initializing HTTP server on port %s", config.webserver.port.v.s); + // Don't start web server if port is not set + if(strlen(config.webserver.port.v.s) == 0) + { + log_warn("Not starting web server as webserver.port is empty. API will not be available!"); + return; + } /* Initialize the library */ + log_web("Initializing HTTP server on port %s", config.webserver.port.v.s); unsigned int features = MG_FEATURES_FILES | MG_FEATURES_IPV6 | MG_FEATURES_CACHE; @@ -597,6 +603,7 @@ void FTL_rewrite_pattern(char *filename, size_t filename_buf_len) void http_terminate(void) { + // The server may have never been started if(!ctx) return; diff --git a/test/pihole.toml b/test/pihole.toml index 57b0c38de..867e2406e 100644 --- a/test/pihole.toml +++ b/test/pihole.toml @@ -530,14 +530,14 @@ # Comma-separated list of ports to listen on. It is possible to specify an IP address # to bind to. In this case, an IP address and a colon must be prepended to the port # number. For example, to bind to the loopback interface on port 80 (IPv4) and to all - # interfaces port 80 (IPv4), use "127.0.0.1:80,80". "[::]:808 can be used to + # interfaces port 8080 (IPv4), use "127.0.0.1:80,8080". "[::]:80" can be used to # listen to IPv6 connections to port 80. IPv6 addresses of network interfaces can be - # specified as well, e.g. "[::1]:80 for the IPv6 loopback interface. [::]:80 will + # specified as well, e.g. "[::1]:80" for the IPv6 loopback interface. [::]:80 will # bind to port 80 IPv6 only. # In order to use port 80 for all interfaces, both IPv4 and IPv6, use either the - # configuration "80,[::]:80" (create one socket for IPv4 and one for IPv6 only), - # or "+80" (create one socket for both, IPv4 and IPv6). The + notation to use IPv4 - # and IPv6 will only work if no network interface is specified. Depending on your + # configuration "80,[::]:80" (create one socket for IPv4 and one for IPv6 only), or + # "+80" (create one socket for both, IPv4 and IPv6). The + notation to use IPv4 and + # IPv6 will only work if no network interface is specified. Depending on your # operating system version and IPv6 network environment, some configurations might not # work as expected, so you have to test to find the configuration most suitable for # your needs. In case "+80" does not work for your environment, you need to use @@ -547,7 +547,8 @@ # non-encrypted ports, it is allowed to append letter 'r' (as in redirect). Redirected # ports will redirect all their traffic to the first configured SSL port. For example, # if webserver.port is "80r,443s", then all HTTP traffic coming at port 80 will be - # redirected to HTTPS port 443. + # redirected to HTTPS port 443. If this value is not set (empty string), the web + # server will not be started and, hence, the API will not be available. # # Possible values are: # comma-separated list of <[ip_address:]port> From 21d393b4beaef8210625b8f9086d5892cc8e139d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 4 Dec 2023 23:14:26 +0100 Subject: [PATCH 151/221] Ensure CPU usage check is really run only every ten seconds Signed-off-by: DL6ER --- src/gc.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/gc.c b/src/gc.c index 98b5b9da3..9be9fb1ea 100644 --- a/src/gc.c +++ b/src/gc.c @@ -493,6 +493,7 @@ void *GC_thread(void *val) time_t lastGCrun = time(NULL) - time(NULL)%GCinterval; lastRateLimitCleaner = time(NULL); time_t lastResourceCheck = 0; + time_t lastCPUcheck = 0; // Remember disk usage unsigned int LastLogStorageUsage = 0; @@ -523,8 +524,11 @@ void *GC_thread(void *val) // Calculate average CPU usage // This is done once every ten seconds to get averaged values - if(now - lastResourceCheck >= CPU_AVERAGE_INTERVAL) + if(now - lastCPUcheck >= CPU_AVERAGE_INTERVAL) + { + lastCPUcheck = now; calc_cpu_usage(CPU_AVERAGE_INTERVAL); + } // Check available resources if(now - lastResourceCheck >= RCinterval) From 0ef89ab3eba3b3509b7f40843cafadefccf1cc8a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 6 Dec 2023 23:27:20 +0100 Subject: [PATCH 152/221] Implement special POST :batchDelete callbacks for /api/groups, /api/domains/, /api/clients, and /api/lists Signed-off-by: DL6ER --- src/api/api.c | 140 ++++++----- src/api/docs/content/specs/clients.yaml | 55 +++- src/api/docs/content/specs/domains.yaml | 58 +++++ src/api/docs/content/specs/groups.yaml | 61 +++++ src/api/docs/content/specs/lists.yaml | 50 ++++ src/api/docs/content/specs/main.yaml | 12 + src/api/list.c | 213 +++++++++++++++- src/api/stats_database.c | 2 +- src/database/gravity-db.c | 319 ++++++++++++++++++------ src/database/gravity-db.h | 2 +- src/webserver/http-common.h | 10 +- 11 files changed, 762 insertions(+), 160 deletions(-) diff --git a/src/api/api.c b/src/api/api.c index 98eedf323..bb26c2a41 100644 --- a/src/api/api.c +++ b/src/api/api.c @@ -30,74 +30,78 @@ static struct { bool require_auth; enum http_method methods; } api_request[] = { - // URI ARGUMENTS FUNCTION OPTIONS AUTH ALLOWED METHODS - // domains json fifo + // URI ARGUMENTS FUNCTION OPTIONS AUTH ALLOWED METHODS + // flags fifo ID // Note: The order of appearance matters here, more specific URIs have to // appear *before* less specific URIs: 1. "/a/b/c", 2. "/a/b", 3. "/a" - { "/api/auth/sessions", "", api_auth_sessions, { false, true, 0 }, true, HTTP_GET }, - { "/api/auth/session", "/{id}", api_auth_session_delete, { false, true, 0 }, true, HTTP_DELETE }, - { "/api/auth/app", "", generateAppPw, { false, true, 0 }, true, HTTP_GET }, - { "/api/auth/totp", "", generateTOTP, { false, true, 0 }, true, HTTP_GET }, - { "/api/auth", "", api_auth, { false, true, 0 }, false, HTTP_GET | HTTP_POST | HTTP_DELETE }, - { "/api/dns/blocking", "", api_dns_blocking, { false, true, 0 }, true, HTTP_GET | HTTP_POST }, - { "/api/clients/_suggestions", "", api_client_suggestions, { false, true, 0 }, true, HTTP_GET }, - { "/api/clients", "/{client}", api_list, { false, true, 0 }, true, HTTP_GET | HTTP_PUT | HTTP_DELETE }, - { "/api/clients", "", api_list, { false, true, 0 }, true, HTTP_POST }, - { "/api/domains", "/{type}/{kind}/{domain}", api_list, { false, true, 0 }, true, HTTP_GET | HTTP_PUT | HTTP_DELETE }, - { "/api/domains", "/{type}/{kind}", api_list, { false, true, 0 }, true, HTTP_POST }, - { "/api/search", "/{domain}", api_search, { false, true, 0 }, true, HTTP_GET }, - { "/api/groups", "/{name}", api_list, { false, true, 0 }, true, HTTP_GET | HTTP_PUT | HTTP_DELETE }, - { "/api/groups", "", api_list, { false, true, 0 }, true, HTTP_POST }, - { "/api/lists", "/{list}", api_list, { false, true, 0 }, true, HTTP_GET | HTTP_PUT | HTTP_DELETE }, - { "/api/lists", "", api_list, { false, true, 0 }, true, HTTP_POST }, - { "/api/info/client", "", api_info_client, { false, true, 0 }, false, HTTP_GET }, - { "/api/info/login", "", api_info_login, { false, true, 0 }, false, HTTP_GET }, - { "/api/info/system", "", api_info_system, { false, true, 0 }, true, HTTP_GET }, - { "/api/info/database", "", api_info_database, { false, true, 0 }, true, HTTP_GET }, - { "/api/info/sensors", "", api_info_sensors, { false, true, 0 }, true, HTTP_GET }, - { "/api/info/host", "", api_info_host, { false, true, 0 }, true, HTTP_GET }, - { "/api/info/ftl", "", api_info_ftl, { false, true, 0 }, true, HTTP_GET }, - { "/api/info/version", "", api_info_version, { false, true, 0 }, true, HTTP_GET }, - { "/api/info/messages/count", "", api_info_messages_count, { false, true, 0 }, true, HTTP_GET }, - { "/api/info/messages", "/{message_id}", api_info_messages, { false, true, 0 }, true, HTTP_DELETE }, - { "/api/info/messages", "", api_info_messages, { false, true, 0 }, true, HTTP_GET }, - { "/api/info/metrics", "", api_info_metrics, { false, true, 0 }, true, HTTP_GET }, - { "/api/logs/dnsmasq", "", api_logs, { false, true, FIFO_DNSMASQ }, true, HTTP_GET }, - { "/api/logs/ftl", "", api_logs, { false, true, FIFO_FTL }, true, HTTP_GET }, - { "/api/logs/webserver", "", api_logs, { false, true, FIFO_WEBSERVER }, true, HTTP_GET }, - { "/api/history/clients", "", api_history_clients, { false, true, 0 }, true, HTTP_GET }, - { "/api/history/database/clients", "", api_history_database_clients, { false, true, 0 }, true, HTTP_GET }, - { "/api/history/database", "", api_history_database, { false, true, 0 }, true, HTTP_GET }, - { "/api/history", "", api_history, { false, true, 0 }, true, HTTP_GET }, - { "/api/queries/suggestions", "", api_queries_suggestions, { false, true, 0 }, true, HTTP_GET }, - { "/api/queries", "", api_queries, { false, true, 0 }, true, HTTP_GET }, - { "/api/stats/summary", "", api_stats_summary, { false, true, 0 }, true, HTTP_GET }, - { "/api/stats/query_types", "", api_stats_query_types, { false, true, 0 }, true, HTTP_GET }, - { "/api/stats/upstreams", "", api_stats_upstreams, { false, true, 0 }, true, HTTP_GET }, - { "/api/stats/top_domains", "", api_stats_top_domains, { false, true, 0 }, true, HTTP_GET }, - { "/api/stats/top_clients", "", api_stats_top_clients, { false, true, 0 }, true, HTTP_GET }, - { "/api/stats/recent_blocked", "", api_stats_recentblocked, { false, true, 0 }, true, HTTP_GET }, - { "/api/stats/database/top_domains", "", api_stats_database_top_items, { true, true, 0 }, true, HTTP_GET }, - { "/api/stats/database/top_clients", "", api_stats_database_top_items, { false, true, 0 }, true, HTTP_GET }, - { "/api/stats/database/summary", "", api_stats_database_summary, { false, true, 0 }, true, HTTP_GET }, - { "/api/stats/database/query_types", "", api_stats_database_query_types, { false, true, 0 }, true, HTTP_GET }, - { "/api/stats/database/upstreams", "", api_stats_database_upstreams, { false, true, 0 }, true, HTTP_GET }, - { "/api/config", "", api_config, { false, true, 0 }, true, HTTP_GET | HTTP_PATCH }, - { "/api/config", "/{element}", api_config, { false, true, 0 }, true, HTTP_GET }, - { "/api/config", "/{element}/{value}", api_config, { false, true, 0 }, true, HTTP_DELETE | HTTP_PUT }, - { "/api/network/gateway", "", api_network_gateway, { false, true, 0 }, true, HTTP_GET }, - { "/api/network/interfaces", "", api_network_interfaces, { false, true, 0 }, true, HTTP_GET }, - { "/api/network/devices", "", api_network_devices, { false, true, 0 }, true, HTTP_GET }, - { "/api/network/devices", "/{device_id}", api_network_devices, { false, true, 0 }, true, HTTP_DELETE }, - { "/api/endpoints", "", api_endpoints, { false, true, 0 }, true, HTTP_GET }, - { "/api/teleporter", "", api_teleporter, { false, false, 0 }, true, HTTP_GET | HTTP_POST }, - { "/api/dhcp/leases", "", api_dhcp_leases_GET, { false, true, 0 }, true, HTTP_GET }, - { "/api/dhcp/leases", "/{ip}", api_dhcp_leases_DELETE, { false, true, 0 }, true, HTTP_DELETE }, - { "/api/action/gravity", "", api_action_gravity, { false, true, 0 }, true, HTTP_POST }, - { "/api/action/restartdns", "", api_action_restartDNS, { false, true, 0 }, true, HTTP_POST }, - { "/api/action/flush/logs", "", api_action_flush_logs, { false, true, 0 }, true, HTTP_POST }, - { "/api/action/flush/arp", "", api_action_flush_arp, { false, true, 0 }, true, HTTP_POST }, - { "/api/docs", "", api_docs, { false, true, 0 }, false, HTTP_GET }, + { "/api/auth/sessions", "", api_auth_sessions, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/auth/session", "/{id}", api_auth_session_delete, { API_PARSE_JSON, 0 }, true, HTTP_DELETE }, + { "/api/auth/app", "", generateAppPw, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/auth/totp", "", generateTOTP, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/auth", "", api_auth, { API_PARSE_JSON, 0 }, false, HTTP_GET | HTTP_POST | HTTP_DELETE }, + { "/api/dns/blocking", "", api_dns_blocking, { API_PARSE_JSON, 0 }, true, HTTP_GET | HTTP_POST }, + { "/api/clients/_suggestions", "", api_client_suggestions, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/clients", "/{client}", api_list, { API_PARSE_JSON, 0 }, true, HTTP_GET | HTTP_PUT | HTTP_DELETE }, + { "/api/clients", "", api_list, { API_PARSE_JSON, 0 }, true, HTTP_POST }, + { "/api/clients:batchDelete", "", api_list, { API_PARSE_JSON | API_BATCHDELETE, 0 }, true, HTTP_POST }, + { "/api/domains", "/{type}/{kind}/{domain}", api_list, { API_PARSE_JSON, 0 }, true, HTTP_GET | HTTP_PUT | HTTP_DELETE }, + { "/api/domains", "/{type}/{kind}", api_list, { API_PARSE_JSON, 0 }, true, HTTP_POST }, + { "/api/domains:batchDelete", "", api_list, { API_PARSE_JSON | API_BATCHDELETE, 0 }, true, HTTP_POST }, + { "/api/search", "/{domain}", api_search, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/groups", "/{name}", api_list, { API_PARSE_JSON, 0 }, true, HTTP_GET | HTTP_PUT | HTTP_DELETE }, + { "/api/groups", "", api_list, { API_PARSE_JSON, 0 }, true, HTTP_POST }, + { "/api/groups:batchDelete", "", api_list, { API_PARSE_JSON | API_BATCHDELETE, 0 }, true, HTTP_POST }, + { "/api/lists", "/{list}", api_list, { API_PARSE_JSON, 0 }, true, HTTP_GET | HTTP_PUT | HTTP_DELETE }, + { "/api/lists", "", api_list, { API_PARSE_JSON, 0 }, true, HTTP_POST }, + { "/api/lists:batchDelete", "", api_list, { API_PARSE_JSON | API_BATCHDELETE, 0 }, true, HTTP_POST }, + { "/api/info/client", "", api_info_client, { API_PARSE_JSON, 0 }, false, HTTP_GET }, + { "/api/info/login", "", api_info_login, { API_PARSE_JSON, 0 }, false, HTTP_GET }, + { "/api/info/system", "", api_info_system, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/info/database", "", api_info_database, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/info/sensors", "", api_info_sensors, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/info/host", "", api_info_host, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/info/ftl", "", api_info_ftl, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/info/version", "", api_info_version, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/info/messages/count", "", api_info_messages_count, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/info/messages", "/{message_id}", api_info_messages, { API_PARSE_JSON, 0 }, true, HTTP_DELETE }, + { "/api/info/messages", "", api_info_messages, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/info/metrics", "", api_info_metrics, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/logs/dnsmasq", "", api_logs, { API_PARSE_JSON, FIFO_DNSMASQ }, true, HTTP_GET }, + { "/api/logs/ftl", "", api_logs, { API_PARSE_JSON, FIFO_FTL }, true, HTTP_GET }, + { "/api/logs/webserver", "", api_logs, { API_PARSE_JSON, FIFO_WEBSERVER }, true, HTTP_GET }, + { "/api/history/clients", "", api_history_clients, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/history/database/clients", "", api_history_database_clients, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/history/database", "", api_history_database, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/history", "", api_history, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/queries/suggestions", "", api_queries_suggestions, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/queries", "", api_queries, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/stats/summary", "", api_stats_summary, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/stats/query_types", "", api_stats_query_types, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/stats/upstreams", "", api_stats_upstreams, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/stats/top_domains", "", api_stats_top_domains, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/stats/top_clients", "", api_stats_top_clients, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/stats/recent_blocked", "", api_stats_recentblocked, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/stats/database/top_domains", "", api_stats_database_top_items, { API_DOMAINS | API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/stats/database/top_clients", "", api_stats_database_top_items, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/stats/database/summary", "", api_stats_database_summary, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/stats/database/query_types", "", api_stats_database_query_types, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/stats/database/upstreams", "", api_stats_database_upstreams, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/config", "", api_config, { API_PARSE_JSON, 0 }, true, HTTP_GET | HTTP_PATCH }, + { "/api/config", "/{element}", api_config, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/config", "/{element}/{value}", api_config, { API_PARSE_JSON, 0 }, true, HTTP_DELETE | HTTP_PUT }, + { "/api/network/gateway", "", api_network_gateway, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/network/interfaces", "", api_network_interfaces, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/network/devices", "", api_network_devices, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/network/devices", "/{device_id}", api_network_devices, { API_PARSE_JSON, 0 }, true, HTTP_DELETE }, + { "/api/endpoints", "", api_endpoints, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/teleporter", "", api_teleporter, { API_FLAG_NONE, 0 }, true, HTTP_GET | HTTP_POST }, + { "/api/dhcp/leases", "", api_dhcp_leases_GET, { API_PARSE_JSON, 0 }, true, HTTP_GET }, + { "/api/dhcp/leases", "/{ip}", api_dhcp_leases_DELETE, { API_PARSE_JSON, 0 }, true, HTTP_DELETE }, + { "/api/action/gravity", "", api_action_gravity, { API_PARSE_JSON, 0 }, true, HTTP_POST }, + { "/api/action/restartdns", "", api_action_restartDNS, { API_PARSE_JSON, 0 }, true, HTTP_POST }, + { "/api/action/flush/logs", "", api_action_flush_logs, { API_PARSE_JSON, 0 }, true, HTTP_POST }, + { "/api/action/flush/arp", "", api_action_flush_arp, { API_PARSE_JSON, 0 }, true, HTTP_POST }, + { "/api/docs", "", api_docs, { API_PARSE_JSON, 0 }, false, HTTP_GET }, }; int api_handler(struct mg_connection *conn, void *ignored) @@ -113,7 +117,7 @@ int api_handler(struct mg_connection *conn, void *ignored) double_time(), { false, NULL, NULL, NULL, 0u }, { false }, - { false, false, 0 } + { API_FLAG_NONE, 0 } }; log_debug(DEBUG_API, "Requested API URI: %s -> %s %s ? %s (Content-Type %s)", @@ -149,7 +153,7 @@ int api_handler(struct mg_connection *conn, void *ignored) continue; } - if(api_request[i].opts.parse_json) + if(api_request[i].opts.flags & API_PARSE_JSON) { // Allocate memory for the payload api.payload.raw = calloc(MAX_PAYLOAD_BYTES, sizeof(char)); diff --git a/src/api/docs/content/specs/clients.yaml b/src/api/docs/content/specs/clients.yaml index 557a68a29..d7619c5bd 100644 --- a/src/api/docs/content/specs/clients.yaml +++ b/src/api/docs/content/specs/clients.yaml @@ -149,7 +149,7 @@ components: Creates a new client in the `clients` object. The `{client}` itself is specified in the request body (POST JSON). Clients may be described either by their IP addresses (IPv4 and IPv6 are supported), - IP subnets (CIDR notation, like `192.168.2.0/24`), their MAC addresses (like `12:34:56:78:9A:BC`), by their hostnames (like `localhost`), or by the interface they are connected to (prefaced with a colon, like `:eth0`).

    + IP subnets (CIDR notation, like `192.168.2.0/24`), their MAC addresses (like `12:34:56:78:9A:BC`), by their hostnames (like `localhost`), or by the interface they are connected to (prefaced with a colon, like `:eth0`). Note that client recognition by IP addresses (incl. subnet ranges) is preferred over MAC address, host name or interface recognition as the two latter will only be available after some time. Furthermore, MAC address recognition only works for devices at most one networking hop away from your Pi-hole. @@ -199,6 +199,59 @@ components: allOf: - $ref: 'common.yaml#/components/schemas/took' - $ref: 'common.yaml#/components/errors/unauthorized' + batchDelete: + post: + summary: Delete multiple clients + tags: + - "Client management" + operationId: "batchDelete_clients" + description: | + Deletes multiple clients in the `clients` object. The `{client}`s themselves are specified in the request body (POST JSON). + + Clients may be described either by their IP addresses (IPv4 and IPv6 are supported), + IP subnets (CIDR notation, like `192.168.2.0/24`), their MAC addresses (like `12:34:56:78:9A:BC`), by their hostnames (like `localhost`), or by the interface they are connected to (prefaced with a colon, like `:eth0`).

    + + *Note:* There will be no content on success. + requestBody: + description: Callback payload + content: + application/json: + schema: + type: array + items: + type: object + properties: + item: + type: string + description: client IP / MAC / hostname / interface + example: + - "item": "192.168.2.5" + - "item": "::1" + - "item": "12:34:56:78:9A:BC" + - "item": "localhost" + - "item": ":eth0" + responses: + '204': + description: Items deleted + '400': + description: Bad request + content: + application/json: + schema: + allOf: + - $ref: 'common.yaml#/components/errors/bad_request' + - $ref: 'common.yaml#/components/schemas/took' + examples: + no_payload: + $ref: 'clients.yaml#/components/examples/errors/bad_request/no_payload' + '401': + description: Unauthorized + content: + application/json: + schema: + allOf: + - $ref: 'common.yaml#/components/errors/unauthorized' + - $ref: 'common.yaml#/components/schemas/took' schemas: clients: get: diff --git a/src/api/docs/content/specs/domains.yaml b/src/api/docs/content/specs/domains.yaml index 773cbeba3..b67cf0e6e 100644 --- a/src/api/docs/content/specs/domains.yaml +++ b/src/api/docs/content/specs/domains.yaml @@ -212,6 +212,64 @@ components: allOf: - $ref: 'common.yaml#/components/errors/unauthorized' - $ref: 'common.yaml#/components/schemas/took' + batchDelete: + summary: Delete multiple domains + post: + summary: Delete multiple domains + tags: + - "Domain management" + operationId: "batchDelete_domains" + description: | + *Note:* There will be no content on success. + requestBody: + description: Callback payload + content: + application/json: + schema: + type: array + items: + type: object + properties: + item: + type: string + description: Domain to delete + example: "example.com" + type: + type: string + description: Type of domain to delete + enum: + - "allow" + - "deny" + example: "allow" + kind: + type: string + description: Kind of domain to delete + enum: + - "exact" + - "regex" + example: "exact" + responses: + '204': + description: Items deleted + '400': + description: Bad request + content: + application/json: + schema: + allOf: + - $ref: 'common.yaml#/components/errors/bad_request' + - $ref: 'common.yaml#/components/schemas/took' + examples: + no_payload: + $ref: 'domains.yaml#/components/examples/errors/bad_request/no_payload' + '401': + description: Unauthorized + content: + application/json: + schema: + allOf: + - $ref: 'common.yaml#/components/errors/unauthorized' + - $ref: 'common.yaml#/components/schemas/took' schemas: domains: get: diff --git a/src/api/docs/content/specs/groups.yaml b/src/api/docs/content/specs/groups.yaml index c70571c68..499a238c8 100644 --- a/src/api/docs/content/specs/groups.yaml +++ b/src/api/docs/content/specs/groups.yaml @@ -165,6 +165,67 @@ components: allOf: - $ref: 'common.yaml#/components/errors/unauthorized' - $ref: 'common.yaml#/components/schemas/took' + batchDelete: + post: + summary: Delete multiple groups + tags: + - "Group management" + operationId: "batchDelete_groups" + description: | + Deletes multiple groups in the `groups` object. The `{groups}` themselves are specified in the request body (POST JSON). + + On success, a new resource is created at `/groups/{name}`. + + The `database_error` with message `UNIQUE constraint failed` error indicates that a group with the same name already exists. + requestBody: + description: Callback payload + content: + application/json: + schema: + type: array + items: + type: object + properties: + item: + type: string + description: group name + example: + - "item": "test1" + - "item": "test2" + responses: + '201': + description: Created item + content: + application/json: + schema: + allOf: + - $ref: 'groups.yaml#/components/schemas/groups/get' # identical to GET + - $ref: 'groups.yaml#/components/schemas/lists_processed' + - $ref: 'common.yaml#/components/schemas/took' + headers: + Location: + $ref: 'common.yaml#/components/headers/Location' + '400': + description: Bad request + content: + application/json: + schema: + allOf: + - $ref: 'common.yaml#/components/errors/bad_request' + - $ref: 'common.yaml#/components/schemas/took' + examples: + no_payload: + $ref: 'groups.yaml#/components/examples/errors/bad_request/no_payload' + duplicate: + $ref: 'groups.yaml#/components/examples/errors/database_error/duplicate' + '401': + description: Unauthorized + content: + application/json: + schema: + allOf: + - $ref: 'common.yaml#/components/errors/unauthorized' + - $ref: 'common.yaml#/components/schemas/took' schemas: groups: get: diff --git a/src/api/docs/content/specs/lists.yaml b/src/api/docs/content/specs/lists.yaml index f6536a764..55a95425a 100644 --- a/src/api/docs/content/specs/lists.yaml +++ b/src/api/docs/content/specs/lists.yaml @@ -170,6 +170,56 @@ components: allOf: - $ref: 'common.yaml#/components/errors/unauthorized' - $ref: 'common.yaml#/components/schemas/took' + batchDelete: + post: + summary: Delete lists + tags: + - "List management" + operationId: "batchDelete_lists" + description: | + Deletes multiple lists in the `lists` object. The `{list}`s themselves are specified in the request body (POST JSON). + + On success, a new resource is created at `/lists/{list}`. + + The `database_error` with message `UNIQUE constraint failed` error indicates that this list already exists. + requestBody: + description: Callback payload + content: + application/json: + schema: + $ref: 'lists.yaml#/components/schemas/lists/post' + responses: + '201': + description: Created item + content: + application/json: + schema: + allOf: + - $ref: 'lists.yaml#/components/schemas/lists/get' + - $ref: 'lists.yaml#/components/schemas/lists_processed' + - $ref: 'common.yaml#/components/schemas/took' + headers: + Location: + $ref: 'common.yaml#/components/headers/Location' + '400': + description: Bad request + content: + application/json: + schema: + allOf: + - $ref: 'common.yaml#/components/errors/bad_request' + - $ref: 'common.yaml#/components/schemas/took' + examples: + no_payload: + $ref: 'lists.yaml#/components/examples/errors/bad_request/no_payload' + '401': + description: Unauthorized + content: + application/json: + schema: + allOf: + - $ref: 'common.yaml#/components/errors/unauthorized' + - $ref: 'common.yaml#/components/schemas/took' schemas: lists: get: diff --git a/src/api/docs/content/specs/main.yaml b/src/api/docs/content/specs/main.yaml index 3e8e1b0d3..1762aed4f 100644 --- a/src/api/docs/content/specs/main.yaml +++ b/src/api/docs/content/specs/main.yaml @@ -142,18 +142,27 @@ paths: /domains/{type}/{kind}: $ref: 'domains.yaml#/components/paths/type_kind' + /domains:batchDelete: + $ref: 'domains.yaml#/components/paths/batchDelete' + /groups/{name}: $ref: 'groups.yaml#/components/paths/name' /groups: $ref: 'groups.yaml#/components/paths/direct' + /groups:batchDelete: + $ref: 'groups.yaml#/components/paths/batchDelete' + /clients/{client}: $ref: 'clients.yaml#/components/paths/client' /clients: $ref: 'clients.yaml#/components/paths/direct' + /clients:batchDelete: + $ref: 'clients.yaml#/components/paths/batchDelete' + /clients/_suggestions: $ref: 'clients.yaml#/components/paths/suggestions' @@ -163,6 +172,9 @@ paths: /lists: $ref: 'lists.yaml#/components/paths/direct' + /lists:batchDelete: + $ref: 'lists.yaml#/components/paths/batchDelete' + /info/client: $ref: 'info.yaml#/components/paths/client' diff --git a/src/api/list.c b/src/api/list.c index f01858464..a7cef1249 100644 --- a/src/api/list.c +++ b/src/api/list.c @@ -503,7 +503,7 @@ static int api_list_write(struct ftl_conn *api, cJSON_AddItemToArray(okay ? success : errors, details); } - // Inform the resolver that it needs to reload the domainlists + // Inform the resolver that it needs to reload gravity set_event(RELOAD_GRAVITY); int response_code = 201; // 201 - Created @@ -525,21 +525,195 @@ static int api_list_remove(struct ftl_conn *api, const char *item) { const char *sql_msg = NULL; - if(gravityDB_delFromTable(listtype, item, &sql_msg)) + cJSON *array = api->payload.json; + bool allocated_json = false; + + // If this is not a :batchDelete call, then the item is specified in the + // URI, not in the payload. Create a JSON array with the item and use + // that instead + const bool isBatchDelete = api->opts.flags & API_BATCHDELETE; + + // If this is a domain callback, we need to translate type/kind into an + // integer for use in the database + if(listtype == GRAVITY_DOMAINLIST_ALLOW_EXACT || + listtype == GRAVITY_DOMAINLIST_DENY_EXACT || + listtype == GRAVITY_DOMAINLIST_ALLOW_REGEX || + listtype == GRAVITY_DOMAINLIST_DENY_REGEX) { - // Inform the resolver that it needs to reload the domainlists + int type = -1; + switch (listtype) + { + case GRAVITY_DOMAINLIST_ALLOW_EXACT: + type = 0; + break; + case GRAVITY_DOMAINLIST_DENY_EXACT: + type = 1; + break; + case GRAVITY_DOMAINLIST_ALLOW_REGEX: + type = 2; + break; + case GRAVITY_DOMAINLIST_DENY_REGEX: + type = 3; + case GRAVITY_GROUPS: + case GRAVITY_ADLISTS: + case GRAVITY_CLIENTS: + // No type required for these tables + break; + // Aggregate types cannot be handled by this routine + case GRAVITY_GRAVITY: + case GRAVITY_ANTIGRAVITY: + case GRAVITY_DOMAINLIST_ALLOW_ALL: + case GRAVITY_DOMAINLIST_DENY_ALL: + case GRAVITY_DOMAINLIST_ALL_EXACT: + case GRAVITY_DOMAINLIST_ALL_REGEX: + case GRAVITY_DOMAINLIST_ALL_ALL: + default: + return false; + } + + // Create new JSON array with the item and type: + // array = [{"item": "example.com", "type": 0}] + array = cJSON_CreateArray(); + cJSON *obj = cJSON_CreateObject(); + cJSON_AddItemToObject(obj, "item", cJSON_CreateStringReference(item)); + cJSON_AddItemToObject(obj, "type", cJSON_CreateNumber(type)); + cJSON_AddItemToArray(array, obj); + allocated_json = true; + } + else if(isBatchDelete && listtype == GRAVITY_DOMAINLIST_ALL_ALL) + { + // Loop over all items and parse type/kind for each item + cJSON *it = NULL; + cJSON_ArrayForEach(it, array) + { + if(!cJSON_IsObject(it)) + { + return send_json_error(api, 400, + "bad_request", + "Invalid request: Batch delete requires an array of objects", + NULL); + } + + // Check if item is a string + cJSON *json_item = cJSON_GetObjectItemCaseSensitive(it, "item"); + if(!cJSON_IsString(json_item)) + { + return send_json_error(api, 400, + "bad_request", + "Invalid request: Batch delete requires an array of objects with \"item\" as string", + NULL); + } + + // Check if type and kind are both present and strings + cJSON *json_type = cJSON_GetObjectItemCaseSensitive(it, "type"); + cJSON *json_kind = cJSON_GetObjectItemCaseSensitive(it, "kind"); + if(!cJSON_IsString(json_type) || !cJSON_IsString(json_kind)) + { + return send_json_error(api, 400, + "bad_request", + "Invalid request: Batch delete requires an array of objects with \"type\" and \"kind\" as string", + NULL); + } + + // Parse type and kind + // 0 = allow exact + // 1 = deny exact + // 2 = allow regex + // 3 = deny regex + int type = -1; + if(strcasecmp(json_type->valuestring, "allow") == 0) + { + if(strcasecmp(json_kind->valuestring, "exact") == 0) + type = 0; + else if(strcasecmp(json_kind->valuestring, "regex") == 0) + type = 2; + } + else if(strcasecmp(json_type->valuestring, "deny") == 0) + { + if(strcasecmp(json_kind->valuestring, "exact") == 0) + type = 1; + else if(strcasecmp(json_kind->valuestring, "regex") == 0) + type = 3; + } + + // Check if type/kind combination is valid + if(type == -1) + { + return send_json_error(api, 400, + "bad_request", + "Invalid request: Batch delete requires an valid combination of \"type\" and \"kind\" for each object", + NULL); + } + + // Replace type/kind with integer type + // array = [{"item": "example.com", "type": 0}] + cJSON_DeleteItemFromObject(it, "type"); + cJSON_DeleteItemFromObject(it, "kind"); + cJSON_AddNumberToObject(it, "type", type); + } + } + else if(!isBatchDelete) + { + // Create array with object (used for clients, groups, lists) + // array = [{"item": }] + array = cJSON_CreateArray(); + cJSON *obj = cJSON_CreateObject(); + cJSON_AddItemToObject(obj, "item", cJSON_CreateStringReference(item)); + cJSON_AddItemToArray(array, obj); + allocated_json = true; + } + + // Verify that the payload is an array of objects each containing an + // item + if(isBatchDelete) + { + cJSON *it = NULL; + cJSON_ArrayForEach(it, array) + { + if(!cJSON_IsObject(it)) + { + return send_json_error(api, 400, + "bad_request", + "Invalid request: Batch delete requires an array of objects", + NULL); + } + + // Check if item is a string + cJSON *json_item = cJSON_GetObjectItemCaseSensitive(it, "item"); + if(!cJSON_IsString(json_item)) + { + return send_json_error(api, 400, + "bad_request", + "Invalid request: Batch delete requires an array of objects with \"item\" as string", + NULL); + } + } + } + + // From here on, we can assume the JSON payload is valid + if(gravityDB_delFromTable(listtype, array, &sql_msg)) + { + // Inform the resolver that it needs to reload gravity set_event(RELOAD_GRAVITY); + // Free memory allocated above + if(allocated_json) + cJSON_free(array); + // Send empty reply with code 204 No Content cJSON *json = JSON_NEW_OBJECT(); JSON_SEND_OBJECT_CODE(json, 204); } else { + // Free memory allocated above + if(allocated_json) + cJSON_free(array); + // Send error reply return send_json_error(api, 400, "database_error", - "Could not remove domain from database table", + "Could not remove entries from table", sql_msg); } } @@ -548,21 +722,40 @@ int api_list(struct ftl_conn *api) { enum gravity_list_type listtype; bool can_modify = false; + bool batchDelete = false; if((api->item = startsWith("/api/groups", api)) != NULL) { listtype = GRAVITY_GROUPS; can_modify = true; } + else if((api->item = startsWith("/api/groups:batchDelete", api)) != NULL) + { + listtype = GRAVITY_GROUPS; + can_modify = true; + batchDelete = true; + } else if((api->item = startsWith("/api/lists", api)) != NULL) { listtype = GRAVITY_ADLISTS; can_modify = true; } + else if((api->item = startsWith("/api/lists:batchDelete", api)) != NULL) + { + listtype = GRAVITY_ADLISTS; + can_modify = true; + batchDelete = true; + } else if((api->item = startsWith("/api/clients", api)) != NULL) { listtype = GRAVITY_CLIENTS; can_modify = true; } + else if((api->item = startsWith("/api/clients:batchDelete", api)) != NULL) + { + listtype = GRAVITY_CLIENTS; + can_modify = true; + batchDelete = true; + } else if((api->item = startsWith("/api/domains/allow/exact", api)) != NULL) { listtype = GRAVITY_DOMAINLIST_ALLOW_EXACT; @@ -603,6 +796,12 @@ int api_list(struct ftl_conn *api) { listtype = GRAVITY_DOMAINLIST_ALL_ALL; } + else if((api->item = startsWith("/api/domains:batchDelete", api)) != NULL) + { + listtype = GRAVITY_DOMAINLIST_ALL_ALL; + can_modify = true; + batchDelete = true; + } else { return send_json_error(api, 400, @@ -643,7 +842,7 @@ int api_list(struct ftl_conn *api) return ret; } } - else if(can_modify && api->method == HTTP_POST) + else if(can_modify && api->method == HTTP_POST && !batchDelete) { // Add item to list identified by payload if(api->item != NULL && strlen(api->item) != 0) @@ -651,7 +850,7 @@ int api_list(struct ftl_conn *api) return send_json_error(api, 400, "uri_error", "Invalid request: Specify item in payload, not as URI parameter", - NULL); + api->item); } else { @@ -664,7 +863,7 @@ int api_list(struct ftl_conn *api) return ret; } } - else if(can_modify && api->method == HTTP_DELETE) + else if(can_modify && (api->method == HTTP_DELETE || (api->method == HTTP_POST && batchDelete))) { // Delete item from list // We would not actually need the SHM lock here, however, we do diff --git a/src/api/stats_database.c b/src/api/stats_database.c index aef2fed60..f6fdcf179 100644 --- a/src/api/stats_database.c +++ b/src/api/stats_database.c @@ -181,7 +181,7 @@ int api_stats_database_top_items(struct ftl_conn *api) // Get options from API struct bool blocked = false; // Can be overwritten by query string - const bool domains = api->opts.domains; + const bool domains = api->opts.flags & API_DOMAINS; // Get parameters from query string if(api->request->query_string != NULL) diff --git a/src/database/gravity-db.c b/src/database/gravity-db.c index 12558f0b4..ed508fd7c 100644 --- a/src/database/gravity-db.c +++ b/src/database/gravity-db.c @@ -1746,15 +1746,15 @@ bool gravityDB_addToTable(const enum gravity_list_type listtype, tablerow *row, { if(strcasecmp("allow", row->type) == 0 && strcasecmp("exact", row->kind) == 0) - oldtype = 0; + oldtype = 0; else if(strcasecmp("deny", row->type) == 0 && strcasecmp("exact", row->kind) == 0) - oldtype = 1; + oldtype = 1; else if(strcasecmp("allow", row->type) == 0 && strcasecmp("regex", row->kind) == 0) - oldtype = 2; + oldtype = 2; else if(strcasecmp("deny", row->type) == 0 && - strcasecmp("regex", row->kind) == 0) + strcasecmp("regex", row->kind) == 0) oldtype = 3; else { @@ -1838,7 +1838,7 @@ bool gravityDB_addToTable(const enum gravity_list_type listtype, tablerow *row, return okay; } -bool gravityDB_delFromTable(const enum gravity_list_type listtype, const char* argument, const char **message) +bool gravityDB_delFromTable(const enum gravity_list_type listtype, const cJSON* array, const char **message) { if(gravity_db == NULL) { @@ -1846,123 +1846,282 @@ bool gravityDB_delFromTable(const enum gravity_list_type listtype, const char* a return false; } - int type = -1; - switch (listtype) + // Return early if passed JSON argument is not an array + if(!cJSON_IsArray(array)) { - case GRAVITY_DOMAINLIST_ALLOW_EXACT: - type = 0; - break; - case GRAVITY_DOMAINLIST_DENY_EXACT: - type = 1; - break; - case GRAVITY_DOMAINLIST_ALLOW_REGEX: - type = 2; - break; - case GRAVITY_DOMAINLIST_DENY_REGEX: - type = 3; - break; + *message = "Argument is not an array"; + log_err("gravityDB_delFromTable(%d): %s", + listtype, *message); + return false; + } - case GRAVITY_GROUPS: - case GRAVITY_ADLISTS: - case GRAVITY_CLIENTS: - // No type required for these tables - break; + const bool isDomain = listtype == GRAVITY_DOMAINLIST_ALLOW_EXACT || + listtype == GRAVITY_DOMAINLIST_DENY_EXACT || + listtype == GRAVITY_DOMAINLIST_ALLOW_REGEX || + listtype == GRAVITY_DOMAINLIST_DENY_REGEX || + listtype == GRAVITY_DOMAINLIST_ALL_ALL; // batch delete - // Aggregate types cannot be handled by this routine - case GRAVITY_GRAVITY: - case GRAVITY_ANTIGRAVITY: - case GRAVITY_DOMAINLIST_ALLOW_ALL: - case GRAVITY_DOMAINLIST_DENY_ALL: - case GRAVITY_DOMAINLIST_ALL_EXACT: - case GRAVITY_DOMAINLIST_ALL_REGEX: - case GRAVITY_DOMAINLIST_ALL_ALL: - default: - return false; + // Begin transaction + const char *querystr = "BEGIN TRANSACTION;"; + int rc = sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL); + if(rc != SQLITE_OK) + { + *message = sqlite3_errmsg(gravity_db); + log_err("gravityDB_delFromTable(%d): SQL error exec(\"%s\"): %s", + listtype, querystr, *message); + return false; } - // Prepare SQLite statement + // Create temporary table for JSON argument + if(isDomain) + // Create temporary table for domains to be deleted + querystr = "CREATE TEMPORARY TABLE deltable (type INT, item TEXT);"; + else + querystr = "CREATE TEMPORARY TABLE deltable (item TEXT);"; + sqlite3_stmt* stmt = NULL; - const char *querystr[3] = {NULL, NULL, NULL}; - if(listtype == GRAVITY_GROUPS) - querystr[0] = "DELETE FROM \"group\" WHERE name = :argument;"; - else if(listtype == GRAVITY_ADLISTS) + rc = sqlite3_prepare_v2(gravity_db, querystr, -1, &stmt, NULL); + if( rc != SQLITE_OK ) { - // This is actually a three-step deletion to satisfy foreign-key constraints - querystr[0] = "DELETE FROM gravity WHERE adlist_id = (SELECT id FROM adlist WHERE address = :argument);"; - querystr[1] = "DELETE FROM antigravity WHERE adlist_id = (SELECT id FROM adlist WHERE address = :argument);"; - querystr[2] = "DELETE FROM adlist WHERE address = :argument;"; + *message = sqlite3_errmsg(gravity_db); + log_err("gravityDB_delFromTable(%d) - SQL error prepare(\"%s\"): %s", + listtype, querystr, *message); + // Rollback transaction + querystr = "ROLLBACK TRANSACTION;"; + rc = sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL); + if(rc != SQLITE_OK) + { + *message = sqlite3_errmsg(gravity_db); + log_err("gravityDB_delFromTable(%d): SQL error exec(\"%s\"): %s", + listtype, querystr, *message); + } + return false; } - else if(listtype == GRAVITY_CLIENTS) - querystr[0] = "DELETE FROM client WHERE ip = :argument;"; - else // domainlist - querystr[0] = "DELETE FROM domainlist WHERE domain = :argument AND type = :type;"; - bool okay = true; - for(unsigned int i = 0; i < ArraySize(querystr); i++) + // Execute statement + if((rc = sqlite3_step(stmt)) != SQLITE_DONE) { - // Finish if no more queries - if(querystr[i] == NULL) - break; + *message = sqlite3_errmsg(gravity_db); + log_err("gravityDB_delFromTable(%d) - SQL error step(\"%s\"): %s", + listtype, querystr, *message); + sqlite3_reset(stmt); + sqlite3_finalize(stmt); + // Rollback transaction + querystr = "ROLLBACK TRANSACTION;"; + rc = sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL); + if(rc != SQLITE_OK) + { + *message = sqlite3_errmsg(gravity_db); + log_err("gravityDB_delFromTable(%d): SQL error exec(\"%s\"): %s", + listtype, querystr, *message); + } + return false; + } + + // Finalize statement + sqlite3_reset(stmt); + sqlite3_finalize(stmt); - // We need to perform a second SQL request - int rc = sqlite3_prepare_v2(gravity_db, querystr[i], -1, &stmt, NULL); - if( rc != SQLITE_OK ) + // Prepare statement for inserting items into virtual table + if(isDomain) + querystr = "INSERT INTO deltable (type, item) VALUES (:type, :item);"; + else + querystr = "INSERT INTO deltable (item) VALUES (:item);"; + + rc = sqlite3_prepare_v2(gravity_db, querystr, -1, &stmt, NULL); + if( rc != SQLITE_OK ) + { + *message = sqlite3_errmsg(gravity_db); + log_err("gravityDB_delFromTable(%d) - SQL error prepare(\"%s\"): %s", + listtype, querystr, *message); + // Rollback transaction + querystr = "ROLLBACK TRANSACTION;"; + rc = sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL); + if(rc != SQLITE_OK) { *message = sqlite3_errmsg(gravity_db); - log_err("gravityDB_delFromTable(%d, %s) - SQL error prepare %u (%i): %s", - type, argument, i, rc, *message); + log_err("gravityDB_delFromTable(%d): SQL error exec(\"%s\"): %s", + listtype, querystr, *message); + } + return false; + } + + // Loop over all domains in the JSON array + cJSON *it = NULL; + cJSON_ArrayForEach(it, array) + { + // Bind type to prepared statement + cJSON *type = cJSON_GetObjectItemCaseSensitive(it, "type"); + const int type_idx = sqlite3_bind_parameter_index(stmt, ":type"); + if(type_idx > 0 && (!cJSON_IsNumber(type) || (rc = sqlite3_bind_int(stmt, type_idx, type->valueint)) != SQLITE_OK)) + { + *message = sqlite3_errmsg(gravity_db); + log_err("gravityDB_delFromTable(%d): Failed to bind type (error %d) - %s", + type->valueint, rc, *message); + sqlite3_reset(stmt); + sqlite3_finalize(stmt); + // Rollback transaction + querystr = "ROLLBACK TRANSACTION;"; + rc = sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL); + if(rc != SQLITE_OK) + { + *message = sqlite3_errmsg(gravity_db); + log_err("gravityDB_delFromTable(%d): SQL error exec(\"%s\"): %s", + type->valueint, querystr, *message); + } return false; } - // Bind domain to prepared statement (if requested) - const int arg_idx = sqlite3_bind_parameter_index(stmt, ":argument"); - if(arg_idx > 0 && (rc = sqlite3_bind_text(stmt, arg_idx, argument, -1, SQLITE_STATIC)) != SQLITE_OK) + // Bind item to prepared statement + cJSON *item = cJSON_GetObjectItemCaseSensitive(it, "item"); + const int item_idx = sqlite3_bind_parameter_index(stmt, ":item"); + if(item_idx > 0 && (!cJSON_IsString(item) || (rc = sqlite3_bind_text(stmt, item_idx, item->valuestring, -1, SQLITE_STATIC)) != SQLITE_OK)) { *message = sqlite3_errmsg(gravity_db); - log_err("gravityDB_delFromTable(%d, %s): Failed to bind argument %u (error %d) - %s", - type, argument, i, rc, *message); + log_err("gravityDB_delFromTable(%d): Failed to bind item (error %d) - %s", + listtype, rc, *message); sqlite3_reset(stmt); sqlite3_finalize(stmt); + // Rollback transaction + querystr = "ROLLBACK TRANSACTION;"; + rc = sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL); + if(rc != SQLITE_OK) + { + *message = sqlite3_errmsg(gravity_db); + log_err("gravityDB_delFromTable(%d): SQL error exec(\"%s\"): %s", + listtype, querystr, *message); + } return false; } - // Bind type to prepared statement (if requested) - const int type_idx = sqlite3_bind_parameter_index(stmt, ":type"); - if(type_idx > 0 && (rc = sqlite3_bind_int(stmt, type_idx, type)) != SQLITE_OK) + // Execute statement + if((rc = sqlite3_step(stmt)) != SQLITE_DONE) { *message = sqlite3_errmsg(gravity_db); - log_err("gravityDB_delFromTable(%d, %s): Failed to bind type (2) (error %d) - %s", - type, argument, rc, *message); + log_err("gravityDB_delFromTable(%d) - SQL error step(\"%s\"): %s", + listtype, querystr, *message); sqlite3_reset(stmt); sqlite3_finalize(stmt); + // Rollback transaction + querystr = "ROLLBACK TRANSACTION;"; + rc = sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL); + if(rc != SQLITE_OK) + { + *message = sqlite3_errmsg(gravity_db); + log_err("gravityDB_delFromTable(%d): SQL error exec(\"%s\"): %s", + listtype, querystr, *message); + } return false; } + // Reset statement + sqlite3_reset(stmt); + // Debug output if(config.debug.api.v.b) { - log_debug(DEBUG_API, "SQL: %s", querystr[i]); - if(arg_idx > 0) - log_debug(DEBUG_API, " :argument = \"%s\"", argument); + log_debug(DEBUG_API, "SQL: %s", querystr); + if(item_idx > 0) + log_debug(DEBUG_API, " :item = \"%s\"", item->valuestring); if(type_idx > 0) - log_debug(DEBUG_API, " :type = \"%i\"", type); + log_debug(DEBUG_API, " :type = %i", cJSON_IsNumber(type) ? type->valueint : -1); } + } - // Perform step - okay = false; - if((rc = sqlite3_step(stmt)) == SQLITE_DONE) + // Finalize statement + sqlite3_finalize(stmt); + + // Prepare SQL for deleting items from the requested table + const char *querystrs[4] = {NULL, NULL, NULL, NULL}; + if(listtype == GRAVITY_GROUPS) + querystrs[0] = "DELETE FROM \"group\" WHERE name IN (SELECT item FROM deltable);"; + else if(listtype == GRAVITY_ADLISTS) + { + // This is actually a three-step deletion to satisfy foreign-key constraints + querystrs[0] = "DELETE FROM gravity WHERE adlist_id = (SELECT id FROM adlist WHERE address IN (SELECT item FROM deltable));"; + querystrs[1] = "DELETE FROM antigravity WHERE adlist_id = (SELECT id FROM adlist WHERE address IN (SELECT item FROM deltable));"; + querystrs[2] = "DELETE FROM adlist WHERE address IN (SELECT item FROM deltable);"; + } + else if(listtype == GRAVITY_CLIENTS) + querystrs[0] = "DELETE FROM client WHERE ip IN (SELECT item FROM deltable);"; + else // domainlist + { + querystrs[0] = "DELETE FROM domainlist WHERE domain IN (SELECT item FROM deltable WHERE type = 0) AND type = 0;"; + querystrs[1] = "DELETE FROM domainlist WHERE domain IN (SELECT item FROM deltable WHERE type = 1) AND type = 1;"; + querystrs[2] = "DELETE FROM domainlist WHERE domain IN (SELECT item FROM deltable WHERE type = 2) AND type = 2;"; + querystrs[3] = "DELETE FROM domainlist WHERE domain IN (SELECT item FROM deltable WHERE type = 3) AND type = 3;"; + } + + bool okay = true; + for(unsigned int i = 0; i < ArraySize(querystrs); i++) + { + // Finish if no more queries + if(querystrs[i] == NULL) + break; + + // Execute statement + rc = sqlite3_exec(gravity_db, querystrs[i], NULL, NULL, NULL); + if(rc != SQLITE_OK) { - // Item removed - okay = true; + *message = sqlite3_errmsg(gravity_db); + log_err("gravityDB_delFromTable(%d): SQL error exec(\"%s\"): %s", + listtype, querystrs[i], *message); + okay = false; + + // Rollback transaction + querystr = "ROLLBACK TRANSACTION;"; + rc = sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL); + if(rc != SQLITE_OK) + { + *message = sqlite3_errmsg(gravity_db); + log_err("gravityDB_delFromTable(%d): SQL error exec: %s", + listtype, *message); + } + + break; } - else + } + + // Drop temporary table + querystr = "DROP TABLE deltable;"; + rc = sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL); + if(rc != SQLITE_OK) + { + *message = sqlite3_errmsg(gravity_db); + log_err("gravityDB_delFromTable(%d): SQL error exec(\"%s\"): %s", + listtype, querystr, *message); + okay = false; + + // Rollback transaction + querystr = "ROLLBACK TRANSACTION;"; + rc = sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL); + if(rc != SQLITE_OK) { *message = sqlite3_errmsg(gravity_db); + log_err("gravityDB_delFromTable(%d): SQL error exec(\"%s\"): %s", + listtype, querystr, *message); } + } - // Finalize statement - sqlite3_reset(stmt); - sqlite3_finalize(stmt); + // Commit transaction + querystr = "COMMIT TRANSACTION;"; + rc = sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL); + if(rc != SQLITE_OK) + { + *message = sqlite3_errmsg(gravity_db); + log_err("gravityDB_delFromTable(%d): SQL error exec(\"%s\"): %s", + listtype, querystr, *message); + okay = false; + + // Rollback transaction + querystr = "ROLLBACK TRANSACTION;"; + rc = sqlite3_exec(gravity_db, querystr, NULL, NULL, NULL); + if(rc != SQLITE_OK) + { + *message = sqlite3_errmsg(gravity_db); + log_err("gravityDB_delFromTable(%d): SQL error exec(\"%s\"): %s", + listtype, querystr, *message); + } } return okay; diff --git a/src/database/gravity-db.h b/src/database/gravity-db.h index 8b77fdecf..7024cdec1 100644 --- a/src/database/gravity-db.h +++ b/src/database/gravity-db.h @@ -70,7 +70,7 @@ bool gravityDB_readTableGetRow(const enum gravity_list_type listtype, tablerow * void gravityDB_readTableFinalize(void); bool gravityDB_addToTable(const enum gravity_list_type listtype, tablerow *row, const char **message, const enum http_method method); -bool gravityDB_delFromTable(const enum gravity_list_type listtype, const char* domain_name, const char **message); +bool gravityDB_delFromTable(const enum gravity_list_type listtype, const cJSON* array, const char **message); bool gravityDB_edit_groups(const enum gravity_list_type listtype, cJSON *groups, const tablerow *row, const char **message); diff --git a/src/webserver/http-common.h b/src/webserver/http-common.h index d10ce72bb..a710dba2a 100644 --- a/src/webserver/http-common.h +++ b/src/webserver/http-common.h @@ -33,9 +33,15 @@ enum http_method { HTTP_OPTIONS = 1 << 5, }; +enum api_flags { + API_FLAG_NONE = 0, + API_DOMAINS = 1 << 0, + API_PARSE_JSON = 1 << 1, + API_BATCHDELETE = 1 << 2, +}; + struct api_options { - bool domains :1; - bool parse_json :1; + enum api_flags flags; enum fifo_logs which; }; From 38a6443adaa4ae1350ff3d516a1b8fd9fb7eb459 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 6 Dec 2023 23:44:14 +0100 Subject: [PATCH 153/221] Change priorities such that special domains (Firefox and Apple at this time) can be explicitly allowed for some clients (per group assignments) while they stay blocked for all others in the network Signed-off-by: DL6ER --- src/dnsmasq_interface.c | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index 071e73d2f..335d6f1ce 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -1396,22 +1396,6 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c break; } - // Not in FTL's cache. Check if this is a special domain - if(special_domain(query, domainstr)) - { - // Set DNS cache properties - dns_cache->blocking_status = SPECIAL_DOMAIN; - dns_cache->force_reply = force_next_DNS_reply; - - // Adjust counters - query_blocked(query, domain, client, QUERY_SPECIAL_DOMAIN); - - // Debug output - log_debug(DEBUG_QUERIES, "Special domain: %s is %s", domainstr, blockingreason); - - return true; - } - // Skip all checks and continue if we hit already at least one whitelist in the chain if(query->flags.allowed) { @@ -1419,6 +1403,8 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c return false; } + // when we reach this point: the query is not in FTL's cache (for this client) + // Make a local copy of the domain string. The string memory may get // reorganized in the following. We cannot expect domainstr to remain // valid for all time. @@ -1432,6 +1418,22 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c if(!query->flags.allowed) query->flags.allowed = in_regex(domainstr, dns_cache, client->id, REGEX_ALLOW); + // Check if this is a special domain + if(!query->flags.allowed && special_domain(query, domainstr)) + { + // Set DNS cache properties + dns_cache->blocking_status = SPECIAL_DOMAIN; + dns_cache->force_reply = force_next_DNS_reply; + + // Adjust counters + query_blocked(query, domain, client, QUERY_SPECIAL_DOMAIN); + + // Debug output + log_debug(DEBUG_QUERIES, "Special domain: %s is %s", domainstr, blockingreason); + + return true; + } + // Check blacklist (exact + regex) and gravity for queried domain unsigned char new_status = QUERY_UNKNOWN; bool db_okay = true; From dc1b1d1e0833ac5c1d3d1dc8146a118d9288382a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 7 Dec 2023 16:58:22 +0100 Subject: [PATCH 154/221] Rename src/{ => config}/setupVars.{c,h} Signed-off-by: DL6ER --- src/CMakeLists.txt | 2 -- src/api/auth.c | 2 +- src/api/dns.c | 2 +- src/api/history.c | 2 +- src/api/info.c | 2 +- src/api/stats.c | 2 +- src/config/CMakeLists.txt | 2 ++ src/config/config.c | 2 +- src/config/dnsmasq_config.c | 2 +- src/config/legacy_reader.c | 2 +- src/{ => config}/setupVars.c | 2 +- src/{ => config}/setupVars.h | 0 src/config/toml_reader.c | 2 +- src/dnsmasq_interface.c | 2 +- src/files.c | 2 +- src/main.c | 2 +- 16 files changed, 15 insertions(+), 15 deletions(-) rename src/{ => config}/setupVars.c (99%) rename src/{ => config}/setupVars.h (100%) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 3428cedb2..4b8b48460 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -219,8 +219,6 @@ set(sources regex_r.h resolve.c resolve.h - setupVars.c - setupVars.h shmem.c shmem.h signals.c diff --git a/src/api/auth.c b/src/api/auth.c index d0f89128f..14b8843af 100644 --- a/src/api/auth.c +++ b/src/api/auth.c @@ -16,7 +16,7 @@ #include "log.h" #include "config/config.h" // get_password_hash() -#include "setupVars.h" +#include "config/setupVars.h" // (un)lock_shm() #include "shmem.h" // getrandom() diff --git a/src/api/dns.c b/src/api/dns.c index 68566759a..525c727ea 100644 --- a/src/api/dns.c +++ b/src/api/dns.c @@ -13,7 +13,7 @@ #include "webserver/json_macros.h" #include "api.h" // {s,g}et_blockingstatus() -#include "setupVars.h" +#include "config/setupVars.h" // set_blockingmode_timer() #include "timers.h" #include "shmem.h" diff --git a/src/api/history.c b/src/api/history.c index ada248479..8a33d7cdb 100644 --- a/src/api/history.c +++ b/src/api/history.c @@ -19,7 +19,7 @@ // config struct #include "../config/config.h" // read_setupVarsconf() -#include "../setupVars.h" +#include "../config/setupVars.h" // get_aliasclient_list() #include "../database/aliasclients.h" diff --git a/src/api/info.c b/src/api/info.c index dd2d0c7fe..774e32d93 100644 --- a/src/api/info.c +++ b/src/api/info.c @@ -15,7 +15,7 @@ // sysinfo() #include // get_blockingstatus() -#include "setupVars.h" +#include "config/setupVars.h" // counters #include "shmem.h" // get_FTL_db_filesize() diff --git a/src/api/stats.c b/src/api/stats.c index 226a0c183..e68be4882 100644 --- a/src/api/stats.c +++ b/src/api/stats.c @@ -15,7 +15,7 @@ #include "../shmem.h" #include "../datastructure.h" // read_setupVarsconf() -#include "../setupVars.h" +#include "../config/setupVars.h" // logging routines #include "../log.h" // config struct diff --git a/src/config/CMakeLists.txt b/src/config/CMakeLists.txt index 99c08375d..39f5cfb6c 100644 --- a/src/config/CMakeLists.txt +++ b/src/config/CMakeLists.txt @@ -21,6 +21,8 @@ set(sources legacy_reader.h password.c password.h + setupVars.c + setupVars.h toml_writer.c toml_writer.h toml_reader.c diff --git a/src/config/config.c b/src/config/config.c index b8360efe4..4b16f50d2 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -12,7 +12,7 @@ #include "config/config.h" #include "config/toml_reader.h" #include "config/toml_writer.h" -#include "setupVars.h" +#include "config/setupVars.h" #include "log.h" #include "log.h" // readFTLlegacy() diff --git a/src/config/dnsmasq_config.c b/src/config/dnsmasq_config.c index cc262bc77..a3f9ed368 100644 --- a/src/config/dnsmasq_config.c +++ b/src/config/dnsmasq_config.c @@ -23,7 +23,7 @@ // directory_exists() #include "files.h" // trim_whitespace() -#include "setupVars.h" +#include "config/setupVars.h" // run_dnsmasq_main() #include "args.h" // optind diff --git a/src/config/legacy_reader.c b/src/config/legacy_reader.c index 9ae6e664b..6e865f78c 100644 --- a/src/config/legacy_reader.c +++ b/src/config/legacy_reader.c @@ -11,7 +11,7 @@ #include "FTL.h" #include "legacy_reader.h" #include "config.h" -#include "setupVars.h" +#include "config/setupVars.h" #include "log.h" // nice() #include diff --git a/src/setupVars.c b/src/config/setupVars.c similarity index 99% rename from src/setupVars.c rename to src/config/setupVars.c index e00405717..d54d17b47 100644 --- a/src/setupVars.c +++ b/src/config/setupVars.c @@ -11,7 +11,7 @@ #include "FTL.h" #include "log.h" #include "config/config.h" -#include "setupVars.h" +#include "config/setupVars.h" #include "datastructure.h" unsigned int setupVarsElements = 0; diff --git a/src/setupVars.h b/src/config/setupVars.h similarity index 100% rename from src/setupVars.h rename to src/config/setupVars.h diff --git a/src/config/toml_reader.c b/src/config/toml_reader.c index 7bad6af9a..a9f090819 100644 --- a/src/config/toml_reader.c +++ b/src/config/toml_reader.c @@ -10,7 +10,7 @@ #include "FTL.h" #include "toml_reader.h" -#include "setupVars.h" +#include "config/setupVars.h" #include "log.h" // getprio(), setprio() #include diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index d880db64a..db3c67184 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -20,7 +20,7 @@ #include "database/database-thread.h" #include "datastructure.h" #include "database/gravity-db.h" -#include "setupVars.h" +#include "config/setupVars.h" #include "daemon.h" #include "timers.h" #include "gc.h" diff --git a/src/files.c b/src/files.c index 0f36def4f..afa07b1ab 100644 --- a/src/files.c +++ b/src/files.c @@ -11,7 +11,7 @@ #include "FTL.h" #include "files.h" #include "config/config.h" -#include "setupVars.h" +#include "config/setupVars.h" #include "log.h" // opendir(), readdir() diff --git a/src/main.c b/src/main.c index 6a9898380..bc3a0bab9 100644 --- a/src/main.c +++ b/src/main.c @@ -11,7 +11,7 @@ #include "FTL.h" #include "daemon.h" #include "log.h" -#include "setupVars.h" +#include "config/setupVars.h" #include "args.h" #include "config/config.h" #include "database/common.h" From 9bcee72512f91862a05eda9c21d8ce12f07185c3 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 7 Dec 2023 16:59:55 +0100 Subject: [PATCH 155/221] Add migration code from setupVars.conf:GRAVITY_TMPDIR to pihole.toml:files.gravity_tmp Signed-off-by: DL6ER --- src/config/setupVars.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/config/setupVars.c b/src/config/setupVars.c index d54d17b47..625d1bfa1 100644 --- a/src/config/setupVars.c +++ b/src/config/setupVars.c @@ -443,6 +443,7 @@ void importsetupVarsConf(void) get_conf_bool_from_setupVars("DHCP_RAPID_COMMIT", &config.dhcp.rapidCommit); get_conf_bool_from_setupVars("queryLogging", &config.dns.queryLogging); + get_conf_string_from_setupVars("GRAVITY_TMPDIR", &config.files.gravity_tmp); } char* __attribute__((pure)) find_equals(char *s) From 6a42fc575758d30267ab8fbb00c9b447171c0bed Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 7 Dec 2023 20:54:49 +0100 Subject: [PATCH 156/221] Tests: Add CI test for allowing special domains per group Signed-off-by: DL6ER --- test/gravity.db.sql | 3 +++ test/test_suite.bats | 12 ++++++++++++ 2 files changed, 15 insertions(+) diff --git a/test/gravity.db.sql b/test/gravity.db.sql index a8c1438f5..6a36c1207 100644 --- a/test/gravity.db.sql +++ b/test/gravity.db.sql @@ -228,6 +228,8 @@ INSERT INTO domainlist VALUES(16,3,'^regex-notMultiple.ftl$;querytype=!ANY,HTTPS /* Other special domains */ INSERT INTO domainlist VALUES(17,1,'blacklisted-group-disabled.com',1,1559928803,1559928803,'Entry disabled by a group'); +INSERT INTO domainlist VALUES(18,0,'mask.icloud.com',1,1559928803,1559928803,'Allowing special domain'); +DELETE FROM domainlist_by_group WHERE domainlist_id = 18 AND group_id = 0; INSERT INTO adlist VALUES(1,0,'https://pi-hole.net/block.txt',1,1559928803,1559928803,'Fake block-list',1559928803,2000,2,1,0); INSERT INTO adlist VALUES(2,1,'https://pi-hole.net/allow.txt',1,1559928803,1559928803,'Fake allow-list',1559928803,2000,2,1,0); @@ -261,6 +263,7 @@ DELETE FROM client_by_group WHERE client_id = 2 AND group_id = 0; INSERT INTO client_by_group VALUES(2,2); INSERT INTO adlist_by_group VALUES(1,2); INSERT INTO domainlist_by_group VALUES(6,2); +INSERT INTO domainlist_by_group VALUES(18,2); /* mask.icloud.com */ INSERT INTO client (id,ip) VALUES(3,'127.0.0.3'); INSERT INTO "group" VALUES(3,1,'Third test group',1559928803,1559928803,'A group associated with client IP 127.0.0.3'); diff --git a/test/test_suite.bats b/test/test_suite.bats index e74d845f5..085779133 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -428,6 +428,18 @@ [[ ${lines[@]} == *"status: SERVFAIL"* ]] } +@test "Special domain: NXDOMAIN is returned" { + run bash -c "dig A mask.icloud.com @127.0.0.1" + printf "%s\n" "${lines[@]}" + [[ ${lines[@]} == *"status: NXDOMAIN"* ]] +} + +@test "Special domain: Record is returned when explicitly allowed" { + run bash -c "dig A mask.icloud.com -b 127.0.0.2 @127.0.0.1" + printf "%s\n" "${lines[@]}" + [[ ${lines[@]} == *"status: NOERROR"* ]] +} + @test "ABP-style matching working as expected" { run bash -c "dig A special.gravity.ftl @127.0.0.1 +short" printf "%s\n" "${lines[@]}" From 0d0cdae84f3f5886860f9e7fa68b17bbd3b0c2d7 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 7 Dec 2023 17:11:10 +0100 Subject: [PATCH 157/221] Fix missing declaration Signed-off-by: DL6ER --- src/dnsmasq/cache.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/dnsmasq/cache.c b/src/dnsmasq/cache.c index 133737da2..d2a50185c 100644 --- a/src/dnsmasq/cache.c +++ b/src/dnsmasq/cache.c @@ -15,6 +15,7 @@ */ #include "dnsmasq.h" +#include "dnsmasq_interface.h" #include "webserver/webserver.h" static struct crec *cache_head = NULL, *cache_tail = NULL, **hash_table = NULL; From 19cfed227f3702f599f385d07687d17a97aaf4b8 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 7 Dec 2023 17:11:54 +0100 Subject: [PATCH 158/221] Fix possible deferencing of NULL pointers Signed-off-by: DL6ER --- src/config/cli.c | 2 +- src/vector.c | 8 ++++++++ src/webserver/webserver.c | 9 ++++++++- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/config/cli.c b/src/config/cli.c index c27c9928b..3f2263b71 100644 --- a/src/config/cli.c +++ b/src/config/cli.c @@ -518,7 +518,7 @@ int get_config_from_CLI(const char *key, const bool quiet) // Use return status if this is a boolean value // and we are in quiet mode - if(quiet && conf_item->t == CONF_BOOL) + if(quiet && conf_item != NULL && conf_item->t == CONF_BOOL) return conf_item->v.b ? EXIT_SUCCESS : EXIT_FAILURE; return EXIT_SUCCESS; diff --git a/src/vector.c b/src/vector.c index cd3bfb646..2abb617f8 100644 --- a/src/vector.c +++ b/src/vector.c @@ -19,6 +19,14 @@ sqlite3_stmt_vec *new_sqlite3_stmt_vec(unsigned int initial_size) log_debug(DEBUG_VECTORS, "Initializing new sqlite3_stmt* vector with size %u", initial_size); sqlite3_stmt_vec *v = calloc(1, sizeof(sqlite3_stmt_vec)); + if(v == NULL) + { + log_err("Memory allocation failed in new_sqlite3_stmt_vec(%u)", + initial_size); + return NULL; + } + + // Initialize vector v->capacity = initial_size; // Calloc ensures they are all set to zero which is the default state v->items = calloc(initial_size, sizeof(sqlite3_stmt *) * initial_size); diff --git a/src/webserver/webserver.c b/src/webserver/webserver.c index 215843646..f9844f4f2 100644 --- a/src/webserver/webserver.c +++ b/src/webserver/webserver.c @@ -577,7 +577,14 @@ void FTL_rewrite_pattern(char *filename, size_t filename_buf_len) // Try full path with ".lp" appended filename_lp = append_to_path(filename, ".lp"); - if(filename_lp != NULL && file_readable(filename_lp)) + if(filename_lp == NULL) + { + //Failed to allocate memory for filename!"); + return; + } + + // Check if the file exists. If so, rewrite the filename and return + if(file_readable(filename_lp)) { log_debug(DEBUG_API, "Rewriting Lua page: %s ==> %s", filename, filename_lp); strncpy(filename, filename_lp, filename_buf_len); From d802f0e301d1102e8fdd20dde99a93b7b36ab260 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 7 Dec 2023 22:27:14 +0100 Subject: [PATCH 159/221] Reduce number of warnings coming from third-party modules we use shown during compilation Signed-off-by: DL6ER --- src/CMakeLists.txt | 5 +++++ src/lua/CMakeLists.txt | 2 +- src/tre-regex/CMakeLists.txt | 2 +- src/webserver/civetweb/CMakeLists.txt | 2 +- src/webserver/civetweb/mod_lua.inl | 4 ++++ src/zip/miniz/miniz.c | 2 +- 6 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 3428cedb2..3f7e0dd82 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -154,6 +154,11 @@ set(EXTRAWARN "${EXTRAWARN_GCC6} \ ${EXTRAWARN_GCC8} \ ${EXTRAWARN_GCC12} \ ${EXTRAWARN_GCC13}") + +# Remove extra spaces from EXTRAWARN +string(REGEX REPLACE " +" " " EXTRAWARN "${EXTRAWARN}") + +# Separate EXTRAWARN into a list of arguments separate_arguments(EXTRAWARN) # -Wxor-used-as-pow diff --git a/src/lua/CMakeLists.txt b/src/lua/CMakeLists.txt index 1deb1f388..908b5a7e1 100644 --- a/src/lua/CMakeLists.txt +++ b/src/lua/CMakeLists.txt @@ -65,7 +65,7 @@ set(sources ) add_library(lua OBJECT ${sources}) -target_compile_options(lua PRIVATE -Wno-maybe-uninitialized) +target_compile_options(lua PRIVATE -Wno-maybe-uninitialized -Wno-unused-variable -Wno-unused-value) # LUA_USE_POSIX: ensures recommended POSIX functions are used instead of # (partially obsoleted) standard C functions diff --git a/src/tre-regex/CMakeLists.txt b/src/tre-regex/CMakeLists.txt index 01d866d80..f4a8ba96f 100644 --- a/src/tre-regex/CMakeLists.txt +++ b/src/tre-regex/CMakeLists.txt @@ -27,4 +27,4 @@ set(sources ) add_library(tre-regex OBJECT ${sources}) -target_compile_options(tre-regex PRIVATE -Wno-maybe-uninitialized -Wno-unused-value) +target_compile_options(tre-regex PRIVATE -Wno-maybe-uninitialized -Wno-unused-value -Wno-empty-body) diff --git a/src/webserver/civetweb/CMakeLists.txt b/src/webserver/civetweb/CMakeLists.txt index d3ddaf3b6..6a6cf9228 100644 --- a/src/webserver/civetweb/CMakeLists.txt +++ b/src/webserver/civetweb/CMakeLists.txt @@ -18,6 +18,7 @@ set(sources ) add_library(civetweb OBJECT ${sources}) +target_compile_options(civetweb PRIVATE -Wno-unused-but-set-variable -Wno-unused-variable) # We can remove the NO_SSL later on. It adds additional constraints to the build system (availablity of libSSL-dev) # NO_CGI = no CGI support (we don't need it) # NO_SSL_DL NO_SSL = no SSL support (for now) @@ -42,6 +43,5 @@ else() target_compile_definitions(civetweb PRIVATE NO_SSL) endif() - include_directories(${PROJECT_SOURCE_DIR}/src/lua /usr/local/include) target_include_directories(civetweb PRIVATE ${PROJECT_SOURCE_DIR}/src) diff --git a/src/webserver/civetweb/mod_lua.inl b/src/webserver/civetweb/mod_lua.inl index b3733e8bd..e9d90ca55 100644 --- a/src/webserver/civetweb/mod_lua.inl +++ b/src/webserver/civetweb/mod_lua.inl @@ -2793,7 +2793,11 @@ lua_error_handler(lua_State *L) static void prepare_lua_environment(struct mg_context *ctx, struct mg_connection *conn, +#if defined(USE_WEBSOCKET) struct lua_websock_data *ws_conn_list, +#else + void *ws_conn_list, +#endif lua_State *L, const char *script_name, int lua_env_type) diff --git a/src/zip/miniz/miniz.c b/src/zip/miniz/miniz.c index 54b85dbf1..1195e620c 100644 --- a/src/zip/miniz/miniz.c +++ b/src/zip/miniz/miniz.c @@ -3182,7 +3182,7 @@ static int mz_stat64(const char *path, struct __stat64 *buffer) #define MZ_DELETE_FILE remove #else -#pragma message("Using fopen, ftello, fseeko, stat() etc. path for file I/O - this path may not support large files.") +//#pragma message("Using fopen, ftello, fseeko, stat() etc. path for file I/O - this path may not support large files.") #ifndef MINIZ_NO_TIME #include #endif From 66cc0e87c3e1886e5187cab0066e43588c4bb06d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 7 Dec 2023 22:30:24 +0100 Subject: [PATCH 160/221] Reduce code complexity slightly Signed-off-by: DL6ER --- src/api/config.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/api/config.c b/src/api/config.c index 30706bf2c..ac01cfb27 100644 --- a/src/api/config.c +++ b/src/api/config.c @@ -605,8 +605,7 @@ static int api_config_get(struct ftl_conn *api) } // Release allocated memory - if(requested_path != NULL) - free_config_path(requested_path); + free_config_path(requested_path); cJSON *json = JSON_NEW_OBJECT(); @@ -852,8 +851,7 @@ static int api_config_put_delete(struct ftl_conn *api) if(min_level < 2) { // Release allocated memory - if(requested_path != NULL) - free_config_path(requested_path); + free_config_path(requested_path); return send_json_error(api, 400, "bad_request", @@ -899,8 +897,7 @@ static int api_config_put_delete(struct ftl_conn *api) { char *key = strdup(new_item->k); free_config(&newconf); - if(requested_path != NULL) - free_config_path(requested_path); + free_config_path(requested_path); return send_json_error_free(api, 400, "bad_request", "Config items set via environment variables cannot be changed via the API", @@ -965,8 +962,7 @@ static int api_config_put_delete(struct ftl_conn *api) } // Release allocated memory - if(requested_path != NULL) - free_config_path(requested_path); + free_config_path(requested_path); // Error 404 if not found if(!found || message != NULL) From 43722cc7a1317214c2fa4d9ce885e709329c2840 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 7 Dec 2023 22:31:58 +0100 Subject: [PATCH 161/221] Use fixed string lengths where available Signed-off-by: DL6ER --- src/api/network.c | 2 +- src/api/queries.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/api/network.c b/src/api/network.c index 4a4e4d1ef..68e8113cb 100644 --- a/src/api/network.c +++ b/src/api/network.c @@ -40,7 +40,7 @@ static bool getDefaultInterface(char iface[IF_NAMESIZE], in_addr_t *gw) // Parse /proc/net/route - the kernel's IPv4 routing table while(fgets(buf, sizeof(buf), file)) { - if(sscanf(buf, "%s %lx %lx %x %*i %*i %i", iface_r, &dest_r, &gw_r, &flags, &metric) != 5) + if(sscanf(buf, "%15s %lx %lx %x %*i %*i %i", iface_r, &dest_r, &gw_r, &flags, &metric) != 5) continue; // Only analyze routes which are UP and whose diff --git a/src/api/queries.c b/src/api/queries.c index 1d0adbbbc..4272cca30 100644 --- a/src/api/queries.c +++ b/src/api/queries.c @@ -279,7 +279,7 @@ int api_queries(struct ftl_conn *api) // Start building database query string char querystr[QUERYSTRBUFFERLEN] = { 0 }; - sprintf(querystr, "%s FROM %s q %s", QUERYSTR, disk ? "disk.query_storage" : "query_storage", JOINSTR); + snprintf(querystr, QUERYSTRBUFFERLEN, "%s FROM %s q %s", QUERYSTR, disk ? "disk.query_storage" : "query_storage", JOINSTR); int draw = 0; char domainname[512] = { 0 }; From b52258c2a17eda2de108699544bd78fea6858edd Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 7 Dec 2023 22:33:56 +0100 Subject: [PATCH 162/221] Fix resource/memory leak on error in teleporter ZIP processing Signed-off-by: DL6ER --- src/zip/teleporter.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/zip/teleporter.c b/src/zip/teleporter.c index e66161627..d541a8e40 100644 --- a/src/zip/teleporter.c +++ b/src/zip/teleporter.c @@ -190,10 +190,10 @@ const char *generate_teleporter_zip(mz_zip_archive *zip, char filename[128], voi { // Loop over all files and add them to the ZIP archive DIR *dir; - struct dirent *ent; if((dir = opendir(directory)) != NULL) { // Loop over all files in the directory + struct dirent *ent; while((ent = readdir(dir)) != NULL) { // Skip "." and ".." @@ -677,6 +677,7 @@ bool write_teleporter_zip_to_disk(void) { log_err("Failed to write %zu bytes to %s: %s", size, filename, strerror(errno)); free_teleporter_zip(&zip); + fclose(fp); return false; } fclose(fp); @@ -722,6 +723,7 @@ bool read_teleporter_zip_from_disk(const char *filename) log_err("Failed to read %zu bytes from %s: %s", size, filename, strerror(errno)); fclose(fp); + free(ptr); return false; } fclose(fp); From 742110d7ae9f7f022b720f7d1606fa785a4dc30d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 7 Dec 2023 22:34:47 +0100 Subject: [PATCH 163/221] Fix possible resouce leak on errors in the GZIP routines Signed-off-by: DL6ER --- src/zip/gzip.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/zip/gzip.c b/src/zip/gzip.c index bf15ea862..1bb491816 100644 --- a/src/zip/gzip.c +++ b/src/zip/gzip.c @@ -29,7 +29,7 @@ static bool deflate_buffer(const unsigned char *buffer_uncompressed, const mz_ul // space for the GZIP header and footer *size_compressed = compressBound(size_uncompressed) + 14; *buffer_compressed = malloc(*size_compressed); - if(buffer_compressed == NULL) + if(*buffer_compressed == NULL) { log_warn("Failed to allocate %lu bytes of memory", (unsigned long)*size_compressed); return false; @@ -308,6 +308,7 @@ bool inflate_file(const char *infilename, const char *outfilename, bool verbose) if(outfile == NULL) { log_warn("Failed to open %s: %s", outfilename, strerror(errno)); + fclose(infile); return false; } @@ -322,12 +323,15 @@ bool inflate_file(const char *infilename, const char *outfilename, bool verbose) { log_warn("Failed to allocate %lu bytes of memory", (unsigned long)size_compressed); fclose(infile); + fclose(outfile); return false; } if(fread(buffer_compressed, 1, size_compressed, infile) != size_compressed) { log_warn("Failed to read %lu bytes from %s", (unsigned long)size_compressed, infilename); fclose(infile); + fclose(outfile); + free(buffer_compressed); return false; } fclose(infile); @@ -389,6 +393,7 @@ bool deflate_file(const char *infilename, const char *outfilename, bool verbose) if(outfile == NULL) { log_warn("Failed to open %s for writing: %s", outfilename, strerror(errno)); + fclose(infile); return false; } @@ -403,12 +408,14 @@ bool deflate_file(const char *infilename, const char *outfilename, bool verbose) { log_warn("Failed to allocate %lu bytes of memory", (unsigned long)size_uncompressed); fclose(infile); + fclose(outfile); return false; } if(fread(buffer_uncompressed, 1, size_uncompressed, infile) != size_uncompressed) { log_warn("Failed to read %lu bytes from %s", (unsigned long)size_uncompressed, infilename); fclose(infile); + fclose(outfile); free(buffer_uncompressed); return false; } From dd5f1c1ffa1ead4e6543424a3d050216b1b45fb7 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 7 Dec 2023 22:35:19 +0100 Subject: [PATCH 164/221] Fix wrong format in printf() in disabled debugging code Signed-off-by: DL6ER --- src/tools/dhcp-discover.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tools/dhcp-discover.c b/src/tools/dhcp-discover.c index 00fc0e1ca..85e994b4f 100644 --- a/src/tools/dhcp-discover.c +++ b/src/tools/dhcp-discover.c @@ -659,8 +659,8 @@ static void get_dhcp_offer(const int sock, const uint32_t xid, const char *iface cli_bold(), iface, cli_normal(), valid_responses, responses); #ifdef DEBUG - printf(" Responses seen while scanning: %d\n", responses); - printf(" Responses meant for this machine: %d\n\n", valid_responses); + printf(" Responses seen while scanning: %u\n", responses); + printf(" Responses meant for this machine: %u\n\n", valid_responses); #endif } From 1ff6c173950ef081bad3a7f5da6874cf136794c5 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 7 Dec 2023 22:35:58 +0100 Subject: [PATCH 165/221] Condition 'web_layout!=NULL' is always true Signed-off-by: DL6ER --- src/setupVars.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/setupVars.c b/src/setupVars.c index e00405717..e9bf44772 100644 --- a/src/setupVars.c +++ b/src/setupVars.c @@ -242,7 +242,7 @@ static void get_conf_weblayout_from_setupVars(void) // If the property is set to false and different than "boxed", the property // is disabled. This is consistent with the code in AdminLTE when writing // this code - if(web_layout != NULL && strcasecmp(web_layout, "boxed") != 0) + if(strcasecmp(web_layout, "boxed") != 0) config.webserver.interface.boxed.v.b = false; // Free memory, harmless to call if read_setupVarsconf() didn't return a result From 895f326dac61a96f7d95b9b7febed02c44018e6c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 7 Dec 2023 22:36:47 +0100 Subject: [PATCH 166/221] Reduce variable scope Signed-off-by: DL6ER --- src/syscalls/vfprintf.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/syscalls/vfprintf.c b/src/syscalls/vfprintf.c index 7a5ffae9f..c7a82de68 100644 --- a/src/syscalls/vfprintf.c +++ b/src/syscalls/vfprintf.c @@ -41,10 +41,9 @@ static void itoa(int n, char s[]) // Reverse string s in place int j; - char c; int len = strlen(s); for (i = 0, j = len-1; i Date: Thu, 7 Dec 2023 22:38:11 +0100 Subject: [PATCH 167/221] Local variable 'hostname' shadows outer function with the same name Signed-off-by: DL6ER --- src/resolve.c | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/resolve.c b/src/resolve.c index 28733cc74..0d2a18603 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -146,7 +146,7 @@ bool __attribute__((pure)) resolve_this_name(const char *ipaddr) char *resolveHostname(const char *addr) { // Get host name - char *hostname = NULL; + char *hostn = NULL; // Check if we want to resolve host names if(!resolve_this_name(addr)) @@ -163,18 +163,18 @@ char *resolveHostname(const char *addr) // if so, return "hidden" as hostname if(strcmp(addr, "0.0.0.0") == 0) { - hostname = strdup("hidden"); - log_debug(DEBUG_RESOLVER, "---> \"%s\" (privacy settings)", hostname); - return hostname; + hostn = strdup("hidden"); + log_debug(DEBUG_RESOLVER, "---> \"%s\" (privacy settings)", hostn); + return hostn; } // Check if this is the internal client // if so, return "hidden" as hostname if(strcmp(addr, "::") == 0) { - hostname = strdup("pi.hole"); - log_debug(DEBUG_RESOLVER, "---> \"%s\" (special)", hostname); - return hostname; + hostn = strdup("pi.hole"); + log_debug(DEBUG_RESOLVER, "---> \"%s\" (special)", hostn); + return hostn; } // Check if we want to resolve host names @@ -286,14 +286,14 @@ char *resolveHostname(const char *addr) if(valid_hostname(host, addr)) { // Return hostname copied to new memory location - hostname = strdup(host); + hostn = strdup(host); } else { - hostname = strdup("[invalid host name]"); + hostn = strdup("[invalid host name]"); } - log_debug(DEBUG_RESOLVER, " ---> \"%s\" (found internally)", hostname); + log_debug(DEBUG_RESOLVER, " ---> \"%s\" (found internally)", hostn); } else log_debug(DEBUG_RESOLVER, " ---> \"\" (not found internally: %s", gai_strerror(ret)); @@ -321,7 +321,7 @@ char *resolveHostname(const char *addr) // If no host name was found before, try again with system-configured // resolvers (necessary for docker and friends) - if(hostname == NULL) + if(hostn == NULL) { // Try to resolve address ret = getnameinfo((struct sockaddr*)&ss, sizeof(ss), host, sizeof(host), NULL, 0, NI_NAMEREQD); @@ -333,19 +333,19 @@ char *resolveHostname(const char *addr) if(valid_hostname(host, addr)) { // Return hostname copied to new memory location - hostname = strdup(host); + hostn = strdup(host); } else { - hostname = strdup("[invalid host name]"); + hostn = strdup("[invalid host name]"); } - log_debug(DEBUG_RESOLVER, " ---> \"%s\" (found externally)", hostname); + log_debug(DEBUG_RESOLVER, " ---> \"%s\" (found externally)", hostn); } else { // No hostname found (empty PTR) - hostname = strdup(""); + hostn = strdup(""); if(config.debug.resolver.v.b) log_debug(DEBUG_RESOLVER, " ---> \"\" (not found externally: %s)", gai_strerror(ret)); @@ -353,7 +353,7 @@ char *resolveHostname(const char *addr) } // Return result - return hostname; + return hostn; } // Resolve upstream destination host names From dc9dc0d6cf85b32f935ba253ffd9812665699276 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 7 Dec 2023 22:39:44 +0100 Subject: [PATCH 168/221] regextest is always true in these cases Signed-off-by: DL6ER --- src/regex.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/regex.c b/src/regex.c index 874101af3..e15ca1d81 100644 --- a/src/regex.c +++ b/src/regex.c @@ -470,14 +470,14 @@ static int match_regex(const char *input, DNSCacheData* dns_cache, const int cli log_info(" %s%s%s matches", cli_bold(), regex->string, cli_normal()); } - else if(regextest && regexid == REGEX_DENY) + else if(regexid == REGEX_DENY) { // Database-sourced regular expression log_info(" %s%s%s matches (regex blacklist, DB ID %i)", cli_bold(), regex->string, cli_normal(), regex->database_id); } - else if(regextest && regexid == REGEX_ALLOW) + else if(regexid == REGEX_ALLOW) { // Database-sourced regular expression log_info(" %s%s%s matches (regex whitelist, DB ID %i)", From d996697e29912cfc9be325b9d47bcf180d3cca31 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 7 Dec 2023 22:41:45 +0100 Subject: [PATCH 169/221] Clarify calculation precedence for '&' and '?' Signed-off-by: DL6ER --- src/dnsmasq_interface.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index 071e73d2f..f616a3403 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -110,7 +110,7 @@ void FTL_hook(unsigned int flags, const char *name, union all_addr *addr, char * { // Extract filename from path const char *path = short_path(file); - const char *types = flags & F_RR ? querystr(arg, type) : "?"; + const char *types = (flags & F_RR) ? querystr(arg, type) : "?"; log_debug(DEBUG_FLAGS, "Processing FTL hook from %s:%d (type: %s, name: \"%s\", id: %i)...", path, line, types, name, id); print_flags(flags); From d75b142d00e4e7b07139b82d7a5851bbda19e33e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 7 Dec 2023 22:43:22 +0100 Subject: [PATCH 170/221] Local variable 'hostname' shadows outer function Signed-off-by: DL6ER --- src/dnsmasq_interface.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index f616a3403..d4b88834f 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -352,7 +352,7 @@ size_t _FTL_make_answer(struct dns_header *header, char *limit, const size_t len return 0; // Are we replying to pi.hole / / pi.hole. / . ? - const bool hostname = strcmp(blockingreason, HOSTNAME) == 0; + const bool hostn = strcmp(blockingreason, HOSTNAME) == 0; int trunc = 0; // Add CNAME answer record if requested @@ -382,9 +382,9 @@ size_t _FTL_make_answer(struct dns_header *header, char *limit, const size_t len config.dns.blocking.mode.v.blocking_mode == MODE_IP_NODATA_AAAA || forced_ip) { - if(hostname && config.dns.reply.host.force4.v.b) + if(hostn && config.dns.reply.host.force4.v.b) memcpy(&addr, &config.dns.reply.host.v4.v.in_addr, sizeof(addr.addr4)); - else if(!hostname && config.dns.reply.blocking.force4.v.b) + else if(!hostn && config.dns.reply.blocking.force4.v.b) memcpy(&addr, &config.dns.reply.blocking.v4.v.in_addr, sizeof(addr.addr4)); else memcpy(&addr, &next_iface.addr4, sizeof(addr.addr4)); @@ -401,7 +401,7 @@ size_t _FTL_make_answer(struct dns_header *header, char *limit, const size_t len // Add A resource record header->ancount = htons(ntohs(header->ancount) + 1); if(add_resource_record(header, limit, &trunc, sizeof(struct dns_header), - &p, hostname ? daemon->local_ttl : config.dns.blockTTL.v.ui, + &p, hostn ? daemon->local_ttl : config.dns.blockTTL.v.ui, NULL, T_A, C_IN, (char*)"4", &addr.addr4)) log_query(flags & ~F_IPV6, name, &addr, (char*)blockingreason, 0); } @@ -417,9 +417,9 @@ size_t _FTL_make_answer(struct dns_header *header, char *limit, const size_t len else if(config.dns.blocking.mode.v.blocking_mode == MODE_IP || forced_ip) { - if(hostname && config.dns.reply.host.force6.v.b) + if(hostn && config.dns.reply.host.force6.v.b) memcpy(&addr, &config.dns.reply.host.v6.v.in6_addr, sizeof(addr.addr6)); - else if(!hostname && config.dns.reply.blocking.force6.v.b) + else if(!hostn && config.dns.reply.blocking.force6.v.b) memcpy(&addr, &config.dns.reply.blocking.v6.v.in6_addr, sizeof(addr.addr6)); else memcpy(&addr, &next_iface.addr6, sizeof(addr.addr6)); @@ -436,7 +436,7 @@ size_t _FTL_make_answer(struct dns_header *header, char *limit, const size_t len // Add AAAA resource record header->ancount = htons(ntohs(header->ancount) + 1); if(add_resource_record(header, limit, &trunc, sizeof(struct dns_header), - &p, hostname ? daemon->local_ttl : config.dns.blockTTL.v.ui, + &p, hostn ? daemon->local_ttl : config.dns.blockTTL.v.ui, NULL, T_AAAA, C_IN, (char*)"6", &addr.addr6)) log_query(flags & ~F_IPV4, name, &addr, (char*)blockingreason, 0); } From d5f68f0022ec8ebded857f0d6896deff1277ea55 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 7 Dec 2023 22:45:10 +0100 Subject: [PATCH 171/221] sscanf() without field width limits can crash with huge input data Signed-off-by: DL6ER --- src/database/network-table.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/database/network-table.c b/src/database/network-table.c index c5e269200..fd6aea392 100644 --- a/src/database/network-table.c +++ b/src/database/network-table.c @@ -1083,7 +1083,7 @@ static bool add_local_interfaces_to_network_table(sqlite3 *db, time_t now, unsig // Try to read IPv4 address // We need a special rule here to avoid "inet6 ..." being accepted as IPv4 address - if(sscanf(linebuffer, " inet%*[ ]%[0-9.] brd", ipaddr) == 1) + if(sscanf(linebuffer, " inet%*[ ]%127[0-9.] brd", ipaddr) == 1) { // Obtained an IPv4 address ipaddr[sizeof(ipaddr)-1] = '\0'; @@ -1091,7 +1091,7 @@ static bool add_local_interfaces_to_network_table(sqlite3 *db, time_t now, unsig else { // Try to read IPv6 address - if(sscanf(linebuffer, " inet6%*[ ]%[0-9a-fA-F:] scope", ipaddr) == 1) + if(sscanf(linebuffer, " inet6%*[ ]%127[0-9a-fA-F:] scope", ipaddr) == 1) { // Obtained an IPv6 address ipaddr[sizeof(ipaddr)-1] = '\0'; From a455a40d018bb5397db924f7f7c778712d4b6445 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 7 Dec 2023 22:45:51 +0100 Subject: [PATCH 172/221] Reduce variable scope Signed-off-by: DL6ER --- src/database/network-table.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/database/network-table.c b/src/database/network-table.c index fd6aea392..8c0e9a1fe 100644 --- a/src/database/network-table.c +++ b/src/database/network-table.c @@ -1250,7 +1250,6 @@ void parse_neighbor_cache(sqlite3* db) // Prepare buffers char *linebuffer = NULL; size_t linebuffersize = 0u; - char ip[128], hwaddr[128], iface[128]; unsigned int entries = 0u, additional_entries = 0u; time_t now = time(NULL); @@ -1314,6 +1313,7 @@ void parse_neighbor_cache(sqlite3* db) break; // Analyze line + char ip[128], hwaddr[128], iface[128]; int num = sscanf(linebuffer, "%99s dev %99s lladdr %99s", ip, iface, hwaddr); From 835fcde32785dd754e72d267d61b9d7e85fbed1a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 7 Dec 2023 22:46:47 +0100 Subject: [PATCH 173/221] Condition 'hwaddr!=NULL' is always true and reduce amount of allocated memory for synthesized MAC addresses from 324 to 18 bytes Signed-off-by: DL6ER --- src/database/gravity-db.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/database/gravity-db.c b/src/database/gravity-db.c index 12558f0b4..fbb3a1c6c 100644 --- a/src/database/gravity-db.c +++ b/src/database/gravity-db.c @@ -398,7 +398,7 @@ static bool get_client_groupids(clientsData* client) { log_debug(DEBUG_CLIENTS, "--> No result."); } - else if(hwaddr != NULL && strlen(hwaddr) > 3 && strncasecmp(hwaddr, "ip-", 3) == 0) + else if(strlen(hwaddr) > 3 && strncasecmp(hwaddr, "ip-", 3) == 0) { free(hwaddr); hwaddr = NULL; @@ -406,7 +406,7 @@ static bool get_client_groupids(clientsData* client) log_debug(DEBUG_CLIENTS, "Skipping mock-device hardware address lookup"); } // Set MAC address from database information if available and the MAC address is not already set - else if(hwaddr != NULL && client->hwlen != 6) + else if(client->hwlen != 6) { // Proper MAC parsing unsigned char data[6]; @@ -425,9 +425,8 @@ static bool get_client_groupids(clientsData* client) // MAC address fallback: Try to synthesize MAC address from internal buffer if(hwaddr == NULL && client->hwlen == 6) { - const size_t strlen = sizeof("AA:BB:CC:DD:EE:FF"); - hwaddr = calloc(18, strlen); - snprintf(hwaddr, strlen, "%02X:%02X:%02X:%02X:%02X:%02X", + hwaddr = calloc(18, sizeof(char)); // 18 == sizeof("AA:BB:CC:DD:EE:FF") + snprintf(hwaddr, 18, "%02X:%02X:%02X:%02X:%02X:%02X", client->hwaddr[0], client->hwaddr[1], client->hwaddr[2], client->hwaddr[3], client->hwaddr[4], client->hwaddr[5]); From a2df36ec47d57a58334b6530603a91c20e2e58b7 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 7 Dec 2023 22:48:11 +0100 Subject: [PATCH 174/221] Add missing legacy config import code for NICE -> misc.nice Signed-off-by: DL6ER --- src/config/legacy_reader.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/config/legacy_reader.c b/src/config/legacy_reader.c index d10ef9138..14466cdc0 100644 --- a/src/config/legacy_reader.c +++ b/src/config/legacy_reader.c @@ -337,6 +337,10 @@ const char *readFTLlegacy(struct config *conf) // 2.0) had the range -infinity..15. buffer = parseFTLconf(fp, "NICE"); + value = 0; + if(buffer != NULL && sscanf(buffer, "%i", &value) && value >= -20 && value <= 19) + conf->misc.nice.v.i = value; + // MAXNETAGE // IP addresses (and associated host names) older than the specified number // of days are removed to avoid dead entries in the network overview table From 198b2727834b562c5ebec253db48faee4ac8b7d8 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 7 Dec 2023 22:48:40 +0100 Subject: [PATCH 175/221] Condition 'fp!=NULL' is always true Signed-off-by: DL6ER --- src/config/legacy_reader.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/config/legacy_reader.c b/src/config/legacy_reader.c index 14466cdc0..33e492e3c 100644 --- a/src/config/legacy_reader.c +++ b/src/config/legacy_reader.c @@ -580,8 +580,8 @@ const char *readFTLlegacy(struct config *conf) // Release memory releaseConfigMemory(); - if(fp != NULL) - fclose(fp); + // Close file + fclose(fp); return path; } From dae83272775525be5461047d2acfd3d67a10c29f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 7 Dec 2023 22:50:29 +0100 Subject: [PATCH 176/221] Password-related code improvements and fixes Signed-off-by: DL6ER --- src/config/password.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/config/password.c b/src/config/password.c index 1b01922dd..578081ec0 100644 --- a/src/config/password.c +++ b/src/config/password.c @@ -91,6 +91,11 @@ static char * __attribute__((malloc)) base64_encode(const uint8_t *data, const s out_len = base64_encode_update(&ctx, encoded, length, data); out_len += base64_encode_final(&ctx, encoded + out_len); + // Lenght check + if(out_len > BASE64_ENCODE_LENGTH(length) + BASE64_ENCODE_FINAL_LENGTH) + log_warn("Base64 encoding may have failed: Output buffer too small? (%zu > %zu)", + out_len, BASE64_ENCODE_LENGTH(length) + BASE64_ENCODE_FINAL_LENGTH); + return encoded; } @@ -259,7 +264,7 @@ static bool parse_PHC_string(const char *phc, size_t *s_cost, size_t *t_cost, ui // Decode salt and hash size_t salt_len = 0; *salt = base64_decode(salt_base64, &salt_len); - if(salt == NULL) + if(*salt == NULL) { // Error log_err("Error while decoding salt: %s", strerror(errno)); @@ -275,7 +280,7 @@ static bool parse_PHC_string(const char *phc, size_t *s_cost, size_t *t_cost, ui size_t hash_len = 0; *hash = base64_decode(hash_base64, &hash_len); - if(hash == NULL) + if(*hash == NULL) { // Error log_err("Error while decoding hash: %s", strerror(errno)); @@ -376,10 +381,8 @@ enum password_result verify_password(const char *password, const char *pwhash, c // Free allocated memory free(supplied); - if(salt != NULL) - free(salt); - if(config_hash != NULL) - free(config_hash); + free(salt); + free(config_hash); // Successful logins do not count against rate-limiting if(result) @@ -408,11 +411,10 @@ enum password_result verify_password(const char *password, const char *pwhash, c writeFTLtoml(true); free(new_hash); } - } - // Successful logins do not count against rate-limiting - if(result) + // Successful logins do not count against rate-limiting num_password_attempts--; + } return result ? PASSWORD_CORRECT : PASSWORD_INCORRECT; } From b82db05411848087b59d7d93ee579ed7c6520d8a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 7 Dec 2023 22:51:19 +0100 Subject: [PATCH 177/221] When using void pointers in calculations, the behaviour may be undefined Signed-off-by: DL6ER --- src/config/config.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/config/config.c b/src/config/config.c index e7389ba66..aff5ce9f8 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -196,7 +196,7 @@ struct conf_item *get_conf_item(struct config *conf, const unsigned int n) } // Return n-th config element - return (void*)conf + n*sizeof(struct conf_item); + return (struct conf_item *)conf + n; } struct conf_item *get_debug_item(struct config *conf, const enum debug_flag debug) @@ -209,7 +209,7 @@ struct conf_item *get_debug_item(struct config *conf, const enum debug_flag debu } // Return n-th config element - return (void*)&conf->debug + debug*sizeof(struct conf_item); + return (struct conf_item *)&conf->debug + debug; } unsigned int __attribute__ ((pure)) config_path_depth(char **paths) From cc8d6294c088085b8cc4c0a3eed12b913609eaa0 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 7 Dec 2023 22:52:55 +0100 Subject: [PATCH 178/221] Slightly simplify args code, argc is always > 1 so it cannot be < 2 Signed-off-by: DL6ER --- src/args.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/args.c b/src/args.c index 7a7895bd7..6378278c8 100644 --- a/src/args.c +++ b/src/args.c @@ -337,7 +337,7 @@ void parse_args(int argc, char* argv[]) (strcmp(argv[1], "--read-x509") == 0 || strcmp(argv[1], "--read-x509-key") == 0)) { - if(argc < 2 || argc > 4) + if(argc > 4) { printf("Usage: %s %s [] []\n", argv[0], argv[1]); printf("Example: %s %s /etc/pihole/tls.pem\n", argv[0], argv[1]); From 34cfc9c4654bdaebdc7f8ba2b3cb275d3aa5bab7 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 7 Dec 2023 22:55:17 +0100 Subject: [PATCH 179/221] Remove undocumented audit feature Signed-off-by: DL6ER --- src/api/stats.c | 45 ++++++++++++------------------------- src/database/gravity-db.c | 47 +-------------------------------------- src/database/gravity-db.h | 1 - src/zip/teleporter.c | 3 +-- test/gravity.db.sql | 9 -------- 5 files changed, 16 insertions(+), 89 deletions(-) diff --git a/src/api/stats.c b/src/api/stats.c index 226a0c183..c9a1b4d42 100644 --- a/src/api/stats.c +++ b/src/api/stats.c @@ -20,8 +20,6 @@ #include "../log.h" // config struct #include "../config/config.h" -// in_auditlist() -#include "../database/gravity-db.h" // overTime data #include "../overTime.h" // enum REGEX @@ -140,7 +138,6 @@ int api_stats_summary(struct ftl_conn *api) int api_stats_top_domains(struct ftl_conn *api) { int count = 10; - bool audit = false; const int domains = counters->domains; int *temparray = calloc(2*domains, sizeof(int*)); if(temparray == NULL) @@ -174,9 +171,6 @@ int api_stats_top_domains(struct ftl_conn *api) // Does the user request a non-default number of replies? // Note: We do not accept zero query requests here get_int_var(api->request->query_string, "count", &count); - - // Apply Audit Log filtering? - get_bool_var(api->request->query_string, "audit", &audit); } // Lock shared memory @@ -231,30 +225,19 @@ int api_stats_top_domains(struct ftl_conn *api) if(domain == NULL) continue; - // Skip this domain if there is a filter on it (but only if not in audit mode) - if(!audit) + // Skip this domain if there is a filter on it + bool skip_domain = false; + for(unsigned int j = 0; j < excludeDomains; j++) { - // Check if this domain should be skipped - bool skip_domain = false; - for(unsigned int j = 0; j < excludeDomains; j++) + cJSON *item = cJSON_GetArrayItem(config.webserver.api.excludeDomains.v.json, j); + if(strcmp(getstr(domain->domainpos), item->valuestring) == 0) { - cJSON *item = cJSON_GetArrayItem(config.webserver.api.excludeDomains.v.json, j); - if(strcmp(getstr(domain->domainpos), item->valuestring) == 0) - { - skip_domain = true; - break; - } + skip_domain = true; + break; } - if(skip_domain) - continue; } - - // Skip this domain if already audited - if(audit && in_auditlist(getstr(domain->domainpos)) > 0) - { - log_debug(DEBUG_API, "API: %s has been audited.", getstr(domain->domainpos)); + if(skip_domain) continue; - } // Hidden domain, probably due to privacy level. Skip this in the top lists if(strcmp(getstr(domain->domainpos), HIDDEN_DOMAIN) == 0) @@ -288,9 +271,9 @@ int api_stats_top_domains(struct ftl_conn *api) cJSON *json = JSON_NEW_OBJECT(); JSON_ADD_ITEM_TO_OBJECT(json, "domains", top_domains); - const int blocked_queries = get_blocked_count(); + const int blocked_count = get_blocked_count(); JSON_ADD_NUMBER_TO_OBJECT(json, "total_queries", counters->queries); - JSON_ADD_NUMBER_TO_OBJECT(json, "blocked_queries", blocked_queries); + JSON_ADD_NUMBER_TO_OBJECT(json, "blocked_queries", blocked_count); JSON_SEND_OBJECT_UNLOCK(json); } @@ -411,8 +394,8 @@ int api_stats_top_clients(struct ftl_conn *api) cJSON *json = JSON_NEW_OBJECT(); JSON_ADD_ITEM_TO_OBJECT(json, "clients", top_clients); - const int blocked_queries = get_blocked_count(); - JSON_ADD_NUMBER_TO_OBJECT(json, "blocked_queries", blocked_queries); + const int blocked_count = get_blocked_count(); + JSON_ADD_NUMBER_TO_OBJECT(json, "blocked_queries", blocked_count); JSON_ADD_NUMBER_TO_OBJECT(json, "total_queries", counters->queries); JSON_SEND_OBJECT_UNLOCK(json); } @@ -526,8 +509,8 @@ int api_stats_upstreams(struct ftl_conn *api) cJSON *json = JSON_NEW_OBJECT(); JSON_ADD_ITEM_TO_OBJECT(json, "upstreams", top_upstreams); - const int forwarded_queries = get_forwarded_count(); - JSON_ADD_NUMBER_TO_OBJECT(json, "forwarded_queries", forwarded_queries); + const int forwarded_count = get_forwarded_count(); + JSON_ADD_NUMBER_TO_OBJECT(json, "forwarded_queries", forwarded_count); JSON_ADD_NUMBER_TO_OBJECT(json, "total_queries", counters->queries); JSON_SEND_OBJECT_UNLOCK(json); } diff --git a/src/database/gravity-db.c b/src/database/gravity-db.c index fbb3a1c6c..70d25aa88 100644 --- a/src/database/gravity-db.c +++ b/src/database/gravity-db.c @@ -46,7 +46,6 @@ sqlite3_stmt_vec *blacklist_stmt = NULL; // Private variables static sqlite3 *gravity_db = NULL; static sqlite3_stmt* table_stmt = NULL; -static sqlite3_stmt* auditlist_stmt = NULL; bool gravityDB_opened = false; static bool gravity_abp_format = false; @@ -174,35 +173,6 @@ bool gravityDB_open(void) return false; } - // Prepare audit statement - log_debug(DEBUG_DATABASE, "gravityDB_open(): Preparing audit query"); - - // We support adding audit domains with a wildcard character (*) - // Example 1: google.de - // matches only google.de - // Example 2: *.google.de - // matches all subdomains of google.de - // BUT NOT google.de itself - // Example 3: *google.de - // matches 'google.de' and all of its subdomains but - // also other domains ending in google.de, like - // abcgoogle.de - rc = sqlite3_prepare_v3(gravity_db, - "SELECT domain, " - "CASE WHEN substr(domain, 1, 1) = '*' " // Does the database string start in '*' ? - "THEN '*' || substr(:input, - length(domain) + 1) " // If so: Crop the input domain and prepend '*' - "ELSE :input " // If not: Use input domain directly for comparison - "END matcher " - "FROM domain_audit WHERE matcher = domain" // Match where (modified) domain equals the database domain - ";", -1, SQLITE_PREPARE_PERSISTENT, &auditlist_stmt, NULL); - - if( rc != SQLITE_OK ) - { - log_err("gravityDB_open(\"SELECT EXISTS(... domain_audit ...)\") - SQL error prepare: %s", sqlite3_errstr(rc)); - gravityDB_close(); - return false; - } - // Set SQLite3 busy timeout to a user-defined value (defaults to 1 second) // to avoid immediate failures when the gravity database is still busy // writing the changes to disk @@ -983,10 +953,6 @@ void gravityDB_close(void) free_sqlite3_stmt_vec(&gravity_stmt); free_sqlite3_stmt_vec(&antigravity_stmt); - // Finalize audit list statement - sqlite3_finalize(auditlist_stmt); - auditlist_stmt = NULL; - // Close table sqlite3_close(gravity_db); gravity_db = NULL; @@ -1187,7 +1153,7 @@ static enum db_result domain_in_list(const char *domain, sqlite3_stmt *stmt, con // Bind domain to prepared statement // SQLITE_STATIC: Use the string without first duplicating it internally. // We can do this as domain has dynamic scope that exceeds that of the binding. - // We need to bind the domain only once even to the prepared audit statement as: + // We need to bind the domain only once: // When the same named SQL parameter is used more than once, second and // subsequent occurrences have the same index as the first occurrence. // (https://www.sqlite.org/c3ref/bind_blob.html) @@ -1503,17 +1469,6 @@ enum db_result in_denylist(const char *domain, DNSCacheData *dns_cache, clientsD return domain_in_list(domain, stmt, "blacklist", &dns_cache->domainlist_id); } -bool in_auditlist(const char *domain) -{ - // If audit list statement is not ready and cannot be initialized (e.g. no access - // to the database), we return false (not in audit list) to prevent an FTL crash - if(auditlist_stmt == NULL) - return false; - - // We check the domain_audit table for the given domain - return domain_in_list(domain, auditlist_stmt, "auditlist", NULL) == FOUND; -} - bool gravityDB_get_regex_client_groups(clientsData* client, const unsigned int numregex, const regexData *regex, const unsigned char type, const char* table) { diff --git a/src/database/gravity-db.h b/src/database/gravity-db.h index 8b77fdecf..512ac80af 100644 --- a/src/database/gravity-db.h +++ b/src/database/gravity-db.h @@ -59,7 +59,6 @@ cJSON *gen_abp_patterns(const char *domain, const bool antigravity); enum db_result in_gravity(const char *domain, clientsData *client, const bool antigravity, int* domain_id); enum db_result in_denylist(const char *domain, DNSCacheData *dns_cache, clientsData *client); enum db_result in_allowlist(const char *domain, DNSCacheData *dns_cache, clientsData *client); -bool in_auditlist(const char *domain); bool gravityDB_get_regex_client_groups(clientsData* client, const unsigned int numregex, const regexData *regex, const unsigned char type, const char* table); diff --git a/src/zip/teleporter.c b/src/zip/teleporter.c index d541a8e40..ee0cf0b7a 100644 --- a/src/zip/teleporter.c +++ b/src/zip/teleporter.c @@ -48,8 +48,7 @@ static const char *gravity_tables[] = { "domainlist", "domainlist_by_group", "client", - "client_by_group", - "domain_audit" + "client_by_group" }; // Tables to copy from the FTL database to the Teleporter database diff --git a/test/gravity.db.sql b/test/gravity.db.sql index a8c1438f5..21e970b37 100644 --- a/test/gravity.db.sql +++ b/test/gravity.db.sql @@ -66,13 +66,6 @@ CREATE TABLE info INSERT INTO "info" VALUES('version','12'); -CREATE TABLE domain_audit -( - id INTEGER PRIMARY KEY AUTOINCREMENT, - domain TEXT UNIQUE NOT NULL, - date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)) -); - CREATE TABLE domainlist_by_group ( domainlist_id INTEGER NOT NULL REFERENCES domainlist (id), @@ -251,8 +244,6 @@ INSERT INTO info VALUES('updated',0); INSERT INTO "group" VALUES(1,0,'Test group',1559928803,1559928803,'A disabled test group'); INSERT INTO domainlist_by_group VALUES(15,1); -INSERT INTO domain_audit VALUES(1,'google.com',1559928803); - INSERT INTO client (id,ip) VALUES(1,'127.0.0.1'); INSERT INTO client (id,ip) VALUES(2,'127.0.0.2'); From 651d3d6512b6287b9d353194a9af76d8ec17ba5b Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 7 Dec 2023 22:58:54 +0100 Subject: [PATCH 180/221] Lenght -1 means users wants to get all queries (no server-side pagination) Signed-off-by: DL6ER --- src/api/queries.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/api/queries.c b/src/api/queries.c index 4272cca30..b63edb58f 100644 --- a/src/api/queries.c +++ b/src/api/queries.c @@ -758,7 +758,8 @@ int api_queries(struct ftl_conn *api) continue; // Check if we have reached the limit - if(added >= (unsigned int)length) + // Length may be set to -1 to indicate we want everything. + if(length > 0 && added >= (unsigned int)length) { if(filtering) { @@ -785,8 +786,6 @@ int api_queries(struct ftl_conn *api) } else if(length > 0 && added >= (unsigned int)length) { - // Length may be set to -1 to indicate we want - // everything. // Skip everything AFTER we added the requested number // of queries if length is > 0. break; From 86213b34b99c573fff5e70008aaecfb96026844e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 7 Dec 2023 22:59:51 +0100 Subject: [PATCH 181/221] Reduce variable scope Signed-off-by: DL6ER --- src/api/network.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/api/network.c b/src/api/network.c index 68e8113cb..9d26ddc21 100644 --- a/src/api/network.c +++ b/src/api/network.c @@ -31,15 +31,15 @@ static bool getDefaultInterface(char iface[IF_NAMESIZE], in_addr_t *gw) unsigned long dest_r = 0, gw_r = 0; unsigned int flags = 0u; int metric = 0, minmetric = __INT_MAX__; - char iface_r[IF_NAMESIZE] = { 0 }; - char buf[1024] = { 0 }; FILE *file; if((file = fopen("/proc/net/route", "r"))) { // Parse /proc/net/route - the kernel's IPv4 routing table + char buf[1024] = { 0 }; while(fgets(buf, sizeof(buf), file)) { + char iface_r[IF_NAMESIZE] = { 0 }; if(sscanf(buf, "%15s %lx %lx %x %*i %*i %i", iface_r, &dest_r, &gw_r, &flags, &metric) != 5) continue; From c7a6a881034ebc3ee720d500d2fc04476d1e9630 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 8 Dec 2023 04:56:16 +0100 Subject: [PATCH 182/221] If we have an exact match for pihole-FTL --config
    • "); - output_html_string(p->out, azCol[i]); - raw_printf(p->out,""); + output_html_string(azCol[i]); + oputz("
    "); - output_html_string(p->out, azArg[i] ? azArg[i] : p->nullValue); - raw_printf(p->out,""); + output_html_string(azArg[i] ? azArg[i] : p->nullValue); + oputz("