diff --git a/src/database/query-table.c b/src/database/query-table.c index 7955a0db5..88fa97230 100644 --- a/src/database/query-table.c +++ b/src/database/query-table.c @@ -1205,7 +1205,7 @@ void DB_read_queries(void) query->domainID = domainID; query->clientID = clientID; query->upstreamID = upstreamID; - query->cacheID = findCacheID(domainID, clientID, query->type, true); + query->cacheID = -1; query->id = counters->queries; query->response = 0; query->flags.response_calculated = reply_time_avail; @@ -1263,6 +1263,7 @@ void DB_read_queries(void) // Set ID of the domainlist entry that was the reason for permitting/blocking this query // We assume the value in this field is said ID when it is not a CNAME-related domain // (checked above) and the value of additional_info is not NULL (0 bytes storage size) + query->cacheID = findCacheID(domainID, clientID, query->type, true); DNSCacheData *cache = getDNSCache(query->cacheID, true); // Only load if // a) we have a cache entry @@ -1539,7 +1540,7 @@ bool queries_to_database(void) } // Get cache entry for this query - const int cacheID = query->cacheID >= 0 ? query->cacheID : findCacheID(query->domainID, query->clientID, query->type, false); + const unsigned int cacheID = query->cacheID > -1 ? query->cacheID : findCacheID(query->domainID, query->clientID, query->type, false); DNSCacheData *cache = getDNSCache(cacheID, true); // ADDITIONAL_INFO diff --git a/src/datastructure.c b/src/datastructure.c index 341f9361b..1911f95ac 100644 --- a/src/datastructure.c +++ b/src/datastructure.c @@ -196,20 +196,10 @@ int _findUpstreamID(const char *upstreamString, const in_port_t port, int line, static int get_next_free_domainID(void) { - // Compare content of domain against known domain IP addresses - for(unsigned int domainID = 0; domainID < counters->domains; domainID++) - { - // Get domain pointer - const domainsData *domain = getDomain(domainID, false); - - // Check if the returned pointer is valid before trying to access it - if(domain == NULL) - continue; - - // Check if the magic byte is set - if(domain->magic == 0x00) - return domainID; - } + // First, try to obtain a previously recycled domain ID + unsigned int domainID = 0; + if(get_next_recycled_ID(DOMAINS, &domainID)) + return domainID; // If we did not return until here, then we need to allocate a new domain ID return counters->domains; @@ -289,20 +279,10 @@ int _findDomainID(const char *domainString, const bool count, int line, const ch static int get_next_free_clientID(void) { - // Compare content of client against known client IP addresses - for(unsigned int clientID = 0; clientID < counters->clients; clientID++) - { - // Get client pointer - const clientsData *client = getClient(clientID, false); - - // Check if the returned pointer is valid before trying to access it - if(client == NULL) - continue; - - // Check if the magic byte is unset - if(client->magic == 0x00) - return clientID; - } + // First, try to obtain a previously recycled client ID + unsigned int clientID = 0; + if(get_next_recycled_ID(CLIENTS, &clientID)) + return clientID; // If we did not return until here, then we need to allocate a new client ID return counters->clients; @@ -459,20 +439,10 @@ void change_clientcount(clientsData *client, int total, int blocked, int overTim static int get_next_free_cacheID(void) { - // Compare content of cache against known cache IP addresses - for(unsigned int cacheID = 0; cacheID < counters->dns_cache_size; cacheID++) - { - // Get cache pointer - const DNSCacheData *cache = getDNSCache(cacheID, false); - - // Check if the returned pointer is valid before trying to access it - if(cache == NULL) - continue; - - // Check if the magic byte is set - if(cache->magic == 0x00) - return cacheID; - } + // First, try to obtain a previously recycled cache ID + unsigned int cacheID = 0; + if(get_next_recycled_ID(DNS_CACHE, &cacheID)) + return cacheID; // If we did not return until here, then we need to allocate a new cache ID return counters->dns_cache_size; @@ -1186,7 +1156,7 @@ void _query_set_status(queriesData *query, const enum query_status new_status, c new_status != QUERY_RETRIED && new_status != QUERY_RETRIED_DNSSEC) { - const int cacheID = findCacheID(query->domainID, query->clientID, query->type, true); + const unsigned int cacheID = query->cacheID > 0 ? query->cacheID : findCacheID(query->domainID, query->clientID, query->type, true); DNSCacheData *dns_cache = getDNSCache(cacheID, true); if(dns_cache != NULL && dns_cache->blocking_status != new_status) { diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index 6dd31083f..b1504f3b2 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -1425,12 +1425,7 @@ static bool FTL_check_blocking(const unsigned int queryID, const unsigned int do } // Get cache pointer - // When this function is called with a different domain than the one - // already stored in the query, we have to re-lookup the cache ID. - // This can happen when a CNAME chain is followed and analyzed - const int cacheID = query->domainID == domainID && query->clientID == clientID ? - query->cacheID : findCacheID(domainID, clientID, query->type, true); - DNSCacheData *dns_cache = getDNSCache(cacheID, true); + DNSCacheData *dns_cache = getDNSCache(query->cacheID, true); if(dns_cache == NULL) { log_err("No memory available, skipping query analysis"); @@ -1448,14 +1443,20 @@ static bool FTL_check_blocking(const unsigned int queryID, const unsigned int do dns_cache->list_id = -1; } + // Check if the cache record we have applies to the current query + // If not, ensure we re-check the domain (happens during CNAME inspection) + enum query_status blocking_status = QUERY_UNKNOWN; + if(query->clientID == clientID && query->domainID == domainID) + blocking_status = dns_cache->blocking_status; + // Memorize blocking status DNS cache for the domain/client combination - cacheStatus = dns_cache->blocking_status; + cacheStatus = blocking_status; log_debug(DEBUG_QUERIES, "Set global cache status to %d", cacheStatus); // Skip the entire chain of tests if we already know the answer for this // particular client char *domainstr = (char*)getstr(domain->domainpos); - switch(dns_cache->blocking_status) + switch(blocking_status) { case QUERY_UNKNOWN: // New domain/client combination. @@ -1468,7 +1469,7 @@ static bool FTL_check_blocking(const unsigned int queryID, const unsigned int do case QUERY_DENYLIST_CNAME: // Known as exactly denied, we return this result early, skipping // all the lengthy tests below - blockingreason = dns_cache->blocking_status == QUERY_DENYLIST ? "exactly denied" : "exactly denied (CNAME)"; + blockingreason = blocking_status == QUERY_DENYLIST ? "exactly denied" : "exactly denied (CNAME)"; log_debug(DEBUG_QUERIES, "%s is known as %s", domainstr, blockingreason); // Do not block if the entire query is to be permitted @@ -1485,7 +1486,7 @@ static bool FTL_check_blocking(const unsigned int queryID, const unsigned int do case QUERY_GRAVITY_CNAME: // Known as gravity blocked, we return this result early, skipping // all the lengthy tests below - blockingreason = dns_cache->blocking_status == QUERY_GRAVITY ? "gravity blocked" : "gravity blocked (CNAME)"; + blockingreason = blocking_status == QUERY_GRAVITY ? "gravity blocked" : "gravity blocked (CNAME)"; log_debug(DEBUG_QUERIES, "%s is known as %s", domainstr, blockingreason); // Do not block if the entire query is to be permitted @@ -1502,7 +1503,7 @@ static bool FTL_check_blocking(const unsigned int queryID, const unsigned int do case QUERY_REGEX_CNAME: // Known as regex denied, we return this result early, skipping all // the lengthy tests below - blockingreason = dns_cache->blocking_status == QUERY_REGEX ? "regex denied" : "regex denied (CNAME)"; + blockingreason = blocking_status == QUERY_REGEX ? "regex denied" : "regex denied (CNAME)"; log_debug(DEBUG_QUERIES, "%s is known as %s (cache regex ID: %i)", domainstr, blockingreason, dns_cache->list_id); @@ -1533,7 +1534,7 @@ static bool FTL_check_blocking(const unsigned int queryID, const unsigned int do case QUERY_EXTERNAL_BLOCKED_NXRA: case QUERY_EXTERNAL_BLOCKED_EDE15: - switch(dns_cache->blocking_status) + switch(blocking_status) { case QUERY_UNKNOWN: case QUERY_GRAVITY: @@ -1573,7 +1574,7 @@ static bool FTL_check_blocking(const unsigned int queryID, const unsigned int do domainstr, blockingreason, (unsigned long)(dns_cache->expires - time(NULL))); force_next_DNS_reply = dns_cache->force_reply; - query_blocked(query, domain, client, dns_cache->blocking_status); + query_blocked(query, domain, client, blocking_status); return true; break; @@ -1802,8 +1803,8 @@ bool FTL_CNAME(const char *dst, const char *src, const int id) else if(query->status == QUERY_REGEX) { // Get parent and child DNS cache entries - const int parent_cacheID = query->cacheID; - const int child_cacheID = findCacheID(child_domainID, clientID, query->type, false); + const unsigned int parent_cacheID = query->cacheID > -1 ? query->cacheID : findCacheID(parent_domainID, clientID, query->type, false); + const unsigned int child_cacheID = findCacheID(child_domainID, clientID, query->type, false); // Get cache pointers DNSCacheData *parent_cache = getDNSCache(parent_cacheID, true); diff --git a/src/gc.c b/src/gc.c index 74043f277..552108dbb 100644 --- a/src/gc.c +++ b/src/gc.c @@ -116,6 +116,9 @@ static void recycle(void) // Remove client from lookup table lookup_remove(CLIENTS_LOOKUP, clientID, client->hash); + // Add ID of recycled client to recycle table + set_next_recycled_ID(CLIENTS, clientID); + // Wipe client's memory memset(client, 0, sizeof(clientsData)); @@ -152,6 +155,9 @@ static void recycle(void) // Remove domain from lookup table lookup_remove(DOMAINS_LOOKUP, domainID, domain->hash); + // Add ID of recycled domain to recycle table + set_next_recycled_ID(DOMAINS, domainID); + // Wipe domain's memory memset(domain, 0, sizeof(domainsData)); @@ -174,6 +180,9 @@ static void recycle(void) // Remove cache entry from lookup table lookup_remove(DNS_CACHE_LOOKUP, cacheID, cache->hash); + // Add ID of recycled domain to recycle table + set_next_recycled_ID(DNS_CACHE, cacheID); + // Wipe cache entry's memory memset(cache, 0, sizeof(DNSCacheData)); @@ -222,7 +231,7 @@ static void recycle(void) counters->domains_MAX + free_domains - counters->domains, counters->domains_MAX, counters->dns_cache_MAX + free_cache - counters->dns_cache_size, counters->dns_cache_MAX); - log_debug(DEBUG_GC, "Recycled additional %u clients, %u domains, and %u cache records (scanned %u queries)", + log_debug(DEBUG_GC, "Recycled %u clients, %u domains, and %u cache records (scanned %u queries)", clients_recycled, domains_recycled, cache_recycled, counters->queries); } } @@ -619,6 +628,7 @@ void *GC_thread(void *val) { lock_shm(); lookup_find_hash_collisions(); + print_recycle_list_fullness(); unlock_shm(); } diff --git a/src/shmem.c b/src/shmem.c index d01dc5349..773582a3e 100644 --- a/src/shmem.c +++ b/src/shmem.c @@ -54,6 +54,7 @@ #define SHARED_CLIENTS_LOOKUP_NAME "FTL-clients-lookup" #define SHARED_DOMAINS_LOOKUP_NAME "FTL-domains-lookup" #define SHARED_DNS_CACHE_LOOKUP_NAME "FTL-dns-cache-lookup" +#define SHARED_RECYCLER_NAME "FTL-recycler" // Allocation step for FTL-strings bucket. This is somewhat special as we use // this as a general-purpose storage which should always be large enough. If, @@ -82,6 +83,7 @@ static SharedMemory shm_fifo_log = { 0 }; static SharedMemory shm_clients_lookup = { 0 }; static SharedMemory shm_domains_lookup = { 0 }; static SharedMemory shm_dns_cache_lookup = { 0 }; +static SharedMemory shm_recycler = { 0 }; static SharedMemory *sharedMemories[] = { &shm_lock, &shm_strings, @@ -97,7 +99,8 @@ static SharedMemory *sharedMemories[] = { &shm_lock, &shm_fifo_log, &shm_clients_lookup, &shm_domains_lookup, - &shm_dns_cache_lookup }; + &shm_dns_cache_lookup, + &shm_recycler }; // Variable size array structs static queriesData *queries = NULL; @@ -109,6 +112,7 @@ fifologData *fifo_log = NULL; struct lookup_table *clients_lookup = NULL; struct lookup_table *domains_lookup = NULL; struct lookup_table *dns_cache_lookup = NULL; +struct recycler_tables *recycler = NULL; static void **global_pointers[] = {(void**)&queries, (void**)&clients, @@ -118,7 +122,8 @@ static void **global_pointers[] = {(void**)&queries, (void**)&fifo_log, (void**)&clients_lookup, (void**)&domains_lookup, - (void**)&dns_cache_lookup }; + (void**)&dns_cache_lookup, + (void**)&recycler}; typedef struct { struct { @@ -578,6 +583,13 @@ bool init_shmem() dns_cache_lookup = (struct lookup_table*)shm_dns_cache_lookup.ptr; counters->dns_cache_lookup_MAX = size; + /****************************** shared recycler struct ******************************/ + // Try to create shared memory object + create_shm(SHARED_RECYCLER_NAME, &shm_recycler, sizeof(struct recycler_tables)); + if(shm_recycler.ptr == NULL) + return false; + recycler = (struct recycler_tables*)shm_recycler.ptr; + return true; } @@ -1317,3 +1329,145 @@ double __attribute__((pure)) get_qps(void) // Return the computed value divided by N (the number of slots) return qps / QPS_AVGLEN; } + +/** + * @brief Retrieves the recycle table based on the specified memory type. + * + * This function returns a pointer to the appropriate recycle table + * corresponding to the given memory type. The memory types can be + * CLIENTS, DOMAINS, or DNS_CACHE. If the memory type does not match + * any of these, the function returns NULL. + * + * @param type The memory type for which the recycle table is requested. + * It can be one of the following: + * - CLIENTS: Recycle table for clients. + * - DOMAINS: Recycle table for domains. + * - DNS_CACHE: Recycle table for DNS cache. + * @param name A pointer to a string that will be set to the name of the + * recycle table corresponding to the given memory type. + * + * @return A pointer to the recycle table corresponding to the given + * memory type, or NULL if the memory type is not recognized. + */ +static struct recycle_table *get_recycle_table(const enum memory_type type, const char **name) +{ + if(type == CLIENTS) + { + *name = "clients"; + return &recycler->client; + } + else if(type == DOMAINS) + { + *name = "domains"; + return &recycler->domain; + } + else if(type == DNS_CACHE) + { + *name = "dns_cache"; + return &recycler->dns_cache; + } + + return NULL; +} + +/** + * @brief Sets the next recycled ID for a given memory type. + * + * This function adds a new ID to the recycle table for the specified memory type. + * If the recycle table is full or the memory type is invalid, the function will + * log an appropriate message and return false. + * + * @param type The memory type for which the ID is being set. + * @param id The ID to be added to the recycle table. + * @return true if the ID was successfully added to the recycle table, false otherwise. + */ +bool set_next_recycled_ID(const enum memory_type type, const unsigned int id) +{ + // Get the correct table + const char *name = NULL; + struct recycle_table *rp = get_recycle_table(type, &name); + + if(rp == NULL) + { + log_err("set_next_recycled_ID(): Invalid memory type %i", type); + return false; + } + + // Check if we already have the maximum number of recycled entries + if(rp->count >= RECYCLE_ARRAY_LEN) + { + // This is not strictly an error, but it is worth noting if in + // debug mode as increasing RECYCLE_ARRAY_LEN may be useful in + // this environment + log_debug(DEBUG_SHMEM, "set_next_recycled_ID(): Recycle table[%s] is full", name); + return false; + } + + log_debug(DEBUG_GC, "RECYCLE[%s][%u] = %u SET", name, rp->count, id); + + // Set the id of the recycled entry and increment the count + rp->id[rp->count] = id; + rp->count++; + + return true; +} + +/** + * @brief Retrieves the next recycled ID from the recycle table for the specified memory type. + * + * This function fetches the next available recycled ID from the recycle table associated with the given memory type. + * If there are no recycled IDs available or the memory type is invalid, the function returns false. + * + * @param type The memory type for which to retrieve the recycled ID. + * @param id A pointer to an unsigned int where the retrieved recycled ID will be stored. + * @return true if a recycled ID was successfully retrieved, false otherwise. + */ +bool get_next_recycled_ID(const enum memory_type type, unsigned int *id) +{ + // Get the correct table + const char *name = NULL; + struct recycle_table *rp = get_recycle_table(type, &name); + + if(rp == NULL) + { + log_err("get_next_recycled_ID(): Invalid memory type %i", type); + return false; + } + + + // Check if we have any recycled entries + if(rp->count == 0) + { + log_debug(DEBUG_GC, "RECYCLE[%s] is empty", name); + return false; + } + + // Take one away from the array + rp->count--; + + // Get the ID of the recycled entry and decrement the count + *id = rp->id[rp->count]; + + // Unset the ID of the element just used + rp->id[rp->count] = 0; + + log_debug(DEBUG_GC, "RECYCLE[%s][%u] = %u TAKE", name, rp->count, *id); + + return true; +} + +/** + * @brief Logs the fullness of various recycle lists. + * + * This function logs the fullness of the recycle lists for clients, domains, + * and DNS cache. It provides the current count, the maximum capacity, and the + * percentage of fullness for each list. + * + */ +void print_recycle_list_fullness(void) +{ + log_info("Recycle list fullness:"); + log_info(" Clients: %u/%u (%.2f%%)", recycler->client.count, RECYCLE_ARRAY_LEN, (double)recycler->client.count / RECYCLE_ARRAY_LEN * 100.0); + log_info(" Domains: %u/%u (%.2f%%)", recycler->domain.count, RECYCLE_ARRAY_LEN, (double)recycler->domain.count / RECYCLE_ARRAY_LEN * 100.0); + log_info(" DNS Cache: %u/%u (%.2f%%)", recycler->dns_cache.count, RECYCLE_ARRAY_LEN, (double)recycler->dns_cache.count / RECYCLE_ARRAY_LEN * 100.0); +} diff --git a/src/shmem.h b/src/shmem.h index 40d14cf85..4f87de895 100644 --- a/src/shmem.h +++ b/src/shmem.h @@ -104,6 +104,35 @@ static bool realloc_shm(SharedMemory *sharedMemory, const size_t size1, const si /// /// \param sharedMemory the shared memory struct static void delete_shm(SharedMemory *sharedMemory); + +// Number of elements in the recycle arrays +// Default: 65535 (which is 2^16 - 1) +// Total RAM estimate of struct recycler_tables is ~ RECYCLE_ARRAY_LEN * 12 bytes +// (roughly 786 KB for the default value) +#define RECYCLE_ARRAY_LEN 65535u + +/** + * struct recycle_table - Structure to hold recycling information. + * @var recycle_table::size: The size of the recycle table. + * @var recycle_table::id: An array of recycled IDs. + */ +struct recycle_table { + unsigned int count; + unsigned int id[RECYCLE_ARRAY_LEN]; +}; + + +/** + * struct recycler_table - Structure to hold multiple recycle tables. + * @var recycler_tables::client: Recycle table for clients. + * @var recycler_tables::domain: Recycle table for domains. + * @var recycler_tables::DNScache: Recycle table for DNS cache. + */ +struct recycler_tables { + struct recycle_table client; + struct recycle_table domain; + struct recycle_table dns_cache; +}; #endif #if defined(SHMEM_PRIVATE) || defined(LOOKUP_TABLE_PRIVATE) @@ -168,4 +197,9 @@ void update_qps(const time_t timestamp); void reset_qps(const time_t timestamp); double get_qps(void) __attribute__((pure)); +// Recycler table functions +bool set_next_recycled_ID(const enum memory_type type, const unsigned int id); +bool get_next_recycled_ID(const enum memory_type type, unsigned int *id); +void print_recycle_list_fullness(void); + #endif //SHARED_MEMORY_SERVER_H diff --git a/src/signals.c b/src/signals.c index d02c81ef3..4017605fa 100644 --- a/src/signals.c +++ b/src/signals.c @@ -25,8 +25,6 @@ #include "timers.h" // struct config #include "config/config.h" -// lookup_find_hash_collisions -#include "lookup-table.h" #define BINARY_NAME "pihole-FTL"