Rename query_storage.regex_id to query_storage.list_id as it is alrea…

…dy now used to also store exact matching domainlist entries by their ID. This commit further extends this to also store the (first) matching anti-/gravity list (if available)

Signed-off-by: DL6ER <[email protected]>

src/api/docs/content/specs/queries.yaml

@@ -216,9 +216,9 @@ components:
                   time:
                     type: number
                     description: Time until the response was received (ms, negative if N/A)
-              regex_id:
+              list_id:
                 type: integer
-                description: ID of regex (`NULL` if N/A)
+                description: ID of corresponding database table (adlist for anti-/gravity, else domainlist) (`NULL` if N/A)
                 nullable: true
               upstream:
                 type: string
@@ -237,7 +237,7 @@ components:
               reply:
                 type: "IP"
                 time: 19
-              regex_id: NULL
+              list_id: NULL
               upstream: "localhost#5353"
               dbid: 112421354
             - time: 1581907871.583821
@@ -252,7 +252,7 @@ components:
               reply:
                 type: "IP"
                 time: 12.3
-              regex_id: NULL
+              list_id: NULL
               upstream: "localhost#5353"
               dbid: 112421355
         cursor:

src/api/queries.c

@@ -174,7 +174,7 @@ int api_queries_suggestions(struct ftl_conn *api)
 	JSON_SEND_OBJECT(json);
 }
 
-#define QUERYSTR "SELECT q.id,timestamp,q.type,status,d.domain,f.forward,additional_info,reply_type,reply_time,dnssec,c.ip,c.name,a.content,regex_id"
+#define QUERYSTR "SELECT q.id,timestamp,q.type,status,d.domain,f.forward,additional_info,reply_type,reply_time,dnssec,c.ip,c.name,a.content,list_id"
 // JOIN: Only return rows where there is a match in BOTH tables
 // LEFT JOIN: Return all rows from the left table, and the matched rows from the right table
 #define JOINSTR "JOIN client_by_id c ON q.client = c.id JOIN domain_by_id d ON q.domain = d.id LEFT JOIN forward_by_id f ON q.forward = f.id LEFT JOIN addinfo_by_id a ON a.id = q.additional_info"
@@ -215,8 +215,8 @@ static void querystr_finish(char *querystr, const char *sort_col, const char *so
 			sort_col_sql = "q.reply_time";
 		else if(strcasecmp(sort_col, "dnssec") == 0)
 			sort_col_sql = "q.dnssec";
-		else if(strcasecmp(sort_col, "regex_id") == 0)
-			sort_col_sql = "regex_id";
+		else if(strcasecmp(sort_col, "list_id") == 0)
+			sort_col_sql = "list_id";
 
 		// ... and the sort direction
 		if(strcasecmp(sort_dir, "asc") == 0 || strcasecmp(sort_dir, "ascending") == 0)
@@ -828,11 +828,11 @@ int api_queries(struct ftl_conn *api)
 			JSON_ADD_NULL_TO_OBJECT(client, "name");
 		JSON_ADD_ITEM_TO_OBJECT(item, "client", client);
 
-		// Add regex_id if it exists
+		// Add list_id if it exists
 		if(sqlite3_column_type(read_stmt, 13) == SQLITE_INTEGER)
-			JSON_ADD_NUMBER_TO_OBJECT(item, "regex_id", sqlite3_column_int(read_stmt, 13)); // regex_id
+			JSON_ADD_NUMBER_TO_OBJECT(item, "list_id", sqlite3_column_int(read_stmt, 13)); // list_id
 		else
-			JSON_ADD_NULL_TO_OBJECT(item, "regex_id");
+			JSON_ADD_NULL_TO_OBJECT(item, "list_id");
 
 		const unsigned char *cname = NULL;
 		switch(query.status)

src/database/common.c

@@ -543,6 +543,26 @@ void db_init(void)
 		dbversion = db_get_int(db, DB_VERSION);
 	}
 
+	// Update to version 17 if lower
+	if(dbversion < 17)
+	{
+		// Update to version 17: Rename regex_id column to regex_id_old
+		log_info("Updating long-term database to version 17");
+		if(!rename_query_storage_column_regex_id(db))
+		{
+			log_info("regex_id cannot be renamed to list_id, database not available");
+			dbclose(&db);
+			return;
+		}
+		// Get updated version
+		dbversion = db_get_int(db, DB_VERSION);
+	}
+
+	// Last check after all migrations, if this happens, it will cause the
+	// CI to fail the tests
+	if(dbversion != MEMDB_VERSION)
+		log_err("Database version %i does not match MEMDB_VERSION %i", dbversion, MEMDB_VERSION);
+
 	lock_shm();
 	import_aliasclients(db);
 	unlock_shm();

src/database/gravity-db.c

@@ -1262,7 +1262,7 @@ enum db_result in_allowlist(const char *domain, DNSCacheData *dns_cache, clients
 	// We have to check both the exact whitelist (using a prepared database statement)
 	// as well the compiled regex whitelist filters to check if the current domain is
 	// whitelisted.
-	return domain_in_list(domain, stmt, "whitelist", &dns_cache->domainlist_id);
+	return domain_in_list(domain, stmt, "whitelist", &dns_cache->list_id);
 }
 
 cJSON *gen_abp_patterns(const char *domain, const bool antigravity)
@@ -1466,7 +1466,7 @@ enum db_result in_denylist(const char *domain, DNSCacheData *dns_cache, clientsD
 	if(stmt == NULL)
 		stmt = blacklist_stmt->get(blacklist_stmt, client->id);
 
-	return domain_in_list(domain, stmt, "blacklist", &dns_cache->domainlist_id);
+	return domain_in_list(domain, stmt, "blacklist", &dns_cache->list_id);
 }
 
 bool gravityDB_get_regex_client_groups(clientsData* client, const unsigned int numregex, const regexData *regex,

src/database/query-table.c

@@ -784,6 +784,29 @@ bool add_ftl_table_description(sqlite3 *db)
 	return true;
 }
 
+bool rename_query_storage_column_regex_id(sqlite3 *db)
+{
+	// Start transaction of database update
+	SQL_bool(db, "BEGIN TRANSACTION");
+
+	// Rename column regex_id to list_id
+	SQL_bool(db, "ALTER TABLE query_storage RENAME COLUMN regex_id TO list_id;");
+
+	// The VIEW queries is automatically updated by SQLite3
+
+	// Update database version to 17
+	if(!db_set_FTL_property(db, DB_VERSION, 17))
+	{
+		log_err("rename_query_storage_column_regex_id(): Failed to update database version!");
+		return false;
+	}
+
+	// Finish transaction
+	SQL_bool(db, "COMMIT");
+
+	return true;
+}
+
 bool optimize_queries_table(sqlite3 *db)
 {
 	// Start transaction of database update
@@ -1131,7 +1154,7 @@ void DB_read_queries(void)
 			//  a) we have a cache entry
 			//  b) the value of additional_info is not NULL (0 bytes storage size)
 			if(cache != NULL && sqlite3_column_bytes(stmt, 7) != 0)
-				cache->domainlist_id = sqlite3_column_int(stmt, 7);
+				cache->list_id = sqlite3_column_int(stmt, 7);
 		}
 
 		// Increment status counters
@@ -1485,15 +1508,15 @@ bool queries_to_database(void)
 				break;
 			}
 		}
-		else if(cache != NULL && query->status == QUERY_REGEX)
+		else if(cache != NULL && cache->list_id > -1)
 		{
 			// Restore regex ID if applicable
-			sqlite3_bind_int(query_stmt, 9, ADDINFO_REGEX_ID);
-			sqlite3_bind_int(query_stmt, 10, cache->domainlist_id);
+			sqlite3_bind_int(query_stmt, 9, ADDINFO_LIST_ID);
+			sqlite3_bind_int(query_stmt, 10, cache->list_id);
 
 			// Execute prepared addinfo statement and check if successful
-			sqlite3_bind_int(addinfo_stmt, 1, ADDINFO_REGEX_ID);
-			sqlite3_bind_int(addinfo_stmt, 2, cache->domainlist_id);
+			sqlite3_bind_int(addinfo_stmt, 1, ADDINFO_LIST_ID);
+			sqlite3_bind_int(addinfo_stmt, 2, cache->list_id);
 			rc = sqlite3_step(addinfo_stmt);
 			sqlite3_clear_bindings(addinfo_stmt);
 			sqlite3_reset(addinfo_stmt);
@@ -1524,9 +1547,9 @@ bool queries_to_database(void)
 		// DNSSEC
 		sqlite3_bind_int(query_stmt, 13, query->dnssec);
 
-		// REGEX_ID
-		if(cache != NULL && cache->domainlist_id > -1)
-			sqlite3_bind_int(query_stmt, 14, cache->domainlist_id);
+		// LIST_ID
+		if(cache != NULL && cache->list_id > -1)
+			sqlite3_bind_int(query_stmt, 14, cache->list_id);
 		else
 			// Not applicable, setting NULL
 			sqlite3_bind_null(query_stmt, 14);

src/database/query-table.h

@@ -23,20 +23,21 @@
                                                        "client TEXT NOT NULL, " \
                                                        "forward TEXT );"
 
-#define CREATE_QUERY_STORAGE_TABLE_V13 "CREATE TABLE query_storage ( id INTEGER PRIMARY KEY AUTOINCREMENT, " \
-                                                              "timestamp INTEGER NOT NULL, " \
-                                                              "type INTEGER NOT NULL, " \
-                                                              "status INTEGER NOT NULL, " \
-                                                              "domain INTEGER NOT NULL, " \
-                                                              "client INTEGER NOT NULL, " \
-                                                              "forward INTEGER, " \
-                                                              "additional_info INTEGER, " \
-                                                              "reply_type INTEGER, " \
-                                                              "reply_time REAL, " \
-                                                              "dnssec INTEGER, " \
-                                                              "regex_id INTEGER );"
+#define MEMDB_VERSION 17
+#define CREATE_QUERY_STORAGE_TABLE "CREATE TABLE query_storage ( id INTEGER PRIMARY KEY AUTOINCREMENT, " \
+                                                                "timestamp INTEGER NOT NULL, " \
+                                                                "type INTEGER NOT NULL, " \
+                                                                "status INTEGER NOT NULL, " \
+                                                                "domain INTEGER NOT NULL, " \
+                                                                "client INTEGER NOT NULL, " \
+                                                                "forward INTEGER, " \
+                                                                "additional_info INTEGER, " \
+                                                                "reply_type INTEGER, " \
+                                                                "reply_time REAL, " \
+                                                                "dnssec INTEGER, " \
+                                                                "list_id INTEGER );"
 
-#define CREATE_QUERIES_VIEW_V13 "CREATE VIEW queries AS " \
+#define CREATE_QUERIES_VIEW "CREATE VIEW queries AS " \
                                     "SELECT id, timestamp, type, status, " \
                                       "CASE typeof(domain) " \
                                         "WHEN 'integer' THEN (SELECT domain FROM domain_by_id d WHERE d.id = q.domain) ELSE domain END domain," \
@@ -46,7 +47,7 @@
                                         "WHEN 'integer' THEN (SELECT forward FROM forward_by_id f WHERE f.id = q.forward) ELSE forward END forward," \
                                       "CASE typeof(additional_info) "\
                                         "WHEN 'integer' THEN (SELECT content FROM addinfo_by_id a WHERE a.id = q.additional_info) ELSE additional_info END additional_info, " \
-                                      "reply_type, reply_time, dnssec, regex_id FROM query_storage q"
+                                      "reply_type, reply_time, dnssec, list_id FROM query_storage q"
 
 // Version 1
 #define CREATE_QUERIES_TIMESTAMP_INDEX		"CREATE INDEX idx_queries_timestamp ON queries (timestamp);"
@@ -62,8 +63,7 @@
 #define CREATE_QUERY_STORAGE_REPLY_TYPE_INDEX		"CREATE INDEX idx_query_storage_reply_type ON query_storage (reply_type);"
 #define CREATE_QUERY_STORAGE_REPLY_TIME_INDEX		"CREATE INDEX idx_query_storage_reply_time ON query_storage (reply_time);"
 #define CREATE_QUERY_STORAGE_DNSSEC_INDEX		"CREATE INDEX idx_query_storage_dnssec ON query_storage (dnssec);"
-//#define CREATE_QUERY_STORAGE_TTL_INDEX		"CREATE INDEX idx_query_storage_ttl ON query_storage (ttl);"
-//#define CREATE_QUERY_STORAGE_REGEX_ID_INDEX		"CREATE INDEX idx_query_storage_regex_id ON query_storage (regex_id);"
+#define CREATE_QUERY_STORAGE_LIST_ID_INDEX		"CREATE INDEX idx_query_storage_list_id ON query_storage (list_id);"
 
 #define CREATE_DOMAINS_BY_ID "CREATE TABLE domain_by_id (id INTEGER PRIMARY KEY, domain TEXT NOT NULL);"
 #define CREATE_CLIENTS_BY_ID "CREATE TABLE client_by_id (id INTEGER PRIMARY KEY, ip TEXT NOT NULL, name TEXT);"
@@ -77,12 +77,12 @@
 
 #ifdef QUERY_TABLE_PRIVATE
 const char *table_creation[] = {
-	CREATE_QUERY_STORAGE_TABLE_V13,
+	CREATE_QUERY_STORAGE_TABLE,
 	CREATE_DOMAINS_BY_ID,
 	CREATE_CLIENTS_BY_ID,
 	CREATE_FORWARD_BY_ID,
 	CREATE_ADDINFO_BY_ID,
-	CREATE_QUERIES_VIEW_V13,
+	CREATE_QUERIES_VIEW,
 };
 const char *index_creation[] = {
 	CREATE_QUERY_STORAGE_ID_INDEX,
@@ -96,8 +96,7 @@ const char *index_creation[] = {
 	CREATE_QUERY_STORAGE_REPLY_TYPE_INDEX,
 	CREATE_QUERY_STORAGE_REPLY_TIME_INDEX,
 	CREATE_QUERY_STORAGE_DNSSEC_INDEX,
-//	CREATE_QUERY_STORAGE_TTL_INDEX,
-//	CREATE_QUERY_STORAGE_REGEX_ID_INDEX
+	CREATE_QUERY_STORAGE_LIST_ID_INDEX
 	CREATE_DOMAIN_BY_ID_DOMAIN_INDEX,
 	CREATE_CLIENTS_BY_ID_IPNAME_INDEX,
 	CREATE_FORWARD_BY_ID_FORWARD_INDEX,
@@ -128,5 +127,6 @@ bool create_addinfo_table(sqlite3 *db);
 bool add_query_storage_columns(sqlite3 *db);
 bool add_query_storage_column_regex_id(sqlite3 *db);
 bool add_ftl_table_description(sqlite3 *db);
+bool rename_query_storage_column_regex_id(sqlite3 *db);
 
 #endif //QUERY_TABLE_PRIVATE_H

src/datastructure.c

@@ -433,7 +433,7 @@ int _findCacheID(const int domainID, const int clientID, const enum query_type q
 	dns_cache->clientID = clientID;
 	dns_cache->query_type = query_type;
 	dns_cache->force_reply = 0u;
-	dns_cache->domainlist_id = -1; // -1 = not set
+	dns_cache->list_id = -1; // -1 = not set
 
 	// Increase counter by one
 	counters->dns_cache_size++;

src/datastructure.h

@@ -112,7 +112,7 @@ typedef struct {
 	enum query_type query_type;
 	int domainID;
 	int clientID;
-	int domainlist_id;
+	int list_id;
 	char *cname_target;
 } DNSCacheData;
 

src/dnsmasq_interface.c

@@ -1139,16 +1139,20 @@ static bool check_domain_blocked(const char *domain, const int clientID,
 	}
 
 	// Check domain against antigravity
-	int domain_id = -1;
-	const enum db_result antigravity = in_gravity(domain, client, true, &domain_id);
+	int list_id = -1;
+	const enum db_result antigravity = in_gravity(domain, client, true, &list_id);
 	if(antigravity == FOUND)
 	{
-		log_debug(DEBUG_QUERIES, "Allowing query due to antigravity match (ID %i)", domain_id);
+		log_debug(DEBUG_QUERIES, "Allowing query due to antigravity match (list ID %i)", list_id);
+
+		// Store ID of the matching antigravity list
+		dns_cache->list_id = list_id;
+
 		return false;
 	}
 
 	// Check domains against gravity domains
-	const enum db_result gravity = in_gravity(domain, client, false, &domain_id);
+	const enum db_result gravity = in_gravity(domain, client, false, &list_id);
 	if(gravity == FOUND)
 	{
 		// Set new status
@@ -1158,6 +1162,11 @@ static bool check_domain_blocked(const char *domain, const int clientID,
 		// Mark domain as gravity blocked for this client
 		set_dnscache_blockingstatus(dns_cache, client, GRAVITY_BLOCKED, domain);
 
+		log_debug(DEBUG_QUERIES, "Blocking query due to gravity match (list ID %i)", list_id);
+
+		// Store ID of the matching gravity list
+		dns_cache->list_id = list_id;
+
 		// We block this domain
 		return true;
 	}
@@ -1218,7 +1227,7 @@ static bool check_domain_blocked(const char *domain, const int clientID,
 		cname_target = dns_cache->cname_target;
 
 		// Store ID of this regex (fork-private)
-		last_regex_idx = dns_cache->domainlist_id;
+		last_regex_idx = dns_cache->list_id;
 
 		// We block this domain
 		return true;
@@ -1353,14 +1362,14 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c
 			// the lengthy tests below
 			blockingreason = "regex denied";
 			log_debug(DEBUG_QUERIES, "%s is known as %s (cache regex ID: %i)",
-			          domainstr, blockingreason, dns_cache->domainlist_id);
+			          domainstr, blockingreason, dns_cache->list_id);
 
 			// Do not block if the entire query is to be permitted as something
 			// along the CNAME path hit the whitelist
 			if(!query->flags.allowed)
 			{
 				force_next_DNS_reply = dns_cache->force_reply;
-				last_regex_idx = dns_cache->domainlist_id;
+				last_regex_idx = dns_cache->list_id;
 				query_blocked(query, domain, client, QUERY_REGEX);
 				return true;
 			}
@@ -1472,7 +1481,7 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c
 		if(config.debug.queries.v.b)
 		{
 			log_debug(DEBUG_QUERIES, "Blocking %s as %s is %s (domainlist ID: %i)",
-			          domainstr, blockedDomain, blockingreason, dns_cache->domainlist_id);
+			          domainstr, blockedDomain, blockingreason, dns_cache->list_id);
 			if(force_next_DNS_reply != 0)
 				log_debug(DEBUG_QUERIES, "Forcing next reply to %s", get_query_reply_str(force_next_DNS_reply));
 		}
@@ -1487,7 +1496,7 @@ static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const c
 		// Debug output
 		// client is guaranteed to be non-NULL above
 		log_debug(DEBUG_QUERIES, "DNS cache: %s/%s is %s (domainlist ID: %i)", getstr(client->ippos),
-		          domainstr, query->flags.allowed ? "whitelisted" : "not blocked", dns_cache->domainlist_id);
+		          domainstr, query->flags.allowed ? "whitelisted" : "not blocked", dns_cache->list_id);
 	}
 
 	free(domainstr);
@@ -1597,8 +1606,8 @@ bool _FTL_CNAME(const char *dst, const char *src, const int id, const char* file
 
 			// Propagate ID of responsible regex up from the child to the parent
 			// domain (but only if set)
-			if(parent_cache != NULL && child_cache != NULL && child_cache->domainlist_id != -1)
-				parent_cache->domainlist_id = child_cache->domainlist_id;
+			if(parent_cache != NULL && child_cache != NULL && child_cache->list_id != -1)
+				parent_cache->list_id = child_cache->list_id;
 
 			// Set status
 			query_set_status(query, QUERY_REGEX_CNAME);

src/enums.h

@@ -283,7 +283,7 @@ enum ptr_type {
 
 enum addinfo_type {
 	ADDINFO_CNAME_DOMAIN = 1,
-	ADDINFO_REGEX_ID
+	ADDINFO_LIST_ID
 } __attribute__ ((packed));
 
 enum listening_mode {

src/regex.c

@@ -548,7 +548,7 @@ bool in_regex(const char *domain, DNSCacheData *dns_cache, const int clientID, c
 	if(regex_id != -1)
 	{
 		// We found a match
-		dns_cache->domainlist_id = regex_id;
+		dns_cache->list_id = regex_id;
 		return true;
 	}