Skip to content

Commit

Permalink
Merge pull request #1806 from pi-hole/tweak/even_better_database
Browse files Browse the repository at this point in the history 
Use WAL, remove (and strip) SQLite3 shared-cache support and improve attached database handling
  • Loading branch information
@DL6ER
DL6ER authored Jan 6, 2024
2 parents 2fef4e2 + d77547d commit c2af66d
Show file tree
Hide file tree
Showing 12 changed files with 180 additions and 198 deletions.
10 changes: 8 additions & 2 deletions src/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,14 +24,20 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR})
# SQLITE_DEFAULT_MEMSTATUS=0: This setting causes the sqlite3_status() interfaces that track memory usage to be disabled. This helps the sqlite3_malloc() routines run much faster, and since SQLite uses sqlite3_malloc() internally, this helps to make the entire library faster.
# SQLITE_OMIT_DEPRECATED: Omitting deprecated interfaces and features will not help SQLite to run any faster. It will reduce the library footprint, however. And it is the right thing to do.
# SQLITE_OMIT_PROGRESS_CALLBACK: The progress handler callback counter must be checked in the inner loop of the bytecode engine. By omitting this interface, a single conditional is removed from the inner loop of the bytecode engine, helping SQL statements to run slightly faster.
# SQLITE_OMIT_SHARED_CACHE: This option builds SQLite without support for shared cache mode. The sqlite3_enable_shared_cache() is omitted along with a fair amount of logic within the B-Tree subsystem associated with shared cache management. This compile-time option is recommended most applications as it results in improved performance and reduced library footprint.
# SQLITE_DEFAULT_FOREIGN_KEYS=1: This macro determines whether enforcement of foreign key constraints is enabled or disabled by default for new database connections.
# SQLITE_DQS=0: This setting disables the double-quoted string literal misfeature.
# SQLITE_ENABLE_DBPAGE_VTAB: Enables the SQLITE_DBPAGE virtual table. Warning: writing to the SQLITE_DBPAGE virtual table can very easily cause unrecoverably database corruption.
# SQLITE_TEMP_STORE=2: Store temporary tables in memory for reduced IO and higher performance (can be overwritten by the user at runtime).
# SQLITE_USE_URI=1: The advantage of using a URI filename is that query parameters on the URI can be used to control details of the newly created database connection.
# HAVE_READLINE: Enable readline support to allow easy editing, history and auto-completion
# SQLITE_DEFAULT_CACHE_SIZE=-16384: Allow up to 16 MiB of cache to be used by SQLite3 (default is 2000 kiB)
set(SQLITE_DEFINES "-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_DEFAULT_FOREIGN_KEYS=1 -DSQLITE_DQS=0 -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_TEMP_STORE=2 -DSQLITE_USE_URI=1 -DHAVE_READLINE -DSQLITE_DEFAULT_CACHE_SIZE=16384")
# SQLITE_DEFAULT_SYNCHRONOUS=1: Use normal synchronous mode (default is 2)
# SQLITE_LIKE_DOESNT_MATCH_BLOBS: This option causes the LIKE operator to only match BLOB values against BLOB values and TEXT values against TEXT values. This compile-time option makes SQLite run more efficiently when processing queries that use the LIKE operator.
# HAVE_MALLOC_USABLE_SIZE: This option causes SQLite to try to use the malloc_usable_size() function to obtain the actual size of memory allocations from the underlying malloc() system interface. Applications are encouraged to use HAVE_MALLOC_USABLE_SIZE whenever possible.
# HAVE_FDATASYNC: This option causes SQLite to try to use the fdatasync() system call to sync the database file to disk when committing a transaction. Syncing using fdatasync() is faster than syncing using fsync() as fdatasync() does not wait for the file metadata to be written to disk.
# SQLITE_DEFAULT_WORKER_THREADS=4: This option sets the default number of worker threads to use when doing parallel sorting and indexing. The default is 0 which means to use a single thread. The default for SQLITE_MAX_WORKER_THREADS is 8.
# SQLITE_MAX_PREPARE_RETRY=200: This option sets the maximum number of automatic re-preparation attempts that can occur after encountering a schema change. This can be caused by running ANALYZE which is done periodically by FTL.
set(SQLITE_DEFINES "-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_OMIT_SHARED_CACHE -DSQLITE_DEFAULT_FOREIGN_KEYS=1 -DSQLITE_DQS=0 -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_TEMP_STORE=2 -DHAVE_READLINE -DSQLITE_DEFAULT_CACHE_SIZE=16384 -DSQLITE_DEFAULT_SYNCHRONOUS=1 -DSQLITE_LIKE_DOESNT_MATCH_BLOBS -DHAVE_MALLOC_USABLE_SIZE -DHAVE_FDATASYNC -DSQLITE_DEFAULT_WORKER_THREADS=4 -DSQLITE_MAX_PREPARE_RETRY=200")

# Code hardening and debugging improvements
# -fstack-protector-strong: The program will be resistant to having its stack overflowed
Expand Down
9 changes: 9 additions & 0 deletions src/FTL.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@
// Number of elements in an array
#define ArraySize(X) (sizeof(X)/sizeof(X[0]))

// Constant socket buffer length
#define SOCKETBUFFERLEN 1024

// How often do we garbage collect (to ensure we only have data fitting to the MAXLOGAGE defined above)? [seconds]
Expand Down Expand Up @@ -133,6 +134,14 @@
// Special exit code used to signal that FTL wants to restart
#define RESTART_FTL_CODE 22

// How often should the database be analyzed?
// Default: 604800 (once per week)
#define DATABASE_ANALYZE_INTERVAL 604800

// How often should we update client vendor's from the MAC vendor database?
// Default: 2592000 (once per month)
#define DATABASE_MACVENDOR_INTERVAL 2592000

// Use out own syscalls handling functions that will detect possible errors
// and report accordingly in the log. This will make debugging FTL crash
// caused by insufficient memory or by code bugs (not properly dealing
Expand Down
2 changes: 1 addition & 1 deletion src/api/info.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ int api_info_database(struct ftl_conn *api)
JSON_ADD_ITEM_TO_OBJECT(json, "owner", owner);

// Add number of queries in on-disk database
const int queries_in_database = get_number_of_queries_in_DB(NULL, "query_storage", true);
const int queries_in_database = get_number_of_queries_in_DB(NULL, "query_storage");
JSON_ADD_NUMBER_TO_OBJECT(json, "queries", queries_in_database);

// Add SQLite library version
Expand Down
61 changes: 9 additions & 52 deletions src/api/queries.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,8 @@ static int add_strings_to_array(struct ftl_conn *api, cJSON *array, const char *
"Could not read from in-memory database",
NULL);
}
sqlite3_stmt *stmt;

sqlite3_stmt *stmt = NULL;
int rc = sqlite3_prepare_v2(memdb, querystr, -1, &stmt, NULL);
if( rc != SQLITE_OK )
{
Expand Down Expand Up @@ -438,29 +438,24 @@ int api_queries(struct ftl_conn *api)
}
}

// Get connection to in-memory database
sqlite3 *db = get_memdb();

// Finish preparing query string
querystr_finish(querystr, sort_col, sort_dir);

// Attach disk database if necessary
const char *message = "";
if(disk && !attach_disk_database(&message))
// Get connection to in-memory database
sqlite3 *memdb = get_memdb();
if(memdb == NULL)
{
return send_json_error(api, 500,
"internal_error",
"Internal server error, cannot attach disk database",
message);
return send_json_error(api, 500, // 500 Internal error
"database_error",
"Could not read from in-memory database",
NULL);
}

// Prepare SQLite3 statement
sqlite3_stmt *read_stmt = NULL;
int rc = sqlite3_prepare_v2(db, querystr, -1, &read_stmt, NULL);
int rc = sqlite3_prepare_v2(memdb, querystr, -1, &read_stmt, NULL);
if( rc != SQLITE_OK )
{
if(disk)
detach_disk_database(NULL);
return send_json_error(api, 500,
"internal_error",
"Internal server error, failed to prepare read SQL query",
Expand All @@ -484,8 +479,6 @@ int api_queries(struct ftl_conn *api)
{
sqlite3_reset(read_stmt);
sqlite3_finalize(read_stmt);
if(disk)
detach_disk_database(NULL);
return send_json_error(api, 500,
"internal_error",
"Internal server error, failed to bind timestamp:from to SQL query",
Expand All @@ -501,8 +494,6 @@ int api_queries(struct ftl_conn *api)
{
sqlite3_reset(read_stmt);
sqlite3_finalize(read_stmt);
if(disk)
detach_disk_database(NULL);
return send_json_error(api, 500,
"internal_error",
"Internal server error, failed to bind timestamp:until to SQL query",
Expand All @@ -518,8 +509,6 @@ int api_queries(struct ftl_conn *api)
{
sqlite3_reset(read_stmt);
sqlite3_finalize(read_stmt);
if(disk)
detach_disk_database(NULL);
return send_json_error(api, 500,
"internal_error",
"Internal server error, failed to bind domain to SQL query",
Expand All @@ -535,8 +524,6 @@ int api_queries(struct ftl_conn *api)
{
sqlite3_reset(read_stmt);
sqlite3_finalize(read_stmt);
if(disk)
detach_disk_database(NULL);
return send_json_error(api, 500,
"internal_error",
"Internal server error, failed to bind cip to SQL query",
Expand All @@ -552,8 +539,6 @@ int api_queries(struct ftl_conn *api)
{
sqlite3_reset(read_stmt);
sqlite3_finalize(read_stmt);
if(disk)
detach_disk_database(NULL);
return send_json_error(api, 500,
"internal_error",
"Internal server error, failed to bind client to SQL query",
Expand All @@ -569,8 +554,6 @@ int api_queries(struct ftl_conn *api)
{
sqlite3_reset(read_stmt);
sqlite3_finalize(read_stmt);
if(disk)
detach_disk_database(NULL);
return send_json_error(api, 500,
"internal_error",
"Internal server error, failed to bind upstream to SQL query",
Expand All @@ -595,8 +578,6 @@ int api_queries(struct ftl_conn *api)
{
sqlite3_reset(read_stmt);
sqlite3_finalize(read_stmt);
if(disk)
detach_disk_database(NULL);
return send_json_error(api, 500,
"internal_error",
"Internal server error, failed to bind type to SQL query",
Expand All @@ -605,8 +586,6 @@ int api_queries(struct ftl_conn *api)
}
else
{
if(disk)
detach_disk_database(NULL);
return send_json_error(api, 400,
"bad_request",
"Requested type is invalid",
Expand All @@ -631,8 +610,6 @@ int api_queries(struct ftl_conn *api)
{
sqlite3_reset(read_stmt);
sqlite3_finalize(read_stmt);
if(disk)
detach_disk_database(NULL);
return send_json_error(api, 500,
"internal_error",
"Internal server error, failed to bind status to SQL query",
Expand All @@ -641,8 +618,6 @@ int api_queries(struct ftl_conn *api)
}
else
{
if(disk)
detach_disk_database(NULL);
return send_json_error(api, 400,
"bad_request",
"Requested status is invalid",
Expand All @@ -667,8 +642,6 @@ int api_queries(struct ftl_conn *api)
{
sqlite3_reset(read_stmt);
sqlite3_finalize(read_stmt);
if(disk)
detach_disk_database(NULL);
return send_json_error(api, 500,
"internal_error",
"Internal server error, failed to bind reply to SQL query",
Expand All @@ -677,8 +650,6 @@ int api_queries(struct ftl_conn *api)
}
else
{
if(disk)
detach_disk_database(NULL);
return send_json_error(api, 400,
"bad_request",
"Requested reply is invalid",
Expand All @@ -703,8 +674,6 @@ int api_queries(struct ftl_conn *api)
{
sqlite3_reset(read_stmt);
sqlite3_finalize(read_stmt);
if(disk)
detach_disk_database(NULL);
return send_json_error(api, 500,
"internal_error",
"Internal server error, failed to bind dnssec to SQL query",
Expand All @@ -713,8 +682,6 @@ int api_queries(struct ftl_conn *api)
}
else
{
if(disk)
detach_disk_database(NULL);
return send_json_error(api, 400,
"bad_request",
"Requested dnssec is invalid",
Expand All @@ -731,8 +698,6 @@ int api_queries(struct ftl_conn *api)
{
sqlite3_reset(read_stmt);
sqlite3_finalize(read_stmt);
if(disk)
detach_disk_database(NULL);
return send_json_error(api, 500,
"internal_error",
"Internal server error, failed to bind count to SQL query",
Expand Down Expand Up @@ -901,13 +866,5 @@ int api_queries(struct ftl_conn *api)
// Finalize statements
sqlite3_finalize(read_stmt);

if(disk && !detach_disk_database(&message))
{
return send_json_error(api, 500,
"internal_error",
"Internal server error, cannot detach disk database",
message);
}

JSON_SEND_OBJECT(json);
}
10 changes: 8 additions & 2 deletions src/database/common.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -251,9 +251,15 @@ void SQLite3LogCallback(void *pArg, int iErrCode, const char *zMsg)
generate_backtrace();

if(iErrCode == SQLITE_WARNING)
log_warn("SQLite3 message: %s (%d)", zMsg, iErrCode);
log_warn("SQLite3: %s (%d)", zMsg, iErrCode);
else if(iErrCode == SQLITE_NOTICE || iErrCode == SQLITE_SCHEMA)
// SQLITE_SCHEMA is returned when the database schema has changed
// This is not necessarily an error, as sqlite3_step() will re-prepare
// the statement and try again. If it cannot, it will return an error
// and this will be handled over there.
log_debug(DEBUG_ANY, "SQLite3: %s (%d)", zMsg, iErrCode);
else
log_err("SQLite3 message: %s (%d)", zMsg, iErrCode);
log_err("SQLite3: %s (%d)", zMsg, iErrCode);
}

void db_init(void)
Expand Down
48 changes: 47 additions & 1 deletion src/database/database-thread.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,27 @@ static bool delete_old_queries_in_DB(sqlite3 *db)
return true;
}

static bool analyze_database(sqlite3 *db)
{
// Optimize the database by running ANALYZE
// The ANALYZE command gathers statistics about tables and indices and
// stores the collected information in internal tables of the database
// where the query optimizer can access the information and use it to
// help make better query planning choices.

// Measure time
struct timespec start, end;
clock_gettime(CLOCK_MONOTONIC, &start);
SQL_bool(db, "ANALYZE;");
clock_gettime(CLOCK_MONOTONIC, &end);

// Print final message
log_info("Optimized database in %.3f seconds",
(double)(end.tv_sec - start.tv_sec) + 1e-9*(end.tv_nsec - start.tv_nsec));

return true;
}

#define DBOPEN_OR_AGAIN() { if(!db) db = dbopen(false, false); if(!db) { thread_sleepms(DB, 5000); continue; } }
#define BREAK_IF_KILLED() { if(killed) break; }
#define DBCLOSE_OR_BREAK() { dbclose(&db); BREAK_IF_KILLED(); }
Expand All @@ -72,6 +93,17 @@ void *DB_thread(void *val)
time_t before = time(NULL);
time_t lastDBsave = before - before%config.database.DBinterval.v.ui;

// Other timestamps, made independent from the exact time FTL was
// started
time_t lastAnalyze = before - before % DATABASE_ANALYZE_INTERVAL;
time_t lastMACVendor = before - before % DATABASE_MACVENDOR_INTERVAL;

// Add some randomness (up to ome hour) to these timestamps to avoid
// them running at the same time. This is not a security feature, so
// using rand() is fine.
lastAnalyze += rand() % 3600;
lastMACVendor += rand() % 3600;

// This thread runs until shutdown of the process. We keep this thread
// running when pihole-FTL.db is corrupted because reloading of privacy
// level, and the gravity database (initially and after gravity)
Expand Down Expand Up @@ -135,16 +167,30 @@ void *DB_thread(void *val)
set_event(PARSE_NEIGHBOR_CACHE);
}

// Intermediate cancellation-point
if(killed)
break;

// Optimize database once per week
if(now - lastAnalyze >= DATABASE_ANALYZE_INTERVAL)
{
DBOPEN_OR_AGAIN();
analyze_database(db);
lastAnalyze = now;
DBCLOSE_OR_BREAK();
}

// Intermediate cancellation-point
if(killed)
break;

// Update MAC vendor strings once a month (the MAC vendor
// database is not updated very often)
if(now % 2592000L == 0)
if(now - lastMACVendor >= DATABASE_MACVENDOR_INTERVAL)
{
DBOPEN_OR_AGAIN();
updateMACVendorRecords(db);
lastMACVendor = now;
DBCLOSE_OR_BREAK();
}

Expand Down
19 changes: 4 additions & 15 deletions src/database/message-table.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@
#include "gc.h"
// get_filesystem_details()
#include "files.h"
// get_memdb()
#include "database/query-table.h"

static const char *get_message_type_str(const enum message_type type)
{
Expand Down Expand Up @@ -214,23 +216,10 @@ bool create_message_table(sqlite3 *db)
// Flush message table
bool flush_message_table(void)
{
// Return early if database is known to be broken
if(FTLDBerror())
return false;

sqlite3 *db;
// Open database connection
if((db = dbopen(false, false)) == NULL)
{
log_err("flush_message_table() - Failed to open DB");
return false;
}
sqlite3 *memdb = get_memdb();

// Flush message table
SQL_bool(db, "DELETE FROM message;");

// Close database connection
dbclose(&db);
SQL_bool(memdb, "DELETE FROM disk.message;");

return true;
}
Expand Down
Loading

0 comments on commit c2af66d

Please sign in to comment.