diff --git a/src/api/auth.c b/src/api/auth.c index 22bbd9de2..f09f3c59f 100644 --- a/src/api/auth.c +++ b/src/api/auth.c @@ -25,6 +25,8 @@ #include "config/password.h" // database session functions #include "database/session-table.h" +// FTLDBerror() +#include "database/common.h" static uint16_t max_sessions = 0; static struct session *auth_data = NULL; @@ -51,7 +53,9 @@ void init_api(void) log_crit("Could not allocate memory for API sessions, check config value of webserver.api.max_sessions"); exit(EXIT_FAILURE); } - restore_db_sessions(auth_data, max_sessions); + + if(!FTLDBerror()) + restore_db_sessions(auth_data, max_sessions); } void free_api(void) diff --git a/src/api/teleporter.c b/src/api/teleporter.c index 16e157566..9ef8ba2c3 100644 --- a/src/api/teleporter.c +++ b/src/api/teleporter.c @@ -461,6 +461,11 @@ static bool import_json_table(cJSON *json, struct teleporter_files *file) return false; } + // Set busy timeout to 1 second to access the database in a + // multi-threaded environment + if(sqlite3_busy_timeout(db, 1000) != SQLITE_OK) + log_warn("import_json_table(%s): Unable to set busy timeout: %s", file->filename, sqlite3_errmsg(db)); + // Disable foreign key constraints if(sqlite3_exec(db, "PRAGMA foreign_keys = OFF;", NULL, NULL, NULL) != SQLITE_OK) { @@ -834,6 +839,9 @@ static int process_received_tar_gz(struct ftl_conn *api, struct upload_data *dat // Free allocated memory free_upload_data(data); + // Migrate the config to v6 + migrate_config_v6(); + // Signal FTL we want to restart for re-import api->ftl.restart_reason = "Teleporter (TAR.GZ) import"; api->ftl.restart = true; diff --git a/src/args.c b/src/args.c index 8c9208f98..d1966be85 100644 --- a/src/args.c +++ b/src/args.c @@ -309,7 +309,6 @@ void parse_args(int argc, char *argv[]) } } - // Set config option through CLI if(argc == 2 && strcmp(argv[1], "--totp") == 0) { @@ -321,7 +320,6 @@ void parse_args(int argc, char *argv[]) exit(printTOTP()); } - // Create teleporter archive through CLI if(argc == 2 && strcmp(argv[1], "--teleporter") == 0) { @@ -602,6 +600,15 @@ void parse_args(int argc, char *argv[]) exit(EXIT_SUCCESS); } + + // Set config option through CLI + if(argc == 3 && strcmp(argv[1], "migrate") == 0 && strcmp(argv[2], "v6") == 0) + { + cli_mode = true; + log_ctrl(false, true); + exit(migrate_config_v6() ? EXIT_SUCCESS : EXIT_FAILURE); + } + // start from 1, as argv[0] is the executable name for(int i = 1; i < argc; i++) { diff --git a/src/config/config.c b/src/config/config.c index cdacd61d2..bc3578859 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -1611,6 +1611,103 @@ static void reset_config_default(struct conf_item *conf_item) } } + +/** + * @brief Determine and set default webserver ports if not imported from setupVars.conf. + * + * @param conf Pointer to the configuration structure. + */ +static void get_web_port(struct config *conf) +{ + // Determine default webserver ports if not imported from setupVars.conf + if(config.webserver.port.f & FLAG_CONF_IMPORTED) + { + log_info("Webserver ports already imported from setupVars.conf, skipping default port detection"); + return; + } + + // else: + // Check if ports 80/TCP and 443/TCP are already in use + const in_port_t http_port = port_in_use(80) ? 8080 : 80; + const in_port_t https_port = port_in_use(443) ? 8443 : 443; + + // Create a string with the default ports + // Allocate memory for the string + char *ports = calloc(32, sizeof(char)); + if(ports == NULL) + { + log_err("Unable to allocate memory for default ports string"); + return; + } + // Create the string + snprintf(ports, 32, "%d,%ds", http_port, https_port); + + // Append IPv6 ports if IPv6 is enabled + const bool have_ipv6 = ipv6_enabled(); + if(have_ipv6) + snprintf(ports + strlen(ports), 32 - strlen(ports), + ",[::]:%d,[::]:%ds", http_port, https_port); + + // Set default values for webserver ports + if(conf->webserver.port.t == CONF_STRING_ALLOCATED) + free(conf->webserver.port.v.s); + conf->webserver.port.v.s = ports; + conf->webserver.port.t = CONF_STRING_ALLOCATED; + + log_info("Config initialized with webserver ports %d (HTTP) and %d (HTTPS), IPv6 support is %s", + http_port, https_port, have_ipv6 ? "enabled" : "disabled"); +} + +bool migrate_config_v6(void) +{ + // Check if MIGRATION_TARGET_V6 exists and is a directory + // Ideally, this directory should be created by the installer but users + // may have deleted it manually and it is necessary for restoring + // Teleporter files + create_migration_target_v6(); + + // If the migration target does not exist, we need to migrate the config + log_info("Migrating config to Pi-hole v6.0 format"); + + // Initialize config with default values + initConfig(&config); + + // If no previous config file could be read, we are likely either running + // for the first time or we are upgrading from a version prior to v6.0 + // In this case, we try to read the legacy config files + const char *path = ""; + if((path = readFTLlegacy(&config)) != NULL) + { + const char *target = MIGRATION_TARGET_V6"/pihole-FTL.conf"; + log_info("Moving %s to %s", path, target); + if(rename(path, target) != 0) + log_warn("Unable to move %s to %s: %s", path, target, strerror(errno)); + } + else + log_info("No legacy config file found, using defaults"); + + // Import bits and pieces from legacy config files + // setupVars.conf + importsetupVarsConf(); + // 04-pihole-static-dhcp.conf + read_legacy_dhcp_static_config(); + // 05-pihole-custom-cname.conf + read_legacy_cnames_config(); + // custom.list + read_legacy_custom_hosts_config(); + + // Determine and set default webserver ports if not imported from + // setupVars.conf + get_web_port(&config); + + // Initialize the TOML config file + writeFTLtoml(true); + write_dnsmasq_config(&config, false, NULL); + write_custom_list(); + + return true; +} + bool readFTLconf(struct config *conf, const bool rewrite) { // Initialize config with default values @@ -1649,33 +1746,6 @@ bool readFTLconf(struct config *conf, const bool rewrite) if(!rewrite) return false; - // Check if MIGRATION_TARGET_V6 exists and is a directory - // Ideally, this directory should be created by the installer but users - // may have deleted it manually and it is necessary for restoring - // Teleporter files - create_migration_target_v6(); - - // If no previous config file could be read, we are likely either running - // for the first time or we are upgrading from a version prior to v6.0 - // In this case, we try to read the legacy config files - const char *path = ""; - if((path = readFTLlegacy(conf)) != NULL) - { - const char *target = MIGRATION_TARGET_V6"/pihole-FTL.conf"; - log_info("Moving %s to %s", path, target); - if(rename(path, target) != 0) - log_warn("Unable to move %s to %s: %s", path, target, strerror(errno)); - } - // Import bits and pieces from legacy config files - // setupVars.conf - importsetupVarsConf(); - // 04-pihole-static-dhcp.conf - read_legacy_dhcp_static_config(); - // 05-pihole-custom-cname.conf - read_legacy_cnames_config(); - // custom.list - read_legacy_custom_hosts_config(); - // When we reach this point but the FTL TOML config file exists, it may // contain errors such as syntax errors, etc. We move it into a // ".broken" location so it can be revisited later @@ -1686,39 +1756,9 @@ bool readFTLconf(struct config *conf, const bool rewrite) rename(GLOBALTOMLPATH, new_name); } - // Determine default webserver ports if not imported from setupVars.conf - if(!(config.webserver.port.f & FLAG_CONF_IMPORTED)) - { - // Check if ports 80/TCP and 443/TCP are already in use - const in_port_t http_port = port_in_use(80) ? 8080 : 80; - const in_port_t https_port = port_in_use(443) ? 8443 : 443; - - // Create a string with the default ports - // Allocate memory for the string - char *ports = calloc(32, sizeof(char)); - if(ports == NULL) - { - log_err("Unable to allocate memory for default ports string"); - return false; - } - // Create the string - snprintf(ports, 32, "%d,%ds", http_port, https_port); - - // Append IPv6 ports if IPv6 is enabled - const bool have_ipv6 = ipv6_enabled(); - if(have_ipv6) - snprintf(ports + strlen(ports), 32 - strlen(ports), - ",[::]:%d,[::]:%ds", http_port, https_port); - - // Set default values for webserver ports - if(conf->webserver.port.t == CONF_STRING_ALLOCATED) - free(conf->webserver.port.v.s); - conf->webserver.port.v.s = ports; - conf->webserver.port.t = CONF_STRING_ALLOCATED; - - log_info("Config initialized with webserver ports %d (HTTP) and %d (HTTPS), IPv6 support is %s", - http_port, https_port, have_ipv6 ? "enabled" : "disabled"); - } + // Determine and set default webserver ports if not imported from + // setupVars.conf + get_web_port(&config); // Initialize the TOML config file writeFTLtoml(true); diff --git a/src/config/config.h b/src/config/config.h index cb679039a..16b48f11c 100644 --- a/src/config/config.h +++ b/src/config/config.h @@ -356,6 +356,7 @@ extern struct config config; // Defined in config.c void set_debug_flags(struct config *conf); void set_all_debug(struct config *conf, const bool status); +bool migrate_config_v6(void); bool readFTLconf(struct config *conf, const bool rewrite); bool getLogFilePath(void); struct conf_item *get_conf_item(struct config *conf, const unsigned int n); diff --git a/src/database/common.c b/src/database/common.c index 31a76d2eb..6d60315ac 100644 --- a/src/database/common.c +++ b/src/database/common.c @@ -127,10 +127,6 @@ sqlite3* _dbopen(const bool readonly, const bool create, const char *func, const int dbquery(sqlite3* db, const char *format, ...) { - // Return early if the database is known to be broken - if(FTLDBerror()) - return SQLITE_ERROR; - va_list args; va_start(args, format); char *query = sqlite3_vmprintf(format, args); @@ -311,6 +307,7 @@ void db_init(void) { log_warn("Database not available, please ensure the database is unlocked when starting pihole-FTL !"); dbclose(&db); + DBerror = true; return; } else @@ -328,6 +325,7 @@ void db_init(void) { log_err("Counter table not initialized, database not available"); dbclose(&db); + DBerror = true; return; } // Get updated version @@ -343,6 +341,7 @@ void db_init(void) { log_err("Network table not initialized, database not available"); dbclose(&db); + DBerror = true; return; } // Get updated version @@ -358,6 +357,7 @@ void db_init(void) { log_err("Unable to unify clients in network table, database not available"); dbclose(&db); + DBerror = true; return; } // Get updated version @@ -373,6 +373,7 @@ void db_init(void) { log_err("Network-addresses table not initialized, database not available"); dbclose(&db); + DBerror = true; return; } // Get updated version @@ -388,6 +389,7 @@ void db_init(void) { log_err("Message table not initialized, database not available"); dbclose(&db); + DBerror = true; return; } // Get updated version @@ -403,6 +405,7 @@ void db_init(void) { log_err("Column additional_info not initialized, database not available"); dbclose(&db); + DBerror = true; return; } // Get updated version @@ -418,6 +421,7 @@ void db_init(void) { log_err("Network addresses table not initialized, database not available"); dbclose(&db); + DBerror = true; return; } // Get updated version @@ -433,6 +437,7 @@ void db_init(void) { log_err("Aliasclients table not initialized, database not available"); dbclose(&db); + DBerror = true; return; } // Get updated version @@ -448,6 +453,7 @@ void db_init(void) { log_info("Queries table not optimized, database not available"); dbclose(&db); + DBerror = true; return; } @@ -469,6 +475,7 @@ void db_init(void) { log_info("Link table for additional_info not generated, database not available"); dbclose(&db); + DBerror = true; return; } @@ -490,6 +497,7 @@ void db_init(void) { log_info("Additional records not generated, database not available"); dbclose(&db); + DBerror = true; return; } // Get updated version @@ -505,6 +513,7 @@ void db_init(void) { log_info("Additional records not generated, database not available"); dbclose(&db); + DBerror = true; return; } // Get updated version @@ -520,6 +529,7 @@ void db_init(void) { log_info("FTL table description cannot be added, database not available"); dbclose(&db); + DBerror = true; return; } // Get updated version @@ -535,6 +545,7 @@ void db_init(void) { log_info("Session table cannot be created, database not available"); dbclose(&db); + DBerror = true; return; } // Get updated version @@ -550,6 +561,7 @@ void db_init(void) { log_info("Session table cannot be updated, database not available"); dbclose(&db); + DBerror = true; return; } // Get updated version @@ -565,6 +577,7 @@ void db_init(void) { log_info("regex_id cannot be renamed to list_id, database not available"); dbclose(&db); + DBerror = true; return; } // Get updated version @@ -580,6 +593,7 @@ void db_init(void) { log_info("Session table cannot be updated, database not available"); dbclose(&db); + DBerror = true; return; } // Get updated version @@ -595,6 +609,7 @@ void db_init(void) { log_info("Session table cannot be updated, database not available"); dbclose(&db); + DBerror = true; return; } // Get updated version @@ -610,6 +625,7 @@ void db_init(void) { log_info("Network addresses network_id index cannot be added, database not available"); dbclose(&db); + DBerror = true; return; } // Get updated version @@ -625,6 +641,7 @@ void db_init(void) { log_info("Additional column 'ede' in the query_storage table cannot be added, database not available"); dbclose(&db); + DBerror = true; return; } // Get updated version @@ -642,7 +659,12 @@ void db_init(void) // Last check after all migrations, if this happens, it will cause the // CI to fail the tests if(dbversion != MEMDB_VERSION) - log_err("Expected query database version %d but found %d", MEMDB_VERSION, dbversion); + { + log_err("Expected query database version %d but found %d, database not available", MEMDB_VERSION, dbversion); + dbclose(&db); + DBerror = true; + return; + } lock_shm(); import_aliasclients(db); diff --git a/src/database/gravity-db.c b/src/database/gravity-db.c index 30ee99269..c9627126f 100644 --- a/src/database/gravity-db.c +++ b/src/database/gravity-db.c @@ -2743,6 +2743,12 @@ bool gravity_updated(void) return false; } + // Set busy timeout to 1 second to access the database in a + // multi-threaded environment and other threads may be writing to the + // database (e.g. Teleporter restoring a backup) + if(sqlite3_busy_timeout(db, 1000) != SQLITE_OK) + log_warn("gravity_updated(): %s - Failed to set busy timeout", config.files.gravity.v.s); + // Get *updated* timestamp from gravity database const char *querystr = "SELECT value FROM info WHERE property = 'updated';"; rc = sqlite3_prepare_v2(db, querystr, -1, &query_stmt, NULL); diff --git a/src/database/query-table.c b/src/database/query-table.c index 756617c77..1231ce238 100644 --- a/src/database/query-table.c +++ b/src/database/query-table.c @@ -124,15 +124,14 @@ bool init_memory_database(void) } } - // Attach disk database - if(!attach_database(_memdb, NULL, config.files.database.v.s, "disk")) - return false; + // Attach disk database. This may fail if the database is unavailable + const bool attached = attach_database(_memdb, NULL, config.files.database.v.s, "disk"); // Enable WAL mode for the on-disk database (pihole-FTL.db) if // configured (default is yes). User may not want to enable WAL // mode if the database is on a network share as all processes // accessing the database must be on the same host in WAL mode. - if(config.database.useWAL.v.b) + if(config.database.useWAL.v.b && attached) { // Change journal mode to WAL // - WAL is significantly faster in most scenarios. @@ -149,7 +148,7 @@ bool init_memory_database(void) return false; } } - else + else if(attached) { // Unlike the other journaling modes, PRAGMA journal_mode=WAL is // persistent. If a process sets WAL mode, then closes and @@ -224,7 +223,11 @@ bool init_memory_database(void) return false; } - load_queries_from_disk(); + // Attach disk database + if(attached) + load_queries_from_disk(); + else + log_err("init_memory_database(): Failed to attach disk database"); // Everything went well return true; @@ -352,6 +355,10 @@ bool attach_database(sqlite3* db, const char **message, const char *path, const bool okay = false; sqlite3_stmt *stmt = NULL; + // Only try to attach database if it is not known to be broken + if(FTLDBerror()) + return false; + log_debug(DEBUG_DATABASE, "ATTACH %s AS %s", path, alias); // ATTACH database file on-disk @@ -583,6 +590,11 @@ bool export_queries_to_disk(bool final) { bool okay = false; const double time = double_time() - (final ? 0.0 : REPLY_TIMEOUT); + + // Only try to export to database if it is known to not be broken + if(FTLDBerror()) + return false; + const char *querystr = "INSERT INTO disk.query_storage SELECT * FROM query_storage WHERE id > ? AND timestamp < ?"; log_debug(DEBUG_DATABASE, "Storing queries on disk WHERE id > %lu (max is %lu) and timestamp < %f", @@ -1064,6 +1076,10 @@ void DB_read_queries(void) "dnssec "\ "FROM queries WHERE timestamp >= ?"; + // Only try to import from database if it is known to not be broken + if(FTLDBerror()) + return; + log_info("Parsing queries in database"); // Prepare SQLite3 statement @@ -1382,12 +1398,22 @@ void DB_read_queries(void) log_info("Imported %u queries from the long-term database", counters->queries); } -void update_disk_db_idx(void) +void init_disk_db_idx(void) { // Query the database to get the maximum database ID is important to avoid // starting counting from zero (would result in a UNIQUE constraint violation) const char *querystr = "SELECT MAX(id) FROM disk.query_storage"; + // If the disk database is broken, we cannot import queries from it, + // however, as we will also never export any queries, we can safely + // assume any index + if(FTLDBerror()) + { + last_disk_db_idx = 0; + last_mem_db_idx = 0; + return; + } + // Prepare SQLite3 statement sqlite3_stmt *stmt = NULL; sqlite3 *memdb = get_memdb(); @@ -1397,7 +1423,7 @@ void update_disk_db_idx(void) if(rc == SQLITE_OK && (rc = sqlite3_step(stmt)) == SQLITE_ROW) last_disk_db_idx = sqlite3_column_int64(stmt, 0); else - log_err("update_disk_db_idx(): Failed to get MAX(id) from disk.query_storage: %s", + log_err("init_disk_db_idx(): Failed to get MAX(id) from disk.query_storage: %s", sqlite3_errstr(rc)); // Finalize statement @@ -1416,6 +1442,10 @@ bool queries_to_database(void) unsigned int added = 0, updated = 0; sqlite3_int64 idx = 0; + // Only try to export to database if it is known to not be broken + if(FTLDBerror()) + return false; + // Skip, we never store nor count queries recorded while have been in // maximum privacy mode in the database if(config.misc.privacylevel.v.privacy_level >= PRIVACY_MAXIMUM) diff --git a/src/database/query-table.h b/src/database/query-table.h index 470fddbec..691ba1e5e 100644 --- a/src/database/query-table.h +++ b/src/database/query-table.h @@ -118,7 +118,7 @@ bool export_queries_to_disk(bool final); bool delete_old_queries_from_db(const bool use_memdb, const double mintime); bool add_additional_info_column(sqlite3 *db); void DB_read_queries(void); -void update_disk_db_idx(void); +void init_disk_db_idx(void); bool queries_to_database(void); bool optimize_queries_table(sqlite3 *db); diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index c24ec298f..3a9433443 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -3118,7 +3118,8 @@ void FTL_fork_and_bind_sockets(struct passwd *ent_pw, bool dnsmasq_start) log_crit("Cannot initialize in-memory database."); // Flush messages stored in the long-term database - flush_message_table(); + if(!FTLDBerror()) + flush_message_table(); // Verify checksum of this binary early on to ensure that the binary is // not corrupted and that the binary is not tampered with. We can only @@ -3127,7 +3128,7 @@ void FTL_fork_and_bind_sockets(struct passwd *ent_pw, bool dnsmasq_start) verify_FTL(false); // Initialize in-memory database starting index - update_disk_db_idx(); + init_disk_db_idx(); // Handle real-time signals in this process (and its children) // Helper processes are already split from the main instance diff --git a/src/files.c b/src/files.c index cbc6daac9..7489dc77f 100644 --- a/src/files.c +++ b/src/files.c @@ -454,6 +454,11 @@ bool chown_pihole(const char *path, struct passwd *pwd) // Change ownership of file to pihole user if(chown(path, pwd->pw_uid, pwd->pw_gid) < 0) { + if(errno == ENOENT) // ENOENT = No such file or directory + { + log_debug(DEBUG_CONFIG, "File \"%s\" does not exist, not changing ownership", path); + return true; + } log_warn("Failed to change ownership of \"%s\" to %s:%s (%u:%u): %s", path, pwd->pw_name, grp_name, pwd->pw_uid, pwd->pw_gid, errno == EPERM ? "Insufficient permissions (CAP_CHOWN required)" : strerror(errno)); diff --git a/src/zip/teleporter.c b/src/zip/teleporter.c index 0185edb8b..07b5b04ad 100644 --- a/src/zip/teleporter.c +++ b/src/zip/teleporter.c @@ -85,6 +85,11 @@ static bool create_teleporter_database(const char *filename, const char **tables snprintf(attach_stmt, sizeof(attach_stmt), "ATTACH DATABASE '%s' AS \"disk\";", filename); + // Set busy timeout to 1 second to access the database in a + // multi-threaded environment + if(sqlite3_busy_timeout(db, 1000) != SQLITE_OK) + log_warn("Failed to set busy timeout during creation of in-memory Teleporter database: %s", sqlite3_errmsg(db)); + if(sqlite3_exec(db, attach_stmt, NULL, NULL, &err) != SQLITE_OK) { log_warn("Failed to attach database \"%s\" to in-memory database: %s", filename, err);