Merge pull request #2308 from pi-hole/development

Pi-hole FTL v6.0.4

Author: PromoFaux

CMakeLists.txt

@@ -16,6 +16,6 @@ set(CMAKE_C_STANDARD 17)
 
 project(PIHOLE_FTL C)
 
-set(DNSMASQ_VERSION pi-hole-v2.91rc5)
+set(DNSMASQ_VERSION pi-hole-v2.91rc5+1)
 
 add_subdirectory(src)

patch/civetweb.sh

@@ -23,4 +23,7 @@ patch -p1 < patch/civetweb/0001-Log-debug-messages-to-webserver.log-when-debug.w
 echo "Applying patch 0001-Expose-bound-to-addresses-from-CivetWeb-to-the-front.patch"
 patch -p1 < patch/civetweb/0001-Expose-bound-to-addresses-from-CivetWeb-to-the-front.patch
 
+echo "Applying patch 0001-Increase-niceness-of-all-civetweb-threads-as-DNS-ope.patch"
+patch -p1 < patch/civetweb/0001-Increase-niceness-of-all-civetweb-threads-as-DNS-ope.patch
+
 echo "ALL PATCHES APPLIED OKAY"

patch/civetweb/0001-Expose-bound-to-addresses-from-CivetWeb-to-the-front.patch

@@ -14,9 +14,9 @@ index fa908e54..66fcab01 100644
 --- a/src/webserver/civetweb/civetweb.c
 +++ b/src/webserver/civetweb/civetweb.c
 @@ -3339,6 +3339,7 @@ mg_get_server_ports(const struct mg_context *ctx,
- 		ports[cnt].is_ssl = ctx->listening_sockets[i].is_ssl;
  		ports[cnt].is_redirect = ctx->listening_sockets[i].ssl_redir;
  		ports[cnt].is_optional = ctx->listening_sockets[i].is_optional;
+ 		ports[cnt].is_bound = ctx->listening_sockets[i].sock != INVALID_SOCKET;
 +		memcpy(&ports[cnt].addr, &ctx->listening_sockets[i].lsa, sizeof(ports[cnt].addr));
 
  		if (ctx->listening_sockets[i].lsa.sa.sa_family == AF_INET) {
@@ -35,7 +35,7 @@ index eee958b4..6dcfa457 100644
  #define CIVETWEB_VERSION_MAJOR (1)
  #define CIVETWEB_VERSION_MINOR (17)
 @@ -721,6 +723,10 @@ struct mg_server_port {
- 	int _reserved2;
+ 	int is_bound;    /* bound: 0 = no, 1 = yes, relevant for optional ports */
  	int _reserved3;
  	int _reserved4;
 +   union {

patch/civetweb/0001-Increase-niceness-of-all-civetweb-threads-as-DNS-ope.patch

@@ -0,0 +1,36 @@
+From ed07ef026f1ba4ad26b8740e6280263f808aeb44 Mon Sep 17 00:00:00 2001
+From: DL6ER <dl6er@dl6er.de>
+Date: Mon, 3 Mar 2025 17:03:19 +0100
+Subject: [PATCH] Increase niceness of all civetweb threads as DNS operation is
+ more important than the API and dashboard
+
+Signed-off-by: DL6ER <dl6er@dl6er.de>
+---
+ src/webserver/civetweb/civetweb.c | 4 ++++
+ 1 file changed, 4 insertions(+)
+
+diff --git a/src/webserver/civetweb/civetweb.c b/src/webserver/civetweb/civetweb.c
+index da254bb4..b1e5b9a6 100644
+--- a/src/webserver/civetweb/civetweb.c
++++ b/src/webserver/civetweb/civetweb.c
+@@ -244,6 +244,7 @@ static void DEBUG_TRACE_FUNC(const char *func,
+ 
+ #else
+ #include "log.h"
++#include <sys/resource.h>
+ #define DEBUG_TRACE(fmt, ...)                                                  \
+ 	if(debug_flags[DEBUG_WEBSERVER]) {\
+ 		log_web("DEBUG: " fmt " (%s:%d)", ##__VA_ARGS__, short_path(__FILE__), __LINE__); }
+@@ -2890,6 +2891,9 @@ mg_set_thread_name(const char *name)
+ 	 */
+ 	(void)prctl(PR_SET_NAME, threadName, 0, 0, 0);
+ #endif
++
++	// Pi-hole modification: Increase niceness of threads
++	setpriority(PRIO_PROCESS, gettid(), 5);
+ }
+ #else /* !defined(NO_THREAD_NAME) */
+ static void
+-- 
+2.43.0
+

src/CMakeLists.txt

@@ -32,15 +32,15 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR})
 # SQLITE_DEFAULT_FOREIGN_KEYS=1: This macro determines whether enforcement of foreign key constraints is enabled or disabled by default for new database connections.
 # SQLITE_DQS=0: This setting disables the double-quoted string literal misfeature.
 # SQLITE_ENABLE_DBPAGE_VTAB: Enables the SQLITE_DBPAGE virtual table. Warning: writing to the SQLITE_DBPAGE virtual table can very easily cause unrecoverably database corruption.
-# SQLITE_TEMP_STORE=2: Store temporary tables in memory for reduced IO and higher performance (can be overwritten by the user at runtime).
+# SQLITE_TEMP_STORE=1: This option sets the default value for the temp_store pragma to 1. This means that temporary tables and indices are always stored on disk. This is the default setting.
 # SQLITE_DEFAULT_CACHE_SIZE=-16384: Allow up to 16 MiB of cache to be used by SQLite3 (default is 2000 kiB)
 # SQLITE_DEFAULT_SYNCHRONOUS=1: Use normal synchronous mode (default is 2)
 # SQLITE_LIKE_DOESNT_MATCH_BLOBS: This option causes the LIKE operator to only match BLOB values against BLOB values and TEXT values against TEXT values. This compile-time option makes SQLite run more efficiently when processing queries that use the LIKE operator.
 # HAVE_MALLOC_USABLE_SIZE: This option causes SQLite to try to use the malloc_usable_size() function to obtain the actual size of memory allocations from the underlying malloc() system interface. Applications are encouraged to use HAVE_MALLOC_USABLE_SIZE whenever possible.
 # HAVE_FDATASYNC: This option causes SQLite to try to use the fdatasync() system call to sync the database file to disk when committing a transaction. Syncing using fdatasync() is faster than syncing using fsync() as fdatasync() does not wait for the file metadata to be written to disk.
-# SQLITE_DEFAULT_WORKER_THREADS=4: This option sets the default number of worker threads to use when doing parallel sorting and indexing. The default is 0 which means to use a single thread. The default for SQLITE_MAX_WORKER_THREADS is 8.
+# SQLITE_DEFAULT_WORKER_THREADS=0: This option sets the default number of worker threads to use when doing parallel sorting and indexing. The default is 0 which means to use a single thread. Do not increase this value as it, ironically, can cause performance degradation and definitely increases total memory usage.
 # SQLITE_MAX_PREPARE_RETRY=200: This option sets the maximum number of automatic re-preparation attempts that can occur after encountering a schema change. This can be caused by running ANALYZE which is done periodically by FTL.
-set(SQLITE_DEFINES "-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_OMIT_SHARED_CACHE -DSQLITE_DEFAULT_FOREIGN_KEYS=1 -DSQLITE_DQS=0 -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_TEMP_STORE=2 -DSQLITE_DEFAULT_CACHE_SIZE=16384 -DSQLITE_DEFAULT_SYNCHRONOUS=1 -DSQLITE_LIKE_DOESNT_MATCH_BLOBS -DHAVE_MALLOC_USABLE_SIZE -DHAVE_FDATASYNC -DSQLITE_DEFAULT_WORKER_THREADS=4 -DSQLITE_MAX_PREPARE_RETRY=200")
+set(SQLITE_DEFINES "-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_OMIT_SHARED_CACHE -DSQLITE_DEFAULT_FOREIGN_KEYS=1 -DSQLITE_DQS=0 -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_TEMP_STORE=1 -DSQLITE_DEFAULT_CACHE_SIZE=16384 -DSQLITE_DEFAULT_SYNCHRONOUS=1 -DSQLITE_LIKE_DOESNT_MATCH_BLOBS -DHAVE_MALLOC_USABLE_SIZE -DHAVE_FDATASYNC -DSQLITE_DEFAULT_WORKER_THREADS=0 -DSQLITE_MAX_PREPARE_RETRY=200")
 
 # Code hardening and debugging improvements
 # -fstack-protector-strong: The program will be resistant to having its stack overflowed

src/config/config.c

@@ -1008,10 +1008,10 @@ static void initConfig(struct config *conf)
 	conf->webserver.port.c = validate_stub; // Type-based checking + civetweb syntax checking
 
 	conf->webserver.threads.k = "webserver.threads";
-	conf->webserver.threads.h = "Maximum number of worker threads allowed.\n The Pi-hole web server handles each incoming connection in a separate thread. Therefore, the value of this option is effectively the number of concurrent HTTP connections that can be handled. Any other connections are queued until they can be processed by a unoccupied thread.\n The default value of 0 means that the number of threads is automatically determined by the number of online CPU cores minus 1 (e.g., launching up to 8-1 = 7 threads on 8 cores). Any other value specifies the number of threads explicitly. A hard-coded maximum of 64 threads is enforced for this option.\n The total number of threads you see may be lower than the configured value as threads are only created when needed due to incoming connections.";
+	conf->webserver.threads.h = "Maximum number of worker threads allowed.\n The Pi-hole web server handles each incoming connection in a separate thread. Therefore, the value of this option is effectively the number of concurrent HTTP connections that can be handled. Any other connections are queued until they can be processed by a unoccupied thread.\n The total number of threads you see may be lower than the configured value as threads are only created when needed due to incoming connections.\n The value 0 means the number of threads is 50 (as per default settings of CivetWeb) for backwards-compatible behavior.";
 	conf->webserver.threads.t = CONF_UINT;
 	conf->webserver.threads.f = FLAG_RESTART_FTL;
-	conf->webserver.threads.d.ui = 0;
+	conf->webserver.threads.d.ui = 50;
 	conf->webserver.threads.c = validate_stub; // Only type-based checking
 
 	conf->webserver.headers.k = "webserver.headers";
@@ -1062,7 +1062,7 @@ static void initConfig(struct config *conf)
 	conf->webserver.paths.webhome.t = CONF_STRING;
 	conf->webserver.paths.webhome.f = FLAG_RESTART_FTL;
 	conf->webserver.paths.webhome.d.s = (char*)"/admin/";
-	conf->webserver.paths.webhome.c = validate_filepath;
+	conf->webserver.paths.webhome.c = validate_filepath_two_slash;
 
 	// sub-struct interface
 	conf->webserver.interface.boxed.k = "webserver.interface.boxed";
@@ -1317,7 +1317,7 @@ static void initConfig(struct config *conf)
 	conf->misc.addr2line.c = validate_stub; // Only type-based checking
 
 	conf->misc.etc_dnsmasq_d.k = "misc.etc_dnsmasq_d";
-	conf->misc.etc_dnsmasq_d.h = "Should FTL load additional dnsmasq configuration files from /etc/dnsmasq.d/?";
+	conf->misc.etc_dnsmasq_d.h = "Should FTL load additional dnsmasq configuration files from /etc/dnsmasq.d/?\n Warning: This is an advanced setting and should only be used with care.\n Incorrectly formatted or config files specifying options which can only be defined once can result in conflicts with the automatic configuration of Pi-hole (see "DNSMASQ_PH_CONFIG") and may stop DNS resolution from working.";
 	conf->misc.etc_dnsmasq_d.t = CONF_BOOL;
 	conf->misc.etc_dnsmasq_d.f = FLAG_RESTART_FTL;
 	conf->misc.etc_dnsmasq_d.d.b = false;
@@ -1740,7 +1740,8 @@ bool migrate_config_v6(void)
 
 	// Initialize the TOML config file
 	writeFTLtoml(true);
-	write_dnsmasq_config(&config, false, NULL);
+	char errbuf[ERRBUF_SIZE] = { 0 };
+	write_dnsmasq_config(&config, false, errbuf);
 	write_custom_list();
 
 	return true;
@@ -1768,7 +1769,8 @@ bool readFTLconf(struct config *conf, const bool rewrite)
 			if(rewrite)
 			{
 				writeFTLtoml(true);
-				write_dnsmasq_config(conf, false, NULL);
+				char errbuf[ERRBUF_SIZE] = { 0 };
+				write_dnsmasq_config(conf, false, errbuf);
 				write_custom_list();
 			}
 			return true;
@@ -1800,7 +1802,8 @@ bool readFTLconf(struct config *conf, const bool rewrite)
 
 	// Initialize the TOML config file
 	writeFTLtoml(true);
-	write_dnsmasq_config(conf, false, NULL);
+	char errbuf[ERRBUF_SIZE] = { 0 };
+	write_dnsmasq_config(conf, false, errbuf);
 	write_custom_list();
 
 	return false;

src/config/dnsmasq_config.c

@@ -244,7 +244,7 @@ static void write_config_header(FILE *fp, const char *description)
 	CONFIG_CENTER(fp, HEADER_WIDTH, "%s", "################################################################################");
 }
 
-bool __attribute__((const)) write_dnsmasq_config(struct config *conf, bool test_config, char errbuf[ERRBUF_SIZE])
+bool __attribute__((nonnull(1,3))) write_dnsmasq_config(struct config *conf, bool test_config, char errbuf[ERRBUF_SIZE])
 {
 	// Early config checks
 	if(conf->dhcp.active.v.b)
@@ -296,9 +296,9 @@ bool __attribute__((const)) write_dnsmasq_config(struct config *conf, bool test_
 		}
 
 		// Check if the DHCP range is valid (start needs to be smaller than end)
-		if(ntohl(conf->dhcp.start.v.in_addr.s_addr) >= ntohl(conf->dhcp.end.v.in_addr.s_addr))
+		if(ntohl(conf->dhcp.start.v.in_addr.s_addr) > ntohl(conf->dhcp.end.v.in_addr.s_addr))
 		{
-			strncpy(errbuf, "DHCP range start address is larger than or equal to the end address", ERRBUF_SIZE);
+			strncpy(errbuf, "DHCP range start address is larger than the end address", ERRBUF_SIZE);
 			log_err("Unable to update dnsmasq configuration: %s", errbuf);
 			return false;
 		}
@@ -703,10 +703,11 @@ bool __attribute__((const)) write_dnsmasq_config(struct config *conf, bool test_
 
 	if(directory_exists("/etc/dnsmasq.d") && conf->misc.etc_dnsmasq_d.v.b)
 	{
-		// Load additional user scripts from /etc/dnsmasq.d if the
+		// Load additional user configs from /etc/dnsmasq.d if the
 		// directory exists (it may not, e.g., in a container)
-		fputs("# Load additional user scripts\n", pihole_conf);
-		fputs("conf-dir=/etc/dnsmasq.d\n", pihole_conf);
+		// Load only files ending in .conf
+		fputs("# Load additional user configs\n", pihole_conf);
+		fputs("conf-dir=/etc/dnsmasq.d,*.conf\n", pihole_conf);
 		fputs("\n", pihole_conf);
 	}
 

src/config/dnsmasq_config.h

@@ -14,7 +14,7 @@
 
 #define ERRBUF_SIZE 1024
 
-bool write_dnsmasq_config(struct config *conf, bool test_config, char errbuf[ERRBUF_SIZE]);
+bool write_dnsmasq_config(struct config *conf, bool test_config, char errbuf[ERRBUF_SIZE]) __attribute__((nonnull(1,3)));
 int get_lineno_from_string(const char *string);
 char *get_dnsmasq_line(const unsigned int lineno);
 bool read_legacy_dhcp_static_config(void);

src/config/env.c

@@ -122,10 +122,10 @@ void printFTLenv(void)
 			else
 			{
 				if(item->error != NULL)
-					log_err("  %s %s %s",
+					log_err("  %s %s %s, using default instead",
 						cli_cross(), item->key, item->error);
 				else
-					log_err("  %s %s is invalid",
+					log_err("  %s %s is invalid, using default instead",
 					        cli_cross(), item->key);
 
 				if(item->error_allocated)
@@ -613,6 +613,20 @@ bool __attribute__((nonnull(1,2,3))) readEnvValue(struct conf_item *conf_item, s
 		}
 	}
 
+	// Validate enum value if it is valid and validation function is defined
+	char errbuf[VALIDATOR_ERRBUF_LEN] = { 0 };
+	if(conf_item->c != NULL && !conf_item->c(&conf_item->v, conf_item->k, errbuf))
+	{
+		log_debug(DEBUG_CONFIG, "Config item validation failed for %s: %s", conf_item->k, errbuf);
+		item->error = strdup(errbuf);
+		item->error_allocated = true;
+		item->valid = false;
+		return false;
+	}
+
+
+	log_debug(DEBUG_CONFIG, "Env var %s passed validation successfully", conf_item->k);
+
 	return true;
 }
 

src/config/password.c

@@ -93,8 +93,8 @@ static char * __attribute__((malloc)) double_sha256_password(const char *passwor
 
 bool get_secure_randomness(uint8_t *buffer, const size_t length)
 {
-	ssize_t result;
-
+	ssize_t result = -1;
+	const char *generator = "getrandom";
 	// First try to get randomness in non-blocking mode and print a warning when not enough entropy is available right now
 	do {
 		result = getrandom(buffer, length, GRND_NONBLOCK);
@@ -103,32 +103,51 @@ bool get_secure_randomness(uint8_t *buffer, const size_t length)
 	// If not enough entropy is available right now, try again in blocking mode
 	if (result < 0 && errno == EAGAIN)
 	{
-		log_warn("getrandom() failed in get_secure_randomness(): Not enough entropy available right now, retrying in blocking mode");
+		log_warn("Not enough entropy available right now for generating secure randomness, retrying in blocking mode");
 		// Sleep for 1 second to give the kernel some time to gather entropy
 		sleepms(1000);
 	}
 	else
 	{
 		// If the first try was successful, return the result
 		if (result >= 0)
-			return true;
+			goto random_success;
 	}
 	do {
 		result = getrandom(buffer, length, 0);
 	} while (result < 0 && errno == EINTR);
 
-	if (result < 0)
+	if(result < 0)
 	{
-		const int err = errno;
-		log_err("getrandom() failed in get_secure_randomness(): %s", strerror(errno));
-		errno = err;
-		return false;
+		log_debug(DEBUG_API, "Getting secure randomness failed (%s), trying /dev/urandom", strerror(errno));
+		generator = "/dev/urandom";
+		result = getrandom_fallback(buffer, length, 0);
+		if(result < 0 || result < (ssize_t)length)
+		{
+			log_debug(DEBUG_API, "Fallback failed, trying internal DRBG generator");
+			generator = "internal DRBG";
+			result = drbg_random(buffer, length);
+			if(result < 0)
+			{
+				// Warning will be printed by drbg_random()
+				return false;
+			}
+
+			log_debug(DEBUG_API, "Internal DRBG generator successfully used");
+		}
+		else
+			log_debug(DEBUG_API, "Fallback to /dev/urandom successful");
 	}
-	else if((size_t)result != length)
+
+	// Check if enough bytes were generated
+	if(result != (ssize_t)length)
 	{
-		log_err("getrandom() failed in get_secure_randomness(): Not enough bytes generated (%zu != %zu)", (size_t)result, length);
+		log_err("Randomness generator (%s) failed: not enough bytes generated (%zd != %zu)", generator, result, length);
 		return false;
 	}
+
+random_success:
+	log_debug(DEBUG_ANY, "Generated %zd bytes of secure randomness", result);
 	return true;
 }
 
@@ -185,7 +204,7 @@ static char * __attribute__((malloc)) balloon_password(const char *password,
 {
 	// Parameter check
 	if(password == NULL || salt == NULL)
-		return NULL;
+		return strdup("");
 
 	struct timespec start, end;
 	// Record starting time
@@ -370,7 +389,7 @@ char * __attribute__((malloc)) create_password(const char *password)
 	// genrandom() returns cryptographically secure random data
 	uint8_t salt[SALT_LEN] = { 0 };
 	if(!get_secure_randomness(salt, sizeof(salt)))
-		return NULL;
+		return strdup("");
 
 	// Generate balloon PHC-encoded password hash
 	return balloon_password(password, salt, true);

src/config/setupVars.c

@@ -288,15 +288,26 @@ static void get_conf_string_array_from_setupVars_regex(const char *key, struct c
 				p++;
 			}
 
-			// Add ^ and $ to the string
+			// Add ^ and $ to the string, but only add ^ if the
+			// string does not start with *. In the latter case, we
+			// only add $ to the string (and remove the *).
 			char *regex2 = calloc(strlen(regex) + 3, sizeof(char));
 			if(regex2 == NULL)
 			{
 				log_warn("get_conf_string_array_from_setupVars(%s) failed: Could not allocate memory for regex2", key);
 				free(regex);
 				continue;
 			}
-			sprintf(regex2, "^%s$", regex);
+			if(regex[0] != '*')
+			{
+				// Add ^ and $ to the string
+				sprintf(regex2, "^%s$", regex);
+			}
+			else
+			{
+				// Skip the * and add $ to the string
+				sprintf(regex2, "%s$", regex + 1);
+			}
 
 			// Free memory
 			free(regex);
@@ -305,7 +316,7 @@ static void get_conf_string_array_from_setupVars_regex(const char *key, struct c
 			cJSON *item = cJSON_CreateString(regex2);
 			cJSON_AddItemToArray(conf_item->v.json, item);
 
-			log_info("setupVars.conf:%s -> Setting %s[%u] = %s\n",
+			log_info("setupVars.conf:%s -> Setting %s[%u] = %s",
 			         key, conf_item->k, i, item->valuestring);
 
 			// Free memory

src/config/validator.c

@@ -270,6 +270,21 @@ bool validate_filepath(union conf_value *val, const char *key, char err[VALIDATO
 	return true;
 }
 
+// Validate a file path that needs to have both a slash at the beginning and at
+// the end
+bool validate_filepath_two_slash(union conf_value *val, const char *key, char err[VALIDATOR_ERRBUF_LEN])
+{
+	// Check if the path starts and ends with a slash
+	if(strlen(val->s) < 1 || val->s[0] != '/' || val->s[strlen(val->s) - 1] != '/')
+	{
+		snprintf(err, VALIDATOR_ERRBUF_LEN, "%s: file path does not start and end with a slash (\"%s\")", key, val->s);
+		return false;
+	}
+
+	// Check if the path contains only valid characters
+	return validate_filepath(val, key, err);
+}
+
 // Validate file path (empty allowed)
 bool validate_filepath_empty(union conf_value *val, const char *key, char err[VALIDATOR_ERRBUF_LEN])
 {

src/config/validator.h

@@ -21,6 +21,7 @@ bool validate_dns_domain(union conf_value *val, const char *key, char err[VALIDA
 bool validate_cidr(union conf_value *val, const char *key, char err[VALIDATOR_ERRBUF_LEN]);
 bool validate_domain(union conf_value *val, const char *key, char err[VALIDATOR_ERRBUF_LEN]);
 bool validate_filepath(union conf_value *val, const char *key, char err[VALIDATOR_ERRBUF_LEN]);
+bool validate_filepath_two_slash(union conf_value *val, const char *key, char err[VALIDATOR_ERRBUF_LEN]);
 bool validate_filepath_empty(union conf_value *val, const char *key, char err[VALIDATOR_ERRBUF_LEN]);
 bool validate_filepath_dash(union conf_value *val, const char *key, char err[VALIDATOR_ERRBUF_LEN]);
 bool validate_regex_array(union conf_value *val, const char *key, char err[VALIDATOR_ERRBUF_LEN]);