Pi-hole Core 6.0.1 (#5912)

Signed-off-by: DL6ER <dl6er@dl6er.de>

.github/workflows/merge-conflict.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check if PRs are have merge conflicts
-        uses: eps1lon/actions-label-merge-conflict@v3.0.2
+        uses: eps1lon/actions-label-merge-conflict@v3.0.3
         with:
           dirtyLabel: "PR: Merge Conflict"
           repoToken: "${{ secrets.GITHUB_TOKEN }}"

.github/workflows/stale.yml

@@ -17,7 +17,7 @@ jobs:
       issues: write
 
     steps:
-      - uses: actions/stale@v9.0.0
+      - uses: actions/stale@v9.1.0
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           days-before-stale: 30

.github/workflows/stale_pr.yml

@@ -17,7 +17,7 @@ jobs:
       pull-requests: write
 
     steps:
-      - uses: actions/stale@v9.0.0
+      - uses: actions/stale@v9.1.0
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           # Do not automatically mark PR/issue as stale

.github/workflows/test.yml

@@ -77,7 +77,7 @@ jobs:
         uses: actions/checkout@v4.2.2
 
       - name: Set up Python 3.10
-        uses: actions/setup-python@v5.3.0
+        uses: actions/setup-python@v5.4.0
         with:
           python-version: "3.10"
 

advanced/Scripts/api.sh

@@ -165,15 +165,17 @@ GetFTLData() {
   # get the data from querying the API as well as the http status code
   response=$(curl -skS -w "%{http_code}" -X GET "${API_URL}$1" -H "Accept: application/json" -H "sid: ${SID}" )
 
-  # status are the last 3 characters
-  status="${response#"${response%???}"}"
-  # data is everything from response without the last 3 characters
-  data="${response%???}"
-
   if [ "${2}" = "raw" ]; then
     # return the raw response
     echo "${response}"
   else
+
+    # status are the last 3 characters
+    # not using ${response#"${response%???}"}" here because it's extremely slow on big responses
+    status=$(printf "%s" "${response}" | tail -c 3)
+    # data is everything from response without the last 3 characters
+    data="${response%???}"
+
     # return only the data
     if [ "${status}" = 200 ]; then
         # response OK
@@ -264,7 +266,8 @@ apiFunc() {
   response=$(GetFTLData "$1" raw)
 
   # status are the last 3 characters
-  status="${response#"${response%???}"}"
+  # not using ${response#"${response%???}"}" here because it's extremely slow on big responses
+  status=$(printf "%s" "${response}" | tail -c 3)
   # data is everything from response without the last 3 characters
   data="${response%???}"
 

advanced/Scripts/piholeDebug.sh

@@ -44,6 +44,14 @@ fi
 # shellcheck disable=SC1091
 . /etc/pihole/versions
 
+# Read the value of an FTL config key. The value is printed to stdout.
+get_ftl_conf_value() {
+    local key=$1
+
+    # Obtain setting from FTL directly
+    pihole-FTL --config "${key}"
+}
+
 # FAQ URLs for use in showing the debug log
 FAQ_HARDWARE_REQUIREMENTS="${COL_CYAN}https://docs.pi-hole.net/main/prerequisites/${COL_NC}"
 FAQ_HARDWARE_REQUIREMENTS_PORTS="${COL_CYAN}https://docs.pi-hole.net/main/prerequisites/#ports${COL_NC}"
@@ -61,10 +69,10 @@ DNSMASQ_D_DIRECTORY="/etc/dnsmasq.d"
 PIHOLE_DIRECTORY="/etc/pihole"
 PIHOLE_SCRIPTS_DIRECTORY="/opt/pihole"
 BIN_DIRECTORY="/usr/local/bin"
-RUN_DIRECTORY="/run"
 LOG_DIRECTORY="/var/log/pihole"
-HTML_DIRECTORY="/var/www/html"
+HTML_DIRECTORY="$(get_ftl_conf_value "webserver.paths.webroot")"
-WEB_GIT_DIRECTORY="${HTML_DIRECTORY}/admin"
+WEBHOME_PATH="$(get_ftl_conf_value "webserver.paths.webhome")"
+WEB_GIT_DIRECTORY="${HTML_DIRECTORY}${WEBHOME_PATH}"
 SHM_DIRECTORY="/dev/shm"
 ETC="/etc"
 
@@ -79,14 +87,6 @@ PIHOLE_FTL_CONF_FILE="${PIHOLE_DIRECTORY}/pihole.toml"
 PIHOLE_DNSMASQ_CONF_FILE="${PIHOLE_DIRECTORY}/dnsmasq.conf"
 PIHOLE_VERSIONS_FILE="${PIHOLE_DIRECTORY}/versions"
 
-# Read the value of an FTL config key. The value is printed to stdout.
-get_ftl_conf_value() {
-    local key=$1
-
-    # Obtain setting from FTL directly
-    pihole-FTL --config "${key}"
-}
-
 PIHOLE_GRAVITY_DB_FILE="$(get_ftl_conf_value "files.gravity")"
 
 PIHOLE_FTL_DB_FILE="$(get_ftl_conf_value "files.database")"
@@ -94,7 +94,7 @@ PIHOLE_FTL_DB_FILE="$(get_ftl_conf_value "files.database")"
 PIHOLE_COMMAND="${BIN_DIRECTORY}/pihole"
 PIHOLE_COLTABLE_FILE="${BIN_DIRECTORY}/COL_TABLE"
 
-FTL_PID="${RUN_DIRECTORY}/pihole-FTL.pid"
+FTL_PID="$(get_ftl_conf_value "files.pid")"
 
 PIHOLE_LOG="${LOG_DIRECTORY}/pihole.log"
 PIHOLE_LOG_GZIPS="${LOG_DIRECTORY}/pihole.log.[0-9].*"
@@ -202,7 +202,7 @@ compare_local_version_to_git_version() {
         if git status &> /dev/null; then
             # The current version the user is on
             local local_version
-            local_version=$(git describe --tags --abbrev=0);
+            local_version=$(git describe --tags --abbrev=0 2> /dev/null);
             # What branch they are on
             local local_branch
             local_branch=$(git rev-parse --abbrev-ref HEAD);
@@ -213,7 +213,13 @@ compare_local_version_to_git_version() {
             local local_status
             local_status=$(git status -s)
             # echo this information out to the user in a nice format
-            log_write "${TICK} Version: ${local_version}"
+            if [ ${local_version} ]; then
+              log_write "${TICK} Version: ${local_version}"
+            elif [ -n "${DOCKER_VERSION}" ]; then
+              log_write "${TICK} Version: Pi-hole Docker Container ${COL_BOLD}${DOCKER_VERSION}${COL_NC}"
+            else
+              log_write "${CROSS} Version: not detected"
+            fi
 
             # Print the repo upstreams
             remotes=$(git remote -v)
@@ -346,6 +352,9 @@ os_check() {
             fi
         done
 
+        # If it is a docker container, we can assume the OS is supported
+        [ -n "${DOCKER_VERSION}" ] && valid_os=true && valid_version=true
+
         local finalmsg
         if [ "$valid_os" = true ]; then
             log_write "${TICK} Distro:  ${COL_GREEN}${detected_os^}${COL_NC}"
@@ -489,13 +498,25 @@ run_and_print_command() {
 }
 
 hardware_check() {
+    # Note: the checks are skipped if Pi-hole is running in a docker container
+
+    local skip_msg="${INFO} Not enough permissions inside Docker container ${COL_YELLOW}(skipped)${COL_NC}"
+
     echo_current_diagnostic "System hardware configuration"
-    # Store the output of the command in a variable
+    if [ -n "${DOCKER_VERSION}" ]; then
-    run_and_print_command "lshw -short"
+        log_write "${skip_msg}"
+    else
+        # Store the output of the command in a variable
+        run_and_print_command "lshw -short"
+    fi
 
     echo_current_diagnostic "Processor details"
-    # Store the output of the command in a variable
+    if [ -n "${DOCKER_VERSION}" ]; then
-    run_and_print_command "lscpu"
+        log_write "${skip_msg}"
+    else
+        # Store the output of the command in a variable
+        run_and_print_command "lscpu"
+    fi
 }
 
 disk_usage() {
@@ -808,26 +829,24 @@ dig_at() {
 process_status(){
     # Check to make sure Pi-hole's services are running and active
     echo_current_diagnostic "Pi-hole processes"
+
     # Local iterator
     local i
+
     # For each process,
     for i in "${PIHOLE_PROCESSES[@]}"; do
+        local status_of_process
+
         # If systemd
         if command -v systemctl &> /dev/null; then
             # get its status via systemctl
-            local status_of_process
             status_of_process=$(systemctl is-active "${i}")
         else
             # Otherwise, use the service command and mock the output of `systemctl is-active`
-            local status_of_process
 
-            # If DOCKER_VERSION is set, the output is slightly different (s6 init system on Docker)
+            # If it is a docker container, there is no systemctl or service. Do nothing.
             if [ -n "${DOCKER_VERSION}" ]; then
-                if service "${i}" status | grep -E '^up' &> /dev/null; then
+                :
-                    status_of_process="active"
-                else
-                    status_of_process="inactive"
-                fi
             else
             # non-Docker system
                 if service "${i}" status | grep -E 'is\srunning' &> /dev/null; then
@@ -837,8 +856,12 @@ process_status(){
                 fi
             fi
         fi
+
         # and print it out to the user
-        if [[ "${status_of_process}" == "active" ]]; then
+        if [ -n "${DOCKER_VERSION}" ]; then
+            # If it's a Docker container, the test was skipped
+            log_write "${INFO} systemctl/service not installed inside docker container ${COL_YELLOW}(skipped)${COL_NC}"
+        elif [[ "${status_of_process}" == "active" ]]; then
             # If it's active, show it in green
             log_write "${TICK} ${COL_GREEN}${i}${COL_NC} daemon is ${COL_GREEN}${status_of_process}${COL_NC}"
         else
@@ -855,6 +878,8 @@ ftl_full_status(){
     if command -v systemctl &> /dev/null; then
       FTL_status=$(systemctl status --full --no-pager pihole-FTL.service)
       log_write "   ${FTL_status}"
+    elif [ -n "${DOCKER_VERSION}" ]; then
+      log_write "${INFO} systemctl/service not installed inside docker container ${COL_YELLOW}(skipped)${COL_NC}"
     else
       log_write "${INFO} systemctl:  command not found"
     fi
@@ -1112,7 +1137,7 @@ show_FTL_db_entries() {
 }
 
 check_dhcp_servers() {
-    echo_current_diagnostic "Discovering active DHCP servers (takes 10 seconds)"
+    echo_current_diagnostic "Discovering active DHCP servers (takes 6 seconds)"
 
     OLD_IFS="$IFS"
     IFS=$'\n'
@@ -1196,7 +1221,7 @@ database_integrity_check(){
     local database="${1}"
 
     log_write "${INFO} Checking integrity of ${database} ... (this can take several minutes)"
-    result="$(pihole-FTL "${database}" "PRAGMA integrity_check" 2>&1 & spinner)"
+    result="$(pihole-FTL sqlite3 -ni "${database}" "PRAGMA integrity_check" 2>&1 & spinner)"
     if [[ ${result} = "ok" ]]; then
       log_write "${TICK} Integrity of ${database} intact"
 
@@ -1317,19 +1342,16 @@ upload_to_tricorder() {
         curl_to_tricorder
         # If we're not running in automated mode,
     else
-        # if not being called from the web interface
+        echo ""
-        if [[ ! "${WEBCALL}" ]]; then
+        # give the user a choice of uploading it or not
-            echo ""
+        # Users can review the log file locally (or the output of the script since they are the same) and try to self-diagnose their problem
-            # give the user a choice of uploading it or not
+        read -r -p "[?] Would you like to upload the log? [y/N] " response
-            # Users can review the log file locally (or the output of the script since they are the same) and try to self-diagnose their problem
+        case ${response} in
-            read -r -p "[?] Would you like to upload the log? [y/N] " response
+            # If they say yes, run our function for uploading the log
-            case ${response} in
+            [yY][eE][sS]|[yY]) curl_to_tricorder;;
-                # If they say yes, run our function for uploading the log
+            # If they choose no, just exit out of the script
-                [yY][eE][sS]|[yY]) curl_to_tricorder;;
+            *) log_write "    * Log will ${COL_GREEN}NOT${COL_NC} be uploaded to tricorder.\\n    * A local copy of the debug log can be found at: ${COL_CYAN}${PIHOLE_DEBUG_LOG}${COL_NC}\\n";exit;
-                # If they choose no, just exit out of the script
+        esac
-                *) log_write "    * Log will ${COL_GREEN}NOT${COL_NC} be uploaded to tricorder.\\n    * A local copy of the debug log can be found at: ${COL_CYAN}${PIHOLE_DEBUG_LOG}${COL_NC}\\n";exit;
-            esac
-        fi
     fi
     # Check if tricorder.pi-hole.net is reachable and provide token
     # along with some additional useful information
@@ -1349,13 +1371,8 @@ upload_to_tricorder() {
     # If no token was generated
     else
         # Show an error and some help instructions
-        # Skip this if being called from web interface and automatic mode was not chosen (users opt-out to upload)
+        log_write "${CROSS} ${COL_RED}There was an error uploading your debug log.${COL_NC}"
-        if [[ "${WEBCALL}" ]] && [[ ! "${AUTOMATED}" ]]; then
+        log_write "   * Please try again or contact the Pi-hole team for assistance."
-            :
-        else
-            log_write "${CROSS} ${COL_RED}There was an error uploading your debug log.${COL_NC}"
-            log_write "   * Please try again or contact the Pi-hole team for assistance."
-        fi
     fi
     # Finally, show where the log file is no matter the outcome of the function so users can look at it
     log_write "   * A local copy of the debug log can be found at: ${COL_CYAN}${PIHOLE_DEBUG_LOG}${COL_NC}\\n"

advanced/Scripts/update.sh

@@ -107,6 +107,9 @@ main() {
     web_update=false
     FTL_update=false
 
+    # Perform an OS check to ensure we're on an appropriate operating system
+    os_check
+
     # Install packages used by this installation script (necessary if users have removed e.g. git from their systems)
     package_manager_detect
     build_dependency_package

advanced/Templates/gravity.db.sql

@@ -67,6 +67,11 @@ CREATE TABLE info
 );
 
 INSERT INTO "info" VALUES('version','19');
+/* This is a flag to indicate if gravity was restored from a backup
+    false = not restored,
+    failed = restoration failed due to no backup
+    other string = restoration successful with the string being the backup file used */
+INSERT INTO "info" VALUES('gravity_restored','false');
 
 CREATE TABLE domainlist_by_group
 (

automated install/basic-install.sh

@@ -388,28 +388,6 @@ os_check() {
     fi
 }
 
-# This function waits for dpkg to unlock, which signals that the previous apt-get command has finished.
-test_dpkg_lock() {
-    i=0
-    printf "  %b Waiting for package manager to finish (up to 30 seconds)\\n" "${INFO}"
-    # fuser is a program to show which processes use the named files, sockets, or filesystems
-    # So while the lock is held,
-    while fuser /var/lib/dpkg/lock >/dev/null 2>&1; do
-        # we wait half a second,
-        sleep 0.5
-        # increase the iterator,
-        ((i = i + 1))
-        # exit if waiting for more then 30 seconds
-        if [[ $i -gt 60 ]]; then
-            printf "  %b %bError: Could not verify package manager finished and released lock. %b\\n" "${CROSS}" "${COL_LIGHT_RED}" "${COL_NC}"
-            printf "       Attempt to install packages manually and retry.\\n"
-            exit 1
-        fi
-    done
-    # and then report success once dpkg is unlocked.
-    return 0
-}
-
 # Compatibility
 package_manager_detect() {
 
@@ -1301,13 +1279,6 @@ installConfigs() {
     # Ensure that permissions are correctly set
     chown -R pihole:pihole /etc/pihole
 
-    # Install list of DNS servers
-    # Format: Name;Primary IPv4;Secondary IPv4;Primary IPv6;Secondary IPv6
-    # Some values may be empty (for example: DNS servers without IPv6 support)
-    echo "${DNS_SERVERS}" >"${PI_HOLE_CONFIG_DIR}/dns-servers.conf"
-    chmod 644 "${PI_HOLE_CONFIG_DIR}/dns-servers.conf"
-    chown pihole:pihole "${PI_HOLE_CONFIG_DIR}/dns-servers.conf"
-
     # Install empty custom.list file if it does not exist
     if [[ ! -r "${PI_HOLE_CONFIG_DIR}/hosts/custom.list" ]]; then
         if ! install -D -T -o pihole -g pihole -m 660 /dev/null "${PI_HOLE_CONFIG_DIR}/hosts/custom.list" &>/dev/null; then
@@ -1458,20 +1429,15 @@ disable_resolved_stublistener() {
     printf "  %b Testing if systemd-resolved is enabled\\n" "${INFO}"
     # Check if Systemd-resolved's DNSStubListener is enabled and active on port 53
     if check_service_active "systemd-resolved"; then
-        # Check if DNSStubListener is enabled
+        # Disable the DNSStubListener to unbind it from port 53
-        printf "  %b %b Testing if systemd-resolved DNSStub-Listener is active" "${OVER}" "${INFO}"
+        # Note that this breaks dns functionality on host until FTL is up and running
-        if (grep -E '#?DNSStubListener=yes' /etc/systemd/resolved.conf &>/dev/null); then
+        printf "%b  %b Disabling systemd-resolved DNSStubListener\\n" "${OVER}" "${TICK}"
-            # Disable the DNSStubListener to unbind it from port 53
+        mkdir -p /etc/systemd/resolved.conf.d
-            # Note that this breaks dns functionality on host until ftl are up and running
+        cat > /etc/systemd/resolved.conf.d/90-pi-hole-disable-stub-listener.conf << EOF
-            printf "%b  %b Disabling systemd-resolved DNSStubListener" "${OVER}" "${TICK}"
+[Resolve]
-            # Make a backup of the original /etc/systemd/resolved.conf
+DNSStubListener=no
-            # (This will need to be restored on uninstallation)
+EOF
-            sed -r -i.orig 's/#?DNSStubListener=yes/DNSStubListener=no/g' /etc/systemd/resolved.conf
+        systemctl reload-or-restart systemd-resolved
-            printf " and restarting systemd-resolved\\n"
-            systemctl reload-or-restart systemd-resolved
-        else
-            printf "%b  %b Systemd-resolved does not need to be restarted\\n" "${OVER}" "${INFO}"
-        fi
     else
         printf "%b  %b Systemd-resolved is not enabled\\n" "${OVER}" "${INFO}"
     fi
@@ -2156,7 +2122,7 @@ get_binary_name() {
         else
             printf "%b  %b Detected 32bit (i686) architecture\\n" "${OVER}" "${TICK}"
         fi
-        l_binary="pihole-FTL-linux-386"
+        l_binary="pihole-FTL-386"
     fi
 
     # Returning a string value via echo
@@ -2300,6 +2266,44 @@ copy_to_install_log() {
     chown pihole:pihole "${installLogLoc}"
 }
 
+disableLighttpd() {
+    # Return early when lighttpd is not active
+    if ! check_service_active lighttpd; then
+        return
+    fi
+
+    local response
+    # Detect if the terminal is interactive
+    if [[ -t 0 ]]; then
+        # The terminal is interactive
+        dialog --no-shadow --keep-tite \
+            --title "Pi-hole v6.0 no longer uses lighttpd" \
+           --yesno "\\n\\nPi-hole v6.0 has its own embedded web server so lighttpd is no longer needed *unless* you have custom configurations.\\n\\nIn this case, you can opt-out of disabling lighttpd and pihole-FTL will try to bind to an alternative port such as 8080.\\n\\nDo you want to disable lighttpd (recommended)?" "${r}" "${c}" && response=0 || response="$?"
+    else
+        # The terminal is non-interactive, assume yes. Lighttpd will be stopped
+        # but keeps being installed and can easily be re-enabled by the user
+        response=0
+    fi
+
+    # If the user does not want to disable lighttpd, return early
+    if [[ "${response}" -ne 0 ]]; then
+        return
+    fi
+
+    # Lighttpd is not needed anymore, so disable it
+    # We keep all the configuration files in place, so the user can re-enable it
+    # if needed
+
+    # Check if lighttpd is installed
+    if is_command lighttpd; then
+        # Stop the lighttpd service
+        stop_service lighttpd
+
+        # Disable the lighttpd service
+        disable_service lighttpd
+    fi
+}
+
 migrate_dnsmasq_configs() {
     # Previously, Pi-hole created a number of files in /etc/dnsmasq.d
     # During migration, their content is copied into the new single source of
@@ -2312,6 +2316,9 @@ migrate_dnsmasq_configs() {
         return 0
     fi
 
+    # Disable lighttpd server during v6 migration
+    disableLighttpd
+
     # Create target directory /etc/pihole/migration_backup_v6
     # and make it owned by pihole:pihole
     mkdir -p "${V6_CONF_MIGRATION_DIR}"
@@ -2326,6 +2333,40 @@ migrate_dnsmasq_configs() {
 
     mv /etc/dnsmasq.d/0{1,2,4,5}-pihole*.conf "${V6_CONF_MIGRATION_DIR}/" 2>/dev/null || true
     mv /etc/dnsmasq.d/06-rfc6761.conf "${V6_CONF_MIGRATION_DIR}/" 2>/dev/null || true
+
+    # Finally, after everything is in place, we can create the new config file
+    # /etc/pihole/pihole.toml
+    # This file will be created with the default settings unless the user has
+    # changed settings via setupVars.conf or the other dnsmasq files moved above
+    # During migration, setupVars.conf is moved to /etc/pihole/migration_backup_v6
+    str="Migrating Pi-hole configuration to version 6"
+    printf "  %b %s..." "${INFO}" "${str}"
+    local FTLoutput FTLstatus
+    FTLoutput=$(pihole-FTL migrate v6)
+    FTLstatus=$?
+    if [[ "${FTLstatus}" -eq 0 ]]; then
+        printf "%b  %b %s\\n" "${OVER}" "${TICK}" "${str}"
+    else
+        printf "%b  %b %s\\n" "${OVER}" "${CROSS}" "${str}"
+    fi
+
+    # Print the output of the FTL migration prefacing every line with four
+    # spaces for alignment
+    printf "%b" "${FTLoutput}" | sed 's/^/    /'
+
+    # Print a blank line for separation
+    printf "\\n"
+}
+
+# Check for availability of either the "service" or "systemctl" commands
+check_service_command() {
+    # Check for the availability of the "service" command
+    if ! is_command service && ! is_command systemctl; then
+        # If neither the "service" nor the "systemctl" command is available, inform the user
+        printf "  %b Neither the service nor the systemctl commands are available\\n" "${CROSS}"
+        printf "      on this machine. This Pi-hole installer cannot continue.\\n"
+        exit 1
+    fi
 }
 
 main() {
@@ -2376,6 +2417,9 @@ main() {
     # Check if SELinux is Enforcing and exit before doing anything else
     checkSelinux
 
+    # Check for availability of either the "service" or "systemctl" commands
+    check_service_command
+
     # Check for supported package managers so that we may install dependencies
     package_manager_detect
 

automated install/uninstall.sh

@@ -94,8 +94,9 @@ removePiholeFiles() {
     echo -e "  ${TICK} Removed config files"
 
     # Restore Resolved
-    if [[ -e /etc/systemd/resolved.conf.orig ]]; then
+    if [[ -e /etc/systemd/resolved.conf.orig ]] || [[ -e /etc/systemd/resolved.conf.d/90-pi-hole-disable-stub-listener.conf ]]; then
-        ${SUDO} cp -p /etc/systemd/resolved.conf.orig /etc/systemd/resolved.conf
+        ${SUDO} cp -p /etc/systemd/resolved.conf.orig /etc/systemd/resolved.conf &> /dev/null || true
+        ${SUDO} rm -f /etc/systemd/resolved.conf.d/90-pi-hole-disable-stub-listener.conf
         systemctl reload-or-restart systemd-resolved
     fi
 

gravity.sh

@@ -30,6 +30,9 @@ PIHOLE_COMMAND="/usr/local/bin/${basename}"
 
 piholeDir="/etc/${basename}"
 
+# Gravity aux files directory
+listsCacheDir="${piholeDir}/listsCache"
+
 # Legacy (pre v5.0) list file locations
 whitelistFile="${piholeDir}/whitelist.txt"
 blacklistFile="${piholeDir}/blacklist.txt"
@@ -44,6 +47,7 @@ gravityDBcopy="${piholeGitDir}/advanced/Templates/gravity_copy.sql"
 
 domainsExtension="domains"
 curl_connect_timeout=10
+etag_support=false
 
 # Check gravity temp directory
 if [ ! -d "${GRAVITY_TMPDIR}" ] || [ ! -w "${GRAVITY_TMPDIR}" ]; then
@@ -58,6 +62,8 @@ gravityDBfile_default="/etc/pihole/gravity.db"
 gravityTEMPfile="${GRAVITYDB}_temp"
 gravityDIR="$(dirname -- "${gravityDBfile}")"
 gravityOLDfile="${gravityDIR}/gravity_old.db"
+gravityBCKdir="${gravityDIR}/gravity_backups"
+gravityBCKfile="${gravityBCKdir}/gravity.db"
 
 fix_owner_permissions() {
   # Fix ownership and permissions for the specified file
@@ -91,11 +97,21 @@ gravity_build_tree() {
 
   if [[ "${status}" -ne 0 ]]; then
     echo -e "\\n  ${CROSS} Unable to build gravity tree in ${gravityTEMPfile}\\n  ${output}"
+    echo -e "  ${INFO} If you have a large amount of domains, make sure your Pi-hole has enough RAM available\\n"
     return 1
   fi
   echo -e "${OVER}  ${TICK} ${str}"
 }
 
+# Rotate gravity backup files
+rotate_gravity_backup() {
+  for i in {9..1}; do
+    if [ -f "${gravityBCKfile}.${i}" ]; then
+      mv "${gravityBCKfile}.${i}" "${gravityBCKfile}.$((i + 1))"
+    fi
+  done
+}
+
 # Copy data from old to new database file and swap them
 gravity_swap_databases() {
   str="Swapping databases"
@@ -111,10 +127,32 @@ gravity_swap_databases() {
   oldAvail=false
   if [ "${availableBlocks}" -gt "$((gravityBlocks * 2))" ] && [ -f "${gravityDBfile}" ]; then
     oldAvail=true
-    mv "${gravityDBfile}" "${gravityOLDfile}"
+    cp "${gravityDBfile}" "${gravityOLDfile}"
-  else
-    rm "${gravityDBfile}"
   fi
+
+  # Drop the gravity and antigravity tables + subsequent VACUUM the current
+  # database for compaction
+  output=$({ printf ".timeout 30000\\nDROP TABLE IF EXISTS gravity;\\nDROP TABLE IF EXISTS antigravity;\\nVACUUM;\\n" | pihole-FTL sqlite3 -ni "${gravityDBfile}"; } 2>&1)
+  status="$?"
+
+  if [[ "${status}" -ne 0 ]]; then
+    echo -e "\\n  ${CROSS} Unable to clean current database for backup\\n  ${output}"
+  else
+    # Check if the backup directory exists
+    if [ ! -d "${gravityBCKdir}" ]; then
+      mkdir -p "${gravityBCKdir}"
+    fi
+
+    # If multiple gravityBCKfile's are present (appended with a number), rotate them
+    # We keep at most 10 backups
+    rotate_gravity_backup
+
+    # Move the old database to the backup location
+    mv "${gravityDBfile}" "${gravityBCKfile}.1"
+  fi
+
+
+  # Move the new database to the correct location
   mv "${gravityTEMPfile}" "${gravityDBfile}"
   echo -e "${OVER}  ${TICK} ${str}"
 
@@ -324,6 +362,54 @@ gravity_CheckDNSResolutionAvailable() {
   echo -e "${OVER}  ${TICK} DNS resolution is available"
 }
 
+# Function: try_restore_backup
+# Description: Attempts to restore the previous Pi-hole gravity database from a
+#              backup file. If a backup exists, it copies the backup to the
+#              gravity database file and prepares a new gravity database. If the
+#              restoration is successful, it returns 0. Otherwise, it returns 1.
+# Returns:
+#   0 - If the backup is successfully restored.
+#   1 - If no backup is available or if the restoration fails.
+try_restore_backup () {
+  local num filename timestamp
+  num=$1
+  filename="${gravityBCKfile}.${num}"
+  # Check if a backup exists
+  if [ -f "${filename}" ]; then
+    echo -e "  ${INFO} Attempting to restore previous database from backup no. ${num}"
+    cp "${filename}" "${gravityDBfile}"
+
+    # If the backup was successfully copied, prepare a new gravity database from
+    # it
+    if [ -f "${gravityDBfile}" ]; then
+      output=$({ pihole-FTL sqlite3 -ni "${gravityTEMPfile}" <<<"${copyGravity}"; } 2>&1)
+      status="$?"
+
+      # Error checking
+      if [[ "${status}" -ne 0 ]]; then
+        echo -e "\\n  ${CROSS} Unable to copy data from ${gravityDBfile} to ${gravityTEMPfile}\\n  ${output}"
+        gravity_Cleanup "error"
+      fi
+
+      # Get the timestamp of the backup file in a human-readable format
+      # Note that this timestamp will be in the server timezone, this may be
+      # GMT, e.g., on a Raspberry Pi where the default timezone has never been
+      # changed
+      timestamp=$(date -r "${filename}" "+%Y-%m-%d %H:%M:%S %Z")
+
+      # Add a record to the info table to indicate that the gravity database was restored
+      pihole-FTL sqlite3 "${gravityTEMPfile}" "INSERT OR REPLACE INTO info (property,value) values ('gravity_restored','${timestamp}');"
+      echo -e "  ${TICK} Successfully restored from backup (${gravityBCKfile}.${num} at ${timestamp})"
+      return 0
+    else
+      echo -e "  ${CROSS} Unable to restore backup no. ${num}"
+    fi
+  fi
+
+  echo -e "  ${CROSS} Backup no. ${num} not available"
+  return 1
+}
+
 # Retrieve blocklist URLs and parse domains from adlist.list
 gravity_DownloadBlocklists() {
   echo -e "  ${INFO} ${COL_BOLD}Neutrino emissions detected${COL_NC}..."
@@ -332,33 +418,7 @@ gravity_DownloadBlocklists() {
     echo -e "  ${INFO} Storing gravity database in ${COL_BOLD}${gravityDBfile}${COL_NC}"
   fi
 
-  # Retrieve source URLs from gravity database
+  local url domain str target compression adlist_type directory success
-  # We source only enabled adlists, SQLite3 stores boolean values as 0 (false) or 1 (true)
-  mapfile -t sources <<<"$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2>/dev/null)"
-  mapfile -t sourceIDs <<<"$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT id FROM vw_adlist;" 2>/dev/null)"
-  mapfile -t sourceTypes <<<"$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT type FROM vw_adlist;" 2>/dev/null)"
-
-  # Parse source domains from $sources
-  mapfile -t sourceDomains <<<"$(
-    # Logic: Split by folder/port
-    awk -F '[/:]' '{
-      # Remove URL protocol & optional username:password@
-      gsub(/(.*:\/\/|.*:.*@)/, "", $0)
-      if(length($1)>0){print $1}
-      else {print "local"}
-    }' <<<"$(printf '%s\n' "${sources[@]}")" 2>/dev/null
-  )"
-
-  local str="Pulling blocklist source list into range"
-  echo -e "${OVER}  ${TICK} ${str}"
-
-  if [[ -z "${sources[*]}" ]] || [[ -z "${sourceDomains[*]}" ]]; then
-    echo -e "  ${INFO} No source list found, or it is empty"
-    echo ""
-    unset sources
-  fi
-
-  local url domain str target compression adlist_type directory
   echo ""
 
   # Prepare new gravity database
@@ -390,10 +450,55 @@ gravity_DownloadBlocklists() {
 
   if [[ "${status}" -ne 0 ]]; then
     echo -e "\\n  ${CROSS} Unable to copy data from ${gravityDBfile} to ${gravityTEMPfile}\\n  ${output}"
-    return 1
+
+    # Try to attempt a backup restore
+    success=false
+    if [[ -d "${gravityBCKdir}" ]]; then
+      for i in {1..10}; do
+        if try_restore_backup "${i}"; then
+          success=true
+          break
+        fi
+      done
+    fi
+
+    # If none of the attempts worked, return 1
+    if [[ "${success}" == false ]]; then
+      pihole-FTL sqlite3 "${gravityTEMPfile}" "INSERT OR REPLACE INTO info (property,value) values ('gravity_restored','failed');"
+      return 1
+    fi
+
+    echo -e "  ${TICK} ${str}"
+  else
+    echo -e "${OVER}  ${TICK} ${str}"
   fi
+
+  # Retrieve source URLs from gravity database
+  # We source only enabled adlists, SQLite3 stores boolean values as 0 (false) or 1 (true)
+  mapfile -t sources <<<"$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2>/dev/null)"
+  mapfile -t sourceIDs <<<"$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT id FROM vw_adlist;" 2>/dev/null)"
+  mapfile -t sourceTypes <<<"$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT type FROM vw_adlist;" 2>/dev/null)"
+
+  # Parse source domains from $sources
+  mapfile -t sourceDomains <<<"$(
+    # Logic: Split by folder/port
+    awk -F '[/:]' '{
+      # Remove URL protocol & optional username:password@
+      gsub(/(.*:\/\/|.*:.*@)/, "", $0)
+      if(length($1)>0){print $1}
+      else {print "local"}
+    }' <<<"$(printf '%s\n' "${sources[@]}")" 2>/dev/null
+  )"
+
+  local str="Pulling blocklist source list into range"
   echo -e "${OVER}  ${TICK} ${str}"
 
+  if [[ -z "${sources[*]}" ]] || [[ -z "${sourceDomains[*]}" ]]; then
+    echo -e "  ${INFO} No source list found, or it is empty"
+    echo ""
+    unset sources
+  fi
+
   # Use compression to reduce the amount of data that is transferred
   # between the Pi-hole and the ad list provider. Use this feature
   # only if it is supported by the locally available version of curl
@@ -404,6 +509,15 @@ gravity_DownloadBlocklists() {
     compression=""
     echo -e "  ${INFO} Libz compression not available\n"
   fi
+
+  # Check if etag is supported by the locally available version of curl
+  # (available as of curl 7.68.0, released Jan 2020)
+  # https://github.com/curl/curl/pull/4543 +
+  # https://github.com/curl/curl/pull/4678
+  if curl --help all | grep -q "etag-save"; then
+    etag_support=true
+  fi
+
   # Loop through $sources and download each one
   for ((i = 0; i < "${#sources[@]}"; i++)); do
     url="${sources[$i]}"
@@ -420,8 +534,8 @@ gravity_DownloadBlocklists() {
     fi
 
     # Save the file as list.#.domain
-    saveLocation="${piholeDir}/list.${id}.${domain}.${domainsExtension}"
+    saveLocation="${listsCacheDir}/list.${id}.${domain}.${domainsExtension}"
-    activeDomains[$i]="${saveLocation}"
+    activeDomains[i]="${saveLocation}"
 
     # Check if we can write to the save location file without actually creating
     # it (in case it doesn't exist)
@@ -488,7 +602,7 @@ compareLists() {
 # Download specified URL and perform checks on HTTP status and file content
 gravity_DownloadBlocklistFromUrl() {
   local url="${1}" adlistID="${2}" saveLocation="${3}" target="${4}" compression="${5}" gravity_type="${6}" domain="${7}"
-  local heisenbergCompensator="" listCurlBuffer str httpCode success="" ip cmd_ext
+  local modifiedOptions="" listCurlBuffer str httpCode success="" ip cmd_ext
   local file_path permissions ip_addr port blocked=false download=true
 
   # Create temp file to store content on disk instead of RAM
@@ -497,12 +611,37 @@ gravity_DownloadBlocklistFromUrl() {
   mv "${listCurlBuffer}" "${listCurlBuffer%.*}.phgpb"
   listCurlBuffer="${listCurlBuffer%.*}.phgpb"
 
-  # Determine if $saveLocation has read permission
+  # For all remote files, we try to determine if the file has changed to skip
-  if [[ -r "${saveLocation}" && $url != "file"* ]]; then
+  # downloading them whenever possible.
-    # Have curl determine if a remote file has been modified since last retrieval
+  if [[ $url != "file"* ]]; then
-    # Uses "Last-Modified" header, which certain web servers do not provide (e.g: raw github urls)
+    # Use the HTTP ETag header to determine if the file has changed if supported
-    # Note: Don't do this for local files, always download them
+    # by curl. Using ETags is supported by raw.githubusercontent.com URLs.
-    heisenbergCompensator="-z ${saveLocation}"
+    if [[ "${etag_support}" == true ]]; then
+      # Save HTTP ETag to the specified file. An ETag is a caching related header,
+      # usually returned in a response. If no ETag is sent by the server, an empty
+      # file is created and can later be used consistently.
+      modifiedOptions="--etag-save ${saveLocation}.etag"
+
+      if [[ -f "${saveLocation}.etag" ]]; then
+        # This option makes a conditional HTTP request for the specific ETag read
+        # from the given file by sending a custom If-None-Match header using the
+        # stored ETag. This way, the server will only send the file if it has
+        # changed since the last request.
+        modifiedOptions="${modifiedOptions} --etag-compare ${saveLocation}.etag"
+      fi
+    fi
+
+    # Add If-Modified-Since header to the request if we did already download the
+    # file once
+    if [[ -f "${saveLocation}" ]]; then
+      # Request a file that has been modified later than the given time and
+      # date. We provide a file here which makes curl use the modification
+      # timestamp (mtime) of this file.
+      # Interstingly, this option is not supported by raw.githubusercontent.com
+      # URLs, however, it is still supported by many older web servers which may
+      # not support the HTTP ETag method so we keep it as a fallback.
+      modifiedOptions="${modifiedOptions} -z ${saveLocation}"
+    fi
   fi
 
   str="Status:"
@@ -632,7 +771,7 @@ gravity_DownloadBlocklistFromUrl() {
 
   if [[ "${download}" == true ]]; then
     # shellcheck disable=SC2086
-    httpCode=$(curl --connect-timeout ${curl_connect_timeout} -s -L ${compression} ${cmd_ext} ${heisenbergCompensator} -w "%{http_code}" "${url}" -o "${listCurlBuffer}" 2>/dev/null)
+    httpCode=$(curl --connect-timeout ${curl_connect_timeout} -s -L ${compression} ${cmd_ext} ${modifiedOptions} -w "%{http_code}" "${url}" -o "${listCurlBuffer}" 2>/dev/null)
   fi
 
   case $url in
@@ -675,9 +814,10 @@ gravity_DownloadBlocklistFromUrl() {
   # Determine if the blocklist was downloaded and saved correctly
   if [[ "${success}" == true ]]; then
     if [[ "${httpCode}" == "304" ]]; then
+      # Set list status to "unchanged/cached"
+      database_adlist_status "${adlistID}" "2"
       # Add domains to database table file
       pihole-FTL "${gravity_type}" parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}"
-      database_adlist_status "${adlistID}" "2"
       done="true"
     # Check if $listCurlBuffer is a non-zero length file
     elif [[ -s "${listCurlBuffer}" ]]; then
@@ -685,10 +825,10 @@ gravity_DownloadBlocklistFromUrl() {
       gravity_ParseFileIntoDomains "${listCurlBuffer}" "${saveLocation}"
       # Remove curl buffer file after its use
       rm "${listCurlBuffer}"
+      # Compare lists if are they identical
+      compareLists "${adlistID}" "${saveLocation}"
       # Add domains to database table file
       pihole-FTL "${gravity_type}" parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}"
-      # Compare lists, are they identical?
-      compareLists "${adlistID}" "${saveLocation}"
       done="true"
     else
       # Fall back to previously cached list if $listCurlBuffer is empty
@@ -701,9 +841,10 @@ gravity_DownloadBlocklistFromUrl() {
     # Determine if cached list has read permission
     if [[ -r "${saveLocation}" ]]; then
       echo -e "  ${CROSS} List download failed: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}"
+      # Set list status to "download-failed/cached"
+      database_adlist_status "${adlistID}" "3"
       # Add domains to database table file
       pihole-FTL "${gravity_type}" parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}"
-      database_adlist_status "${adlistID}" "3"
     else
       echo -e "  ${CROSS} List download failed: ${COL_LIGHT_RED}no cached list available${COL_NC}"
       # Manually reset these two numbers because we do not call parseList here
@@ -764,11 +905,11 @@ gravity_Table_Count() {
 gravity_ShowCount() {
   # Here we use the table "gravity" instead of the view "vw_gravity" for speed.
   # It's safe to replace it here, because right after a gravity run both will show the exactly same number of domains.
-  gravity_Table_Count "gravity" "gravity domains" ""
+  gravity_Table_Count "gravity" "gravity domains"
-  gravity_Table_Count "vw_blacklist" "exact denied domains"
+  gravity_Table_Count "domainlist WHERE type = 1 AND enabled = 1" "exact denied domains"
-  gravity_Table_Count "vw_regex_blacklist" "regex denied filters"
+  gravity_Table_Count "domainlist WHERE type = 3 AND enabled = 1" "regex denied filters"
-  gravity_Table_Count "vw_whitelist" "exact allowed domains"
+  gravity_Table_Count "domainlist WHERE type = 0 AND enabled = 1" "exact allowed domains"
-  gravity_Table_Count "vw_regex_whitelist" "regex allowed filters"
+  gravity_Table_Count "domainlist WHERE type = 2 AND enabled = 1" "regex allowed filters"
 }
 
 # Trap Ctrl-C
@@ -917,11 +1058,33 @@ timeit(){
   elapsed_time=$((end_time - start_time))
 
   # Display the elapsed time
-  printf "  %b--> took %d.%03d seconds%b\n" ${COL_BLUE} $((elapsed_time / 1000)) $((elapsed_time % 1000)) ${COL_NC}
+  printf "  %b--> took %d.%03d seconds%b\n" "${COL_BLUE}" $((elapsed_time / 1000)) $((elapsed_time % 1000)) "${COL_NC}"
 
   return $ret
 }
 
+migrate_to_listsCache_dir() {
+  # If the ${listsCacheDir} directory already exists, this has been done before
+  if [[ -d "${listsCacheDir}" ]]; then
+    return
+  fi
+
+  # If not, we need to migrate the old files to the new directory
+  local str="Migrating the list's cache directory to new location"
+  echo -ne "  ${INFO} ${str}..."
+  mkdir -p "${listsCacheDir}"
+
+  # Move the old files to the new directory
+  if mv "${piholeDir}"/list.* "${listsCacheDir}/" 2>/dev/null; then
+    echo -e "${OVER}  ${TICK} ${str}"
+  else
+    echo -e "${OVER}  ${CROSS} ${str}"
+  fi
+
+  # Update the list's paths in the corresponding .sha1 files to the new location
+  sed -i "s|${piholeDir}/|${listsCacheDir}/|g" "${listsCacheDir}"/*.sha1
+}
+
 helpFunc() {
   echo "Usage: pihole -g
 Update domains from blocklists specified in adlists.list
@@ -997,6 +1160,9 @@ if [[ "${recover_database:-}" == true ]]; then
   timeit database_recovery "$4"
 fi
 
+# Migrate scattered list files to the new cache directory
+migrate_to_listsCache_dir
+
 # Move possibly existing legacy files to the gravity database
 if ! timeit migrate_to_database; then
   echo -e "   ${CROSS} Unable to migrate to database. Please contact support."
@@ -1007,7 +1173,7 @@ if [[ "${forceDelete:-}" == true ]]; then
   str="Deleting existing list cache"
   echo -ne "${INFO} ${str}..."
 
-  rm /etc/pihole/list.* 2>/dev/null || true
+  rm "${listsCacheDir}/list.*" 2>/dev/null || true
   echo -e "${OVER}  ${TICK} ${str}"
 fi
 

pihole

@@ -73,19 +73,17 @@ listFunc() {
 
 debugFunc() {
     local automated
-    local web
     local check_database_integrity
     # Pull off the `debug` leaving passed call augmentation flags in $1
     shift
 
     for value in "$@"; do
         [[ "$value"  == *"-a"* ]] && automated="true"
-        [[ "$value"  == *"-w"* ]] && web="true"
         [[ "$value"  == *"-c"* ]] && check_database_integrity="true"
         [[ "$value" == *"--check_database"* ]] && check_database_integrity="true"
     done
 
-  AUTOMATED=${automated:-} WEBCALL=${web:-} CHECK_DATABASE=${check_database_integrity:-} "${PI_HOLE_SCRIPT_DIR}"/piholeDebug.sh
+  AUTOMATED=${automated:-} CHECK_DATABASE=${check_database_integrity:-} "${PI_HOLE_SCRIPT_DIR}"/piholeDebug.sh
   exit 0
 }
 

test/_ubuntu_23.Dockerfile

@@ -1,18 +0,0 @@
-FROM buildpack-deps:lunar-scm
-
-ENV GITDIR=/etc/.pihole
-ENV SCRIPTDIR=/opt/pihole
-
-RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole
-ADD . $GITDIR
-RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $GITDIR/advanced/Scripts/COL_TABLE $SCRIPTDIR/
-ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR
-ENV DEBIAN_FRONTEND=noninteractive
-
-RUN true && \
-    chmod +x $SCRIPTDIR/*
-
-ENV SKIP_INSTALL=true
-ENV OS_CHECK_DOMAIN_NAME=dev-supportedos.pi-hole.net
-
-#sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \

test/requirements.txt

@@ -2,5 +2,5 @@ pyyaml == 6.0.2
 pytest == 8.3.4
 pytest-xdist == 3.6.1
 pytest-testinfra == 10.1.1
-tox == 4.23.2
+tox == 4.24.1
 pytest-clarity == 1.0.1

test/test_any_automated_install.py

@@ -119,11 +119,6 @@ def test_installPihole_fresh_install_readableFiles(host):
     assert exit_status_success == actual_rc
     check_leases = test_cmd.format("w", "/etc/pihole/dhcp.leases", piholeuser)
     actual_rc = host.run(check_leases).rc
-    # readable dns-servers.conf
-    assert exit_status_success == actual_rc
-    check_servers = test_cmd.format("r", "/etc/pihole/dns-servers.conf", piholeuser)
-    actual_rc = host.run(check_servers).rc
-    assert exit_status_success == actual_rc
     # readable install.log
     check_install = test_cmd.format("r", "/etc/pihole/install.log", piholeuser)
     actual_rc = host.run(check_install).rc