Explicitly log migration process

Signed-off-by: DL6ER <dl6er@dl6er.de>

.github/workflows/merge-conflict.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check if PRs are have merge conflicts
-        uses: eps1lon/actions-label-merge-conflict@v3.0.3
+        uses: eps1lon/actions-label-merge-conflict@v3.0.2
         with:
           dirtyLabel: "PR: Merge Conflict"
           repoToken: "${{ secrets.GITHUB_TOKEN }}"

.github/workflows/stale.yml

@@ -17,7 +17,7 @@ jobs:
       issues: write
 
     steps:
-      - uses: actions/stale@v9.1.0
+      - uses: actions/stale@v9.0.0
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           days-before-stale: 30

.github/workflows/stale_pr.yml

@@ -17,7 +17,7 @@ jobs:
       pull-requests: write
 
     steps:
-      - uses: actions/stale@v9.1.0
+      - uses: actions/stale@v9.0.0
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           # Do not automatically mark PR/issue as stale

.github/workflows/test.yml

@@ -77,7 +77,7 @@ jobs:
         uses: actions/checkout@v4.2.2
 
       - name: Set up Python 3.10
-        uses: actions/setup-python@v5.4.0
+        uses: actions/setup-python@v5.3.0
         with:
           python-version: "3.10"
 

advanced/Scripts/api.sh

@@ -165,17 +165,15 @@ GetFTLData() {
   # get the data from querying the API as well as the http status code
   response=$(curl -skS -w "%{http_code}" -X GET "${API_URL}$1" -H "Accept: application/json" -H "sid: ${SID}" )
 
+  # status are the last 3 characters
+  status="${response#"${response%???}"}"
+  # data is everything from response without the last 3 characters
+  data="${response%???}"
+
   if [ "${2}" = "raw" ]; then
     # return the raw response
     echo "${response}"
   else
-
-    # status are the last 3 characters
-    # not using ${response#"${response%???}"}" here because it's extremely slow on big responses
-    status=$(printf "%s" "${response}" | tail -c 3)
-    # data is everything from response without the last 3 characters
-    data="${response%???}"
-
     # return only the data
     if [ "${status}" = 200 ]; then
         # response OK
@@ -266,8 +264,7 @@ apiFunc() {
   response=$(GetFTLData "$1" raw)
 
   # status are the last 3 characters
-  # not using ${response#"${response%???}"}" here because it's extremely slow on big responses
-  status=$(printf "%s" "${response}" | tail -c 3)
+  status="${response#"${response%???}"}"
   # data is everything from response without the last 3 characters
   data="${response%???}"
 

advanced/Scripts/piholeDebug.sh

@@ -44,14 +44,6 @@ fi
 # shellcheck disable=SC1091
 . /etc/pihole/versions
 
-# Read the value of an FTL config key. The value is printed to stdout.
-get_ftl_conf_value() {
-    local key=$1
-
-    # Obtain setting from FTL directly
-    pihole-FTL --config "${key}"
-}
-
 # FAQ URLs for use in showing the debug log
 FAQ_HARDWARE_REQUIREMENTS="${COL_CYAN}https://docs.pi-hole.net/main/prerequisites/${COL_NC}"
 FAQ_HARDWARE_REQUIREMENTS_PORTS="${COL_CYAN}https://docs.pi-hole.net/main/prerequisites/#ports${COL_NC}"
@@ -69,10 +61,10 @@ DNSMASQ_D_DIRECTORY="/etc/dnsmasq.d"
 PIHOLE_DIRECTORY="/etc/pihole"
 PIHOLE_SCRIPTS_DIRECTORY="/opt/pihole"
 BIN_DIRECTORY="/usr/local/bin"
+RUN_DIRECTORY="/run"
 LOG_DIRECTORY="/var/log/pihole"
-HTML_DIRECTORY="$(get_ftl_conf_value "webserver.paths.webroot")"
-WEBHOME_PATH="$(get_ftl_conf_value "webserver.paths.webhome")"
-WEB_GIT_DIRECTORY="${HTML_DIRECTORY}${WEBHOME_PATH}"
+HTML_DIRECTORY="/var/www/html"
+WEB_GIT_DIRECTORY="${HTML_DIRECTORY}/admin"
 SHM_DIRECTORY="/dev/shm"
 ETC="/etc"
 
@@ -87,6 +79,14 @@ PIHOLE_FTL_CONF_FILE="${PIHOLE_DIRECTORY}/pihole.toml"
 PIHOLE_DNSMASQ_CONF_FILE="${PIHOLE_DIRECTORY}/dnsmasq.conf"
 PIHOLE_VERSIONS_FILE="${PIHOLE_DIRECTORY}/versions"
 
+# Read the value of an FTL config key. The value is printed to stdout.
+get_ftl_conf_value() {
+    local key=$1
+
+    # Obtain setting from FTL directly
+    pihole-FTL --config "${key}"
+}
+
 PIHOLE_GRAVITY_DB_FILE="$(get_ftl_conf_value "files.gravity")"
 
 PIHOLE_FTL_DB_FILE="$(get_ftl_conf_value "files.database")"
@@ -94,7 +94,7 @@ PIHOLE_FTL_DB_FILE="$(get_ftl_conf_value "files.database")"
 PIHOLE_COMMAND="${BIN_DIRECTORY}/pihole"
 PIHOLE_COLTABLE_FILE="${BIN_DIRECTORY}/COL_TABLE"
 
-FTL_PID="$(get_ftl_conf_value "files.pid")"
+FTL_PID="${RUN_DIRECTORY}/pihole-FTL.pid"
 
 PIHOLE_LOG="${LOG_DIRECTORY}/pihole.log"
 PIHOLE_LOG_GZIPS="${LOG_DIRECTORY}/pihole.log.[0-9].*"
@@ -202,7 +202,7 @@ compare_local_version_to_git_version() {
         if git status &> /dev/null; then
             # The current version the user is on
             local local_version
-            local_version=$(git describe --tags --abbrev=0 2> /dev/null);
+            local_version=$(git describe --tags --abbrev=0);
             # What branch they are on
             local local_branch
             local_branch=$(git rev-parse --abbrev-ref HEAD);
@@ -213,13 +213,7 @@ compare_local_version_to_git_version() {
             local local_status
             local_status=$(git status -s)
             # echo this information out to the user in a nice format
-            if [ ${local_version} ]; then
-              log_write "${TICK} Version: ${local_version}"
-            elif [ -n "${DOCKER_VERSION}" ]; then
-              log_write "${TICK} Version: Pi-hole Docker Container ${COL_BOLD}${DOCKER_VERSION}${COL_NC}"
-            else
-              log_write "${CROSS} Version: not detected"
-            fi
+            log_write "${TICK} Version: ${local_version}"
 
             # Print the repo upstreams
             remotes=$(git remote -v)
@@ -352,9 +346,6 @@ os_check() {
             fi
         done
 
-        # If it is a docker container, we can assume the OS is supported
-        [ -n "${DOCKER_VERSION}" ] && valid_os=true && valid_version=true
-
         local finalmsg
         if [ "$valid_os" = true ]; then
             log_write "${TICK} Distro:  ${COL_GREEN}${detected_os^}${COL_NC}"
@@ -498,25 +489,13 @@ run_and_print_command() {
 }
 
 hardware_check() {
-    # Note: the checks are skipped if Pi-hole is running in a docker container
-
-    local skip_msg="${INFO} Not enough permissions inside Docker container ${COL_YELLOW}(skipped)${COL_NC}"
-
     echo_current_diagnostic "System hardware configuration"
-    if [ -n "${DOCKER_VERSION}" ]; then
-        log_write "${skip_msg}"
-    else
-        # Store the output of the command in a variable
-        run_and_print_command "lshw -short"
-    fi
+    # Store the output of the command in a variable
+    run_and_print_command "lshw -short"
 
     echo_current_diagnostic "Processor details"
-    if [ -n "${DOCKER_VERSION}" ]; then
-        log_write "${skip_msg}"
-    else
-        # Store the output of the command in a variable
-        run_and_print_command "lscpu"
-    fi
+    # Store the output of the command in a variable
+    run_and_print_command "lscpu"
 }
 
 disk_usage() {
@@ -829,24 +808,26 @@ dig_at() {
 process_status(){
     # Check to make sure Pi-hole's services are running and active
     echo_current_diagnostic "Pi-hole processes"
-
     # Local iterator
     local i
-
     # For each process,
     for i in "${PIHOLE_PROCESSES[@]}"; do
-        local status_of_process
-
         # If systemd
         if command -v systemctl &> /dev/null; then
             # get its status via systemctl
+            local status_of_process
             status_of_process=$(systemctl is-active "${i}")
         else
             # Otherwise, use the service command and mock the output of `systemctl is-active`
+            local status_of_process
 
-            # If it is a docker container, there is no systemctl or service. Do nothing.
+            # If DOCKER_VERSION is set, the output is slightly different (s6 init system on Docker)
             if [ -n "${DOCKER_VERSION}" ]; then
-                :
+                if service "${i}" status | grep -E '^up' &> /dev/null; then
+                    status_of_process="active"
+                else
+                    status_of_process="inactive"
+                fi
             else
             # non-Docker system
                 if service "${i}" status | grep -E 'is\srunning' &> /dev/null; then
@@ -856,12 +837,8 @@ process_status(){
                 fi
             fi
         fi
-
         # and print it out to the user
-        if [ -n "${DOCKER_VERSION}" ]; then
-            # If it's a Docker container, the test was skipped
-            log_write "${INFO} systemctl/service not installed inside docker container ${COL_YELLOW}(skipped)${COL_NC}"
-        elif [[ "${status_of_process}" == "active" ]]; then
+        if [[ "${status_of_process}" == "active" ]]; then
             # If it's active, show it in green
             log_write "${TICK} ${COL_GREEN}${i}${COL_NC} daemon is ${COL_GREEN}${status_of_process}${COL_NC}"
         else
@@ -878,8 +855,6 @@ ftl_full_status(){
     if command -v systemctl &> /dev/null; then
       FTL_status=$(systemctl status --full --no-pager pihole-FTL.service)
       log_write "   ${FTL_status}"
-    elif [ -n "${DOCKER_VERSION}" ]; then
-      log_write "${INFO} systemctl/service not installed inside docker container ${COL_YELLOW}(skipped)${COL_NC}"
     else
       log_write "${INFO} systemctl:  command not found"
     fi
@@ -1137,7 +1112,7 @@ show_FTL_db_entries() {
 }
 
 check_dhcp_servers() {
-    echo_current_diagnostic "Discovering active DHCP servers (takes 6 seconds)"
+    echo_current_diagnostic "Discovering active DHCP servers (takes 10 seconds)"
 
     OLD_IFS="$IFS"
     IFS=$'\n'
@@ -1221,7 +1196,7 @@ database_integrity_check(){
     local database="${1}"
 
     log_write "${INFO} Checking integrity of ${database} ... (this can take several minutes)"
-    result="$(pihole-FTL sqlite3 -ni "${database}" "PRAGMA integrity_check" 2>&1 & spinner)"
+    result="$(pihole-FTL "${database}" "PRAGMA integrity_check" 2>&1 & spinner)"
     if [[ ${result} = "ok" ]]; then
       log_write "${TICK} Integrity of ${database} intact"
 
@@ -1342,16 +1317,19 @@ upload_to_tricorder() {
         curl_to_tricorder
         # If we're not running in automated mode,
     else
-        echo ""
-        # give the user a choice of uploading it or not
-        # Users can review the log file locally (or the output of the script since they are the same) and try to self-diagnose their problem
-        read -r -p "[?] Would you like to upload the log? [y/N] " response
-        case ${response} in
-            # If they say yes, run our function for uploading the log
-            [yY][eE][sS]|[yY]) curl_to_tricorder;;
-            # If they choose no, just exit out of the script
-            *) log_write "    * Log will ${COL_GREEN}NOT${COL_NC} be uploaded to tricorder.\\n    * A local copy of the debug log can be found at: ${COL_CYAN}${PIHOLE_DEBUG_LOG}${COL_NC}\\n";exit;
-        esac
+        # if not being called from the web interface
+        if [[ ! "${WEBCALL}" ]]; then
+            echo ""
+            # give the user a choice of uploading it or not
+            # Users can review the log file locally (or the output of the script since they are the same) and try to self-diagnose their problem
+            read -r -p "[?] Would you like to upload the log? [y/N] " response
+            case ${response} in
+                # If they say yes, run our function for uploading the log
+                [yY][eE][sS]|[yY]) curl_to_tricorder;;
+                # If they choose no, just exit out of the script
+                *) log_write "    * Log will ${COL_GREEN}NOT${COL_NC} be uploaded to tricorder.\\n    * A local copy of the debug log can be found at: ${COL_CYAN}${PIHOLE_DEBUG_LOG}${COL_NC}\\n";exit;
+            esac
+        fi
     fi
     # Check if tricorder.pi-hole.net is reachable and provide token
     # along with some additional useful information
@@ -1371,8 +1349,13 @@ upload_to_tricorder() {
     # If no token was generated
     else
         # Show an error and some help instructions
-        log_write "${CROSS} ${COL_RED}There was an error uploading your debug log.${COL_NC}"
-        log_write "   * Please try again or contact the Pi-hole team for assistance."
+        # Skip this if being called from web interface and automatic mode was not chosen (users opt-out to upload)
+        if [[ "${WEBCALL}" ]] && [[ ! "${AUTOMATED}" ]]; then
+            :
+        else
+            log_write "${CROSS} ${COL_RED}There was an error uploading your debug log.${COL_NC}"
+            log_write "   * Please try again or contact the Pi-hole team for assistance."
+        fi
     fi
     # Finally, show where the log file is no matter the outcome of the function so users can look at it
     log_write "   * A local copy of the debug log can be found at: ${COL_CYAN}${PIHOLE_DEBUG_LOG}${COL_NC}\\n"

advanced/Scripts/update.sh

@@ -107,9 +107,6 @@ main() {
     web_update=false
     FTL_update=false
 
-    # Perform an OS check to ensure we're on an appropriate operating system
-    os_check
-
     # Install packages used by this installation script (necessary if users have removed e.g. git from their systems)
     package_manager_detect
     build_dependency_package

advanced/Templates/gravity.db.sql

@@ -67,11 +67,6 @@ CREATE TABLE info
 );
 
 INSERT INTO "info" VALUES('version','19');
-/* This is a flag to indicate if gravity was restored from a backup
-    false = not restored,
-    failed = restoration failed due to no backup
-    other string = restoration successful with the string being the backup file used */
-INSERT INTO "info" VALUES('gravity_restored','false');
 
 CREATE TABLE domainlist_by_group
 (

automated install/basic-install.sh

@@ -388,6 +388,28 @@ os_check() {
     fi
 }
 
+# This function waits for dpkg to unlock, which signals that the previous apt-get command has finished.
+test_dpkg_lock() {
+    i=0
+    printf "  %b Waiting for package manager to finish (up to 30 seconds)\\n" "${INFO}"
+    # fuser is a program to show which processes use the named files, sockets, or filesystems
+    # So while the lock is held,
+    while fuser /var/lib/dpkg/lock >/dev/null 2>&1; do
+        # we wait half a second,
+        sleep 0.5
+        # increase the iterator,
+        ((i = i + 1))
+        # exit if waiting for more then 30 seconds
+        if [[ $i -gt 60 ]]; then
+            printf "  %b %bError: Could not verify package manager finished and released lock. %b\\n" "${CROSS}" "${COL_LIGHT_RED}" "${COL_NC}"
+            printf "       Attempt to install packages manually and retry.\\n"
+            exit 1
+        fi
+    done
+    # and then report success once dpkg is unlocked.
+    return 0
+}
+
 # Compatibility
 package_manager_detect() {
 
@@ -1279,6 +1301,13 @@ installConfigs() {
     # Ensure that permissions are correctly set
     chown -R pihole:pihole /etc/pihole
 
+    # Install list of DNS servers
+    # Format: Name;Primary IPv4;Secondary IPv4;Primary IPv6;Secondary IPv6
+    # Some values may be empty (for example: DNS servers without IPv6 support)
+    echo "${DNS_SERVERS}" >"${PI_HOLE_CONFIG_DIR}/dns-servers.conf"
+    chmod 644 "${PI_HOLE_CONFIG_DIR}/dns-servers.conf"
+    chown pihole:pihole "${PI_HOLE_CONFIG_DIR}/dns-servers.conf"
+
     # Install empty custom.list file if it does not exist
     if [[ ! -r "${PI_HOLE_CONFIG_DIR}/hosts/custom.list" ]]; then
         if ! install -D -T -o pihole -g pihole -m 660 /dev/null "${PI_HOLE_CONFIG_DIR}/hosts/custom.list" &>/dev/null; then
@@ -1429,15 +1458,20 @@ disable_resolved_stublistener() {
     printf "  %b Testing if systemd-resolved is enabled\\n" "${INFO}"
     # Check if Systemd-resolved's DNSStubListener is enabled and active on port 53
     if check_service_active "systemd-resolved"; then
-        # Disable the DNSStubListener to unbind it from port 53
-        # Note that this breaks dns functionality on host until FTL is up and running
-        printf "%b  %b Disabling systemd-resolved DNSStubListener\\n" "${OVER}" "${TICK}"
-        mkdir -p /etc/systemd/resolved.conf.d
-        cat > /etc/systemd/resolved.conf.d/90-pi-hole-disable-stub-listener.conf << EOF
-[Resolve]
-DNSStubListener=no
-EOF
-        systemctl reload-or-restart systemd-resolved
+        # Check if DNSStubListener is enabled
+        printf "  %b %b Testing if systemd-resolved DNSStub-Listener is active" "${OVER}" "${INFO}"
+        if (grep -E '#?DNSStubListener=yes' /etc/systemd/resolved.conf &>/dev/null); then
+            # Disable the DNSStubListener to unbind it from port 53
+            # Note that this breaks dns functionality on host until ftl are up and running
+            printf "%b  %b Disabling systemd-resolved DNSStubListener" "${OVER}" "${TICK}"
+            # Make a backup of the original /etc/systemd/resolved.conf
+            # (This will need to be restored on uninstallation)
+            sed -r -i.orig 's/#?DNSStubListener=yes/DNSStubListener=no/g' /etc/systemd/resolved.conf
+            printf " and restarting systemd-resolved\\n"
+            systemctl reload-or-restart systemd-resolved
+        else
+            printf "%b  %b Systemd-resolved does not need to be restarted\\n" "${OVER}" "${INFO}"
+        fi
     else
         printf "%b  %b Systemd-resolved is not enabled\\n" "${OVER}" "${INFO}"
     fi
@@ -2122,7 +2156,7 @@ get_binary_name() {
         else
             printf "%b  %b Detected 32bit (i686) architecture\\n" "${OVER}" "${TICK}"
         fi
-        l_binary="pihole-FTL-386"
+        l_binary="pihole-FTL-linux-386"
     fi
 
     # Returning a string value via echo
@@ -2266,44 +2300,6 @@ copy_to_install_log() {
     chown pihole:pihole "${installLogLoc}"
 }
 
-disableLighttpd() {
-    # Return early when lighttpd is not active
-    if ! check_service_active lighttpd; then
-        return
-    fi
-
-    local response
-    # Detect if the terminal is interactive
-    if [[ -t 0 ]]; then
-        # The terminal is interactive
-        dialog --no-shadow --keep-tite \
-            --title "Pi-hole v6.0 no longer uses lighttpd" \
-           --yesno "\\n\\nPi-hole v6.0 has its own embedded web server so lighttpd is no longer needed *unless* you have custom configurations.\\n\\nIn this case, you can opt-out of disabling lighttpd and pihole-FTL will try to bind to an alternative port such as 8080.\\n\\nDo you want to disable lighttpd (recommended)?" "${r}" "${c}" && response=0 || response="$?"
-    else
-        # The terminal is non-interactive, assume yes. Lighttpd will be stopped
-        # but keeps being installed and can easily be re-enabled by the user
-        response=0
-    fi
-
-    # If the user does not want to disable lighttpd, return early
-    if [[ "${response}" -ne 0 ]]; then
-        return
-    fi
-
-    # Lighttpd is not needed anymore, so disable it
-    # We keep all the configuration files in place, so the user can re-enable it
-    # if needed
-
-    # Check if lighttpd is installed
-    if is_command lighttpd; then
-        # Stop the lighttpd service
-        stop_service lighttpd
-
-        # Disable the lighttpd service
-        disable_service lighttpd
-    fi
-}
-
 migrate_dnsmasq_configs() {
     # Previously, Pi-hole created a number of files in /etc/dnsmasq.d
     # During migration, their content is copied into the new single source of
@@ -2316,9 +2312,6 @@ migrate_dnsmasq_configs() {
         return 0
     fi
 
-    # Disable lighttpd server during v6 migration
-    disableLighttpd
-
     # Create target directory /etc/pihole/migration_backup_v6
     # and make it owned by pihole:pihole
     mkdir -p "${V6_CONF_MIGRATION_DIR}"
@@ -2338,35 +2331,20 @@ migrate_dnsmasq_configs() {
     # /etc/pihole/pihole.toml
     # This file will be created with the default settings unless the user has
     # changed settings via setupVars.conf or the other dnsmasq files moved above
-    # During migration, setupVars.conf is moved to /etc/pihole/migration_backup_v6
     str="Migrating Pi-hole configuration to version 6"
-    printf "  %b %s..." "${INFO}" "${str}"
+    printf "  %b %s...\\n" "${INFO}"
     local FTLoutput FTLstatus
     FTLoutput=$(pihole-FTL migrate v6)
     FTLstatus=$?
     if [[ "${FTLstatus}" -eq 0 ]]; then
-        printf "%b  %b %s\\n" "${OVER}" "${TICK}" "${str}"
+        printf "  %b %s\\n" "${TICK}" "${str}"
     else
-        printf "%b  %b %s\\n" "${OVER}" "${CROSS}" "${str}"
+        printf "  %b %s\\n" "${CROSS}" "${str}"
     fi
 
     # Print the output of the FTL migration prefacing every line with four
     # spaces for alignment
     printf "%b" "${FTLoutput}" | sed 's/^/    /'
-
-    # Print a blank line for separation
-    printf "\\n"
-}
-
-# Check for availability of either the "service" or "systemctl" commands
-check_service_command() {
-    # Check for the availability of the "service" command
-    if ! is_command service && ! is_command systemctl; then
-        # If neither the "service" nor the "systemctl" command is available, inform the user
-        printf "  %b Neither the service nor the systemctl commands are available\\n" "${CROSS}"
-        printf "      on this machine. This Pi-hole installer cannot continue.\\n"
-        exit 1
-    fi
 }
 
 main() {
@@ -2417,9 +2395,6 @@ main() {
     # Check if SELinux is Enforcing and exit before doing anything else
     checkSelinux
 
-    # Check for availability of either the "service" or "systemctl" commands
-    check_service_command
-
     # Check for supported package managers so that we may install dependencies
     package_manager_detect
 

automated install/uninstall.sh

@@ -94,9 +94,8 @@ removePiholeFiles() {
     echo -e "  ${TICK} Removed config files"
 
     # Restore Resolved
-    if [[ -e /etc/systemd/resolved.conf.orig ]] || [[ -e /etc/systemd/resolved.conf.d/90-pi-hole-disable-stub-listener.conf ]]; then
-        ${SUDO} cp -p /etc/systemd/resolved.conf.orig /etc/systemd/resolved.conf &> /dev/null || true
-        ${SUDO} rm -f /etc/systemd/resolved.conf.d/90-pi-hole-disable-stub-listener.conf
+    if [[ -e /etc/systemd/resolved.conf.orig ]]; then
+        ${SUDO} cp -p /etc/systemd/resolved.conf.orig /etc/systemd/resolved.conf
         systemctl reload-or-restart systemd-resolved
     fi
 

gravity.sh

@@ -30,9 +30,6 @@ PIHOLE_COMMAND="/usr/local/bin/${basename}"
 
 piholeDir="/etc/${basename}"
 
-# Gravity aux files directory
-listsCacheDir="${piholeDir}/listsCache"
-
 # Legacy (pre v5.0) list file locations
 whitelistFile="${piholeDir}/whitelist.txt"
 blacklistFile="${piholeDir}/blacklist.txt"
@@ -47,7 +44,6 @@ gravityDBcopy="${piholeGitDir}/advanced/Templates/gravity_copy.sql"
 
 domainsExtension="domains"
 curl_connect_timeout=10
-etag_support=false
 
 # Check gravity temp directory
 if [ ! -d "${GRAVITY_TMPDIR}" ] || [ ! -w "${GRAVITY_TMPDIR}" ]; then
@@ -62,8 +58,6 @@ gravityDBfile_default="/etc/pihole/gravity.db"
 gravityTEMPfile="${GRAVITYDB}_temp"
 gravityDIR="$(dirname -- "${gravityDBfile}")"
 gravityOLDfile="${gravityDIR}/gravity_old.db"
-gravityBCKdir="${gravityDIR}/gravity_backups"
-gravityBCKfile="${gravityBCKdir}/gravity.db"
 
 fix_owner_permissions() {
   # Fix ownership and permissions for the specified file
@@ -97,21 +91,11 @@ gravity_build_tree() {
 
   if [[ "${status}" -ne 0 ]]; then
     echo -e "\\n  ${CROSS} Unable to build gravity tree in ${gravityTEMPfile}\\n  ${output}"
-    echo -e "  ${INFO} If you have a large amount of domains, make sure your Pi-hole has enough RAM available\\n"
     return 1
   fi
   echo -e "${OVER}  ${TICK} ${str}"
 }
 
-# Rotate gravity backup files
-rotate_gravity_backup() {
-  for i in {9..1}; do
-    if [ -f "${gravityBCKfile}.${i}" ]; then
-      mv "${gravityBCKfile}.${i}" "${gravityBCKfile}.$((i + 1))"
-    fi
-  done
-}
-
 # Copy data from old to new database file and swap them
 gravity_swap_databases() {
   str="Swapping databases"
@@ -127,32 +111,10 @@ gravity_swap_databases() {
   oldAvail=false
   if [ "${availableBlocks}" -gt "$((gravityBlocks * 2))" ] && [ -f "${gravityDBfile}" ]; then
     oldAvail=true
-    cp "${gravityDBfile}" "${gravityOLDfile}"
-  fi
-
-  # Drop the gravity and antigravity tables + subsequent VACUUM the current
-  # database for compaction
-  output=$({ printf ".timeout 30000\\nDROP TABLE IF EXISTS gravity;\\nDROP TABLE IF EXISTS antigravity;\\nVACUUM;\\n" | pihole-FTL sqlite3 -ni "${gravityDBfile}"; } 2>&1)
-  status="$?"
-
-  if [[ "${status}" -ne 0 ]]; then
-    echo -e "\\n  ${CROSS} Unable to clean current database for backup\\n  ${output}"
+    mv "${gravityDBfile}" "${gravityOLDfile}"
   else
-    # Check if the backup directory exists
-    if [ ! -d "${gravityBCKdir}" ]; then
-      mkdir -p "${gravityBCKdir}"
-    fi
-
-    # If multiple gravityBCKfile's are present (appended with a number), rotate them
-    # We keep at most 10 backups
-    rotate_gravity_backup
-
-    # Move the old database to the backup location
-    mv "${gravityDBfile}" "${gravityBCKfile}.1"
+    rm "${gravityDBfile}"
   fi
-
-
-  # Move the new database to the correct location
   mv "${gravityTEMPfile}" "${gravityDBfile}"
   echo -e "${OVER}  ${TICK} ${str}"
 
@@ -362,54 +324,6 @@ gravity_CheckDNSResolutionAvailable() {
   echo -e "${OVER}  ${TICK} DNS resolution is available"
 }
 
-# Function: try_restore_backup
-# Description: Attempts to restore the previous Pi-hole gravity database from a
-#              backup file. If a backup exists, it copies the backup to the
-#              gravity database file and prepares a new gravity database. If the
-#              restoration is successful, it returns 0. Otherwise, it returns 1.
-# Returns:
-#   0 - If the backup is successfully restored.
-#   1 - If no backup is available or if the restoration fails.
-try_restore_backup () {
-  local num filename timestamp
-  num=$1
-  filename="${gravityBCKfile}.${num}"
-  # Check if a backup exists
-  if [ -f "${filename}" ]; then
-    echo -e "  ${INFO} Attempting to restore previous database from backup no. ${num}"
-    cp "${filename}" "${gravityDBfile}"
-
-    # If the backup was successfully copied, prepare a new gravity database from
-    # it
-    if [ -f "${gravityDBfile}" ]; then
-      output=$({ pihole-FTL sqlite3 -ni "${gravityTEMPfile}" <<<"${copyGravity}"; } 2>&1)
-      status="$?"
-
-      # Error checking
-      if [[ "${status}" -ne 0 ]]; then
-        echo -e "\\n  ${CROSS} Unable to copy data from ${gravityDBfile} to ${gravityTEMPfile}\\n  ${output}"
-        gravity_Cleanup "error"
-      fi
-
-      # Get the timestamp of the backup file in a human-readable format
-      # Note that this timestamp will be in the server timezone, this may be
-      # GMT, e.g., on a Raspberry Pi where the default timezone has never been
-      # changed
-      timestamp=$(date -r "${filename}" "+%Y-%m-%d %H:%M:%S %Z")
-
-      # Add a record to the info table to indicate that the gravity database was restored
-      pihole-FTL sqlite3 "${gravityTEMPfile}" "INSERT OR REPLACE INTO info (property,value) values ('gravity_restored','${timestamp}');"
-      echo -e "  ${TICK} Successfully restored from backup (${gravityBCKfile}.${num} at ${timestamp})"
-      return 0
-    else
-      echo -e "  ${CROSS} Unable to restore backup no. ${num}"
-    fi
-  fi
-
-  echo -e "  ${CROSS} Backup no. ${num} not available"
-  return 1
-}
-
 # Retrieve blocklist URLs and parse domains from adlist.list
 gravity_DownloadBlocklists() {
   echo -e "  ${INFO} ${COL_BOLD}Neutrino emissions detected${COL_NC}..."
@@ -418,7 +332,33 @@ gravity_DownloadBlocklists() {
     echo -e "  ${INFO} Storing gravity database in ${COL_BOLD}${gravityDBfile}${COL_NC}"
   fi
 
-  local url domain str target compression adlist_type directory success
+  # Retrieve source URLs from gravity database
+  # We source only enabled adlists, SQLite3 stores boolean values as 0 (false) or 1 (true)
+  mapfile -t sources <<<"$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2>/dev/null)"
+  mapfile -t sourceIDs <<<"$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT id FROM vw_adlist;" 2>/dev/null)"
+  mapfile -t sourceTypes <<<"$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT type FROM vw_adlist;" 2>/dev/null)"
+
+  # Parse source domains from $sources
+  mapfile -t sourceDomains <<<"$(
+    # Logic: Split by folder/port
+    awk -F '[/:]' '{
+      # Remove URL protocol & optional username:password@
+      gsub(/(.*:\/\/|.*:.*@)/, "", $0)
+      if(length($1)>0){print $1}
+      else {print "local"}
+    }' <<<"$(printf '%s\n' "${sources[@]}")" 2>/dev/null
+  )"
+
+  local str="Pulling blocklist source list into range"
+  echo -e "${OVER}  ${TICK} ${str}"
+
+  if [[ -z "${sources[*]}" ]] || [[ -z "${sourceDomains[*]}" ]]; then
+    echo -e "  ${INFO} No source list found, or it is empty"
+    echo ""
+    unset sources
+  fi
+
+  local url domain str target compression adlist_type directory
   echo ""
 
   # Prepare new gravity database
@@ -450,55 +390,10 @@ gravity_DownloadBlocklists() {
 
   if [[ "${status}" -ne 0 ]]; then
     echo -e "\\n  ${CROSS} Unable to copy data from ${gravityDBfile} to ${gravityTEMPfile}\\n  ${output}"
-
-    # Try to attempt a backup restore
-    success=false
-    if [[ -d "${gravityBCKdir}" ]]; then
-      for i in {1..10}; do
-        if try_restore_backup "${i}"; then
-          success=true
-          break
-        fi
-      done
-    fi
-
-    # If none of the attempts worked, return 1
-    if [[ "${success}" == false ]]; then
-      pihole-FTL sqlite3 "${gravityTEMPfile}" "INSERT OR REPLACE INTO info (property,value) values ('gravity_restored','failed');"
-      return 1
-    fi
-
-    echo -e "  ${TICK} ${str}"
-  else
-    echo -e "${OVER}  ${TICK} ${str}"
+    return 1
   fi
-
-  # Retrieve source URLs from gravity database
-  # We source only enabled adlists, SQLite3 stores boolean values as 0 (false) or 1 (true)
-  mapfile -t sources <<<"$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2>/dev/null)"
-  mapfile -t sourceIDs <<<"$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT id FROM vw_adlist;" 2>/dev/null)"
-  mapfile -t sourceTypes <<<"$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT type FROM vw_adlist;" 2>/dev/null)"
-
-  # Parse source domains from $sources
-  mapfile -t sourceDomains <<<"$(
-    # Logic: Split by folder/port
-    awk -F '[/:]' '{
-      # Remove URL protocol & optional username:password@
-      gsub(/(.*:\/\/|.*:.*@)/, "", $0)
-      if(length($1)>0){print $1}
-      else {print "local"}
-    }' <<<"$(printf '%s\n' "${sources[@]}")" 2>/dev/null
-  )"
-
-  local str="Pulling blocklist source list into range"
   echo -e "${OVER}  ${TICK} ${str}"
 
-  if [[ -z "${sources[*]}" ]] || [[ -z "${sourceDomains[*]}" ]]; then
-    echo -e "  ${INFO} No source list found, or it is empty"
-    echo ""
-    unset sources
-  fi
-
   # Use compression to reduce the amount of data that is transferred
   # between the Pi-hole and the ad list provider. Use this feature
   # only if it is supported by the locally available version of curl
@@ -509,15 +404,6 @@ gravity_DownloadBlocklists() {
     compression=""
     echo -e "  ${INFO} Libz compression not available\n"
   fi
-
-  # Check if etag is supported by the locally available version of curl
-  # (available as of curl 7.68.0, released Jan 2020)
-  # https://github.com/curl/curl/pull/4543 +
-  # https://github.com/curl/curl/pull/4678
-  if curl --help all | grep -q "etag-save"; then
-    etag_support=true
-  fi
-
   # Loop through $sources and download each one
   for ((i = 0; i < "${#sources[@]}"; i++)); do
     url="${sources[$i]}"
@@ -534,8 +420,8 @@ gravity_DownloadBlocklists() {
     fi
 
     # Save the file as list.#.domain
-    saveLocation="${listsCacheDir}/list.${id}.${domain}.${domainsExtension}"
-    activeDomains[i]="${saveLocation}"
+    saveLocation="${piholeDir}/list.${id}.${domain}.${domainsExtension}"
+    activeDomains[$i]="${saveLocation}"
 
     # Check if we can write to the save location file without actually creating
     # it (in case it doesn't exist)
@@ -602,7 +488,7 @@ compareLists() {
 # Download specified URL and perform checks on HTTP status and file content
 gravity_DownloadBlocklistFromUrl() {
   local url="${1}" adlistID="${2}" saveLocation="${3}" target="${4}" compression="${5}" gravity_type="${6}" domain="${7}"
-  local modifiedOptions="" listCurlBuffer str httpCode success="" ip cmd_ext
+  local heisenbergCompensator="" listCurlBuffer str httpCode success="" ip cmd_ext
   local file_path permissions ip_addr port blocked=false download=true
 
   # Create temp file to store content on disk instead of RAM
@@ -611,37 +497,12 @@ gravity_DownloadBlocklistFromUrl() {
   mv "${listCurlBuffer}" "${listCurlBuffer%.*}.phgpb"
   listCurlBuffer="${listCurlBuffer%.*}.phgpb"
 
-  # For all remote files, we try to determine if the file has changed to skip
-  # downloading them whenever possible.
-  if [[ $url != "file"* ]]; then
-    # Use the HTTP ETag header to determine if the file has changed if supported
-    # by curl. Using ETags is supported by raw.githubusercontent.com URLs.
-    if [[ "${etag_support}" == true ]]; then
-      # Save HTTP ETag to the specified file. An ETag is a caching related header,
-      # usually returned in a response. If no ETag is sent by the server, an empty
-      # file is created and can later be used consistently.
-      modifiedOptions="--etag-save ${saveLocation}.etag"
-
-      if [[ -f "${saveLocation}.etag" ]]; then
-        # This option makes a conditional HTTP request for the specific ETag read
-        # from the given file by sending a custom If-None-Match header using the
-        # stored ETag. This way, the server will only send the file if it has
-        # changed since the last request.
-        modifiedOptions="${modifiedOptions} --etag-compare ${saveLocation}.etag"
-      fi
-    fi
-
-    # Add If-Modified-Since header to the request if we did already download the
-    # file once
-    if [[ -f "${saveLocation}" ]]; then
-      # Request a file that has been modified later than the given time and
-      # date. We provide a file here which makes curl use the modification
-      # timestamp (mtime) of this file.
-      # Interstingly, this option is not supported by raw.githubusercontent.com
-      # URLs, however, it is still supported by many older web servers which may
-      # not support the HTTP ETag method so we keep it as a fallback.
-      modifiedOptions="${modifiedOptions} -z ${saveLocation}"
-    fi
+  # Determine if $saveLocation has read permission
+  if [[ -r "${saveLocation}" && $url != "file"* ]]; then
+    # Have curl determine if a remote file has been modified since last retrieval
+    # Uses "Last-Modified" header, which certain web servers do not provide (e.g: raw github urls)
+    # Note: Don't do this for local files, always download them
+    heisenbergCompensator="-z ${saveLocation}"
   fi
 
   str="Status:"
@@ -771,7 +632,7 @@ gravity_DownloadBlocklistFromUrl() {
 
   if [[ "${download}" == true ]]; then
     # shellcheck disable=SC2086
-    httpCode=$(curl --connect-timeout ${curl_connect_timeout} -s -L ${compression} ${cmd_ext} ${modifiedOptions} -w "%{http_code}" "${url}" -o "${listCurlBuffer}" 2>/dev/null)
+    httpCode=$(curl --connect-timeout ${curl_connect_timeout} -s -L ${compression} ${cmd_ext} ${heisenbergCompensator} -w "%{http_code}" "${url}" -o "${listCurlBuffer}" 2>/dev/null)
   fi
 
   case $url in
@@ -814,10 +675,9 @@ gravity_DownloadBlocklistFromUrl() {
   # Determine if the blocklist was downloaded and saved correctly
   if [[ "${success}" == true ]]; then
     if [[ "${httpCode}" == "304" ]]; then
-      # Set list status to "unchanged/cached"
-      database_adlist_status "${adlistID}" "2"
       # Add domains to database table file
       pihole-FTL "${gravity_type}" parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}"
+      database_adlist_status "${adlistID}" "2"
       done="true"
     # Check if $listCurlBuffer is a non-zero length file
     elif [[ -s "${listCurlBuffer}" ]]; then
@@ -825,10 +685,10 @@ gravity_DownloadBlocklistFromUrl() {
       gravity_ParseFileIntoDomains "${listCurlBuffer}" "${saveLocation}"
       # Remove curl buffer file after its use
       rm "${listCurlBuffer}"
-      # Compare lists if are they identical
-      compareLists "${adlistID}" "${saveLocation}"
       # Add domains to database table file
       pihole-FTL "${gravity_type}" parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}"
+      # Compare lists, are they identical?
+      compareLists "${adlistID}" "${saveLocation}"
       done="true"
     else
       # Fall back to previously cached list if $listCurlBuffer is empty
@@ -841,10 +701,9 @@ gravity_DownloadBlocklistFromUrl() {
     # Determine if cached list has read permission
     if [[ -r "${saveLocation}" ]]; then
       echo -e "  ${CROSS} List download failed: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}"
-      # Set list status to "download-failed/cached"
-      database_adlist_status "${adlistID}" "3"
       # Add domains to database table file
       pihole-FTL "${gravity_type}" parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}"
+      database_adlist_status "${adlistID}" "3"
     else
       echo -e "  ${CROSS} List download failed: ${COL_LIGHT_RED}no cached list available${COL_NC}"
       # Manually reset these two numbers because we do not call parseList here
@@ -905,11 +764,11 @@ gravity_Table_Count() {
 gravity_ShowCount() {
   # Here we use the table "gravity" instead of the view "vw_gravity" for speed.
   # It's safe to replace it here, because right after a gravity run both will show the exactly same number of domains.
-  gravity_Table_Count "gravity" "gravity domains"
-  gravity_Table_Count "domainlist WHERE type = 1 AND enabled = 1" "exact denied domains"
-  gravity_Table_Count "domainlist WHERE type = 3 AND enabled = 1" "regex denied filters"
-  gravity_Table_Count "domainlist WHERE type = 0 AND enabled = 1" "exact allowed domains"
-  gravity_Table_Count "domainlist WHERE type = 2 AND enabled = 1" "regex allowed filters"
+  gravity_Table_Count "gravity" "gravity domains" ""
+  gravity_Table_Count "vw_blacklist" "exact denied domains"
+  gravity_Table_Count "vw_regex_blacklist" "regex denied filters"
+  gravity_Table_Count "vw_whitelist" "exact allowed domains"
+  gravity_Table_Count "vw_regex_whitelist" "regex allowed filters"
 }
 
 # Trap Ctrl-C
@@ -1058,33 +917,11 @@ timeit(){
   elapsed_time=$((end_time - start_time))
 
   # Display the elapsed time
-  printf "  %b--> took %d.%03d seconds%b\n" "${COL_BLUE}" $((elapsed_time / 1000)) $((elapsed_time % 1000)) "${COL_NC}"
+  printf "  %b--> took %d.%03d seconds%b\n" ${COL_BLUE} $((elapsed_time / 1000)) $((elapsed_time % 1000)) ${COL_NC}
 
   return $ret
 }
 
-migrate_to_listsCache_dir() {
-  # If the ${listsCacheDir} directory already exists, this has been done before
-  if [[ -d "${listsCacheDir}" ]]; then
-    return
-  fi
-
-  # If not, we need to migrate the old files to the new directory
-  local str="Migrating the list's cache directory to new location"
-  echo -ne "  ${INFO} ${str}..."
-  mkdir -p "${listsCacheDir}"
-
-  # Move the old files to the new directory
-  if mv "${piholeDir}"/list.* "${listsCacheDir}/" 2>/dev/null; then
-    echo -e "${OVER}  ${TICK} ${str}"
-  else
-    echo -e "${OVER}  ${CROSS} ${str}"
-  fi
-
-  # Update the list's paths in the corresponding .sha1 files to the new location
-  sed -i "s|${piholeDir}/|${listsCacheDir}/|g" "${listsCacheDir}"/*.sha1
-}
-
 helpFunc() {
   echo "Usage: pihole -g
 Update domains from blocklists specified in adlists.list
@@ -1160,9 +997,6 @@ if [[ "${recover_database:-}" == true ]]; then
   timeit database_recovery "$4"
 fi
 
-# Migrate scattered list files to the new cache directory
-migrate_to_listsCache_dir
-
 # Move possibly existing legacy files to the gravity database
 if ! timeit migrate_to_database; then
   echo -e "   ${CROSS} Unable to migrate to database. Please contact support."
@@ -1173,7 +1007,7 @@ if [[ "${forceDelete:-}" == true ]]; then
   str="Deleting existing list cache"
   echo -ne "${INFO} ${str}..."
 
-  rm "${listsCacheDir}/list.*" 2>/dev/null || true
+  rm /etc/pihole/list.* 2>/dev/null || true
   echo -e "${OVER}  ${TICK} ${str}"
 fi
 

pihole

@@ -73,17 +73,19 @@ listFunc() {
 
 debugFunc() {
     local automated
+    local web
     local check_database_integrity
     # Pull off the `debug` leaving passed call augmentation flags in $1
     shift
 
     for value in "$@"; do
         [[ "$value"  == *"-a"* ]] && automated="true"
+        [[ "$value"  == *"-w"* ]] && web="true"
         [[ "$value"  == *"-c"* ]] && check_database_integrity="true"
         [[ "$value" == *"--check_database"* ]] && check_database_integrity="true"
     done
 
-  AUTOMATED=${automated:-} CHECK_DATABASE=${check_database_integrity:-} "${PI_HOLE_SCRIPT_DIR}"/piholeDebug.sh
+  AUTOMATED=${automated:-} WEBCALL=${web:-} CHECK_DATABASE=${check_database_integrity:-} "${PI_HOLE_SCRIPT_DIR}"/piholeDebug.sh
   exit 0
 }
 

test/_ubuntu_23.Dockerfile

@@ -0,0 +1,18 @@
+FROM buildpack-deps:lunar-scm
+
+ENV GITDIR=/etc/.pihole
+ENV SCRIPTDIR=/opt/pihole
+
+RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole
+ADD . $GITDIR
+RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $GITDIR/advanced/Scripts/COL_TABLE $SCRIPTDIR/
+ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN true && \
+    chmod +x $SCRIPTDIR/*
+
+ENV SKIP_INSTALL=true
+ENV OS_CHECK_DOMAIN_NAME=dev-supportedos.pi-hole.net
+
+#sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \

test/requirements.txt

@@ -2,5 +2,5 @@ pyyaml == 6.0.2
 pytest == 8.3.4
 pytest-xdist == 3.6.1
 pytest-testinfra == 10.1.1
-tox == 4.24.1
+tox == 4.23.2
 pytest-clarity == 1.0.1

test/test_any_automated_install.py

@@ -119,6 +119,11 @@ def test_installPihole_fresh_install_readableFiles(host):
     assert exit_status_success == actual_rc
     check_leases = test_cmd.format("w", "/etc/pihole/dhcp.leases", piholeuser)
     actual_rc = host.run(check_leases).rc
+    # readable dns-servers.conf
+    assert exit_status_success == actual_rc
+    check_servers = test_cmd.format("r", "/etc/pihole/dns-servers.conf", piholeuser)
+    actual_rc = host.run(check_servers).rc
+    assert exit_status_success == actual_rc
     # readable install.log
     check_install = test_cmd.format("r", "/etc/pihole/install.log", piholeuser)
     actual_rc = host.run(check_install).rc