Pi-hole Core v6.0.6 (#6118)

Bumps [actions/setup-python](https://github.com/actions/setup-python) from 5.4.0 to 5.5.0.
- [Release notes](https://github.com/actions/setup-python/releases)
- [Commits](https://github.com/actions/setup-python/compare/v5.4.0...v5.5.0)

---
updated-dependencies:
- dependency-name: actions/setup-python
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

.github/workflows/merge-conflict.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check if PRs are have merge conflicts
-        uses: eps1lon/actions-label-merge-conflict@v3.0.2
+        uses: eps1lon/actions-label-merge-conflict@v3.0.3
         with:
           dirtyLabel: "PR: Merge Conflict"
           repoToken: "${{ secrets.GITHUB_TOKEN }}"

.github/workflows/stale.yml

@@ -17,7 +17,7 @@ jobs:
       issues: write
 
     steps:
-      - uses: actions/stale@v9.0.0
+      - uses: actions/stale@v9.1.0
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           days-before-stale: 30

.github/workflows/stale_pr.yml

@@ -17,7 +17,7 @@ jobs:
       pull-requests: write
 
     steps:
-      - uses: actions/stale@v9.0.0
+      - uses: actions/stale@v9.1.0
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           # Do not automatically mark PR/issue as stale

.github/workflows/test.yml

@@ -77,7 +77,7 @@ jobs:
         uses: actions/checkout@v4.2.2
 
       - name: Set up Python 3.10
-        uses: actions/setup-python@v5.3.0
+        uses: actions/setup-python@v5.5.0
         with:
           python-version: "3.10"
 

README.md

@@ -3,13 +3,9 @@
 #
 
 <p align="center">
-  <picture>
+  <img src="https://raw.githubusercontent.com/pi-hole/graphics/refs/heads/master/Vortex/vortex_with_text.svg" alt="Pi-hole website" width="168" height="270">
-    <source media="(prefers-color-scheme: dark)" srcset="https://pi-hole.github.io/graphics/Vortex/Vortex_Vertical_wordmark_darkmode.png">
+  <br>
-    <source media="(prefers-color-scheme: light)" srcset="https://pi-hole.github.io/graphics/Vortex/Vortex_Vertical_wordmark_lightmode.png">
+  <strong>Network-wide ad blocking via your own Linux hardware</strong>
-    <img src="https://pi-hole.github.io/graphics/Vortex/Vortex_Vertical_wordmark_lightmode.png" width="168" height="270" alt="Pi-hole website">
-  </picture>
-    <br>
-    <strong>Network-wide ad blocking via your own Linux hardware</strong>
 </p>
 
 <!-- markdownlint-enable MD033 -->
@@ -140,7 +136,7 @@ The [pihole](https://docs.pi-hole.net/core/pihole-command/) command has all the
 
 Some notable features include:
 
-- [Whitelisting, Blacklisting, and Regex](https://docs.pi-hole.net/core/pihole-command/#whitelisting-blacklisting-and-regex)
+- [Allowlisting, Denylisting (fka Whitelisting, Blacklisting), and Regex](https://docs.pi-hole.net/core/pihole-command/#allowlisting-denylisting-and-regex)
 - [Debugging utility](https://docs.pi-hole.net/core/pihole-command/#debugger)
 - [Viewing the live log file](https://docs.pi-hole.net/core/pihole-command/#tail)
 - [Updating Ad Lists](https://docs.pi-hole.net/core/pihole-command/#gravity)

advanced/Scripts/COL_TABLE

@@ -1,5 +1,6 @@
+#!/usr/bin/env sh
 # Determine if terminal is capable of showing colors
-if ([ -t 1 ] && [ $(tput colors) -ge 8 ]) || [ "${WEBCALL}" ]; then
+if [ -t 1 ] && [ "$(tput colors)" -ge 8 ]; then
   # Bold and underline may not show up on all clients
   # If something MUST be emphasized, use both
   COL_BOLD=''

advanced/Scripts/api.sh

@@ -21,7 +21,7 @@
 TestAPIAvailability() {
 
     # as we are running locally, we can get the port value from FTL directly
-    local chaos_api_list availabilityResponse
+    local chaos_api_list authResponse authStatus authData
 
     # Query the API URLs from FTL using CHAOS TXT local.api.ftl
     # The result is a space-separated enumeration of full URLs
@@ -34,6 +34,12 @@ TestAPIAvailability() {
         exit 1
     fi
 
+    # If an error occurred, the variable starts with ;;
+    if [ "${chaos_api_list#;;}" != "${chaos_api_list}" ]; then
+        echo "Communication error. Is FTL running?"
+        exit 1
+    fi
+
     # Iterate over space-separated list of URLs
     while [ -n "${chaos_api_list}" ]; do
         # Get the first URL
@@ -43,20 +49,29 @@ TestAPIAvailability() {
         API_URL="${API_URL#\"}"
 
         # Test if the API is available at this URL
-        availabilityResponse=$(curl -skS -o /dev/null -w "%{http_code}" "${API_URL}auth")
+        authResponse=$(curl --connect-timeout 2 -skS -w "%{http_code}" "${API_URL}auth")
+
+        # authStatus are the last 3 characters
+        # not using ${authResponse#"${authResponse%???}"}" here because it's extremely slow on big responses
+        authStatus=$(printf "%s" "${authResponse}" | tail -c 3)
+        # data is everything from response without the last 3 characters
+        authData=$(printf %s "${authResponse%???}")
 
         # Test if http status code was 200 (OK) or 401 (authentication required)
-        if [ ! "${availabilityResponse}" = 200 ] && [ ! "${availabilityResponse}" = 401 ]; then
+        if [ ! "${authStatus}" = 200 ] && [ ! "${authStatus}" = 401 ]; then
             # API is not available at this port/protocol combination
             API_PORT=""
         else
             # API is available at this URL combination
 
-            if [ "${availabilityResponse}" = 200 ]; then
+            if [ "${authStatus}" = 200 ]; then
                 # API is available without authentication
                 needAuth=false
             fi
 
+            # Check if 2FA is required
+            needTOTP=$(echo "${authData}"| jq --raw-output .session.totp 2>/dev/null)
+
             break
         fi
 
@@ -102,22 +117,51 @@ LoginAPI() {
             echo "API Authentication: Trying to use CLI password"
         fi
 
-        # Try to authenticate using the CLI password
+        # If we can read the CLI password, we can skip 2FA even when it's required otherwise
-        Authentication "${1}"
+        needTOTP=false
-
     elif [ "${1}" = "verbose" ]; then
         echo "API Authentication: CLI password not available"
     fi
 
+    if [ -z "${password}" ]; then
+        # no password read from CLI file
+        echo "Please enter your password:"
+        # secretly read the password
+        secretRead; printf '\n'
+    fi
 
+    if [ "${needTOTP}" = true ]; then
+        # 2FA required
+        echo "Please enter the correct second factor."
+        echo "(Can be any number if you used the app password)"
+        read -r totp
+    fi
 
-    # If this did not work, ask the user for the password
+    # Try to authenticate using the supplied password (CLI file or user input) and TOTP
-    while [ "${validSession}" = false ] || [ -z "${validSession}" ] ; do
+    Authentication "${1}"
+
+    # Try to login again until the session is valid
+    while [ ! "${validSession}" = true ]  ; do
         echo "Authentication failed. Please enter your Pi-hole password"
 
+        # Print the error message if there is one
+        if  [ ! "${sessionError}" = "null"  ] && [ "${1}" = "verbose" ]; then
+            echo "Error: ${sessionError}"
+        fi
+        # Print the session message if there is one
+        if  [ ! "${sessionMessage}" = "null" ] && [ "${1}" = "verbose" ]; then
+            echo "Error: ${sessionMessage}"
+        fi
+
         # secretly read the password
         secretRead; printf '\n'
 
+        if [ "${needTOTP}" = true ]; then
+            echo "Please enter the correct second factor:"
+            echo "(Can be any number if you used the app password)"
+            read -r totp
+        fi
+
         # Try to authenticate again
         Authentication "${1}"
     done
@@ -125,23 +169,27 @@ LoginAPI() {
 }
 
 Authentication() {
-  sessionResponse="$(curl -skS -X POST "${API_URL}auth" --user-agent "Pi-hole cli " --data "{\"password\":\"${password}\"}" )"
+    sessionResponse="$(curl --connect-timeout 2 -skS -X POST "${API_URL}auth" --user-agent "Pi-hole cli" --data "{\"password\":\"${password}\", \"totp\":${totp:-null}}" )"
 
-  if [ -z "${sessionResponse}" ]; then
+    if [ -z "${sessionResponse}" ]; then
-    echo "No response from FTL server. Please check connectivity"
+        echo "No response from FTL server. Please check connectivity"
-    exit 1
+        exit 1
-  fi
+    fi
-  # obtain validity and session ID from session response
+    # obtain validity, session ID and sessionMessage from session response
-  validSession=$(echo "${sessionResponse}"| jq .session.valid 2>/dev/null)
+    validSession=$(echo "${sessionResponse}"| jq .session.valid 2>/dev/null)
-  SID=$(echo "${sessionResponse}"| jq --raw-output .session.sid 2>/dev/null)
+    SID=$(echo "${sessionResponse}"| jq --raw-output .session.sid 2>/dev/null)
-
+    sessionMessage=$(echo "${sessionResponse}"| jq --raw-output .session.message 2>/dev/null)
-  if [ "${1}" = "verbose" ]; then
+
-    if [ "${validSession}" = true ]; then
+    # obtain the error message from the session response
-      echo "API Authentication: ${COL_GREEN}Success${COL_NC}"
+    sessionError=$(echo "${sessionResponse}"| jq --raw-output .error.message 2>/dev/null)
-    else
+
-      echo "API Authentication: ${COL_RED}Failed${COL_NC}"
+    if [ "${1}" = "verbose" ]; then
+        if [ "${validSession}" = true ]; then
+            echo "API Authentication: ${COL_GREEN}Success${COL_NC}"
+        else
+            echo "API Authentication: ${COL_RED}Failed${COL_NC}"
+        fi
     fi
-  fi
 }
 
 LogoutAPI() {
@@ -165,15 +213,17 @@ GetFTLData() {
   # get the data from querying the API as well as the http status code
   response=$(curl -skS -w "%{http_code}" -X GET "${API_URL}$1" -H "Accept: application/json" -H "sid: ${SID}" )
 
-  # status are the last 3 characters
-  status="${response#"${response%???}"}"
-  # data is everything from response without the last 3 characters
-  data="${response%???}"
-
   if [ "${2}" = "raw" ]; then
     # return the raw response
     echo "${response}"
   else
+
+    # status are the last 3 characters
+    # not using ${response#"${response%???}"}" here because it's extremely slow on big responses
+    status=$(printf "%s" "${response}" | tail -c 3)
+    # data is everything from response without the last 3 characters
+    data="${response%???}"
+
     # return only the data
     if [ "${status}" = 200 ]; then
         # response OK
@@ -264,7 +314,8 @@ apiFunc() {
   response=$(GetFTLData "$1" raw)
 
   # status are the last 3 characters
-  status="${response#"${response%???}"}"
+  # not using ${response#"${response%???}"}" here because it's extremely slow on big responses
+  status=$(printf "%s" "${response}" | tail -c 3)
   # data is everything from response without the last 3 characters
   data="${response%???}"
 

advanced/Scripts/piholeDebug.sh

@@ -44,6 +44,14 @@ fi
 # shellcheck disable=SC1091
 . /etc/pihole/versions
 
+# Read the value of an FTL config key. The value is printed to stdout.
+get_ftl_conf_value() {
+    local key=$1
+
+    # Obtain setting from FTL directly
+    pihole-FTL --config "${key}"
+}
+
 # FAQ URLs for use in showing the debug log
 FAQ_HARDWARE_REQUIREMENTS="${COL_CYAN}https://docs.pi-hole.net/main/prerequisites/${COL_NC}"
 FAQ_HARDWARE_REQUIREMENTS_PORTS="${COL_CYAN}https://docs.pi-hole.net/main/prerequisites/#ports${COL_NC}"
@@ -61,10 +69,10 @@ DNSMASQ_D_DIRECTORY="/etc/dnsmasq.d"
 PIHOLE_DIRECTORY="/etc/pihole"
 PIHOLE_SCRIPTS_DIRECTORY="/opt/pihole"
 BIN_DIRECTORY="/usr/local/bin"
-RUN_DIRECTORY="/run"
 LOG_DIRECTORY="/var/log/pihole"
-HTML_DIRECTORY="/var/www/html"
+HTML_DIRECTORY="$(get_ftl_conf_value "webserver.paths.webroot")"
-WEB_GIT_DIRECTORY="${HTML_DIRECTORY}/admin"
+WEBHOME_PATH="$(get_ftl_conf_value "webserver.paths.webhome")"
+WEB_GIT_DIRECTORY="${HTML_DIRECTORY}${WEBHOME_PATH}"
 SHM_DIRECTORY="/dev/shm"
 ETC="/etc"
 
@@ -79,14 +87,6 @@ PIHOLE_FTL_CONF_FILE="${PIHOLE_DIRECTORY}/pihole.toml"
 PIHOLE_DNSMASQ_CONF_FILE="${PIHOLE_DIRECTORY}/dnsmasq.conf"
 PIHOLE_VERSIONS_FILE="${PIHOLE_DIRECTORY}/versions"
 
-# Read the value of an FTL config key. The value is printed to stdout.
-get_ftl_conf_value() {
-    local key=$1
-
-    # Obtain setting from FTL directly
-    pihole-FTL --config "${key}"
-}
-
 PIHOLE_GRAVITY_DB_FILE="$(get_ftl_conf_value "files.gravity")"
 
 PIHOLE_FTL_DB_FILE="$(get_ftl_conf_value "files.database")"
@@ -94,7 +94,7 @@ PIHOLE_FTL_DB_FILE="$(get_ftl_conf_value "files.database")"
 PIHOLE_COMMAND="${BIN_DIRECTORY}/pihole"
 PIHOLE_COLTABLE_FILE="${BIN_DIRECTORY}/COL_TABLE"
 
-FTL_PID="${RUN_DIRECTORY}/pihole-FTL.pid"
+FTL_PID="$(get_ftl_conf_value "files.pid")"
 
 PIHOLE_LOG="${LOG_DIRECTORY}/pihole.log"
 PIHOLE_LOG_GZIPS="${LOG_DIRECTORY}/pihole.log.[0-9].*"
@@ -202,7 +202,7 @@ compare_local_version_to_git_version() {
         if git status &> /dev/null; then
             # The current version the user is on
             local local_version
-            local_version=$(git describe --tags --abbrev=0);
+            local_version=$(git describe --tags --abbrev=0 2> /dev/null);
             # What branch they are on
             local local_branch
             local_branch=$(git rev-parse --abbrev-ref HEAD);
@@ -213,7 +213,13 @@ compare_local_version_to_git_version() {
             local local_status
             local_status=$(git status -s)
             # echo this information out to the user in a nice format
-            log_write "${TICK} Version: ${local_version}"
+            if [ ${local_version} ]; then
+              log_write "${TICK} Version: ${local_version}"
+            elif [ -n "${DOCKER_VERSION}" ]; then
+              log_write "${TICK} Version: Pi-hole Docker Container ${COL_BOLD}${DOCKER_VERSION}${COL_NC}"
+            else
+              log_write "${CROSS} Version: not detected"
+            fi
 
             # Print the repo upstreams
             remotes=$(git remote -v)
@@ -346,6 +352,9 @@ os_check() {
             fi
         done
 
+        # If it is a docker container, we can assume the OS is supported
+        [ -n "${DOCKER_VERSION}" ] && valid_os=true && valid_version=true
+
         local finalmsg
         if [ "$valid_os" = true ]; then
             log_write "${TICK} Distro:  ${COL_GREEN}${detected_os^}${COL_NC}"
@@ -489,13 +498,25 @@ run_and_print_command() {
 }
 
 hardware_check() {
+    # Note: the checks are skipped if Pi-hole is running in a docker container
+
+    local skip_msg="${INFO} Not enough permissions inside Docker container ${COL_YELLOW}(skipped)${COL_NC}"
+
     echo_current_diagnostic "System hardware configuration"
-    # Store the output of the command in a variable
+    if [ -n "${DOCKER_VERSION}" ]; then
-    run_and_print_command "lshw -short"
+        log_write "${skip_msg}"
+    else
+        # Store the output of the command in a variable
+        run_and_print_command "lshw -short"
+    fi
 
     echo_current_diagnostic "Processor details"
-    # Store the output of the command in a variable
+    if [ -n "${DOCKER_VERSION}" ]; then
-    run_and_print_command "lscpu"
+        log_write "${skip_msg}"
+    else
+        # Store the output of the command in a variable
+        run_and_print_command "lscpu"
+    fi
 }
 
 disk_usage() {
@@ -808,26 +829,24 @@ dig_at() {
 process_status(){
     # Check to make sure Pi-hole's services are running and active
     echo_current_diagnostic "Pi-hole processes"
+
     # Local iterator
     local i
+
     # For each process,
     for i in "${PIHOLE_PROCESSES[@]}"; do
+        local status_of_process
+
         # If systemd
         if command -v systemctl &> /dev/null; then
             # get its status via systemctl
-            local status_of_process
             status_of_process=$(systemctl is-active "${i}")
         else
             # Otherwise, use the service command and mock the output of `systemctl is-active`
-            local status_of_process
 
-            # If DOCKER_VERSION is set, the output is slightly different (s6 init system on Docker)
+            # If it is a docker container, there is no systemctl or service. Do nothing.
             if [ -n "${DOCKER_VERSION}" ]; then
-                if service "${i}" status | grep -E '^up' &> /dev/null; then
+                :
-                    status_of_process="active"
-                else
-                    status_of_process="inactive"
-                fi
             else
             # non-Docker system
                 if service "${i}" status | grep -E 'is\srunning' &> /dev/null; then
@@ -837,8 +856,12 @@ process_status(){
                 fi
             fi
         fi
+
         # and print it out to the user
-        if [[ "${status_of_process}" == "active" ]]; then
+        if [ -n "${DOCKER_VERSION}" ]; then
+            # If it's a Docker container, the test was skipped
+            log_write "${INFO} systemctl/service not installed inside docker container ${COL_YELLOW}(skipped)${COL_NC}"
+        elif [[ "${status_of_process}" == "active" ]]; then
             # If it's active, show it in green
             log_write "${TICK} ${COL_GREEN}${i}${COL_NC} daemon is ${COL_GREEN}${status_of_process}${COL_NC}"
         else
@@ -855,6 +878,8 @@ ftl_full_status(){
     if command -v systemctl &> /dev/null; then
       FTL_status=$(systemctl status --full --no-pager pihole-FTL.service)
       log_write "   ${FTL_status}"
+    elif [ -n "${DOCKER_VERSION}" ]; then
+      log_write "${INFO} systemctl/service not installed inside docker container ${COL_YELLOW}(skipped)${COL_NC}"
     else
       log_write "${INFO} systemctl:  command not found"
     fi
@@ -1112,7 +1137,7 @@ show_FTL_db_entries() {
 }
 
 check_dhcp_servers() {
-    echo_current_diagnostic "Discovering active DHCP servers (takes 10 seconds)"
+    echo_current_diagnostic "Discovering active DHCP servers (takes 6 seconds)"
 
     OLD_IFS="$IFS"
     IFS=$'\n'
@@ -1196,7 +1221,7 @@ database_integrity_check(){
     local database="${1}"
 
     log_write "${INFO} Checking integrity of ${database} ... (this can take several minutes)"
-    result="$(pihole-FTL "${database}" "PRAGMA integrity_check" 2>&1 & spinner)"
+    result="$(pihole-FTL sqlite3 -ni "${database}" "PRAGMA integrity_check" 2>&1 & spinner)"
     if [[ ${result} = "ok" ]]; then
       log_write "${TICK} Integrity of ${database} intact"
 
@@ -1317,19 +1342,16 @@ upload_to_tricorder() {
         curl_to_tricorder
         # If we're not running in automated mode,
     else
-        # if not being called from the web interface
+        echo ""
-        if [[ ! "${WEBCALL}" ]]; then
+        # give the user a choice of uploading it or not
-            echo ""
+        # Users can review the log file locally (or the output of the script since they are the same) and try to self-diagnose their problem
-            # give the user a choice of uploading it or not
+        read -r -p "[?] Would you like to upload the log? [y/N] " response
-            # Users can review the log file locally (or the output of the script since they are the same) and try to self-diagnose their problem
+        case ${response} in
-            read -r -p "[?] Would you like to upload the log? [y/N] " response
+            # If they say yes, run our function for uploading the log
-            case ${response} in
+            [yY][eE][sS]|[yY]) curl_to_tricorder;;
-                # If they say yes, run our function for uploading the log
+            # If they choose no, just exit out of the script
-                [yY][eE][sS]|[yY]) curl_to_tricorder;;
+            *) log_write "    * Log will ${COL_GREEN}NOT${COL_NC} be uploaded to tricorder.\\n    * A local copy of the debug log can be found at: ${COL_CYAN}${PIHOLE_DEBUG_LOG}${COL_NC}\\n";exit;
-                # If they choose no, just exit out of the script
+        esac
-                *) log_write "    * Log will ${COL_GREEN}NOT${COL_NC} be uploaded to tricorder.\\n    * A local copy of the debug log can be found at: ${COL_CYAN}${PIHOLE_DEBUG_LOG}${COL_NC}\\n";exit;
-            esac
-        fi
     fi
     # Check if tricorder.pi-hole.net is reachable and provide token
     # along with some additional useful information
@@ -1349,13 +1371,8 @@ upload_to_tricorder() {
     # If no token was generated
     else
         # Show an error and some help instructions
-        # Skip this if being called from web interface and automatic mode was not chosen (users opt-out to upload)
+        log_write "${CROSS} ${COL_RED}There was an error uploading your debug log.${COL_NC}"
-        if [[ "${WEBCALL}" ]] && [[ ! "${AUTOMATED}" ]]; then
+        log_write "   * Please try again or contact the Pi-hole team for assistance."
-            :
-        else
-            log_write "${CROSS} ${COL_RED}There was an error uploading your debug log.${COL_NC}"
-            log_write "   * Please try again or contact the Pi-hole team for assistance."
-        fi
     fi
     # Finally, show where the log file is no matter the outcome of the function so users can look at it
     log_write "   * A local copy of the debug log can be found at: ${COL_CYAN}${PIHOLE_DEBUG_LOG}${COL_NC}\\n"

advanced/Scripts/update.sh

@@ -107,6 +107,9 @@ main() {
     web_update=false
     FTL_update=false
 
+    # Perform an OS check to ensure we're on an appropriate operating system
+    os_check
+
     # Install packages used by this installation script (necessary if users have removed e.g. git from their systems)
     package_manager_detect
     build_dependency_package
@@ -215,7 +218,7 @@ main() {
     fi
 
     if [[ "${FTL_update}" == true || "${core_update}" == true ]]; then
-        ${PI_HOLE_FILES_DIR}/automated\ install/basic-install.sh --reconfigure --unattended || \
+        ${PI_HOLE_FILES_DIR}/automated\ install/basic-install.sh --repair --unattended || \
             echo -e "${basicError}" && exit 1
     fi
 

advanced/Scripts/version.sh

@@ -12,7 +12,7 @@
 # shellcheck disable=SC3043
 # https://github.com/koalaman/shellcheck/wiki/SC3043#exceptions
 
-# Source the versions file poupulated by updatechecker.sh
+# Source the versions file populated by updatechecker.sh
 cachedVersions="/etc/pihole/versions"
 
 if [ -f ${cachedVersions} ]; then

advanced/Templates/gravity.db.sql

@@ -67,6 +67,11 @@ CREATE TABLE info
 );
 
 INSERT INTO "info" VALUES('version','19');
+/* This is a flag to indicate if gravity was restored from a backup
+    false = not restored,
+    failed = restoration failed due to no backup
+    other string = restoration successful with the string being the backup file used */
+INSERT INTO "info" VALUES('gravity_restored','false');
 
 CREATE TABLE domainlist_by_group
 (

advanced/Templates/pihole-FTL-prestart.sh

@@ -10,25 +10,21 @@ utilsfile="${PI_HOLE_SCRIPT_DIR}/utils.sh"
 FTL_PID_FILE="$(getFTLConfigValue files.pid)"
 
 # Ensure that permissions are set so that pihole-FTL can edit all necessary files
-# shellcheck disable=SC2174
+mkdir -p /var/log/pihole
-mkdir -pm 0640 /var/log/pihole
+chown -R pihole:pihole /etc/pihole/ /var/log/pihole/
-chown -R pihole:pihole /etc/pihole /var/log/pihole
-chmod -R 0640 /var/log/pihole
-chmod -R 0660 /etc/pihole
-
-# Logrotate config file need to be owned by root and must not be writable by group and others
-chown root:root /etc/pihole/logrotate
-chmod 0644 /etc/pihole/logrotate
-
-# allow all users to enter the directories
-chmod 0755 /etc/pihole /var/log/pihole
-
 # allow pihole to access subdirs in /etc/pihole (sets execution bit on dirs)
-# credits https://stackoverflow.com/a/11512211
+find /etc/pihole/ /var/log/pihole/ -type d -exec chmod 0755 {} +
-find /etc/pihole -type d -exec chmod 0755 {} \;
+# Set all files (except TLS-related ones) to u+rw g+r
+find /etc/pihole/ /var/log/pihole/ -type f ! \( -name '*.pem' -o -name '*.crt' \) -exec chmod 0640 {} +
+# Set TLS-related files to a more restrictive u+rw *only* (they may contain private keys)
+find /etc/pihole/ -type f \( -name '*.pem' -o -name '*.crt' \) -exec chmod 0600 {} +
+
+# Logrotate config file need to be owned by root
+chown root:root /etc/pihole/logrotate
 
 # Touch files to ensure they exist (create if non-existing, preserve if existing)
 [ -f "${FTL_PID_FILE}" ] || install -D -m 644 -o pihole -g pihole /dev/null "${FTL_PID_FILE}"
 [ -f /var/log/pihole/FTL.log ] || install -m 640 -o pihole -g pihole /dev/null /var/log/pihole/FTL.log
 [ -f /var/log/pihole/pihole.log ] || install -m 640 -o pihole -g pihole /dev/null /var/log/pihole/pihole.log
+[ -f /var/log/pihole/webserver.log ] || install -m 640 -o pihole -g pihole /dev/null /var/log/pihole/webserver.log
 [ -f /etc/pihole/dhcp.leases ] || install -m 644 -o pihole -g pihole /dev/null /etc/pihole/dhcp.leases

advanced/bash-completion/pihole

@@ -7,7 +7,7 @@ _pihole() {
 
     case "${prev}" in
         "pihole")
-            opts="allow allow-regex allow-wild deny checkout debug disable enable flush help logging query reconfigure regex reloaddns reloadlists status tail uninstall updateGravity updatePihole version wildcard arpflush api"
+            opts="allow allow-regex allow-wild deny checkout debug disable enable flush help logging query repair regex reloaddns reloadlists status tail uninstall updateGravity updatePihole version wildcard arpflush api"
             COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
         ;;
         "allow"|"deny"|"wildcard"|"regex"|"allow-regex"|"allow-wild")

automated install/basic-install.sh

@@ -81,9 +81,7 @@ PI_HOLE_INSTALL_DIR="/opt/pihole"
 PI_HOLE_CONFIG_DIR="/etc/pihole"
 PI_HOLE_BIN_DIR="/usr/local/bin"
 PI_HOLE_V6_CONFIG="${PI_HOLE_CONFIG_DIR}/pihole.toml"
-if [ -z "$useUpdateVars" ]; then
+fresh_install=true
-    useUpdateVars=false
-fi
 
 adlistFile="/etc/pihole/adlists.list"
 # Pi-hole needs an IP address; to begin, these variables are empty since we don't know what the IP is until this script can run
@@ -91,7 +89,6 @@ IPV4_ADDRESS=${IPV4_ADDRESS}
 IPV6_ADDRESS=${IPV6_ADDRESS}
 # Give settings their default values. These may be changed by prompts later in the script.
 QUERY_LOGGING=
-WEBPORT=
 PRIVACY_LEVEL=
 
 # Where old configs go to if a v6 migration is performed
@@ -142,12 +139,12 @@ EOM
 ######## Undocumented Flags. Shhh ########
 # These are undocumented flags; some of which we can use when repairing an installation
 # The runUnattended flag is one example of this
-reconfigure=false
+repair=false
 runUnattended=false
 # Check arguments for the undocumented flags
 for var in "$@"; do
     case "$var" in
-    "--reconfigure") reconfigure=true ;;
+    "--repair") repair=true ;;
     "--unattended") runUnattended=true ;;
     esac
 done
@@ -388,28 +385,6 @@ os_check() {
     fi
 }
 
-# This function waits for dpkg to unlock, which signals that the previous apt-get command has finished.
-test_dpkg_lock() {
-    i=0
-    printf "  %b Waiting for package manager to finish (up to 30 seconds)\\n" "${INFO}"
-    # fuser is a program to show which processes use the named files, sockets, or filesystems
-    # So while the lock is held,
-    while fuser /var/lib/dpkg/lock >/dev/null 2>&1; do
-        # we wait half a second,
-        sleep 0.5
-        # increase the iterator,
-        ((i = i + 1))
-        # exit if waiting for more then 30 seconds
-        if [[ $i -gt 60 ]]; then
-            printf "  %b %bError: Could not verify package manager finished and released lock. %b\\n" "${CROSS}" "${COL_LIGHT_RED}" "${COL_NC}"
-            printf "       Attempt to install packages manually and retry.\\n"
-            exit 1
-        fi
-    done
-    # and then report success once dpkg is unlocked.
-    return 0
-}
-
 # Compatibility
 package_manager_detect() {
 
@@ -1133,7 +1108,7 @@ setPrivacyLevel() {
 
 # A function to display a list of example blocklists for users to select
 chooseBlocklists() {
-    # Back up any existing adlist file, on the off chance that it exists. Useful in case of a reconfigure.
+    # Back up any existing adlist file, on the off chance that it exists.
     if [[ -f "${adlistFile}" ]]; then
         mv "${adlistFile}" "${adlistFile}.old"
     fi
@@ -1182,27 +1157,23 @@ installDefaultBlocklists() {
     echo "https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts" >>"${adlistFile}"
 }
 
-remove_old_dnsmasq_ftl_configs() {
+move_old_dnsmasq_ftl_configs() {
-    # Local, named variables
+    # Create migration directory /etc/pihole/migration_backup_v6
+    # and make it owned by pihole:pihole
+    mkdir -p "${V6_CONF_MIGRATION_DIR}"
+    chown pihole:pihole "${V6_CONF_MIGRATION_DIR}"
+
+    # Move all conf files originally created by Pi-hole into this directory
+    # - 01-pihole.conf
+    # - 02-pihole-dhcp.conf
+    # - 04-pihole-static-dhcp.conf
+    # - 05-pihole-custom-cname.conf
+    # - 06-rfc6761.conf
+    mv /etc/dnsmasq.d/0{1,2,4,5}-pihole*.conf "${V6_CONF_MIGRATION_DIR}/" 2>/dev/null || true
+    mv /etc/dnsmasq.d/06-rfc6761.conf "${V6_CONF_MIGRATION_DIR}/" 2>/dev/null || true
+
+    # If the dnsmasq main config file exists
     local dnsmasq_conf="/etc/dnsmasq.conf"
-    local pihole_01="/etc/dnsmasq.d/01-pihole.conf"
-    local rfc6761_06="/etc/dnsmasq.d/06-rfc6761.conf"
-    local pihole_dhcp_02="/etc/dnsmasq.d/02-pihole-dhcp.conf"
-
-    # pihole-FTL does some fancy stuff with config these days, and so we can remove some old config files
-    if [[ -f "${pihole_01}" ]]; then
-        rm "${pihole_01}"
-    fi
-
-    if [[ -f "${rfc6761_06}" ]]; then
-        rm "${rfc6761_06}"
-    fi
-
-    if [[ -f "${pihole_dhcp_02}" ]]; then
-        rm "${pihole_dhcp_02}"
-    fi
-
-    # If the dnsmasq config file exists
     if [[ -f "${dnsmasq_conf}" ]]; then
         # There should not be anything custom in here for Pi-hole users
         # It is no longer needed, but we'll back it up instead of deleting it just in case
@@ -1233,12 +1204,12 @@ remove_old_pihole_lighttpd_configs() {
         lighty-disable-mod pihole-admin >/dev/null || true
     fi
 
-    if [[ -f "${confavailable}" ]]; then
+    if [[ -f "${confenabled}" || -L "${confenabled}" ]]; then
-        rm "${confavailable}"
+        rm "${confenabled}"
     fi
 
-    if [[ -f "${confenabled}" ]]; then
+    if [[ -f "${confavailable}" ]]; then
-        rm "${confenabled}"
+        rm "${confavailable}"
     fi
 }
 
@@ -1301,13 +1272,6 @@ installConfigs() {
     # Ensure that permissions are correctly set
     chown -R pihole:pihole /etc/pihole
 
-    # Install list of DNS servers
-    # Format: Name;Primary IPv4;Secondary IPv4;Primary IPv6;Secondary IPv6
-    # Some values may be empty (for example: DNS servers without IPv6 support)
-    echo "${DNS_SERVERS}" >"${PI_HOLE_CONFIG_DIR}/dns-servers.conf"
-    chmod 644 "${PI_HOLE_CONFIG_DIR}/dns-servers.conf"
-    chown pihole:pihole "${PI_HOLE_CONFIG_DIR}/dns-servers.conf"
-
     # Install empty custom.list file if it does not exist
     if [[ ! -r "${PI_HOLE_CONFIG_DIR}/hosts/custom.list" ]]; then
         if ! install -D -T -o pihole -g pihole -m 660 /dev/null "${PI_HOLE_CONFIG_DIR}/hosts/custom.list" &>/dev/null; then
@@ -1387,9 +1351,9 @@ stop_service() {
     local str="Stopping ${1} service"
     printf "  %b %s..." "${INFO}" "${str}"
     if is_command systemctl; then
-        systemctl stop "${1}" &>/dev/null || true
+        systemctl -q stop "${1}" || true
     else
-        service "${1}" stop &>/dev/null || true
+        service "${1}" stop >/dev/null || true
     fi
     printf "%b  %b %s...\\n" "${OVER}" "${TICK}" "${str}"
 }
@@ -1402,10 +1366,10 @@ restart_service() {
     # If systemctl exists,
     if is_command systemctl; then
         # use that to restart the service
-        systemctl restart "${1}" &>/dev/null
+        systemctl -q restart "${1}"
     else
         # Otherwise, fall back to the service command
-        service "${1}" restart &>/dev/null
+        service "${1}" restart >/dev/null
     fi
     printf "%b  %b %s...\\n" "${OVER}" "${TICK}" "${str}"
 }
@@ -1418,10 +1382,10 @@ enable_service() {
     # If systemctl exists,
     if is_command systemctl; then
         # use that to enable the service
-        systemctl enable "${1}" &>/dev/null
+        systemctl -q enable "${1}"
     else
         #  Otherwise, use update-rc.d to accomplish this
-        update-rc.d "${1}" defaults &>/dev/null
+        update-rc.d "${1}" defaults >/dev/null
     fi
     printf "%b  %b %s...\\n" "${OVER}" "${TICK}" "${str}"
 }
@@ -1434,10 +1398,10 @@ disable_service() {
     # If systemctl exists,
     if is_command systemctl; then
         # use that to disable the service
-        systemctl disable "${1}" &>/dev/null
+        systemctl -q disable "${1}"
     else
         # Otherwise, use update-rc.d to accomplish this
-        update-rc.d "${1}" disable &>/dev/null
+        update-rc.d "${1}" disable >/dev/null
     fi
     printf "%b  %b %s...\\n" "${OVER}" "${TICK}" "${str}"
 }
@@ -1446,7 +1410,7 @@ check_service_active() {
     # If systemctl exists,
     if is_command systemctl; then
         # use that to check the status of the service
-        systemctl is-enabled "${1}" &>/dev/null
+        systemctl -q is-enabled "${1}" 2>/dev/null
     else
         # Otherwise, fall back to service command
         service "${1}" status &>/dev/null
@@ -1458,20 +1422,15 @@ disable_resolved_stublistener() {
     printf "  %b Testing if systemd-resolved is enabled\\n" "${INFO}"
     # Check if Systemd-resolved's DNSStubListener is enabled and active on port 53
     if check_service_active "systemd-resolved"; then
-        # Check if DNSStubListener is enabled
+        # Disable the DNSStubListener to unbind it from port 53
-        printf "  %b %b Testing if systemd-resolved DNSStub-Listener is active" "${OVER}" "${INFO}"
+        # Note that this breaks dns functionality on host until FTL is up and running
-        if (grep -E '#?DNSStubListener=yes' /etc/systemd/resolved.conf &>/dev/null); then
+        printf "%b  %b Disabling systemd-resolved DNSStubListener\\n" "${OVER}" "${TICK}"
-            # Disable the DNSStubListener to unbind it from port 53
+        mkdir -p /etc/systemd/resolved.conf.d
-            # Note that this breaks dns functionality on host until ftl are up and running
+        cat > /etc/systemd/resolved.conf.d/90-pi-hole-disable-stub-listener.conf << EOF
-            printf "%b  %b Disabling systemd-resolved DNSStubListener" "${OVER}" "${TICK}"
+[Resolve]
-            # Make a backup of the original /etc/systemd/resolved.conf
+DNSStubListener=no
-            # (This will need to be restored on uninstallation)
+EOF
-            sed -r -i.orig 's/#?DNSStubListener=yes/DNSStubListener=no/g' /etc/systemd/resolved.conf
+        systemctl reload-or-restart systemd-resolved
-            printf " and restarting systemd-resolved\\n"
-            systemctl reload-or-restart systemd-resolved
-        else
-            printf "%b  %b Systemd-resolved does not need to be restarted\\n" "${OVER}" "${INFO}"
-        fi
     else
         printf "%b  %b Systemd-resolved is not enabled\\n" "${OVER}" "${INFO}"
     fi
@@ -1510,16 +1469,11 @@ notify_package_updates_available() {
     # Store the list of packages in a variable
     updatesToInstall=$(eval "${PKG_COUNT}")
 
-    if [[ -d "/lib/modules/$(uname -r)" ]]; then
+    if [[ "${updatesToInstall}" -eq 0 ]]; then
-        if [[ "${updatesToInstall}" -eq 0 ]]; then
+        printf "%b  %b %s... up to date!\\n\\n" "${OVER}" "${TICK}" "${str}"
-            printf "%b  %b %s... up to date!\\n\\n" "${OVER}" "${TICK}" "${str}"
-        else
-            printf "%b  %b %s... %s updates available\\n" "${OVER}" "${TICK}" "${str}" "${updatesToInstall}"
-            printf "  %b %bIt is recommended to update your OS after installing the Pi-hole!%b\\n\\n" "${INFO}" "${COL_LIGHT_GREEN}" "${COL_NC}"
-        fi
     else
-        printf "%b  %b %s\\n" "${OVER}" "${CROSS}" "${str}"
+        printf "%b  %b %s... %s updates available\\n" "${OVER}" "${TICK}" "${str}" "${updatesToInstall}"
-        printf "      Kernel update detected. If the install fails, please reboot and try again\\n"
+        printf "  %b %bIt is recommended to update your OS after installing the Pi-hole!%b\\n\\n" "${INFO}" "${COL_LIGHT_GREEN}" "${COL_NC}"
     fi
 }
 
@@ -1728,7 +1682,8 @@ installPihole() {
         exit 1
     fi
 
-    remove_old_dnsmasq_ftl_configs
+    # Move old dnsmasq files to $V6_CONF_MIGRATION_DIR for later migration via migrate_dnsmasq_configs()
+    move_old_dnsmasq_ftl_configs
     remove_old_pihole_lighttpd_configs
 
     # Install config files
@@ -1793,83 +1748,6 @@ checkSelinux() {
     fi
 }
 
-# Installation complete message with instructions for the user
-displayFinalMessage() {
-    # TODO: COME BACK TO THIS, WHAT IS GOING ON?
-    # If the number of arguments is > 0,
-    if [[ "${#1}" -gt 0 ]]; then
-        # set the password to the first argument.
-        pwstring="$1"
-    elif [[ $(pihole-FTL --config webserver.api.pwhash) == '""' ]]; then
-        # Else if the password exists from previous setup, we'll load it later
-        pwstring="unchanged"
-    else
-        # Else, inform the user that there is no set password.
-        pwstring="NOT SET"
-    fi
-
-    # Store a message in a variable and display it
-    additional="View the web interface at http://pi.hole/admin:${WEBPORT} or http://${IPV4_ADDRESS%/*}:${WEBPORT}/admin\\n\\nYour Admin Webpage login password is ${pwstring}"
-
-    # Final completion message to user
-    dialog --no-shadow --keep-tite \
-        --title "Installation Complete!" \
-        --msgbox "Configure your devices to use the Pi-hole as their DNS server using:\
-\\n\\nIPv4:	${IPV4_ADDRESS%/*}\
-\\nIPv6:	${IPV6_ADDRESS:-"Not Configured"}\
-\\nIf you have not done so already, the above IP should be set to static.\
-\\n${additional}" "${r}" "${c}"
-}
-
-update_dialogs() {
-    # If pihole -r "reconfigure" option was selected,
-    if [[ "${reconfigure}" = true ]]; then
-        # set some variables that will be used
-        opt1a="Repair"
-        opt1b="This will retain existing settings"
-        strAdd="You will remain on the same version"
-    else
-        # Otherwise, set some variables with different values
-        opt1a="Update"
-        opt1b="This will retain existing settings."
-        strAdd="You will be updated to the latest version."
-    fi
-    opt2a="Reconfigure"
-    opt2b="Resets Pi-hole and allows re-selecting settings."
-
-    # Display the information to the user
-    UpdateCmd=$(dialog --no-shadow --keep-tite --output-fd 1 \
-        --cancel-label Exit \
-        --title "Existing Install Detected!" \
-        --menu "\\n\\nWe have detected an existing install.\
-\\n\\nPlease choose from the following options:\
-\\n($strAdd)" \
-        "${r}" "${c}" 2 \
-        "${opt1a}" "${opt1b}" \
-        "${opt2a}" "${opt2b}") || result=$?
-
-    case ${result} in
-    "${DIALOG_CANCEL}" | "${DIALOG_ESC}")
-        printf "  %b Cancel was selected, exiting installer%b\\n" "${COL_LIGHT_RED}" "${COL_NC}"
-        exit 1
-        ;;
-    esac
-
-    # Set the variable based on if the user chooses
-    case ${UpdateCmd} in
-    # repair, or
-    "${opt1a}")
-        printf "  %b %s option selected\\n" "${INFO}" "${opt1a}"
-        useUpdateVars=true
-        ;;
-    # reconfigure,
-    "${opt2a}")
-        printf "  %b %s option selected\\n" "${INFO}" "${opt2a}"
-        useUpdateVars=false
-        ;;
-    esac
-}
-
 check_download_exists() {
     # Check if the download exists and we can reach the server
     local status=$(curl --head --silent "https://ftl.pi-hole.net/${1}" | head -n 1)
@@ -1956,10 +1834,10 @@ checkout_pull_branch() {
     return 0
 }
 
-clone_or_update_repos() {
+clone_or_reset_repos() {
-    # If the user wants to reconfigure,
+    # If the user wants to repair/update,
-    if [[ "${reconfigure}" == true ]]; then
+    if [[ "${repair}" == true ]]; then
-        printf "  %b Performing reconfiguration, skipping download of local repos\\n" "${INFO}"
+        printf "  %b Resetting local repos\\n" "${INFO}"
         # Reset the Core repo
         resetRepo ${PI_HOLE_LOCAL_REPO} ||
             {
@@ -1972,7 +1850,7 @@ clone_or_update_repos() {
                 printf "  %b Unable to reset %s, exiting installer%b\\n" "${COL_LIGHT_RED}" "${webInterfaceDir}" "${COL_NC}"
                 exit 1
             }
-    # Otherwise, a repair is happening
+    # Otherwise, a fresh installation is happening
     else
         # so get git files for Core
         getGitFiles ${PI_HOLE_LOCAL_REPO} ${piholeGitUrl} ||
@@ -2035,7 +1913,7 @@ FTLinstall() {
             curl -sSL "https://ftl.pi-hole.net/macvendor.db" -o "${PI_HOLE_CONFIG_DIR}/macvendor.db" || true
 
             # Stop pihole-FTL service if available
-            stop_service pihole-FTL &>/dev/null
+            stop_service pihole-FTL >/dev/null
 
             # Install the new version with the correct permissions
             install -T -m 0755 "${binary}" /usr/bin/pihole-FTL
@@ -2156,7 +2034,7 @@ get_binary_name() {
         else
             printf "%b  %b Detected 32bit (i686) architecture\\n" "${OVER}" "${TICK}"
         fi
-        l_binary="pihole-FTL-linux-386"
+        l_binary="pihole-FTL-386"
     fi
 
     # Returning a string value via echo
@@ -2300,6 +2178,44 @@ copy_to_install_log() {
     chown pihole:pihole "${installLogLoc}"
 }
 
+disableLighttpd() {
+    # Return early when lighttpd is not active
+    if ! check_service_active lighttpd; then
+        return
+    fi
+
+    local response
+    # Detect if the terminal is interactive
+    if [[ -t 0 ]]; then
+        # The terminal is interactive
+        dialog --no-shadow --keep-tite \
+            --title "Pi-hole v6.0 no longer uses lighttpd" \
+           --yesno "\\n\\nPi-hole v6.0 has its own embedded web server so lighttpd is no longer needed *unless* you have custom configurations.\\n\\nIn this case, you can opt-out of disabling lighttpd and pihole-FTL will try to bind to an alternative port such as 8080.\\n\\nDo you want to disable lighttpd (recommended)?" "${r}" "${c}" && response=0 || response="$?"
+    else
+        # The terminal is non-interactive, assume yes. Lighttpd will be stopped
+        # but keeps being installed and can easily be re-enabled by the user
+        response=0
+    fi
+
+    # If the user does not want to disable lighttpd, return early
+    if [[ "${response}" -ne 0 ]]; then
+        return
+    fi
+
+    # Lighttpd is not needed anymore, so disable it
+    # We keep all the configuration files in place, so the user can re-enable it
+    # if needed
+
+    # Check if lighttpd is installed
+    if is_command lighttpd; then
+        # Stop the lighttpd service
+        stop_service lighttpd
+
+        # Disable the lighttpd service
+        disable_service lighttpd
+    fi
+}
+
 migrate_dnsmasq_configs() {
     # Previously, Pi-hole created a number of files in /etc/dnsmasq.d
     # During migration, their content is copied into the new single source of
@@ -2307,44 +2223,47 @@ migrate_dnsmasq_configs() {
     # avoid conflicts with other services on this system
 
     # Exit early if this is already Pi-hole v6.0
-    # We decide this on the presence of the file /etc/pihole/pihole.toml
+    # We decide this on the non-existence of the file /etc/pihole/setupVars.conf (either moved by previous migration or fresh install)
-    if [[ -f "${PI_HOLE_V6_CONFIG}" ]]; then
+    if [[ ! -f "/etc/pihole/setupVars.conf" ]]; then
         return 0
     fi
 
-    # Create target directory /etc/pihole/migration_backup_v6
+    # Disable lighttpd server during v6 migration
-    # and make it owned by pihole:pihole
+    disableLighttpd
-    mkdir -p "${V6_CONF_MIGRATION_DIR}"
-    chown pihole:pihole "${V6_CONF_MIGRATION_DIR}"
 
-    # Move all conf files originally created by Pi-hole into this directory
+    # move_old_dnsmasq_ftl_configs() moved everything is in place,
-    # - 01-pihole.conf
+    # so we can create the new config file /etc/pihole/pihole.toml
-    # - 02-pihole-dhcp.conf
-    # - 04-pihole-static-dhcp.conf
-    # - 05-pihole-custom-cname.conf
-    # - 06-rfc6761.conf
-
-    mv /etc/dnsmasq.d/0{1,2,4,5}-pihole*.conf "${V6_CONF_MIGRATION_DIR}/" 2>/dev/null || true
-    mv /etc/dnsmasq.d/06-rfc6761.conf "${V6_CONF_MIGRATION_DIR}/" 2>/dev/null || true
-
-    # Finally, after everything is in place, we can create the new config file
-    # /etc/pihole/pihole.toml
     # This file will be created with the default settings unless the user has
-    # changed settings via setupVars.conf or the other dnsmasq files moved above
+    # changed settings via setupVars.conf or the other dnsmasq files moved before
+    # During migration, setupVars.conf is moved to /etc/pihole/migration_backup_v6
     str="Migrating Pi-hole configuration to version 6"
-    printf "  %b %s...\\n" "${INFO}"
+    printf "  %b %s..." "${INFO}" "${str}"
     local FTLoutput FTLstatus
     FTLoutput=$(pihole-FTL migrate v6)
     FTLstatus=$?
     if [[ "${FTLstatus}" -eq 0 ]]; then
-        printf "  %b %s\\n" "${TICK}" "${str}"
+        printf "%b  %b %s\\n" "${OVER}" "${TICK}" "${str}"
     else
-        printf "  %b %s\\n" "${CROSS}" "${str}"
+        printf "%b  %b %s\\n" "${OVER}" "${CROSS}" "${str}"
     fi
 
     # Print the output of the FTL migration prefacing every line with four
     # spaces for alignment
     printf "%b" "${FTLoutput}" | sed 's/^/    /'
+
+    # Print a blank line for separation
+    printf "\\n"
+}
+
+# Check for availability of either the "service" or "systemctl" commands
+check_service_command() {
+    # Check for the availability of the "service" command
+    if ! is_command service && ! is_command systemctl; then
+        # If neither the "service" nor the "systemctl" command is available, inform the user
+        printf "  %b Neither the service nor the systemctl commands are available\\n" "${CROSS}"
+        printf "      on this machine. This Pi-hole installer cannot continue.\\n"
+        exit 1
+    fi
 }
 
 main() {
@@ -2395,6 +2314,9 @@ main() {
     # Check if SELinux is Enforcing and exit before doing anything else
     checkSelinux
 
+    # Check for availability of either the "service" or "systemctl" commands
+    check_service_command
+
     # Check for supported package managers so that we may install dependencies
     package_manager_detect
 
@@ -2420,22 +2342,19 @@ main() {
         exit 1
     fi
 
-    # in case of an update (can be a v5 -> v6 or v6 -> v6 update)
+    # in case of an update (can be a v5 -> v6 or v6 -> v6 update) or repair
     if [[ -f "${PI_HOLE_V6_CONFIG}" ]] || [[ -f "/etc/pihole/setupVars.conf" ]]; then
+        # retain settings
+        fresh_install=false
         # if it's running unattended,
         if [[ "${runUnattended}" == true ]]; then
             printf "  %b Performing unattended setup, no dialogs will be displayed\\n" "${INFO}"
-            # Use the setup variables
-            useUpdateVars=true
             # also disable debconf-apt-progress dialogs
             export DEBIAN_FRONTEND="noninteractive"
-        else
-            # If running attended, show the available options (repair/reconfigure)
-            update_dialogs
         fi
     fi
 
-    if [[ "${useUpdateVars}" == false ]]; then
+    if [[ "${fresh_install}" == true ]]; then
         # Display welcome dialogs
         welcomeDialogs
         # Create directory for Pi-hole storage (/etc/pihole/)
@@ -2458,9 +2377,8 @@ main() {
         # Setup adlist file if not exists
         installDefaultBlocklists
     fi
-    # Download or update the scripts by updating the appropriate git repos
+    # Download or reset the appropriate git repos depending on the 'repair' flag
-    clone_or_update_repos
+    clone_or_reset_repos
-
 
     # Create the pihole user
     create_pihole_user
@@ -2490,15 +2408,6 @@ main() {
     # Copy the temp log file into final log location for storage
     copy_to_install_log
 
-    # Add password to web UI if there is none
-    pw=""
-    # If no password is set,
-    if [[ $(pihole-FTL --config webserver.api.pwhash) == '""' ]]; then
-        # generate a random password
-        pw=$(tr -dc _A-Z-a-z-0-9 </dev/urandom | head -c 8)
-        pihole setpassword "${pw}"
-    fi
-
     # Migrate existing install to v6.0
     migrate_dnsmasq_configs
 
@@ -2524,14 +2433,25 @@ main() {
 
     restart_service pihole-FTL
 
-    # write privacy level and logging to pihole.toml
+    if [[ "${fresh_install}" == true ]]; then
-    # needs to be done after FTL service has been started, otherwise pihole.toml does not exist
+        # apply settings to pihole.toml
-    # set on fresh installations by setPrivacyLevel() and setLogging(
+        # needs to be done after FTL service has been started, otherwise pihole.toml does not exist
-    if [ -n "${QUERY_LOGGING}" ]; then
+        # set on fresh installations by setDNS() and setPrivacyLevel() and setLogging()
-        setFTLConfigValue "dns.queryLogging" "${QUERY_LOGGING}"
+
-    fi
+        # Upstreams may be needed in order to run gravity.sh
-    if [ -n "${PRIVACY_LEVEL}" ]; then
+        if [ -n "${PIHOLE_DNS_1}" ]; then
-        setFTLConfigValue "misc.privacylevel" "${PRIVACY_LEVEL}"
+            local string="\"${PIHOLE_DNS_1}\""
+            [ -n "${PIHOLE_DNS_2}" ] && string+=", \"${PIHOLE_DNS_2}\""
+            setFTLConfigValue "dns.upstreams" "[ $string ]"
+        fi
+
+        if [ -n "${QUERY_LOGGING}" ]; then
+            setFTLConfigValue "dns.queryLogging" "${QUERY_LOGGING}"
+        fi
+
+        if [ -n "${PRIVACY_LEVEL}" ]; then
+            setFTLConfigValue "misc.privacylevel" "${PRIVACY_LEVEL}"
+        fi
     fi
 
     # Download and compile the aggregated block list
@@ -2540,28 +2460,35 @@ main() {
     # Update local and remote versions via updatechecker
     /opt/pihole/updatecheck.sh
 
-    # If there is a password
+    if [[ "${fresh_install}" == true ]]; then
-    if ((${#pw} > 0)); then
-        # display the password
-        printf "  %b Web Interface password: %b%s%b\\n" "${INFO}" "${COL_LIGHT_GREEN}" "${pw}" "${COL_NC}"
-        printf "  %b This can be changed using 'pihole setpassword'\\n\\n" "${INFO}"
-    fi
 
-    if [[ "${useUpdateVars}" == false ]]; then
         # Get the Web interface port, return only the first port and strip all non-numeric characters
         WEBPORT=$(getFTLConfigValue webserver.port|cut -d, -f1 | tr -cd '0-9')
 
-        # Display the completion dialog
+        # If this is a fresh install, we will set a random password.
-        displayFinalMessage "${pw}"
+        # Users can change this password after installation if they wish
-
+        pw=$(tr -dc _A-Z-a-z-0-9 </dev/urandom | head -c 8)
-        # If the Web interface was installed,
+        pihole setpassword "${pw}" > /dev/null
-        printf "  %b View the web interface at http://pi.hole:${WEBPORT}/admin or http://%s/admin\\n\\n" "${INFO}" "${IPV4_ADDRESS%/*}:${WEBPORT}"
 
         # Explain to the user how to use Pi-hole as their DNS server
-        printf "  %b You may now configure your devices to use the Pi-hole as their DNS server\\n" "${INFO}"
+        printf "\\n  %b You may now configure your devices to use the Pi-hole as their DNS server\\n" "${INFO}"
         [[ -n "${IPV4_ADDRESS%/*}" ]] && printf "  %b Pi-hole DNS (IPv4): %s\\n" "${INFO}" "${IPV4_ADDRESS%/*}"
         [[ -n "${IPV6_ADDRESS}" ]] && printf "  %b Pi-hole DNS (IPv6): %s\\n" "${INFO}" "${IPV6_ADDRESS}"
         printf "  %b If you have not done so already, the above IP should be set to static.\\n" "${INFO}"
+
+        printf "  %b View the web interface at http://pi.hole:${WEBPORT}/admin or http://%s/admin\\n\\n" "${INFO}" "${IPV4_ADDRESS%/*}:${WEBPORT}"
+        printf "  %b Web Interface password: %b%s%b\\n" "${INFO}" "${COL_LIGHT_GREEN}" "${pw}" "${COL_NC}"
+        printf "  %b This can be changed using 'pihole setpassword'\\n\\n" "${INFO}"
+
+        # Final dialog message to the user
+        dialog --no-shadow --keep-tite \
+            --title "Installation Complete!" \
+            --msgbox "Configure your devices to use the Pi-hole as their DNS server using:\
+\\n\\nIPv4:	${IPV4_ADDRESS%/*}\
+\\nIPv6:	${IPV6_ADDRESS:-"Not Configured"}\
+\\nIf you have not done so already, the above IP should be set to static.\
+\\nView the web interface at http://pi.hole/admin:${WEBPORT} or http://${IPV4_ADDRESS%/*}:${WEBPORT}/admin\\n\\nYour Admin Webpage login password is ${pw}" "${r}" "${c}"
+
         INSTALL_TYPE="Installation"
     else
         INSTALL_TYPE="Update"

automated install/uninstall.sh

@@ -94,8 +94,9 @@ removePiholeFiles() {
     echo -e "  ${TICK} Removed config files"
 
     # Restore Resolved
-    if [[ -e /etc/systemd/resolved.conf.orig ]]; then
+    if [[ -e /etc/systemd/resolved.conf.orig ]] || [[ -e /etc/systemd/resolved.conf.d/90-pi-hole-disable-stub-listener.conf ]]; then
-        ${SUDO} cp -p /etc/systemd/resolved.conf.orig /etc/systemd/resolved.conf
+        ${SUDO} cp -p /etc/systemd/resolved.conf.orig /etc/systemd/resolved.conf &> /dev/null || true
+        ${SUDO} rm -f /etc/systemd/resolved.conf.d/90-pi-hole-disable-stub-listener.conf
         systemctl reload-or-restart systemd-resolved
     fi
 

gravity.sh

@@ -30,6 +30,9 @@ PIHOLE_COMMAND="/usr/local/bin/${basename}"
 
 piholeDir="/etc/${basename}"
 
+# Gravity aux files directory
+listsCacheDir="${piholeDir}/listsCache"
+
 # Legacy (pre v5.0) list file locations
 whitelistFile="${piholeDir}/whitelist.txt"
 blacklistFile="${piholeDir}/blacklist.txt"
@@ -44,6 +47,7 @@ gravityDBcopy="${piholeGitDir}/advanced/Templates/gravity_copy.sql"
 
 domainsExtension="domains"
 curl_connect_timeout=10
+etag_support=false
 
 # Check gravity temp directory
 if [ ! -d "${GRAVITY_TMPDIR}" ] || [ ! -w "${GRAVITY_TMPDIR}" ]; then
@@ -58,6 +62,8 @@ gravityDBfile_default="/etc/pihole/gravity.db"
 gravityTEMPfile="${GRAVITYDB}_temp"
 gravityDIR="$(dirname -- "${gravityDBfile}")"
 gravityOLDfile="${gravityDIR}/gravity_old.db"
+gravityBCKdir="${gravityDIR}/gravity_backups"
+gravityBCKfile="${gravityBCKdir}/gravity.db"
 
 fix_owner_permissions() {
   # Fix ownership and permissions for the specified file
@@ -91,11 +97,21 @@ gravity_build_tree() {
 
   if [[ "${status}" -ne 0 ]]; then
     echo -e "\\n  ${CROSS} Unable to build gravity tree in ${gravityTEMPfile}\\n  ${output}"
+    echo -e "  ${INFO} If you have a large amount of domains, make sure your Pi-hole has enough RAM available\\n"
     return 1
   fi
   echo -e "${OVER}  ${TICK} ${str}"
 }
 
+# Rotate gravity backup files
+rotate_gravity_backup() {
+  for i in {9..1}; do
+    if [ -f "${gravityBCKfile}.${i}" ]; then
+      mv "${gravityBCKfile}.${i}" "${gravityBCKfile}.$((i + 1))"
+    fi
+  done
+}
+
 # Copy data from old to new database file and swap them
 gravity_swap_databases() {
   str="Swapping databases"
@@ -111,10 +127,32 @@ gravity_swap_databases() {
   oldAvail=false
   if [ "${availableBlocks}" -gt "$((gravityBlocks * 2))" ] && [ -f "${gravityDBfile}" ]; then
     oldAvail=true
-    mv "${gravityDBfile}" "${gravityOLDfile}"
+    cp "${gravityDBfile}" "${gravityOLDfile}"
-  else
-    rm "${gravityDBfile}"
   fi
+
+  # Drop the gravity and antigravity tables + subsequent VACUUM the current
+  # database for compaction
+  output=$({ printf ".timeout 30000\\nDROP TABLE IF EXISTS gravity;\\nDROP TABLE IF EXISTS antigravity;\\nVACUUM;\\n" | pihole-FTL sqlite3 -ni "${gravityDBfile}"; } 2>&1)
+  status="$?"
+
+  if [[ "${status}" -ne 0 ]]; then
+    echo -e "\\n  ${CROSS} Unable to clean current database for backup\\n  ${output}"
+  else
+    # Check if the backup directory exists
+    if [ ! -d "${gravityBCKdir}" ]; then
+      mkdir -p "${gravityBCKdir}"
+    fi
+
+    # If multiple gravityBCKfile's are present (appended with a number), rotate them
+    # We keep at most 10 backups
+    rotate_gravity_backup
+
+    # Move the old database to the backup location
+    mv "${gravityDBfile}" "${gravityBCKfile}.1"
+  fi
+
+
+  # Move the new database to the correct location
   mv "${gravityTEMPfile}" "${gravityDBfile}"
   echo -e "${OVER}  ${TICK} ${str}"
 
@@ -324,6 +362,54 @@ gravity_CheckDNSResolutionAvailable() {
   echo -e "${OVER}  ${TICK} DNS resolution is available"
 }
 
+# Function: try_restore_backup
+# Description: Attempts to restore the previous Pi-hole gravity database from a
+#              backup file. If a backup exists, it copies the backup to the
+#              gravity database file and prepares a new gravity database. If the
+#              restoration is successful, it returns 0. Otherwise, it returns 1.
+# Returns:
+#   0 - If the backup is successfully restored.
+#   1 - If no backup is available or if the restoration fails.
+try_restore_backup () {
+  local num filename timestamp
+  num=$1
+  filename="${gravityBCKfile}.${num}"
+  # Check if a backup exists
+  if [ -f "${filename}" ]; then
+    echo -e "  ${INFO} Attempting to restore previous database from backup no. ${num}"
+    cp "${filename}" "${gravityDBfile}"
+
+    # If the backup was successfully copied, prepare a new gravity database from
+    # it
+    if [ -f "${gravityDBfile}" ]; then
+      output=$({ pihole-FTL sqlite3 -ni "${gravityTEMPfile}" <<<"${copyGravity}"; } 2>&1)
+      status="$?"
+
+      # Error checking
+      if [[ "${status}" -ne 0 ]]; then
+        echo -e "\\n  ${CROSS} Unable to copy data from ${gravityDBfile} to ${gravityTEMPfile}\\n  ${output}"
+        gravity_Cleanup "error"
+      fi
+
+      # Get the timestamp of the backup file in a human-readable format
+      # Note that this timestamp will be in the server timezone, this may be
+      # GMT, e.g., on a Raspberry Pi where the default timezone has never been
+      # changed
+      timestamp=$(date -r "${filename}" "+%Y-%m-%d %H:%M:%S %Z")
+
+      # Add a record to the info table to indicate that the gravity database was restored
+      pihole-FTL sqlite3 "${gravityTEMPfile}" "INSERT OR REPLACE INTO info (property,value) values ('gravity_restored','${timestamp}');"
+      echo -e "  ${TICK} Successfully restored from backup (${gravityBCKfile}.${num} at ${timestamp})"
+      return 0
+    else
+      echo -e "  ${CROSS} Unable to restore backup no. ${num}"
+    fi
+  fi
+
+  echo -e "  ${CROSS} Backup no. ${num} not available"
+  return 1
+}
+
 # Retrieve blocklist URLs and parse domains from adlist.list
 gravity_DownloadBlocklists() {
   echo -e "  ${INFO} ${COL_BOLD}Neutrino emissions detected${COL_NC}..."
@@ -332,33 +418,7 @@ gravity_DownloadBlocklists() {
     echo -e "  ${INFO} Storing gravity database in ${COL_BOLD}${gravityDBfile}${COL_NC}"
   fi
 
-  # Retrieve source URLs from gravity database
+  local url domain str target compression adlist_type directory success
-  # We source only enabled adlists, SQLite3 stores boolean values as 0 (false) or 1 (true)
-  mapfile -t sources <<<"$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2>/dev/null)"
-  mapfile -t sourceIDs <<<"$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT id FROM vw_adlist;" 2>/dev/null)"
-  mapfile -t sourceTypes <<<"$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT type FROM vw_adlist;" 2>/dev/null)"
-
-  # Parse source domains from $sources
-  mapfile -t sourceDomains <<<"$(
-    # Logic: Split by folder/port
-    awk -F '[/:]' '{
-      # Remove URL protocol & optional username:password@
-      gsub(/(.*:\/\/|.*:.*@)/, "", $0)
-      if(length($1)>0){print $1}
-      else {print "local"}
-    }' <<<"$(printf '%s\n' "${sources[@]}")" 2>/dev/null
-  )"
-
-  local str="Pulling blocklist source list into range"
-  echo -e "${OVER}  ${TICK} ${str}"
-
-  if [[ -z "${sources[*]}" ]] || [[ -z "${sourceDomains[*]}" ]]; then
-    echo -e "  ${INFO} No source list found, or it is empty"
-    echo ""
-    unset sources
-  fi
-
-  local url domain str target compression adlist_type directory
   echo ""
 
   # Prepare new gravity database
@@ -390,10 +450,55 @@ gravity_DownloadBlocklists() {
 
   if [[ "${status}" -ne 0 ]]; then
     echo -e "\\n  ${CROSS} Unable to copy data from ${gravityDBfile} to ${gravityTEMPfile}\\n  ${output}"
-    return 1
+
+    # Try to attempt a backup restore
+    success=false
+    if [[ -d "${gravityBCKdir}" ]]; then
+      for i in {1..10}; do
+        if try_restore_backup "${i}"; then
+          success=true
+          break
+        fi
+      done
+    fi
+
+    # If none of the attempts worked, return 1
+    if [[ "${success}" == false ]]; then
+      pihole-FTL sqlite3 "${gravityTEMPfile}" "INSERT OR REPLACE INTO info (property,value) values ('gravity_restored','failed');"
+      return 1
+    fi
+
+    echo -e "  ${TICK} ${str}"
+  else
+    echo -e "${OVER}  ${TICK} ${str}"
   fi
+
+  # Retrieve source URLs from gravity database
+  # We source only enabled adlists, SQLite3 stores boolean values as 0 (false) or 1 (true)
+  mapfile -t sources <<<"$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2>/dev/null)"
+  mapfile -t sourceIDs <<<"$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT id FROM vw_adlist;" 2>/dev/null)"
+  mapfile -t sourceTypes <<<"$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT type FROM vw_adlist;" 2>/dev/null)"
+
+  # Parse source domains from $sources
+  mapfile -t sourceDomains <<<"$(
+    # Logic: Split by folder/port
+    awk -F '[/:]' '{
+      # Remove URL protocol & optional username:password@
+      gsub(/(.*:\/\/|.*:.*@)/, "", $0)
+      if(length($1)>0){print $1}
+      else {print "local"}
+    }' <<<"$(printf '%s\n' "${sources[@]}")" 2>/dev/null
+  )"
+
+  local str="Pulling blocklist source list into range"
   echo -e "${OVER}  ${TICK} ${str}"
 
+  if [[ -z "${sources[*]}" ]] || [[ -z "${sourceDomains[*]}" ]]; then
+    echo -e "  ${INFO} No source list found, or it is empty"
+    echo ""
+    unset sources
+  fi
+
   # Use compression to reduce the amount of data that is transferred
   # between the Pi-hole and the ad list provider. Use this feature
   # only if it is supported by the locally available version of curl
@@ -404,6 +509,15 @@ gravity_DownloadBlocklists() {
     compression=""
     echo -e "  ${INFO} Libz compression not available\n"
   fi
+
+  # Check if etag is supported by the locally available version of curl
+  # (available as of curl 7.68.0, released Jan 2020)
+  # https://github.com/curl/curl/pull/4543 +
+  # https://github.com/curl/curl/pull/4678
+  if curl --help all | grep -q "etag-save"; then
+    etag_support=true
+  fi
+
   # Loop through $sources and download each one
   for ((i = 0; i < "${#sources[@]}"; i++)); do
     url="${sources[$i]}"
@@ -420,8 +534,8 @@ gravity_DownloadBlocklists() {
     fi
 
     # Save the file as list.#.domain
-    saveLocation="${piholeDir}/list.${id}.${domain}.${domainsExtension}"
+    saveLocation="${listsCacheDir}/list.${id}.${domain}.${domainsExtension}"
-    activeDomains[$i]="${saveLocation}"
+    activeDomains[i]="${saveLocation}"
 
     # Check if we can write to the save location file without actually creating
     # it (in case it doesn't exist)
@@ -488,7 +602,7 @@ compareLists() {
 # Download specified URL and perform checks on HTTP status and file content
 gravity_DownloadBlocklistFromUrl() {
   local url="${1}" adlistID="${2}" saveLocation="${3}" target="${4}" compression="${5}" gravity_type="${6}" domain="${7}"
-  local heisenbergCompensator="" listCurlBuffer str httpCode success="" ip cmd_ext
+  local modifiedOptions="" listCurlBuffer str httpCode success="" ip cmd_ext
   local file_path permissions ip_addr port blocked=false download=true
 
   # Create temp file to store content on disk instead of RAM
@@ -497,12 +611,37 @@ gravity_DownloadBlocklistFromUrl() {
   mv "${listCurlBuffer}" "${listCurlBuffer%.*}.phgpb"
   listCurlBuffer="${listCurlBuffer%.*}.phgpb"
 
-  # Determine if $saveLocation has read permission
+  # For all remote files, we try to determine if the file has changed to skip
-  if [[ -r "${saveLocation}" && $url != "file"* ]]; then
+  # downloading them whenever possible.
-    # Have curl determine if a remote file has been modified since last retrieval
+  if [[ $url != "file"* ]]; then
-    # Uses "Last-Modified" header, which certain web servers do not provide (e.g: raw github urls)
+    # Use the HTTP ETag header to determine if the file has changed if supported
-    # Note: Don't do this for local files, always download them
+    # by curl. Using ETags is supported by raw.githubusercontent.com URLs.
-    heisenbergCompensator="-z ${saveLocation}"
+    if [[ "${etag_support}" == true ]]; then
+      # Save HTTP ETag to the specified file. An ETag is a caching related header,
+      # usually returned in a response. If no ETag is sent by the server, an empty
+      # file is created and can later be used consistently.
+      modifiedOptions="--etag-save ${saveLocation}.etag"
+
+      if [[ -f "${saveLocation}.etag" ]]; then
+        # This option makes a conditional HTTP request for the specific ETag read
+        # from the given file by sending a custom If-None-Match header using the
+        # stored ETag. This way, the server will only send the file if it has
+        # changed since the last request.
+        modifiedOptions="${modifiedOptions} --etag-compare ${saveLocation}.etag"
+      fi
+    fi
+
+    # Add If-Modified-Since header to the request if we did already download the
+    # file once
+    if [[ -f "${saveLocation}" ]]; then
+      # Request a file that has been modified later than the given time and
+      # date. We provide a file here which makes curl use the modification
+      # timestamp (mtime) of this file.
+      # Interstingly, this option is not supported by raw.githubusercontent.com
+      # URLs, however, it is still supported by many older web servers which may
+      # not support the HTTP ETag method so we keep it as a fallback.
+      modifiedOptions="${modifiedOptions} -z ${saveLocation}"
+    fi
   fi
 
   str="Status:"
@@ -632,7 +771,7 @@ gravity_DownloadBlocklistFromUrl() {
 
   if [[ "${download}" == true ]]; then
     # shellcheck disable=SC2086
-    httpCode=$(curl --connect-timeout ${curl_connect_timeout} -s -L ${compression} ${cmd_ext} ${heisenbergCompensator} -w "%{http_code}" "${url}" -o "${listCurlBuffer}" 2>/dev/null)
+    httpCode=$(curl --connect-timeout ${curl_connect_timeout} -s -L ${compression} ${cmd_ext} ${modifiedOptions} -w "%{http_code}" "${url}" -o "${listCurlBuffer}" 2>/dev/null)
   fi
 
   case $url in
@@ -675,9 +814,10 @@ gravity_DownloadBlocklistFromUrl() {
   # Determine if the blocklist was downloaded and saved correctly
   if [[ "${success}" == true ]]; then
     if [[ "${httpCode}" == "304" ]]; then
+      # Set list status to "unchanged/cached"
+      database_adlist_status "${adlistID}" "2"
       # Add domains to database table file
       pihole-FTL "${gravity_type}" parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}"
-      database_adlist_status "${adlistID}" "2"
       done="true"
     # Check if $listCurlBuffer is a non-zero length file
     elif [[ -s "${listCurlBuffer}" ]]; then
@@ -685,10 +825,10 @@ gravity_DownloadBlocklistFromUrl() {
       gravity_ParseFileIntoDomains "${listCurlBuffer}" "${saveLocation}"
       # Remove curl buffer file after its use
       rm "${listCurlBuffer}"
+      # Compare lists if are they identical
+      compareLists "${adlistID}" "${saveLocation}"
       # Add domains to database table file
       pihole-FTL "${gravity_type}" parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}"
-      # Compare lists, are they identical?
-      compareLists "${adlistID}" "${saveLocation}"
       done="true"
     else
       # Fall back to previously cached list if $listCurlBuffer is empty
@@ -701,9 +841,10 @@ gravity_DownloadBlocklistFromUrl() {
     # Determine if cached list has read permission
     if [[ -r "${saveLocation}" ]]; then
       echo -e "  ${CROSS} List download failed: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}"
+      # Set list status to "download-failed/cached"
+      database_adlist_status "${adlistID}" "3"
       # Add domains to database table file
       pihole-FTL "${gravity_type}" parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}"
-      database_adlist_status "${adlistID}" "3"
     else
       echo -e "  ${CROSS} List download failed: ${COL_LIGHT_RED}no cached list available${COL_NC}"
       # Manually reset these two numbers because we do not call parseList here
@@ -764,11 +905,11 @@ gravity_Table_Count() {
 gravity_ShowCount() {
   # Here we use the table "gravity" instead of the view "vw_gravity" for speed.
   # It's safe to replace it here, because right after a gravity run both will show the exactly same number of domains.
-  gravity_Table_Count "gravity" "gravity domains" ""
+  gravity_Table_Count "gravity" "gravity domains"
-  gravity_Table_Count "vw_blacklist" "exact denied domains"
+  gravity_Table_Count "domainlist WHERE type = 1 AND enabled = 1" "exact denied domains"
-  gravity_Table_Count "vw_regex_blacklist" "regex denied filters"
+  gravity_Table_Count "domainlist WHERE type = 3 AND enabled = 1" "regex denied filters"
-  gravity_Table_Count "vw_whitelist" "exact allowed domains"
+  gravity_Table_Count "domainlist WHERE type = 0 AND enabled = 1" "exact allowed domains"
-  gravity_Table_Count "vw_regex_whitelist" "regex allowed filters"
+  gravity_Table_Count "domainlist WHERE type = 2 AND enabled = 1" "regex allowed filters"
 }
 
 # Trap Ctrl-C
@@ -917,11 +1058,33 @@ timeit(){
   elapsed_time=$((end_time - start_time))
 
   # Display the elapsed time
-  printf "  %b--> took %d.%03d seconds%b\n" ${COL_BLUE} $((elapsed_time / 1000)) $((elapsed_time % 1000)) ${COL_NC}
+  printf "  %b--> took %d.%03d seconds%b\n" "${COL_BLUE}" $((elapsed_time / 1000)) $((elapsed_time % 1000)) "${COL_NC}"
 
   return $ret
 }
 
+migrate_to_listsCache_dir() {
+  # If the ${listsCacheDir} directory already exists, this has been done before
+  if [[ -d "${listsCacheDir}" ]]; then
+    return
+  fi
+
+  # If not, we need to migrate the old files to the new directory
+  local str="Migrating the list's cache directory to new location"
+  echo -ne "  ${INFO} ${str}..."
+  mkdir -p "${listsCacheDir}"
+
+  # Move the old files to the new directory
+  if mv "${piholeDir}"/list.* "${listsCacheDir}/" 2>/dev/null; then
+    echo -e "${OVER}  ${TICK} ${str}"
+  else
+    echo -e "${OVER}  ${CROSS} ${str}"
+  fi
+
+  # Update the list's paths in the corresponding .sha1 files to the new location
+  sed -i "s|${piholeDir}/|${listsCacheDir}/|g" "${listsCacheDir}"/*.sha1 2>/dev/null
+}
+
 helpFunc() {
   echo "Usage: pihole -g
 Update domains from blocklists specified in adlists.list
@@ -997,6 +1160,9 @@ if [[ "${recover_database:-}" == true ]]; then
   timeit database_recovery "$4"
 fi
 
+# Migrate scattered list files to the new cache directory
+migrate_to_listsCache_dir
+
 # Move possibly existing legacy files to the gravity database
 if ! timeit migrate_to_database; then
   echo -e "   ${CROSS} Unable to migrate to database. Please contact support."
@@ -1007,7 +1173,7 @@ if [[ "${forceDelete:-}" == true ]]; then
   str="Deleting existing list cache"
   echo -ne "${INFO} ${str}..."
 
-  rm /etc/pihole/list.* 2>/dev/null || true
+  rm "${listsCacheDir}/list.*" 2>/dev/null || true
   echo -e "${OVER}  ${TICK} ${str}"
 fi
 

pihole

@@ -73,19 +73,17 @@ listFunc() {
 
 debugFunc() {
     local automated
-    local web
     local check_database_integrity
     # Pull off the `debug` leaving passed call augmentation flags in $1
     shift
 
     for value in "$@"; do
         [[ "$value"  == *"-a"* ]] && automated="true"
-        [[ "$value"  == *"-w"* ]] && web="true"
         [[ "$value"  == *"-c"* ]] && check_database_integrity="true"
         [[ "$value" == *"--check_database"* ]] && check_database_integrity="true"
     done
 
-  AUTOMATED=${automated:-} WEBCALL=${web:-} CHECK_DATABASE=${check_database_integrity:-} "${PI_HOLE_SCRIPT_DIR}"/piholeDebug.sh
+  AUTOMATED=${automated:-} CHECK_DATABASE=${check_database_integrity:-} "${PI_HOLE_SCRIPT_DIR}"/piholeDebug.sh
   exit 0
 }
 
@@ -109,11 +107,11 @@ updatePiholeFunc() {
   fi
 }
 
-reconfigurePiholeFunc() {
+repairPiholeFunc() {
   if [ -n "${DOCKER_VERSION}" ]; then
     unsupportedFunc
   else
-    /etc/.pihole/automated\ install/basic-install.sh --reconfigure
+    /etc/.pihole/automated\ install/basic-install.sh --repair
     exit 0;
   fi
 }
@@ -400,7 +398,10 @@ tailFunc() {
 
 piholeCheckoutFunc() {
   if [ -n "${DOCKER_VERSION}" ]; then
-    unsupportedFunc
+    echo -e "${CROSS} Function not supported in Docker images"
+    echo "Please build a custom image following the steps at"
+    echo "https://github.com/pi-hole/docker-pi-hole?tab=readme-ov-file#building-the-image-locally"
+    exit 0
   else
     if [[ "$2" == "-h" ]] || [[ "$2" == "--help" ]]; then
       echo "Switch Pi-hole subsystems to a different GitHub branch
@@ -478,7 +479,7 @@ Debugging Options:
                         Add '-c' or '--check-database' to include a Pi-hole database integrity check
                         Add '-a' to automatically upload the log to tricorder.pi-hole.net
   -f, flush           Flush the Pi-hole log
-  -r, reconfigure     Reconfigure or Repair Pi-hole subsystems
+  -r, repair          Repair Pi-hole subsystems
   -t, tail [arg]      View the live output of the Pi-hole log.
                       Add an optional argument to filter the log
                       (regular expressions are supported)
@@ -535,7 +536,7 @@ case "${1}" in
   "--allow-wild" | "allow-wild"   ) need_root=0;;
   "-f" | "flush"                  ) ;;
   "-up" | "updatePihole"          ) ;;
-  "-r"  | "reconfigure"           ) ;;
+  "-r"  | "repair"                ) ;;
   "-l" | "logging"                ) ;;
   "uninstall"                     ) ;;
   "enable"                        ) need_root=0;;
@@ -578,7 +579,7 @@ case "${1}" in
   "-d" | "debug"                  ) debugFunc "$@";;
   "-f" | "flush"                  ) flushFunc "$@";;
   "-up" | "updatePihole"          ) updatePiholeFunc "$@";;
-  "-r"  | "reconfigure"           ) reconfigurePiholeFunc;;
+  "-r"  | "repair"                ) repairPiholeFunc;;
   "-g" | "updateGravity"          ) updateGravityFunc "$@";;
   "-l" | "logging"                ) piholeLogging "$@";;
   "uninstall"                     ) uninstallFunc;;

test/_ubuntu_23.Dockerfile

@@ -1,18 +0,0 @@
-FROM buildpack-deps:lunar-scm
-
-ENV GITDIR=/etc/.pihole
-ENV SCRIPTDIR=/opt/pihole
-
-RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole
-ADD . $GITDIR
-RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $GITDIR/advanced/Scripts/COL_TABLE $SCRIPTDIR/
-ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR
-ENV DEBIAN_FRONTEND=noninteractive
-
-RUN true && \
-    chmod +x $SCRIPTDIR/*
-
-ENV SKIP_INSTALL=true
-ENV OS_CHECK_DOMAIN_NAME=dev-supportedos.pi-hole.net
-
-#sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \

test/requirements.txt

@@ -1,6 +1,6 @@
 pyyaml == 6.0.2
-pytest == 8.3.4
+pytest == 8.3.5
 pytest-xdist == 3.6.1
 pytest-testinfra == 10.1.1
-tox == 4.23.2
+tox == 4.25.0
 pytest-clarity == 1.0.1

test/test_any_automated_install.py

@@ -89,10 +89,10 @@ def test_installPihole_fresh_install_readableFiles(host):
     export DEBIAN_FRONTEND=noninteractive
     umask 0027
     runUnattended=true
-    useUpdateVars=true
+    fresh_install=false
     source /opt/pihole/basic-install.sh > /dev/null
     runUnattended=true
-    useUpdateVars=true
+    fresh_install=false
     main
     /opt/pihole/pihole-FTL-prestart.sh
     """
@@ -119,11 +119,6 @@ def test_installPihole_fresh_install_readableFiles(host):
     assert exit_status_success == actual_rc
     check_leases = test_cmd.format("w", "/etc/pihole/dhcp.leases", piholeuser)
     actual_rc = host.run(check_leases).rc
-    # readable dns-servers.conf
-    assert exit_status_success == actual_rc
-    check_servers = test_cmd.format("r", "/etc/pihole/dns-servers.conf", piholeuser)
-    actual_rc = host.run(check_servers).rc
-    assert exit_status_success == actual_rc
     # readable install.log
     check_install = test_cmd.format("r", "/etc/pihole/install.log", piholeuser)
     actual_rc = host.run(check_install).rc
@@ -132,10 +127,6 @@ def test_installPihole_fresh_install_readableFiles(host):
     check_localversion = test_cmd.format("r", "/etc/pihole/versions", piholeuser)
     actual_rc = host.run(check_localversion).rc
     assert exit_status_success == actual_rc
-    # readable logrotate
-    check_logrotate = test_cmd.format("r", "/etc/pihole/logrotate", piholeuser)
-    actual_rc = host.run(check_logrotate).rc
-    assert exit_status_success == actual_rc
     # readable macvendor.db
     check_macvendor = test_cmd.format("r", "/etc/pihole/macvendor.db", piholeuser)
     actual_rc = host.run(check_macvendor).rc