Merge pull request 'Add trufflehog to testworkflow' (#17) from trufflehog into master

Reviewed-on: #17

.github/CODEOWNERS

@@ -0,0 +1,5 @@
+# see https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners#codeowners-syntax
+
+# These owners will be the default owners for everything in
+# the repo. Unless a later match takes precedence,
+*       @pi-hole/core-maintainers

.github/dependabot.yml

@@ -8,8 +8,10 @@ updates:
     time: "10:00"
   open-pull-requests-limit: 10
   target-branch: development
-  reviewers:
-    - "pi-hole/core-maintainers"
+  groups:
+    github-actions-dependencies:
+      patterns:
+        - "*"
 - package-ecosystem: pip
   directory: "/test"
   schedule:
@@ -18,5 +20,7 @@ updates:
     time: "10:00"
   open-pull-requests-limit: 10
   target-branch: development
-  reviewers:
-    - "pi-hole/core-maintainers"
+  groups:
+    python-dependencies:
+      patterns:
+        - "*"

.github/release.yml

@@ -2,6 +2,7 @@ changelog:
   exclude:
     labels:
       - internal
+      - dependencies
     authors:
       - dependabot
       - github-actions

.github/workflows/codeql-analysis.yml

@@ -25,16 +25,16 @@ jobs:
     steps:
     -
       name: Checkout repository
-      uses: actions/checkout@v4.2.2
+      uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #v6.0.2
     # Initializes the CodeQL tools for scanning.
     -
       name: Initialize CodeQL
-      uses: github/codeql-action/init@v3
+      uses: github/codeql-action/init@0d579ffd059c29b07949a3cce3983f0780820c98 #v4.32.6
       with:
         languages: 'python'
     -
       name: Autobuild
-      uses: github/codeql-action/autobuild@v3
+      uses: github/codeql-action/autobuild@0d579ffd059c29b07949a3cce3983f0780820c98 #v4.32.6
     -
       name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v3
+      uses: github/codeql-action/analyze@0d579ffd059c29b07949a3cce3983f0780820c98 #v4.32.6

.github/workflows/merge-conflict.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check if PRs are have merge conflicts
-        uses: eps1lon/actions-label-merge-conflict@v3.0.3
+        uses: eps1lon/actions-label-merge-conflict@1df065ebe6e3310545d4f4c4e862e43bdca146f0 #v3.0.3
         with:
           dirtyLabel: "PR: Merge Conflict"
           repoToken: "${{ secrets.GITHUB_TOKEN }}"

.github/workflows/stale.yml

@@ -17,14 +17,14 @@ jobs:
       issues: write
 
     steps:
-      - uses: actions/stale@v9.1.0
+      - uses: actions/stale@b5d41d4e1d5dceea10e7104786b73624c18a190f #v10.2.0
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           days-before-stale: 30
           days-before-close: 5
           stale-issue-message: 'This issue is stale because it has been open 30 days with no activity. Please comment or update this issue or it will be closed in 5 days.'
           stale-issue-label: '${{ env.stale_label }}'
-          exempt-issue-labels: 'Internal, Fixed in next release, Bug: Confirmed, Documentation Needed'
+          exempt-issue-labels: 'Internal, Fixed in next release, Bug: Confirmed, Documentation Needed, never-stale'
           exempt-all-issue-assignees: true
           operations-per-run: 300
           close-issue-reason: 'not_planned'
@@ -40,7 +40,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v4.2.2
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #v6.0.2
       - name: Remove 'stale' label
         run: gh issue edit ${{ github.event.issue.number }} --remove-label ${{ env.stale_label }}
         env:

.github/workflows/stale_pr.yml

@@ -17,7 +17,7 @@ jobs:
       pull-requests: write
 
     steps:
-      - uses: actions/stale@v9.1.0
+      - uses: actions/stale@b5d41d4e1d5dceea10e7104786b73624c18a190f #v10.2.0
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           # Do not automatically mark PR/issue as stale

.github/workflows/sync-back-to-dev.yml

@@ -33,7 +33,7 @@ jobs:
     name: Syncing branches
     steps:
       - name: Checkout
-        uses: actions/checkout@v4.2.2
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #v6.0.2
       - name: Opening pull request
         run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal'
         env:

.github/workflows/test.yml

@@ -18,7 +18,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4.2.2
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #v6.0.2
+        with:
+          fetch-depth: 0 # Differential ShellCheck requires full git history
 
       - name: Check scripts in repository are executable
         run: |
@@ -28,26 +30,30 @@ jobs:
           # If FAIL is 1 then we fail.
           [[ $FAIL == 1 ]] && exit 1 || echo "Scripts are executable!"
 
-      - name: Run shellcheck
-        uses: ludeeus/action-shellcheck@master
+      - name: Differential ShellCheck
+        uses: redhat-plumbers-in-action/differential-shellcheck@d965e66ec0b3b2f821f75c8eff9b12442d9a7d1e #v5.5.6
         with:
-          check_together: 'yes'
-          format: tty
-          severity: error
+          severity: warning
+          display-engine: sarif-fmt
+
+      - name: Secret Scanning with TruffleHog
+        uses: trufflesecurity/trufflehog@821e8b9e5cdf8dc484dd23e06f78941fcf6b9191 #v3.91.2
+        with:
+          extra_args: --results=verified,unknown
 
       - name: Spell-Checking
-        uses: codespell-project/actions-codespell@master
+        uses: codespell-project/actions-codespell@8f01853be192eb0f849a5c7d721450e7a467c579 #v2.2
         with:
           ignore_words_file: .codespellignore
 
       - name: Get editorconfig-checker
-        uses: editorconfig-checker/action-editorconfig-checker@main # tag v1.0.0 is really out of date
+        uses: editorconfig-checker/action-editorconfig-checker@4b6cd6190d435e7e084fb35e36a096e98506f7b9 #v2.1.0
 
       - name: Run editorconfig-checker
         run: editorconfig-checker
 
       - name: Check python code formatting with black
-        uses: psf/black@stable
+        uses: psf/black@c6755bb741b6481d6b3d3bb563c83fa060db96c9 #26.3.1
         with:
           src: "./test"
           options: "--check --diff --color"
@@ -63,23 +69,30 @@ jobs:
           [
             debian_11,
             debian_12,
+            debian_13,
             ubuntu_20,
             ubuntu_22,
             ubuntu_24,
             centos_9,
+            centos_10,
             fedora_40,
             fedora_41,
+            fedora_42,
+            fedora_43,
+            alpine_3_21,
+            alpine_3_22,
+            alpine_3_23,
           ]
     env:
       DISTRO: ${{matrix.distro}}
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4.2.2
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #v6.0.2
 
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v5.4.0
+      - name: Set up Python
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 #v6.2.0
         with:
-          python-version: "3.10"
+          python-version: "3.13"
 
       - name: Install wheel
         run:  pip install wheel

.gitignore

@@ -10,3 +10,6 @@ __pycache__
 .idea/
 *.iml
 .vscode/
+.venv/
+.fleet/
+.cache/

.shellcheckrc

@@ -0,0 +1,6 @@
+external-sources=true # allow shellcheck to read external sources
+disable=SC3043 #disable SC3043: In POSIX sh, local is undefined.
+enable=useless-use-of-cat # disabled by default as of shellcheck 0.11.0
+enable=avoid-negated-conditions # avoid-negated-conditions is optional as of shellcheck 0.11.0
+enable=require-variable-braces
+enable=deprecate-which

README.md

@@ -3,13 +3,9 @@
 #
 
 <p align="center">
-  <picture>
-    <source media="(prefers-color-scheme: dark)" srcset="https://pi-hole.github.io/graphics/Vortex/Vortex_Vertical_wordmark_darkmode.png">
-    <source media="(prefers-color-scheme: light)" srcset="https://pi-hole.github.io/graphics/Vortex/Vortex_Vertical_wordmark_lightmode.png">
-    <img src="https://pi-hole.github.io/graphics/Vortex/Vortex_Vertical_wordmark_lightmode.png" width="168" height="270" alt="Pi-hole website">
-  </picture>
-    <br>
-    <strong>Network-wide ad blocking via your own Linux hardware</strong>
+  <img src="https://raw.githubusercontent.com/pi-hole/graphics/refs/heads/master/Vortex/vortex_with_text.svg" alt="Pi-hole website" width="168" height="270">
+  <br>
+  <strong>Network-wide ad blocking via your own Linux hardware</strong>
 </p>
 
 <!-- markdownlint-enable MD033 -->
@@ -132,7 +128,10 @@ Some of the statistics you can integrate include:
 - Queries cached
 - Unique clients
 
-Access the API via [`telnet`](https://github.com/pi-hole/FTL), the Web (`admin/api.php`) and Command Line (`pihole -c -j`). You can find out [more details over here](https://discourse.pi-hole.net/t/pi-hole-api/1863).
+Access the API using:
+- your browser: http://pi.hole/api/docs
+- `curl`: `curl --connect-timeout 2 -ks "https://pi.hole/api/stats/summary" -H "Accept: application/json"`;
+- the command line - examples: `pihole api config/webserver/port` or `pihole api stats/summary`.
 
 ### The Command-Line Interface
 
@@ -140,7 +139,7 @@ The [pihole](https://docs.pi-hole.net/core/pihole-command/) command has all the
 
 Some notable features include:
 
-- [Whitelisting, Blacklisting, and Regex](https://docs.pi-hole.net/core/pihole-command/#whitelisting-blacklisting-and-regex)
+- [Allowlisting, Denylisting (fka Whitelisting, Blacklisting), and Regex](https://docs.pi-hole.net/core/pihole-command/#allowlisting-denylisting-and-regex)
 - [Debugging utility](https://docs.pi-hole.net/core/pihole-command/#debugger)
 - [Viewing the live log file](https://docs.pi-hole.net/core/pihole-command/#tail)
 - [Updating Ad Lists](https://docs.pi-hole.net/core/pihole-command/#gravity)

advanced/Scripts/COL_TABLE

@@ -1,10 +1,12 @@
+#!/usr/bin/env sh
+# shellcheck disable=SC2034  # Disable warning about unused variables
+
 # Determine if terminal is capable of showing colors
-if ([ -t 1 ] && [ $(tput colors) -ge 8 ]) || [ "${WEBCALL}" ]; then
+# When COL_TABLE is sourced via gravity invoked by FTL, FORCE_COLOR is set to true
+if { [ -t 1 ] && [ "$(tput colors)" -ge 8 ]; } || [ "${FORCE_COLOR}" ]; then
   # Bold and underline may not show up on all clients
   # If something MUST be emphasized, use both
   COL_BOLD=''
-  COL_ULINE=''
-
   COL_NC=''
   COL_GRAY=''
   COL_RED=''
@@ -16,8 +18,6 @@ if ([ -t 1 ] && [ $(tput colors) -ge 8 ]) || [ "${WEBCALL}" ]; then
 else
   # Provide empty variables for `set -u`
   COL_BOLD=""
-  COL_ULINE=""
-
   COL_NC=""
   COL_GRAY=""
   COL_RED=""
@@ -28,22 +28,8 @@ else
   COL_CYAN=""
 fi
 
-# Deprecated variables
-COL_WHITE="${COL_BOLD}"
-COL_BLACK="${COL_NC}"
-COL_LIGHT_BLUE="${COL_BLUE}"
-COL_LIGHT_GREEN="${COL_GREEN}"
-COL_LIGHT_CYAN="${COL_CYAN}"
-COL_LIGHT_RED="${COL_RED}"
-COL_URG_RED="${COL_RED}${COL_BOLD}${COL_ULINE}"
-COL_LIGHT_PURPLE="${COL_PURPLE}"
-COL_BROWN="${COL_YELLOW}"
-COL_LIGHT_GRAY="${COL_GRAY}"
-COL_DARK_GRAY="${COL_GRAY}"
-
 TICK="[${COL_GREEN}✓${COL_NC}]"
 CROSS="[${COL_RED}✗${COL_NC}]"
 INFO="[i]"
 QST="[?]"
-DONE="${COL_GREEN} done!${COL_NC}"
 OVER="\\r"

advanced/Scripts/api.sh

@@ -1,5 +1,4 @@
 #!/usr/bin/env sh
-# shellcheck disable=SC3043 #https://github.com/koalaman/shellcheck/wiki/SC3043#exceptions
 
 # Pi-hole: A black hole for Internet advertisements
 # (c) 2017 Pi-hole, LLC (https://pi-hole.net)
@@ -20,13 +19,20 @@
 
 TestAPIAvailability() {
 
+    local chaos_api_list authResponse authStatus authData apiAvailable DNSport
+
     # as we are running locally, we can get the port value from FTL directly
-    local chaos_api_list availabilityResponse
+    PI_HOLE_SCRIPT_DIR="/opt/pihole"
+    utilsfile="${PI_HOLE_SCRIPT_DIR}/utils.sh"
+    # shellcheck source=./advanced/Scripts/utils.sh
+    . "${utilsfile}"
+
+    DNSport=$(getFTLConfigValue dns.port)
 
     # Query the API URLs from FTL using CHAOS TXT local.api.ftl
     # The result is a space-separated enumeration of full URLs
     # e.g., "http://localhost:80/api/" "https://localhost:443/api/"
-    chaos_api_list="$(dig +short chaos txt local.api.ftl @127.0.0.1)"
+    chaos_api_list="$(dig +short -p "${DNSport}" chaos txt local.api.ftl @127.0.0.1)"
 
     # If the query was not successful, the variable is empty
     if [ -z "${chaos_api_list}" ]; then
@@ -34,6 +40,12 @@ TestAPIAvailability() {
         exit 1
     fi
 
+    # If an error occurred, the variable starts with ;;
+    if [ "${chaos_api_list#;;}" != "${chaos_api_list}" ]; then
+        echo "Communication error. Is FTL running?"
+        exit 1
+    fi
+
     # Iterate over space-separated list of URLs
     while [ -n "${chaos_api_list}" ]; do
         # Get the first URL
@@ -42,39 +54,50 @@ TestAPIAvailability() {
         API_URL="${API_URL%\"}"
         API_URL="${API_URL#\"}"
 
-        # Test if the API is available at this URL
-        availabilityResponse=$(curl -skS -o /dev/null -w "%{http_code}" "${API_URL}auth")
+        # Test if the API is available at this URL, include delimiter for ease in splitting payload
+        authResponse=$(curl --connect-timeout 2 -skS -w ">>%{http_code}" "${API_URL}auth")
+
+        # authStatus is the response http_code, eg. 200, 401.
+        # Shell parameter expansion, remove everything up to and including the >> delim
+        authStatus=${authResponse#*>>}
+        # data is everything from response
+        # Shell parameter expansion, remove the >> delim and everything after
+        authData=${authResponse%>>*}
 
         # Test if http status code was 200 (OK) or 401 (authentication required)
-        if [ ! "${availabilityResponse}" = 200 ] && [ ! "${availabilityResponse}" = 401 ]; then
-            # API is not available at this port/protocol combination
-            API_PORT=""
-        else
-            # API is available at this URL combination
-
-            if [ "${availabilityResponse}" = 200 ]; then
-                # API is available without authentication
-                needAuth=false
-            fi
-
+        if [ "${authStatus}" = 200 ]; then
+            # API is available without authentication
+            apiAvailable=true
+            needAuth=false
             break
-        fi
 
-        # Remove the first URL from the list
-        local last_api_list
-        last_api_list="${chaos_api_list}"
-        chaos_api_list="${chaos_api_list#* }"
+        elif [ "${authStatus}" = 401 ]; then
+            # API is available with authentication
+            apiAvailable=true
+            needAuth=true
+            # Check if 2FA is required
+            needTOTP=$(echo "${authData}"| jq --raw-output .session.totp 2>/dev/null)
+            break
 
-        # If the list did not change, we are at the last element
-        if [ "${last_api_list}" = "${chaos_api_list}" ]; then
-            # Remove the last element
-            chaos_api_list=""
+        else
+            # API is not available at this port/protocol combination
+            apiAvailable=false
+            # Remove the first URL from the list
+            local last_api_list
+            last_api_list="${chaos_api_list}"
+            chaos_api_list="${chaos_api_list#* }"
+
+            # If the list did not change, we are at the last element
+            if [ "${last_api_list}" = "${chaos_api_list}" ]; then
+                # Remove the last element
+                chaos_api_list=""
+            fi
         fi
     done
 
-    # if API_PORT is empty, no working API port was found
-    if [ -n "${API_PORT}" ]; then
-        echo "API not available at: ${API_URL}"
+    # if apiAvailable is false, no working API was found
+    if [ "${apiAvailable}" = false ]; then
+        echo "API not available. Please check FTL.log"
         echo "Exiting."
         exit 1
     fi
@@ -102,22 +125,58 @@ LoginAPI() {
             echo "API Authentication: Trying to use CLI password"
         fi
 
-        # Try to authenticate using the CLI password
-        Authentication "${1}"
-
+        # If we can read the CLI password, we can skip 2FA even when it's required otherwise
+        needTOTP=false
     elif [ "${1}" = "verbose" ]; then
         echo "API Authentication: CLI password not available"
     fi
 
+    if [ -z "${password}" ]; then
+        # no password read from CLI file
+        echo "Please enter your password:"
+        # secretly read the password
+        secretRead; printf '\n'
+    fi
 
+    if [ "${needTOTP}" = true ]; then
+        # 2FA required
+        echo "Please enter the correct second factor."
+        echo "(Can be any number if you used the app password)"
+        read -r totp
+    fi
 
-    # If this did not work, ask the user for the password
-    while [ "${validSession}" = false ] || [ -z "${validSession}" ] ; do
-        echo "Authentication failed. Please enter your Pi-hole password"
+    # Try to authenticate using the supplied password (CLI file or user input) and TOTP
+    Authentication "${1}"
+
+    # Try to login again until the session is valid
+    while [ ! "${validSession}" = true ]  ; do
+
+        # Print the error message if there is one
+        if  [ ! "${sessionError}" = "null"  ] && [ "${1}" = "verbose" ]; then
+            echo "Error: ${sessionError}"
+        fi
+        # Print the session message if there is one
+        if  [ ! "${sessionMessage}" = "null" ] && [ "${1}" = "verbose" ]; then
+            echo "Error: ${sessionMessage}"
+        fi
+
+        if  [ "${1}" = "verbose" ]; then
+            # If we are not in verbose mode, no need to print the error message again
+            echo "Please enter your Pi-hole password"
+        else
+
+            echo "Authentication failed. Please enter your Pi-hole password"
+        fi
 
         # secretly read the password
         secretRead; printf '\n'
 
+        if [ "${needTOTP}" = true ]; then
+            echo "Please enter the correct second factor:"
+            echo "(Can be any number if you used the app password)"
+            read -r totp
+        fi
+
         # Try to authenticate again
         Authentication "${1}"
     done
@@ -125,23 +184,34 @@ LoginAPI() {
 }
 
 Authentication() {
-  sessionResponse="$(curl -skS -X POST "${API_URL}auth" --user-agent "Pi-hole cli " --data "{\"password\":\"${password}\"}" )"
+    sessionResponse="$(curl --connect-timeout 2 -skS -X POST "${API_URL}auth" --user-agent "Pi-hole cli" --data "{\"password\":\"${password}\", \"totp\":${totp:-null}}" )"
 
-  if [ -z "${sessionResponse}" ]; then
-    echo "No response from FTL server. Please check connectivity"
-    exit 1
-  fi
-  # obtain validity and session ID from session response
-  validSession=$(echo "${sessionResponse}"| jq .session.valid 2>/dev/null)
-  SID=$(echo "${sessionResponse}"| jq --raw-output .session.sid 2>/dev/null)
-
-  if [ "${1}" = "verbose" ]; then
-    if [ "${validSession}" = true ]; then
-      echo "API Authentication: ${COL_GREEN}Success${COL_NC}"
-    else
-      echo "API Authentication: ${COL_RED}Failed${COL_NC}"
+    if [ -z "${sessionResponse}" ]; then
+        echo "No response from FTL server. Please check connectivity"
+        exit 1
+    fi
+
+    # obtain validity, session ID, sessionMessage and error message from
+    # session response, apply default values if none returned
+    result=$(echo "${sessionResponse}" | jq -r '
+        (.session.valid // false),
+        (.session.sid // null),
+        (.session.message // null),
+        (.error.message // null)
+    ' 2>/dev/null)
+
+    validSession=$(echo "${result}" | sed -n '1p')
+    SID=$(echo "${result}" | sed -n '2p')
+    sessionMessage=$(echo "${result}" | sed -n '3p')
+    sessionError=$(echo "${result}" | sed -n '4p')
+
+    if [ "${1}" = "verbose" ]; then
+        if [ "${validSession}" = true ]; then
+            echo "API Authentication: ${COL_GREEN}Success${COL_NC}"
+        else
+            echo "API Authentication: ${COL_RED}Failed${COL_NC}"
+        fi
     fi
-  fi
 }
 
 LogoutAPI() {
@@ -179,7 +249,7 @@ GetFTLData() {
     # return only the data
     if [ "${status}" = 200 ]; then
         # response OK
-        echo "${data}"
+        printf %s "${data}"
     else
         # connection lost
         echo "${status}"
@@ -253,14 +323,23 @@ secretRead() {
 }
 
 apiFunc() {
-  local data response status status_col
+  local data response status status_col verbosity
+
+  # Define if the output will be silent (default) or verbose
+  verbosity="silent"
+  if [ "$1" = "verbose" ]; then
+    verbosity="verbose"
+    shift
+  fi
 
   # Authenticate with the API
-  LoginAPI verbose
-  echo ""
+  LoginAPI "${verbosity}"
 
-  echo "Requesting: ${COL_PURPLE}GET ${COL_CYAN}${API_URL}${COL_YELLOW}$1${COL_NC}"
-  echo ""
+  if [ "${verbosity}" = "verbose" ]; then
+    echo ""
+    echo "Requesting: ${COL_PURPLE}GET ${COL_CYAN}${API_URL}${COL_YELLOW}$1${COL_NC}"
+    echo ""
+  fi
 
   # Get the data from the API
   response=$(GetFTLData "$1" raw)
@@ -277,17 +356,21 @@ apiFunc() {
   else
     status_col="${COL_RED}"
   fi
-  echo "Status: ${status_col}${status}${COL_NC}"
+
+  # Only print the status in verbose mode or if the status is not 200
+  if [ "${verbosity}" = "verbose" ] || [ "${status}" != 200 ]; then
+    echo "Status: ${status_col}${status}${COL_NC}"
+  fi
 
   # Output the data. Format it with jq if available and data is actually JSON.
   # Otherwise just print it
-  echo "Data:"
-  if command -v jq >/dev/null && echo "${data}" | jq . >/dev/null 2>&1; then
-    echo "${data}" | jq .
-  else
-    echo "${data}"
+  if [ "${verbosity}" = "verbose" ]; then
+    echo "Data:"
   fi
+  # Attempt to print the data with jq, if it is not valid JSON, or not installed
+  # then print the plain text.
+  echo "${data}" | jq . 2>/dev/null || echo "${data}"
 
   # Delete the session
-  LogoutAPI verbose
+  LogoutAPI "${verbosity}"
 }

advanced/Scripts/database_migration/gravity-db.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# shellcheck disable=SC1090
+
 
 # Pi-hole: A black hole for Internet advertisements
 # (c) 2019 Pi-hole, LLC (https://pi-hole.net)
@@ -13,9 +13,8 @@
 readonly scriptPath="/etc/.pihole/advanced/Scripts/database_migration/gravity"
 
 upgrade_gravityDB(){
-    local database piholeDir version
+    local database version
     database="${1}"
-    piholeDir="${2}"
 
     # Exit early if the database does not exist (e.g. in CI tests)
     if [[ ! -f "${database}" ]]; then
@@ -151,4 +150,10 @@ upgrade_gravityDB(){
         pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/18_to_19.sql"
         version=19
     fi
+    if [[ "$version" == "19" ]]; then
+        # Update views to use new allowlist/denylist names
+        echo -e "  ${INFO} Upgrading gravity database from version 19 to 20"
+        pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/19_to_20.sql"
+        version=20
+    fi
 }

advanced/Scripts/database_migration/gravity/19_to_20.sql

@@ -0,0 +1,43 @@
+.timeout 30000
+
+BEGIN TRANSACTION;
+
+DROP VIEW vw_whitelist;
+CREATE VIEW vw_allowlist AS SELECT domain, domainlist.id AS id, domainlist_by_group.group_id AS group_id
+    FROM domainlist
+    LEFT JOIN domainlist_by_group ON domainlist_by_group.domainlist_id = domainlist.id
+    LEFT JOIN "group" ON "group".id = domainlist_by_group.group_id
+    WHERE domainlist.enabled = 1 AND (domainlist_by_group.group_id IS NULL OR "group".enabled = 1)
+    AND domainlist.type = 0
+    ORDER BY domainlist.id;
+
+DROP VIEW vw_blacklist;
+CREATE VIEW vw_denylist AS SELECT domain, domainlist.id AS id, domainlist_by_group.group_id AS group_id
+    FROM domainlist
+    LEFT JOIN domainlist_by_group ON domainlist_by_group.domainlist_id = domainlist.id
+    LEFT JOIN "group" ON "group".id = domainlist_by_group.group_id
+    WHERE domainlist.enabled = 1 AND (domainlist_by_group.group_id IS NULL OR "group".enabled = 1)
+    AND domainlist.type = 1
+    ORDER BY domainlist.id;
+
+DROP VIEW vw_regex_whitelist;
+CREATE VIEW vw_regex_allowlist AS SELECT domain, domainlist.id AS id, domainlist_by_group.group_id AS group_id
+    FROM domainlist
+    LEFT JOIN domainlist_by_group ON domainlist_by_group.domainlist_id = domainlist.id
+    LEFT JOIN "group" ON "group".id = domainlist_by_group.group_id
+    WHERE domainlist.enabled = 1 AND (domainlist_by_group.group_id IS NULL OR "group".enabled = 1)
+    AND domainlist.type = 2
+    ORDER BY domainlist.id;
+
+DROP VIEW vw_regex_blacklist;
+CREATE VIEW vw_regex_denylist AS SELECT domain, domainlist.id AS id, domainlist_by_group.group_id AS group_id
+    FROM domainlist
+    LEFT JOIN domainlist_by_group ON domainlist_by_group.domainlist_id = domainlist.id
+    LEFT JOIN "group" ON "group".id = domainlist_by_group.group_id
+    WHERE domainlist.enabled = 1 AND (domainlist_by_group.group_id IS NULL OR "group".enabled = 1)
+    AND domainlist.type = 3
+    ORDER BY domainlist.id;
+
+UPDATE info SET value = 20 WHERE property = 'version';
+
+COMMIT;

advanced/Scripts/list.sh

@@ -1,5 +1,4 @@
 #!/usr/bin/env bash
-# shellcheck disable=SC1090
 
 # Pi-hole: A black hole for Internet advertisements
 # (c) 2017 Pi-hole, LLC (https://pi-hole.net)
@@ -10,11 +9,13 @@
 # This file is copyright under the latest version of the EUPL.
 # Please see LICENSE file for your rights under this license.
 
-readonly PI_HOLE_SCRIPT_DIR="/opt/pihole"
-readonly utilsfile="${PI_HOLE_SCRIPT_DIR}/utils.sh"
+PI_HOLE_SCRIPT_DIR="/opt/pihole"
+utilsfile="${PI_HOLE_SCRIPT_DIR}/utils.sh"
+# shellcheck source="./advanced/Scripts/utils.sh"
 source "${utilsfile}"
 
-readonly apifile="${PI_HOLE_SCRIPT_DIR}/api.sh"
+apifile="${PI_HOLE_SCRIPT_DIR}/api.sh"
+# shellcheck source="./advanced/Scripts/api.sh"
 source "${apifile}"
 
 # Determine database location
@@ -39,6 +40,7 @@ typeId=""
 comment=""
 
 colfile="/opt/pihole/COL_TABLE"
+# shellcheck source="./advanced/Scripts/COL_TABLE"
 source ${colfile}
 
 helpFunc() {

advanced/Scripts/piholeARPTable.sh

@@ -1,82 +0,0 @@
-#!/usr/bin/env bash
-# shellcheck disable=SC1090
-
-# Pi-hole: A black hole for Internet advertisements
-# (c) 2019 Pi-hole, LLC (https://pi-hole.net)
-# Network-wide ad blocking via your own hardware.
-#
-# ARP table interaction
-#
-# This file is copyright under the latest version of the EUPL.
-# Please see LICENSE file for your rights under this license.
-
-coltable="/opt/pihole/COL_TABLE"
-if [[ -f ${coltable} ]]; then
-    source ${coltable}
-fi
-
-readonly PI_HOLE_SCRIPT_DIR="/opt/pihole"
-utilsfile="${PI_HOLE_SCRIPT_DIR}/utils.sh"
-source "${utilsfile}"
-
-# Determine database location
-DBFILE=$(getFTLConfigValue "files.database")
-if [ -z "$DBFILE" ]; then
-    DBFILE="/etc/pihole/pihole-FTL.db"
-fi
-
-flushARP(){
-    local output
-    if [[ "${args[1]}" != "quiet" ]]; then
-        echo -ne "  ${INFO} Flushing network table ..."
-    fi
-
-    # Stop FTL to prevent database access
-    if ! output=$(service pihole-FTL stop 2>&1); then
-        echo -e "${OVER}  ${CROSS} Failed to stop FTL"
-        echo "  Output: ${output}"
-        return 1
-    fi
-
-    # Truncate network_addresses table in pihole-FTL.db
-    # This needs to be done before we can truncate the network table due to
-    # foreign key constraints
-    if ! output=$(pihole-FTL sqlite3 -ni "${DBFILE}" "DELETE FROM network_addresses" 2>&1); then
-        echo -e "${OVER}  ${CROSS} Failed to truncate network_addresses table"
-        echo "  Database location: ${DBFILE}"
-        echo "  Output: ${output}"
-        return 1
-    fi
-
-    # Truncate network table in pihole-FTL.db
-    if ! output=$(pihole-FTL sqlite3 -ni "${DBFILE}" "DELETE FROM network" 2>&1); then
-        echo -e "${OVER}  ${CROSS} Failed to truncate network table"
-        echo "  Database location: ${DBFILE}"
-        echo "  Output: ${output}"
-        return 1
-    fi
-
-    # Flush ARP cache of the host
-    if ! output=$(ip -s -s neigh flush all 2>&1); then
-        echo -e "${OVER}  ${CROSS} Failed to flush ARP cache"
-        echo "  Output: ${output}"
-        return 1
-    fi
-
-    # Start FTL again
-    if ! output=$(service pihole-FTL restart 2>&1); then
-        echo -e "${OVER}  ${CROSS} Failed to restart FTL"
-        echo "  Output: ${output}"
-        return 1
-    fi
-
-    if [[ "${args[1]}" != "quiet" ]]; then
-        echo -e "${OVER}  ${TICK} Flushed network table"
-    fi
-}
-
-args=("$@")
-
-case "${args[0]}" in
-    "arpflush"            ) flushARP;;
-esac

advanced/Scripts/piholeCheckout.sh

@@ -10,6 +10,7 @@
 
 readonly PI_HOLE_FILES_DIR="/etc/.pihole"
 SKIP_INSTALL="true"
+# shellcheck source="./automated install/basic-install.sh"
 source "${PI_HOLE_FILES_DIR}/automated install/basic-install.sh"
 
 # webInterfaceGitUrl set in basic-install.sh
@@ -25,7 +26,7 @@ source "${PI_HOLE_FILES_DIR}/automated install/basic-install.sh"
 warning1() {
     echo "  Please note that changing branches severely alters your Pi-hole subsystems"
     echo "  Features that work on the master branch, may not on a development branch"
-    echo -e "  ${COL_LIGHT_RED}This feature is NOT supported unless a Pi-hole developer explicitly asks!${COL_NC}"
+    echo -e "  ${COL_RED}This feature is NOT supported unless a Pi-hole developer explicitly asks!${COL_NC}"
     read -r -p "  Have you read and understood this? [y/N] " response
     case "${response}" in
         [yY][eE][sS]|[yY])
@@ -40,6 +41,22 @@ warning1() {
 }
 
 checkout() {
+
+    local skipFTL additionalFlag
+    skipFTL=false
+    # Check arguments
+    for var in "$@"; do
+        case "$var" in
+            "--skipFTL") skipFTL=true ;;
+        esac
+    done
+
+    if [ "${skipFTL}" == true ]; then
+        additionalFlag="--skipFTL"
+    else
+        additionalFlag=""
+    fi
+
     local corebranches
     local webbranches
 
@@ -54,19 +71,19 @@ checkout() {
 
     # This is unlikely
     if ! is_repo "${PI_HOLE_FILES_DIR}" ; then
-        echo -e "  ${COL_LIGHT_RED}Error: Core Pi-hole repo is missing from system!"
+        echo -e "  ${COL_RED}Error: Core Pi-hole repo is missing from system!"
         echo -e "  Please re-run install script from https://github.com/pi-hole/pi-hole${COL_NC}"
         exit 1;
     fi
 
     if ! is_repo "${webInterfaceDir}" ; then
-        echo -e "  ${COL_LIGHT_RED}Error: Web Admin repo is missing from system!"
+        echo -e "  ${COL_RED}Error: Web Admin repo is missing from system!"
         echo -e "  Please re-run install script from https://github.com/pi-hole/pi-hole${COL_NC}"
         exit 1;
     fi
 
     if [[ -z "${1}" ]]; then
-        echo -e "  ${COL_LIGHT_RED}Invalid option${COL_NC}"
+        echo -e "  ${COL_RED}Invalid option${COL_NC}"
         echo -e "  Try 'pihole checkout --help' for more information."
         exit 1
     fi
@@ -109,7 +126,7 @@ checkout() {
             echo -e "${OVER}  ${CROSS} $str"
             exit 1
         fi
-        corebranches=($(get_available_branches "${PI_HOLE_FILES_DIR}"))
+        mapfile -t corebranches < <(get_available_branches "${PI_HOLE_FILES_DIR}")
 
         if [[ "${corebranches[*]}" == *"master"* ]]; then
             echo -e "${OVER}  ${TICK} $str"
@@ -136,7 +153,7 @@ checkout() {
             echo -e "${OVER}  ${CROSS} $str"
             exit 1
         fi
-        webbranches=($(get_available_branches "${webInterfaceDir}"))
+        mapfile -t webbranches < <(get_available_branches "${webInterfaceDir}")
 
         if [[ "${webbranches[*]}" == *"master"* ]]; then
             echo -e "${OVER}  ${TICK} $str"
@@ -167,7 +184,7 @@ checkout() {
 
         # Check if requested branch is available
         echo -e "  ${INFO} Checking for availability of branch ${COL_CYAN}${2}${COL_NC} on GitHub"
-        ftlbranches=( $(git ls-remote https://github.com/pi-hole/ftl | grep "refs/heads" | cut -d'/' -f3- -) )
+        mapfile -t ftlbranches < <(git ls-remote https://github.com/pi-hole/ftl | grep "refs/heads" | cut -d'/' -f3- -)
         # If returned array is empty -> connectivity issue
         if [[ ${#ftlbranches[@]} -eq 0 ]]; then
             echo -e "  ${CROSS} Unable to fetch branches from GitHub. Please check your Internet connection and try again later."
@@ -209,13 +226,15 @@ checkout() {
             # Update local and remote versions via updatechecker
             /opt/pihole/updatecheck.sh
         else
-            if [ $? -eq 1 ]; then
+            local status
+            status=$?
+            if [ $status -eq 1 ]; then
                 # Binary for requested branch is not available, may still be
                 # int he process of being built or CI build job failed
-                printf "  %b Binary for requested branch is not available, please try again later.\\n" ${CROSS}
+                printf "  %b Binary for requested branch is not available, please try again later.\\n" "${CROSS}"
                 printf "      If the issue persists, please contact Pi-hole Support and ask them to re-generate the binary.\\n"
                 exit 1
-            elif [ $? -eq 2 ]; then
+            elif [ $status -eq 2 ]; then
                 printf "  %b Unable to download from ftl.pi-hole.net. Please check your Internet connection and try again later.\\n" "${CROSS}"
                 exit 1
             else
@@ -232,10 +251,10 @@ checkout() {
     # Force updating everything
     if [[  ! "${1}" == "web" && ! "${1}" == "ftl" ]]; then
         echo -e "  ${INFO} Running installer to upgrade your installation"
-        if "${PI_HOLE_FILES_DIR}/automated install/basic-install.sh" --unattended; then
+        if "${PI_HOLE_FILES_DIR}/automated install/basic-install.sh" --unattended ${additionalFlag}; then
             exit 0
         else
-            echo -e "  ${COL_LIGHT_RED} Error: Unable to complete update, please contact support${COL_NC}"
+            echo -e "  ${COL_RED} Error: Unable to complete update, please contact support${COL_NC}"
             exit 1
         fi
     fi

advanced/Scripts/piholeDebug.sh

@@ -8,7 +8,6 @@
 # This file is copyright under the latest version of the EUPL.
 # Please see LICENSE file for your rights under this license.
 
-# shellcheck source=/dev/null
 
 # -e option instructs bash to immediately exit if any command [1] has a non-zero exit status
 # -u a reference to any variable you haven't previously defined
@@ -27,6 +26,7 @@ PIHOLE_COLTABLE_FILE="${PIHOLE_SCRIPTS_DIRECTORY}/COL_TABLE"
 
 # These provide the colors we need for making the log more readable
 if [[ -f ${PIHOLE_COLTABLE_FILE} ]]; then
+# shellcheck source=./advanced/Scripts/COL_TABLE
     source ${PIHOLE_COLTABLE_FILE}
 else
     COL_NC='\e[0m' # No Color
@@ -41,7 +41,7 @@ else
     #OVER="\r\033[K"
 fi
 
-# shellcheck disable=SC1091
+# shellcheck source=/dev/null
 . /etc/pihole/versions
 
 # Read the value of an FTL config key. The value is printed to stdout.
@@ -169,7 +169,7 @@ initialize_debug() {
     # Display that the debug process is beginning
     log_write "${COL_PURPLE}*** [ INITIALIZING ]${COL_NC}"
     # Timestamp the start of the log
-    log_write "${INFO} $(date "+%Y-%m-%d:%H:%M:%S") debug log has been initialized."
+    log_write "${INFO} $(date "+%Y-%m-%d %H:%M:%S") debug log has been initialized."
     # Uptime of the system
     # credits to https://stackoverflow.com/questions/28353409/bash-format-uptime-to-show-days-hours-minutes
     system_uptime=$(uptime | awk -F'( |,|:)+' '{if ($7=="min") m=$6; else {if ($7~/^day/){if ($9=="min") {d=$6;m=$8} else {d=$6;h=$8;m=$9}} else {h=$6;m=$7}}} {print d+0,"days,",h+0,"hours,",m+0,"minutes"}')
@@ -213,7 +213,7 @@ compare_local_version_to_git_version() {
             local local_status
             local_status=$(git status -s)
             # echo this information out to the user in a nice format
-            if [ ${local_version} ]; then
+            if [ "${local_version}" ]; then
               log_write "${TICK} Version: ${local_version}"
             elif [ -n "${DOCKER_VERSION}" ]; then
               log_write "${TICK} Version: Pi-hole Docker Container ${COL_BOLD}${DOCKER_VERSION}${COL_NC}"
@@ -296,91 +296,12 @@ check_component_versions() {
     check_ftl_version
 }
 
-os_check() {
-    # This function gets a list of supported OS versions from a TXT record at versions.pi-hole.net
-    # and determines whether or not the script is running on one of those systems
-    local remote_os_domain valid_os valid_version detected_os detected_version cmdResult digReturnCode response
-    remote_os_domain=${OS_CHECK_DOMAIN_NAME:-"versions.pi-hole.net"}
-
-    detected_os=$(grep "\bID\b" /etc/os-release | cut -d '=' -f2 | tr -d '"')
-    detected_version=$(grep VERSION_ID /etc/os-release | cut -d '=' -f2 | tr -d '"')
-
-    cmdResult="$(dig -4 +short -t txt "${remote_os_domain}" @ns1.pi-hole.net 2>&1; echo $?)"
-    #Get the return code of the previous command (last line)
-    digReturnCode="${cmdResult##*$'\n'}"
-
-    # Extract dig response
-    response="${cmdResult%%$'\n'*}"
-
-    if [ "${digReturnCode}" -ne 0 ]; then
-        log_write "${INFO} Distro: ${detected_os^}"
-        log_write "${INFO} Version: ${detected_version}"
-        log_write "${CROSS} dig IPv4 return code: ${COL_RED}${digReturnCode}${COL_NC}"
-        log_write "${CROSS} dig response: ${response}"
-        log_write "${INFO} Retrying via IPv6"
-
-        cmdResult="$(dig -6 +short -t txt "${remote_os_domain}" @ns1.pi-hole.net 2>&1; echo $?)"
-        #Get the return code of the previous command (last line)
-        digReturnCode="${cmdResult##*$'\n'}"
-
-        # Extract dig response
-        response="${cmdResult%%$'\n'*}"
-    fi
-    # If also no success via IPv6
-    if [ "${digReturnCode}" -ne 0 ]; then
-        log_write "${CROSS} dig IPv6 return code: ${COL_RED}${digReturnCode}${COL_NC}"
-        log_write "${CROSS} dig response: ${response}"
-        log_write "${CROSS} Error: ${COL_RED}dig command failed - Unable to check OS${COL_NC}"
-    else
-        IFS=" " read -r -a supportedOS < <(echo "${response}" | tr -d '"')
-        for distro_and_versions in "${supportedOS[@]}"
-        do
-            distro_part="${distro_and_versions%%=*}"
-            versions_part="${distro_and_versions##*=}"
-
-            if [[ "${detected_os^^}" =~ ${distro_part^^} ]]; then
-                valid_os=true
-                IFS="," read -r -a supportedVer <<<"${versions_part}"
-                for version in "${supportedVer[@]}"
-                do
-                    if [[ "${detected_version}" =~ $version ]]; then
-                        valid_version=true
-                        break
-                    fi
-                done
-                break
-            fi
-        done
-
-        # If it is a docker container, we can assume the OS is supported
-        [ -n "${DOCKER_VERSION}" ] && valid_os=true && valid_version=true
-
-        local finalmsg
-        if [ "$valid_os" = true ]; then
-            log_write "${TICK} Distro:  ${COL_GREEN}${detected_os^}${COL_NC}"
-
-            if [ "$valid_version" = true ]; then
-                log_write "${TICK} Version: ${COL_GREEN}${detected_version}${COL_NC}"
-                finalmsg="${TICK} ${COL_GREEN}Distro and version supported${COL_NC}"
-            else
-                log_write "${CROSS} Version: ${COL_RED}${detected_version}${COL_NC}"
-                finalmsg="${CROSS} Error: ${COL_RED}${detected_os^} is supported but version ${detected_version} is currently unsupported ${COL_NC}(${FAQ_HARDWARE_REQUIREMENTS})${COL_NC}"
-            fi
-        else
-            log_write "${CROSS} Distro:  ${COL_RED}${detected_os^}${COL_NC}"
-            finalmsg="${CROSS} Error: ${COL_RED}${detected_os^} is not a supported distro ${COL_NC}(${FAQ_HARDWARE_REQUIREMENTS})${COL_NC}"
-        fi
-
-        # Print dig response and the final check result
-        log_write "${TICK} dig return code: ${COL_GREEN}${digReturnCode}${COL_NC}"
-        log_write "${INFO} dig response: ${response}"
-        log_write "${finalmsg}"
-    fi
-}
-
 diagnose_operating_system() {
     # error message in a variable so we can easily modify it later (or reuse it)
     local error_msg="Distribution unknown -- most likely you are on an unsupported platform and may run into issues."
+    local detected_os
+    local detected_version
+
     # Display the current test that is running
     echo_current_diagnostic "Operating system"
 
@@ -389,8 +310,13 @@ diagnose_operating_system() {
 
     # If there is a /etc/*release file, it's probably a supported operating system, so we can
     if ls /etc/*release 1> /dev/null 2>&1; then
-        # display the attributes to the user from the function made earlier
-        os_check
+        # display the attributes to the user
+
+        detected_os=$(grep "\bID\b" /etc/os-release | cut -d '=' -f2 | tr -d '"')
+        detected_version=$(grep VERSION_ID /etc/os-release | cut -d '=' -f2 | tr -d '"')
+
+        log_write "${INFO} Distro: ${detected_os^}"
+        log_write "${INFO} Version: ${detected_version}"
     else
         # If it doesn't exist, it's not a system we currently support and link to FAQ
         log_write "${CROSS} ${COL_RED}${error_msg}${COL_NC} (${FAQ_HARDWARE_REQUIREMENTS})"
@@ -441,7 +367,7 @@ check_firewalld() {
             # test common required service ports
             local firewalld_enabled_services
             firewalld_enabled_services=$(firewall-cmd --list-services)
-            local firewalld_expected_services=("http" "dns" "dhcp" "dhcpv6")
+            local firewalld_expected_services=("http" "https" "dns" "dhcp" "dhcpv6" "ntp")
             for i in "${firewalld_expected_services[@]}"; do
                 if [[ "${firewalld_enabled_services}" =~ ${i} ]]; then
                     log_write "${TICK} ${COL_GREEN}  Allow Service: ${i}${COL_NC}";
@@ -449,30 +375,6 @@ check_firewalld() {
                     log_write "${CROSS} ${COL_RED}  Allow Service: ${i}${COL_NC} (${FAQ_HARDWARE_REQUIREMENTS_FIREWALLD})"
                 fi
             done
-            # check for custom FTL FirewallD zone
-            local firewalld_zones
-            firewalld_zones=$(firewall-cmd --get-zones)
-            if [[ "${firewalld_zones}" =~ "ftl" ]]; then
-                log_write "${TICK} ${COL_GREEN}FTL Custom Zone Detected${COL_NC}";
-                # check FTL custom zone interface: lo
-                local firewalld_ftl_zone_interfaces
-                firewalld_ftl_zone_interfaces=$(firewall-cmd --zone=ftl --list-interfaces)
-                if [[ "${firewalld_ftl_zone_interfaces}" =~ "lo" ]]; then
-                    log_write "${TICK} ${COL_GREEN}  Local Interface Detected${COL_NC}";
-                else
-                    log_write "${CROSS} ${COL_RED}  Local Interface Not Detected${COL_NC} (${FAQ_HARDWARE_REQUIREMENTS_FIREWALLD})"
-                fi
-                # check FTL custom zone port: 4711
-                local firewalld_ftl_zone_ports
-                firewalld_ftl_zone_ports=$(firewall-cmd --zone=ftl --list-ports)
-                if [[ "${firewalld_ftl_zone_ports}" =~ "4711/tcp" ]]; then
-                    log_write "${TICK} ${COL_GREEN}  FTL Port 4711/tcp Detected${COL_NC}";
-                else
-                    log_write "${CROSS} ${COL_RED}  FTL Port 4711/tcp Not Detected${COL_NC} (${FAQ_HARDWARE_REQUIREMENTS_FIREWALLD})"
-                fi
-            else
-                log_write "${CROSS} ${COL_RED}FTL Custom Zone Not Detected${COL_NC} (${FAQ_HARDWARE_REQUIREMENTS_FIREWALLD})"
-            fi
         fi
     else
         log_write "${TICK} ${COL_GREEN}Firewalld service not detected${COL_NC}";
@@ -488,7 +390,9 @@ run_and_print_command() {
     local output
     output=$(${cmd} 2>&1)
     # If the command was successful,
-    if [[ $? -eq 0 ]]; then
+    local return_code
+    return_code=$?
+    if [[ "${return_code}" -eq 0 ]]; then
         # show the output
         log_write "${output}"
     else
@@ -569,16 +473,25 @@ ping_gateway() {
     ping_ipv4_or_ipv6 "${protocol}"
     # Check if we are using IPv4 or IPv6
     # Find the default gateways using IPv4 or IPv6
-    local gateway gateway_addr gateway_iface
+    local gateway gateway_addr gateway_iface default_route
 
     log_write "${INFO} Default IPv${protocol} gateway(s):"
 
-    while IFS= read -r gateway; do
-        log_write "     $(cut -d ' ' -f 3 <<< "${gateway}")%$(cut -d ' ' -f 5 <<< "${gateway}")"
-    done < <(ip -"${protocol}" route | grep default)
+    while IFS= read -r default_route; do
+        gateway_addr=$(jq -r '.gateway' <<< "${default_route}")
+        gateway_iface=$(jq -r '.dev' <<< "${default_route}")
+        log_write "     ${gateway_addr}%${gateway_iface}"
+    done < <(ip -j -"${protocol}" route | jq -c '.[] | select(.dst == "default")')
+
+    # Find the first default route
+    default_route=$(ip -j -"${protocol}" route show default)
+    if echo "$default_route" | grep 'gateway' | grep -q 'dev'; then
+        gateway_addr=$(echo "$default_route" | jq -r -c '.[0].gateway')
+        gateway_iface=$(echo "$default_route" | jq -r -c '.[0].dev')
+    else
+        log_write "     Unable to determine gateway address for IPv${protocol}"
+    fi
 
-    gateway_addr=$(ip -"${protocol}" route | grep default | cut -d ' ' -f 3 | head -n 1)
-    gateway_iface=$(ip -"${protocol}" route | grep default | cut -d ' ' -f 5 | head -n 1)
     # If there was at least one gateway
     if [ -n "${gateway_addr}" ]; then
         # Append the interface to the gateway address if it is a link-local address
@@ -664,18 +577,21 @@ check_required_ports() {
     # Add port 53
     ports_configured+=("53")
 
+    local protocol_type port_number service_name
     # Now that we have the values stored,
     for i in "${!ports_in_use[@]}"; do
         # loop through them and assign some local variables
-        local service_name
-        service_name=$(echo "${ports_in_use[$i]}" | awk '{gsub(/users:\(\("/,"",$7);gsub(/".*/,"",$7);print $7}')
-        local protocol_type
-        protocol_type=$(echo "${ports_in_use[$i]}" | awk '{print $1}')
-        local port_number
-        port_number="$(echo "${ports_in_use[$i]}" | awk '{print $5}')" #  | awk '{gsub(/^.*:/,"",$5);print $5}')
+        read -r protocol_type port_number service_name <<< "$(
+            awk '{
+                p=$1; n=$5; s=$7
+                gsub(/users:\(\("/,"",s)
+                gsub(/".*/,"",s)
+                print p, n, s
+            }' <<< "${ports_in_use[$i]}"
+        )"
 
         # Check if the right services are using the right ports
-        if [[ ${ports_configured[*]} =~ $(echo "${port_number}" | rev | cut -d: -f1 | rev) ]]; then
+        if [[ ${ports_configured[*]} =~ ${port_number##*:} ]]; then
             compare_port_to_service_assigned  "${ftl}" "${service_name}" "${protocol_type}:${port_number}"
         else
             # If it's not a default port that Pi-hole needs, just print it out for the user to see
@@ -743,7 +659,7 @@ dig_at() {
         local record_type="A"
     fi
 
-    # Find a random blocked url that has not been whitelisted and is not ABP style.
+    # Find a random blocked url that has not been allowlisted and is not ABP style.
     # This helps emulate queries to different domains that a user might query
     # It will also give extra assurance that Pi-hole is correctly resolving and blocking domains
     local random_url
@@ -793,7 +709,7 @@ dig_at() {
                 fi
 
               # Check if Pi-hole can use itself to block a domain
-                if local_dig="$(dig +tries=1 +time=2 -"${protocol}" "${random_url}" @"${local_address}" "${record_type}")"; then
+                if local_dig="$(dig +tries=1 +time=2 -"${protocol}" "${random_url}" @"${local_address}" "${record_type}" -p "$(get_ftl_conf_value "dns.port")")"; then
                     # If it can, show success
                     if [[ "${local_dig}" == *"status: NOERROR"* ]]; then
                         local_dig="NOERROR"
@@ -849,7 +765,7 @@ process_status(){
                 :
             else
             # non-Docker system
-                if service "${i}" status | grep -E 'is\srunning' &> /dev/null; then
+                if service "${i}" status | grep -q -E 'is\srunning|started'; then
                     status_of_process="active"
                 else
                     status_of_process="inactive"
@@ -887,42 +803,27 @@ ftl_full_status(){
 
 make_array_from_file() {
     local filename="${1}"
+
+    # If the file is a directory do nothing since it cannot be parsed
+    [[ -d "${filename}" ]] && return
+
     # The second argument can put a limit on how many line should be read from the file
     # Since some of the files are so large, this is helpful to limit the output
     local limit=${2}
     # A local iterator for testing if we are at the limit above
     local i=0
-    # If the file is a directory
-    if [[ -d "${filename}" ]]; then
-        # do nothing since it cannot be parsed
-        :
-    else
-        # Otherwise, read the file line by line
-        while IFS= read -r line;do
-            # Otherwise, strip out comments and blank lines
-            new_line=$(echo "${line}" | sed -e 's/^\s*#.*$//' -e '/^$/d')
-            # If the line still has content (a non-zero value)
-            if [[ -n "${new_line}" ]]; then
 
-                # If the string contains "### CHANGED", highlight this part in red
-                if [[ "${new_line}" == *"### CHANGED"* ]]; then
-                    new_line="${new_line//### CHANGED/${COL_RED}### CHANGED${COL_NC}}"
-                fi
+    # Process the file, strip out comments and blank lines
+    local processed
+    processed=$(sed -e 's/^\s*#.*$//' -e '/^$/d' "${filename}")
 
-                # Finally, write this line to the log
-                log_write "   ${new_line}"
-            fi
-            # Increment the iterator +1
-            i=$((i+1))
-            # but if the limit of lines we want to see is exceeded
-            if [[ -z ${limit} ]]; then
-                # do nothing
-                :
-            elif [[ $i -eq ${limit} ]]; then
-                break
-            fi
-        done < "${filename}"
-    fi
+    while IFS= read -r line; do
+        # If the string contains "### CHANGED", highlight this part in red
+        log_write "   ${line//### CHANGED/${COL_RED}### CHANGED${COL_NC}}"
+        ((i++))
+        # if the limit of lines we want to see is exceeded do nothing
+        [[ -n ${limit} && $i -eq ${limit} ]] && break
+    done <<< "$processed"
 }
 
 parse_file() {
@@ -933,7 +834,6 @@ parse_file() {
     # Get the lines that are in the file(s) and store them in an array for parsing later
     local file_info
     if [[ -f "$filename" ]]; then
-        #shellcheck disable=SC2016
         IFS=$'\r\n' command eval 'file_info=( $(cat "${filename}") )'
     else
         read -r -a file_info <<< "$filename"
@@ -996,38 +896,38 @@ list_files_in_dir() {
     fi
 
     # Store the files found in an array
-    mapfile -t files_found < <(ls "${dir_to_parse}")
+    local files_found=("${dir_to_parse}"/*)
     # For each file in the array,
     for each_file in "${files_found[@]}"; do
-        if [[ -d "${dir_to_parse}/${each_file}" ]]; then
+        if [[ -d "${each_file}" ]]; then
             # If it's a directory, do nothing
             :
-        elif [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_DEBUG_LOG}" ]] || \
-            [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_RAW_BLOCKLIST_FILES}" ]] || \
-            [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_INSTALL_LOG_FILE}" ]] || \
-            [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_LOG}" ]] || \
-            [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_LOG_GZIPS}" ]]; then
+        elif [[ "${each_file}" == "${PIHOLE_DEBUG_LOG}" ]] || \
+            [[ "${each_file}" == "${PIHOLE_RAW_BLOCKLIST_FILES}" ]] || \
+            [[ "${each_file}" == "${PIHOLE_INSTALL_LOG_FILE}" ]] || \
+            [[ "${each_file}" == "${PIHOLE_LOG}" ]] || \
+            [[ "${each_file}" == "${PIHOLE_LOG_GZIPS}" ]]; then
             :
         elif [[ "${dir_to_parse}" == "${DNSMASQ_D_DIRECTORY}" ]]; then
             # in case of the dnsmasq directory include all files in the debug output
-            log_write "\\n${COL_GREEN}$(ls -lhd "${dir_to_parse}"/"${each_file}")${COL_NC}"
-            make_array_from_file "${dir_to_parse}/${each_file}"
+            log_write "\\n${COL_GREEN}$(ls -lhd "${each_file}")${COL_NC}"
+            make_array_from_file "${each_file}"
         else
             # Then, parse the file's content into an array so each line can be analyzed if need be
             for i in "${!REQUIRED_FILES[@]}"; do
-                if [[ "${dir_to_parse}/${each_file}" == "${REQUIRED_FILES[$i]}" ]]; then
+                if [[ "${each_file}" == "${REQUIRED_FILES[$i]}" ]]; then
                     # display the filename
-                    log_write "\\n${COL_GREEN}$(ls -lhd "${dir_to_parse}"/"${each_file}")${COL_NC}"
+                    log_write "\\n${COL_GREEN}$(ls -lhd "${each_file}")${COL_NC}"
                     # Check if the file we want to view has a limit (because sometimes we just need a little bit of info from the file, not the entire thing)
-                    case "${dir_to_parse}/${each_file}" in
+                    case "${each_file}" in
                         # If it's Web server log, give the first and last 25 lines
-                        "${PIHOLE_WEBSERVER_LOG}") head_tail_log "${dir_to_parse}/${each_file}" 25
+                        "${PIHOLE_WEBSERVER_LOG}") head_tail_log "${each_file}" 25
                             ;;
                         # Same for the FTL log
-                        "${PIHOLE_FTL_LOG}") head_tail_log "${dir_to_parse}/${each_file}" 35
+                        "${PIHOLE_FTL_LOG}") head_tail_log "${each_file}" 35
                             ;;
                         # parse the file into an array in case we ever need to analyze it line-by-line
-                        *) make_array_from_file "${dir_to_parse}/${each_file}";
+                        *) make_array_from_file "${each_file}";
                     esac
                 else
                     # Otherwise, do nothing since it's not a file needed for Pi-hole so we don't care about it
@@ -1063,6 +963,7 @@ head_tail_log() {
     local filename="${1}"
     # The number of lines to use for head and tail
     local qty="${2}"
+    local filebasename="${filename##*/}"
     local head_line
     local tail_line
     # Put the current Internal Field Separator into another variable so it can be restored later
@@ -1071,14 +972,14 @@ head_tail_log() {
     IFS=$'\r\n'
     local log_head=()
     mapfile -t log_head < <(head -n "${qty}" "${filename}")
-    log_write "   ${COL_CYAN}-----head of $(basename "${filename}")------${COL_NC}"
+    log_write "   ${COL_CYAN}-----head of ${filebasename}------${COL_NC}"
     for head_line in "${log_head[@]}"; do
         log_write "   ${head_line}"
     done
     log_write ""
     local log_tail=()
     mapfile -t log_tail < <(tail -n "${qty}" "${filename}")
-    log_write "   ${COL_CYAN}-----tail of $(basename "${filename}")------${COL_NC}"
+    log_write "   ${COL_CYAN}-----tail of ${filebasename}------${COL_NC}"
     for tail_line in "${log_tail[@]}"; do
         log_write "   ${tail_line}"
     done
@@ -1105,6 +1006,24 @@ show_db_entries() {
     )
 
     for line in "${entries[@]}"; do
+        # Use gray color for "no". Normal color for "yes"
+        line=${line//--no---/${COL_GRAY}  no   ${COL_NC}}
+        line=${line//--yes--/  yes  }
+
+        # Use red for "deny" and green for "allow"
+        if [ "$title" = "Domainlist" ]; then
+            line=${line//regex-deny/${COL_RED}regex-deny${COL_NC}}
+            line=${line//regex-allow/${COL_GREEN}regex-allow${COL_NC}}
+            line=${line//exact-deny/${COL_RED}exact-deny${COL_NC}}
+            line=${line//exact-allow/${COL_GREEN}exact-allow${COL_NC}}
+        fi
+
+        # Use red for "block" and green for "allow"
+        if [ "$title" = "Adlists" ]; then
+            line=${line//-BLOCK-/${COL_RED} Block ${COL_NC}}
+            line=${line//-ALLOW-/${COL_GREEN} Allow ${COL_NC}}
+        fi
+
         log_write "   ${line}"
     done
 
@@ -1152,15 +1071,15 @@ check_dhcp_servers() {
 }
 
 show_groups() {
-    show_db_entries "Groups" "SELECT id,CASE enabled WHEN '0' THEN '   0' WHEN '1' THEN '      1' ELSE enabled END enabled,name,datetime(date_added,'unixepoch','localtime') date_added,datetime(date_modified,'unixepoch','localtime') date_modified,description FROM \"group\"" "4 7 50 19 19 50"
+    show_db_entries "Groups" "SELECT id,CASE enabled WHEN '0' THEN '--no---' WHEN '1' THEN '--yes--' ELSE enabled END enabled,name,datetime(date_added,'unixepoch','localtime') date_added,datetime(date_modified,'unixepoch','localtime') date_modified,description FROM \"group\"" "4 7 50 19 19 50"
 }
 
 show_adlists() {
-    show_db_entries "Adlists" "SELECT id,CASE enabled WHEN '0' THEN '   0' WHEN '1' THEN '      1' ELSE enabled END enabled,GROUP_CONCAT(adlist_by_group.group_id) group_ids,address,datetime(date_added,'unixepoch','localtime') date_added,datetime(date_modified,'unixepoch','localtime') date_modified,comment FROM adlist LEFT JOIN adlist_by_group ON adlist.id = adlist_by_group.adlist_id GROUP BY id;" "5 7 12 100 19 19 50"
+    show_db_entries "Adlists" "SELECT id,CASE enabled WHEN '0' THEN '--no---' WHEN '1' THEN '--yes--' ELSE enabled END enabled,GROUP_CONCAT(adlist_by_group.group_id) group_ids, CASE type WHEN '0' THEN '-BLOCK-' WHEN '1' THEN '-ALLOW-' ELSE type END type, address,datetime(date_added,'unixepoch','localtime') date_added,datetime(date_modified,'unixepoch','localtime') date_modified,comment FROM adlist LEFT JOIN adlist_by_group ON adlist.id = adlist_by_group.adlist_id GROUP BY id;" "5 7 12 7 100 19 19 50"
 }
 
 show_domainlist() {
-    show_db_entries "Domainlist (0/1 = exact white-/blacklist, 2/3 = regex white-/blacklist)" "SELECT id,CASE type WHEN '0' THEN '0   ' WHEN '1' THEN ' 1  ' WHEN '2' THEN '  2 ' WHEN '3' THEN '   3' ELSE type END type,CASE enabled WHEN '0' THEN '   0' WHEN '1' THEN '      1' ELSE enabled END enabled,GROUP_CONCAT(domainlist_by_group.group_id) group_ids,domain,datetime(date_added,'unixepoch','localtime') date_added,datetime(date_modified,'unixepoch','localtime') date_modified,comment FROM domainlist LEFT JOIN domainlist_by_group ON domainlist.id = domainlist_by_group.domainlist_id GROUP BY id;" "5 4 7 12 100 19 19 50"
+    show_db_entries "Domainlist" "SELECT id,CASE type WHEN '0' THEN 'exact-allow' WHEN '1' THEN 'exact-deny' WHEN '2' THEN 'regex-allow' WHEN '3' THEN 'regex-deny' ELSE type END type,CASE enabled WHEN '0' THEN '--no---' WHEN '1' THEN '--yes--' ELSE enabled END enabled,GROUP_CONCAT(domainlist_by_group.group_id) group_ids,domain,datetime(date_added,'unixepoch','localtime') date_added,datetime(date_modified,'unixepoch','localtime') date_modified,comment FROM domainlist LEFT JOIN domainlist_by_group ON domainlist.id = domainlist_by_group.domainlist_id GROUP BY id;" "5 11 7 12 90 19 19 50"
 }
 
 show_clients() {

advanced/Scripts/piholeLogFlush.sh

@@ -9,17 +9,14 @@
 # Please see LICENSE file for your rights under this license.
 
 colfile="/opt/pihole/COL_TABLE"
+# shellcheck source="./advanced/Scripts/COL_TABLE"
 source ${colfile}
 
 readonly PI_HOLE_SCRIPT_DIR="/opt/pihole"
 utilsfile="${PI_HOLE_SCRIPT_DIR}/utils.sh"
+# shellcheck source="./advanced/Scripts/utils.sh"
 source "${utilsfile}"
 
-# In case we're running at the same time as a system logrotate, use a
-# separate logrotate state file to prevent stepping on each other's
-# toes.
-STATEFILE="/var/lib/logrotate/pihole"
-
 # Determine database location
 DBFILE=$(getFTLConfigValue "files.database")
 if [ -z "$DBFILE" ]; then
@@ -35,88 +32,45 @@ FTLFILE=$(getFTLConfigValue "files.log.ftl")
 if [ -z "$FTLFILE" ]; then
     FTLFILE="/var/log/pihole/FTL.log"
 fi
-
-if [[ "$*" == *"once"* ]]; then
-    # Nightly logrotation
-    if command -v /usr/sbin/logrotate >/dev/null; then
-        # Logrotate once
-
-        if [[ "$*" != *"quiet"* ]]; then
-            echo -ne "  ${INFO} Running logrotate ..."
-        fi
-        /usr/sbin/logrotate --force --state "${STATEFILE}" /etc/pihole/logrotate
-    else
-        # Copy pihole.log over to pihole.log.1
-        # and empty out pihole.log
-        # Note that moving the file is not an option, as
-        # dnsmasq would happily continue writing into the
-        # moved file (it will have the same file handler)
-        if [[ "$*" != *"quiet"* ]]; then
-            echo -ne "  ${INFO} Rotating ${LOGFILE} ..."
-        fi
-        cp -p "${LOGFILE}" "${LOGFILE}.1"
-        echo " " > "${LOGFILE}"
-        chmod 640 "${LOGFILE}"
-        if [[ "$*" != *"quiet"* ]]; then
-            echo -e "${OVER}  ${TICK} Rotated ${LOGFILE} ..."
-        fi
-        # Copy FTL.log over to FTL.log.1
-        # and empty out FTL.log
-        if [[ "$*" != *"quiet"* ]]; then
-            echo -ne "  ${INFO} Rotating ${FTLFILE} ..."
-        fi
-        cp -p "${FTLFILE}" "${FTLFILE}.1"
-        echo " " > "${FTLFILE}"
-        chmod 640 "${FTLFILE}"
-        if [[ "$*" != *"quiet"* ]]; then
-            echo -e "${OVER}  ${TICK} Rotated ${FTLFILE} ..."
-        fi
-    fi
-else
-    # Manual flushing
-
-    # Flush both pihole.log and pihole.log.1 (if existing)
-    if [[ "$*" != *"quiet"* ]]; then
-        echo -ne "  ${INFO} Flushing ${LOGFILE} ..."
-    fi
-    echo " " > "${LOGFILE}"
-    chmod 640 "${LOGFILE}"
-    if [ -f "${LOGFILE}.1" ]; then
-        echo " " > "${LOGFILE}.1"
-        chmod 640 "${LOGFILE}.1"
-    fi
-    if [[ "$*" != *"quiet"* ]]; then
-        echo -e "${OVER}  ${TICK} Flushed ${LOGFILE} ..."
-    fi
-
-    # Flush both FTL.log and FTL.log.1 (if existing)
-    if [[ "$*" != *"quiet"* ]]; then
-        echo -ne "  ${INFO} Flushing ${FTLFILE} ..."
-    fi
-    echo " " > "${FTLFILE}"
-    chmod 640 "${FTLFILE}"
-    if [ -f "${FTLFILE}.1" ]; then
-        echo " " > "${FTLFILE}.1"
-        chmod 640 "${FTLFILE}.1"
-    fi
-    if [[ "$*" != *"quiet"* ]]; then
-        echo -e "${OVER}  ${TICK} Flushed ${FTLFILE} ..."
-    fi
-
-    if [[ "$*" != *"quiet"* ]]; then
-        echo -ne "  ${INFO} Flushing database, DNS resolution temporarily unavailable ..."
-    fi
-
-    # Stop FTL to make sure it doesn't write to the database while we're deleting data
-    service pihole-FTL stop
-
-    # Delete most recent 24 hours from FTL's database, leave even older data intact (don't wipe out all history)
-    deleted=$(pihole-FTL sqlite3 -ni "${DBFILE}" "DELETE FROM query_storage WHERE timestamp >= strftime('%s','now')-86400; select changes() from query_storage limit 1")
-
-    # Restart FTL
-    service pihole-FTL restart
-    if [[ "$*" != *"quiet"* ]]; then
-        echo -e "${OVER}  ${TICK} Deleted ${deleted} queries from long-term query database"
-    fi
+WEBFILE=$(getFTLConfigValue "files.log.webserver")
+if [ -z "$WEBFILE" ]; then
+    WEBFILE="/var/log/pihole/webserver.log"
 fi
 
+# Helper function to handle log flushing for a single file
+flush_log() {
+    local logfile="$1"
+    if [[ "$*" != *"quiet"* ]]; then
+        echo -ne "  ${INFO} Flushing ${logfile} ..."
+    fi
+    echo " " > "${logfile}"
+    chmod 640 "${logfile}"
+    if [ -f "${logfile}.1" ]; then
+        echo " " > "${logfile}.1"
+        chmod 640 "${logfile}.1"
+    fi
+    if [[ "$*" != *"quiet"* ]]; then
+        echo -e "${OVER}  ${TICK} Flushed ${logfile} ..."
+    fi
+}
+
+# Manual flushing
+flush_log "${LOGFILE}"
+flush_log "${FTLFILE}"
+flush_log "${WEBFILE}"
+
+if [[ "$*" != *"quiet"* ]]; then
+    echo -ne "  ${INFO} Flushing database, DNS resolution temporarily unavailable ..."
+fi
+
+# Stop FTL to make sure it doesn't write to the database while we're deleting data
+service pihole-FTL stop
+
+# Delete most recent 24 hours from FTL's database, leave even older data intact (don't wipe out all history)
+deleted=$(pihole-FTL sqlite3 -ni "${DBFILE}" "DELETE FROM query_storage WHERE timestamp >= strftime('%s','now')-86400; select changes() from query_storage limit 1")
+
+# Restart FTL
+service pihole-FTL restart
+if [[ "$*" != *"quiet"* ]]; then
+    echo -e "${OVER}  ${TICK} Deleted ${deleted} queries from long-term query database"
+fi

advanced/Scripts/piholeLogRotate.sh

@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+# Pi-hole: A black hole for Internet advertisements
+# (c) 2025 Pi-hole, LLC (https://pi-hole.net)
+# Network-wide ad blocking via your own hardware.
+#
+# Rotate Pi-hole's log file
+#
+# This file is copyright under the latest version of the EUPL.
+# Please see LICENSE file for your rights under this license.
+
+colfile="/opt/pihole/COL_TABLE"
+# shellcheck source="./advanced/Scripts/COL_TABLE"
+source ${colfile}
+
+readonly PI_HOLE_SCRIPT_DIR="/opt/pihole"
+utilsfile="${PI_HOLE_SCRIPT_DIR}/utils.sh"
+# shellcheck source="./advanced/Scripts/utils.sh"
+source "${utilsfile}"
+
+# In case we're running at the same time as a system logrotate, use a
+# separate logrotate state file to prevent stepping on each other's
+# toes.
+STATEFILE="/var/lib/logrotate/pihole"
+
+
+# Determine log file location
+LOGFILE=$(getFTLConfigValue "files.log.dnsmasq")
+if [ -z "$LOGFILE" ]; then
+    LOGFILE="/var/log/pihole/pihole.log"
+fi
+FTLFILE=$(getFTLConfigValue "files.log.ftl")
+if [ -z "$FTLFILE" ]; then
+    FTLFILE="/var/log/pihole/FTL.log"
+fi
+WEBFILE=$(getFTLConfigValue "files.log.webserver")
+if [ -z "$WEBFILE" ]; then
+    WEBFILE="/var/log/pihole/webserver.log"
+fi
+
+# Helper function to handle log rotation for a single file
+rotate_log() {
+    # This function copies x.log over to x.log.1
+    # and then empties x.log
+    # Note that moving the file is not an option, as
+    # dnsmasq would happily continue writing into the
+    # moved file (it will have the same file handler)
+    local logfile="$1"
+    if [[ "$*" != *"quiet"* ]]; then
+        echo -ne "  ${INFO} Rotating ${logfile} ..."
+    fi
+    cp -p "${logfile}" "${logfile}.1"
+    echo " " > "${logfile}"
+    chmod 640 "${logfile}"
+    if [[ "$*" != *"quiet"* ]]; then
+        echo -e "${OVER}  ${TICK} Rotated ${logfile} ..."
+    fi
+}
+
+# Nightly logrotation
+if command -v /usr/sbin/logrotate >/dev/null; then
+    # Logrotate once
+    if [[ "$*" != *"quiet"* ]]; then
+        echo -ne "  ${INFO} Running logrotate ..."
+    fi
+    mkdir -p "${STATEFILE%/*}"
+    /usr/sbin/logrotate --force --state "${STATEFILE}" /etc/pihole/logrotate
+else
+    # Handle rotation for each log file
+    rotate_log "${LOGFILE}"
+    rotate_log "${FTLFILE}"
+    rotate_log "${WEBFILE}"
+fi

advanced/Scripts/piholeNetworkFlush.sh

@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+
+# Pi-hole: A black hole for Internet advertisements
+# (c) 2019 Pi-hole, LLC (https://pi-hole.net)
+# Network-wide ad blocking via your own hardware.
+#
+# Network table flush
+#
+# This file is copyright under the latest version of the EUPL.
+# Please see LICENSE file for your rights under this license.
+
+coltable="/opt/pihole/COL_TABLE"
+if [[ -f ${coltable} ]]; then
+# shellcheck source="./advanced/Scripts/COL_TABLE"
+    source ${coltable}
+fi
+
+PI_HOLE_SCRIPT_DIR="/opt/pihole"
+utilsfile="${PI_HOLE_SCRIPT_DIR}/utils.sh"
+# shellcheck source=./advanced/Scripts/utils.sh
+source "${utilsfile}"
+
+# Source api functions
+# shellcheck source="./advanced/Scripts/api.sh"
+. "${PI_HOLE_SCRIPT_DIR}/api.sh"
+
+flushNetwork(){
+    local output
+
+    echo -ne "  ${INFO} Flushing network table ..."
+
+    local data status error
+    # Authenticate with FTL
+    LoginAPI
+
+    # send query again
+    data=$(PostFTLData "action/flush/network" "" "status")
+
+    # Separate the status from the data
+    status=$(printf %s "${data#"${data%???}"}")
+    data=$(printf %s "${data%???}")
+
+    # If there is an .error object in the returned data, display it
+    local error
+    error=$(jq --compact-output <<< "${data}" '.error')
+    if [[ $error != "null" && $error != "" ]]; then
+        echo -e "${OVER}  ${CROSS} Failed to flush the network table:"
+        echo -e "      $(jq <<< "${data}" '.error')"
+        LogoutAPI
+        exit 1
+    elif [[ "${status}" == "200" ]]; then
+        echo -e "${OVER}  ${TICK} Flushed network table"
+    fi
+
+    # Delete session
+    LogoutAPI
+}
+
+flushArp(){
+    # Flush ARP cache of the host
+    if ! output=$(ip -s -s neigh flush all 2>&1); then
+        echo -e "${OVER}  ${CROSS} Failed to flush ARP cache"
+        echo "  Output: ${output}"
+        return 1
+    fi
+}
+
+# Process all options (if present)
+while [ "$#" -gt 0 ]; do
+    case "$1" in
+    "--arp" ) doARP=true ;;
+    esac
+    shift
+done
+
+flushNetwork
+
+if [[ "${doARP}" == true ]]; then
+    echo -ne "  ${INFO} Flushing ARP cache"
+    if flushArp; then
+        echo -e "${OVER}  ${TICK} Flushed ARP cache"
+    fi
+fi
+

advanced/Scripts/query.sh

@@ -1,9 +1,4 @@
 #!/usr/bin/env sh
-# shellcheck disable=SC1090
-
-# Ignore warning about `local` being undefinded in POSIX
-# shellcheck disable=SC3043
-# https://github.com/koalaman/shellcheck/wiki/SC3043#exceptions
 
 # Pi-hole: A black hole for Internet advertisements
 # (c) 2023 Pi-hole, LLC (https://pi-hole.net)
@@ -22,9 +17,11 @@ domain=""
 
 # Source color table
 colfile="/opt/pihole/COL_TABLE"
+# shellcheck source="./advanced/Scripts/COL_TABLE"
 . "${colfile}"
 
 # Source api functions
+# shellcheck source="./advanced/Scripts/api.sh"
 . "${PI_HOLE_INSTALL_DIR}/api.sh"
 
 Help() {
@@ -40,19 +37,16 @@ Options:
 }
 
 GenerateOutput() {
-    local data gravity_data lists_data num_gravity num_lists search_type_str
-    local gravity_data_csv lists_data_csv line current_domain url type color
+    local counts data num_gravity num_lists search_type_str
+    local gravity_data_csv lists_data_csv line url type color
     data="${1}"
 
-    # construct a new json for the list results where each object contains the domain and the related type
-    lists_data=$(printf %s "${data}" | jq '.search.domains | [.[] | {domain: .domain, type: .type}]')
-
-    # construct a new json for the gravity results where each object contains the adlist URL and the related domains
-    gravity_data=$(printf %s "${data}" | jq '.search.gravity  | group_by(.address,.type) | map({ address: (.[0].address), type: (.[0].type), domains: [.[] | .domain] })')
-
-    # number of objects in each json
-    num_gravity=$(printf %s "${gravity_data}" | jq length)
-    num_lists=$(printf %s "${lists_data}" | jq length)
+    # Get count of list and gravity matches
+    # Use JQ to count number of entries in lists and gravity
+    # (output is number of list matches then number of gravity matches)
+    counts=$(printf %s "${data}" | jq --raw-output '(.search.domains | length), (.search.gravity | group_by(.address,.type) | length)')
+    num_lists=$(echo "$counts" | sed -n '1p')
+    num_gravity=$(echo "$counts" | sed -n '2p')
 
     if [ "${partial}" = true ]; then
         search_type_str="partially"
@@ -65,7 +59,7 @@ GenerateOutput() {
     if [ "${num_lists}" -gt 0 ]; then
         # Convert the data to a csv, each line is a "domain,type" string
         # not using jq's @csv here as it quotes each value individually
-        lists_data_csv=$(printf %s "${lists_data}" | jq --raw-output '.[] | [.domain, .type] | join(",")')
+        lists_data_csv=$(printf %s "${data}" | jq --raw-output '.search.domains | map([.domain, .type] | join(",")) | join("\n")')
 
         # Generate output for each csv line, separating line in a domain and type substring at the ','
         echo "${lists_data_csv}" | while read -r line; do
@@ -74,11 +68,11 @@ GenerateOutput() {
     fi
 
     # Results from gravity
-    printf "%s\n\n" "Found ${num_gravity} adlists ${search_type_str} matching '${COL_BLUE}${domain}${COL_NC}'."
+    printf "%s\n\n" "Found ${num_gravity} lists ${search_type_str} matching '${COL_BLUE}${domain}${COL_NC}'."
     if [ "${num_gravity}" -gt 0 ]; then
-        # Convert the data to a csv, each line is a "URL,domain,domain,...." string
+        # Convert the data to a csv, each line is a "URL,type,domain,domain,...." string
         # not using jq's @csv here as it quotes each value individually
-        gravity_data_csv=$(printf %s "${gravity_data}" | jq --raw-output '.[] | [.address, .type, .domains[]] | join(",")')
+        gravity_data_csv=$(printf %s "${data}" | jq --raw-output '.search.gravity | group_by(.address,.type) | map([.[0].address, .[0].type, (.[] | .domain)] | join(",")) | join("\n")')
 
         # Generate line-by-line output for each csv line
         echo "${gravity_data_csv}" | while read -r line; do
@@ -100,15 +94,8 @@ GenerateOutput() {
 
             # cut off type, leaving "domain,domain,...."
             line=${line#*,}
-            # print each domain and remove it from the string until nothing is left
-            while [ ${#line} -gt 0 ]; do
-                current_domain=${line%%,*}
-                printf '    - %s\n' "${COL_GREEN}${current_domain}${COL_NC}"
-                # we need to remove the current_domain and the comma in two steps because
-                # the last domain won't have a trailing comma and the while loop wouldn't exit
-                line=${line#"${current_domain}"}
-                line=${line#,}
-            done
+            # Replace commas with newlines and format output
+            echo "${line}" | sed 's/,/\n/g' | sed "s/^/    - ${COL_GREEN}/" | sed "s/$/${COL_NC}/"
             printf "\n\n"
         done
     fi

advanced/Scripts/update.sh

@@ -12,26 +12,31 @@
 
 # Variables
 readonly ADMIN_INTERFACE_GIT_URL="https://github.com/pi-hole/web.git"
-readonly ADMIN_INTERFACE_DIR="/var/www/html/admin"
 readonly PI_HOLE_GIT_URL="https://github.com/pi-hole/pi-hole.git"
 readonly PI_HOLE_FILES_DIR="/etc/.pihole"
 
-# shellcheck disable=SC2034
 SKIP_INSTALL=true
 
 # when --check-only is passed to this script, it will not perform the actual update
 CHECK_ONLY=false
 
-# shellcheck disable=SC1090
+# shellcheck source="./automated install/basic-install.sh"
 source "${PI_HOLE_FILES_DIR}/automated install/basic-install.sh"
-# shellcheck disable=SC1091
+# shellcheck source=./advanced/Scripts/COL_TABLE
 source "/opt/pihole/COL_TABLE"
+# shellcheck source="./advanced/Scripts/utils.sh"
+source "${PI_HOLE_INSTALL_DIR}/utils.sh"
 
 # is_repo() sourced from basic-install.sh
 # make_repo() sourced from basic-install.sh
 # update_repo() source from basic-install.sh
 # getGitFiles() sourced from basic-install.sh
 # FTLcheckUpdate() sourced from basic-install.sh
+# getFTLConfigValue() sourced from utils.sh
+
+# Honour configured paths for the web application.
+ADMIN_INTERFACE_DIR=$(getFTLConfigValue "webserver.paths.webroot")$(getFTLConfigValue "webserver.paths.webhome")
+readonly ADMIN_INTERFACE_DIR
 
 GitCheckUpdateAvail() {
     local directory
@@ -42,7 +47,7 @@ GitCheckUpdateAvail() {
 
     # Fetch latest changes in this repo
     if ! git fetch --quiet origin ; then
-        echo -e "\\n  ${COL_LIGHT_RED}Error: Unable to update local repository. Contact Pi-hole Support.${COL_NC}"
+        echo -e "\\n  ${COL_RED}Error: Unable to update local repository. Contact Pi-hole Support.${COL_NC}"
         exit 1
     fi
 
@@ -71,13 +76,13 @@ GitCheckUpdateAvail() {
 
 
     if [[ "${#LOCAL}" == 0 ]]; then
-        echo -e "\\n  ${COL_LIGHT_RED}Error: Local revision could not be obtained, please contact Pi-hole Support"
+        echo -e "\\n  ${COL_RED}Error: Local revision could not be obtained, please contact Pi-hole Support"
         echo -e "  Additional debugging output:${COL_NC}"
         git status
         exit 1
     fi
     if [[ "${#REMOTE}" == 0 ]]; then
-        echo -e "\\n  ${COL_LIGHT_RED}Error: Remote revision could not be obtained, please contact Pi-hole Support"
+        echo -e "\\n  ${COL_RED}Error: Remote revision could not be obtained, please contact Pi-hole Support"
         echo -e "  Additional debugging output:${COL_NC}"
         git status
         exit 1
@@ -97,8 +102,52 @@ GitCheckUpdateAvail() {
     fi
 }
 
+updateWarnDialog() {
+    # Display the warning dialog
+
+    local core_str web_str ftl_str
+
+    if [[ "${core_update}" == true ]]; then
+        core_str="Core: \\Zb\\Z1update available\\Zn"
+    else
+        core_str="Core: \\Zb\\Z4up to date\\Zn"
+    fi
+    if [[ "${web_update}" == true ]]; then
+        web_str="Web:  \\Zb\\Z1update available\\Zn"
+    else
+        web_str="Web:  \\Zb\\Z4up to date\\Zn"
+    fi
+    if [[ "${FTL_update}" == true ]]; then
+        ftl_str="FTL:  \\Zb\\Z1update available\\Zn"
+    else
+        ftl_str="FTL:  \\Zb\\Z4up to date\\Zn"
+    fi
+    # shellcheck disable=SC2154 # Variables "${r}" "${c}" are defined in the main script
+    dialog --no-shadow --clear --keep-tite \
+        --colors \
+        --backtitle "Updating Pi-hole" \
+        --title "Warning" \
+        --no-button "Exit" --yes-button "Continue" \
+        --defaultno \
+        --yesno "\\nThe following Pi-hole components are going to be updated.\\n\\n\\n\
+        $core_str\\n\
+        $web_str\\n\
+        $ftl_str\\n\\n\\n\
+\\Zb\\Z1IMPORTANT:\\Zn Make a (teleporter) backup of your system!\\n\\n\
+Updates can come with significant changes. Please read the changelog at https://pi-hole.net/blog carefully.\\n\\n\\n\
+Please confirm you want to start the update process." \
+        "${r}" "${c}" && result=0 || result="$?"
+
+    case "${result}" in
+    "${DIALOG_CANCEL}" | "${DIALOG_ESC}")
+        printf "  %b User canceled the update process.\\n" "${INFO}"
+        exit 1
+        ;;
+    esac
+}
+
 main() {
-    local basicError="\\n  ${COL_LIGHT_RED}Unable to complete update, please contact Pi-hole Support${COL_NC}"
+    local basicError="\\n  ${COL_RED}Unable to complete update, please contact Pi-hole Support${COL_NC}"
     local core_update
     local web_update
     local FTL_update
@@ -107,8 +156,6 @@ main() {
     web_update=false
     FTL_update=false
 
-    # Perform an OS check to ensure we're on an appropriate operating system
-    os_check
 
     # Install packages used by this installation script (necessary if users have removed e.g. git from their systems)
     package_manager_detect
@@ -117,7 +164,7 @@ main() {
 
     # This is unlikely
     if ! is_repo "${PI_HOLE_FILES_DIR}" ; then
-        echo -e "\\n  ${COL_LIGHT_RED}Error: Core Pi-hole repo is missing from system!"
+        echo -e "\\n  ${COL_RED}Error: Core Pi-hole repo is missing from system!"
         echo -e "  Please re-run install script from https://pi-hole.net${COL_NC}"
         exit 1;
     fi
@@ -129,11 +176,11 @@ main() {
         echo -e "  ${INFO} Pi-hole Core:\\t${COL_YELLOW}update available${COL_NC}"
     else
         core_update=false
-        echo -e "  ${INFO} Pi-hole Core:\\t${COL_LIGHT_GREEN}up to date${COL_NC}"
+        echo -e "  ${INFO} Pi-hole Core:\\t${COL_GREEN}up to date${COL_NC}"
     fi
 
     if ! is_repo "${ADMIN_INTERFACE_DIR}" ; then
-        echo -e "\\n  ${COL_LIGHT_RED}Error: Web Admin repo is missing from system!"
+        echo -e "\\n  ${COL_RED}Error: Web Admin repo is missing from system!"
         echo -e "  Please re-run install script from https://pi-hole.net${COL_NC}"
         exit 1;
     fi
@@ -143,33 +190,40 @@ main() {
         echo -e "  ${INFO} Web Interface:\\t${COL_YELLOW}update available${COL_NC}"
     else
         web_update=false
-        echo -e "  ${INFO} Web Interface:\\t${COL_LIGHT_GREEN}up to date${COL_NC}"
+        echo -e "  ${INFO} Web Interface:\\t${COL_GREEN}up to date${COL_NC}"
     fi
 
-    local funcOutput
-    funcOutput=$(get_binary_name) #Store output of get_binary_name here
-    local binary
-    binary="pihole-FTL${funcOutput##*pihole-FTL}" #binary name will be the last line of the output of get_binary_name (it always begins with pihole-FTL)
+    # Allow the user to skip this check if they are using a self-compiled FTL binary from an unsupported architecture
+    if [ "${skipFTL}" != true ]; then
+        local funcOutput
+        funcOutput=$(get_binary_name) #Store output of get_binary_name here
+        local binary
+        binary="pihole-FTL${funcOutput##*pihole-FTL}" #binary name will be the last line of the output of get_binary_name (it always begins with pihole-FTL)
 
-    if FTLcheckUpdate "${binary}" &>/dev/null; then
-        FTL_update=true
-        echo -e "  ${INFO} FTL:\\t\\t${COL_YELLOW}update available${COL_NC}"
+        if FTLcheckUpdate "${binary}" &>/dev/null; then
+            FTL_update=true
+            echo -e "  ${INFO} FTL:\\t\\t${COL_YELLOW}update available${COL_NC}"
+        else
+            case $? in
+                1)
+                    echo -e "  ${INFO} FTL:\\t\\t${COL_GREEN}up to date${COL_NC}"
+                    ;;
+                2)
+                    echo -e "  ${INFO} FTL:\\t\\t${COL_RED}Branch is not available.${COL_NC}\\n\\t\\t\\tUse ${COL_GREEN}pihole checkout ftl [branchname]${COL_NC} to switch to a valid branch."
+                    exit 1
+                    ;;
+                3)
+                    echo -e "  ${INFO} FTL:\\t\\t${COL_RED}Something has gone wrong, cannot reach download server${COL_NC}"
+                    exit 1
+                    ;;
+                *)
+                    echo -e "  ${INFO} FTL:\\t\\t${COL_RED}Something has gone wrong, contact support${COL_NC}"
+                    exit 1
+            esac
+            FTL_update=false
+        fi
     else
-        case $? in
-            1)
-                echo -e "  ${INFO} FTL:\\t\\t${COL_LIGHT_GREEN}up to date${COL_NC}"
-                ;;
-            2)
-                echo -e "  ${INFO} FTL:\\t\\t${COL_LIGHT_RED}Branch is not available.${COL_NC}\\n\\t\\t\\tUse ${COL_LIGHT_GREEN}pihole checkout ftl [branchname]${COL_NC} to switch to a valid branch."
-                ;;
-            3)
-                echo -e "  ${INFO} FTL:\\t\\t${COL_LIGHT_RED}Something has gone wrong, cannot reach download server${COL_NC}"
-                exit 1
-                ;;
-            *)
-                echo -e "  ${INFO} FTL:\\t\\t${COL_LIGHT_RED}Something has gone wrong, contact support${COL_NC}"
-                exit 1
-        esac
+        echo -e "  ${INFO} FTL:\\t\\t${COL_YELLOW}--skipFTL set - update check skipped${COL_NC}"
         FTL_update=false
     fi
 
@@ -184,7 +238,7 @@ main() {
     if [[ ! "${ftlBranch}" == "master" && ! "${ftlBranch}" == "development" ]]; then
         # Notify user that they are on a custom branch which might mean they they are lost
         # behind if a branch was merged to development and got abandoned
-        printf "  %b %bWarning:%b You are using FTL from a custom branch (%s) and might be missing future releases.\\n" "${INFO}" "${COL_LIGHT_RED}" "${COL_NC}" "${ftlBranch}"
+        printf "  %b %bWarning:%b You are using FTL from a custom branch (%s) and might be missing future releases.\\n" "${INFO}" "${COL_RED}" "${COL_NC}" "${ftlBranch}"
     fi
 
     if [[ "${core_update}" == false && "${web_update}" == false && "${FTL_update}" == false ]]; then
@@ -198,6 +252,11 @@ main() {
         exit 0
     fi
 
+    # if there is any update, show the warning dialog and ask for confirmation
+    if [[ "${core_update}" == true || "${web_update}" == true || "${FTL_update}" == true ]]; then
+        updateWarnDialog
+    fi
+
     if [[ "${core_update}" == true ]]; then
         echo ""
         echo -e "  ${INFO} Pi-hole core files out of date, updating local repo."
@@ -209,7 +268,7 @@ main() {
         echo ""
         echo -e "  ${INFO} Pi-hole Web Admin files out of date, updating local repo."
         getGitFiles "${ADMIN_INTERFACE_DIR}" "${ADMIN_INTERFACE_GIT_URL}"
-        echo -e "  ${INFO} If you had made any changes in '/var/www/html/admin/', they have been stashed using 'git stash'"
+        echo -e "  ${INFO} If you had made any changes in '${ADMIN_INTERFACE_DIR}', they have been stashed using 'git stash'"
     fi
 
     if [[ "${FTL_update}" == true ]]; then
@@ -218,7 +277,14 @@ main() {
     fi
 
     if [[ "${FTL_update}" == true || "${core_update}" == true ]]; then
-        ${PI_HOLE_FILES_DIR}/automated\ install/basic-install.sh --reconfigure --unattended || \
+        local addionalFlag
+
+        if [[ ${skipFTL} == true ]]; then
+             addionalFlag="--skipFTL"
+        else
+             addionalFlag=""
+        fi
+        ${PI_HOLE_FILES_DIR}/automated\ install/basic-install.sh --repair --unattended ${addionalFlag} || \
             echo -e "${basicError}" && exit 1
     fi
 
@@ -238,8 +304,15 @@ main() {
     exit 0
 }
 
-if [[ "$1" == "--check-only" ]]; then
-    CHECK_ONLY=true
-fi
+CHECK_ONLY=false
+skipFTL=false
+
+# Check arguments
+for var in "$@"; do
+    case "$var" in
+    "--check-only") CHECK_ONLY=true ;;
+    "--skipFTL") skipFTL=true ;;
+    esac
+done
 
 main

advanced/Scripts/updatecheck.sh

@@ -10,46 +10,50 @@
 
 function get_local_branch() {
     # Return active branch
-    cd "${1}" 2>/dev/null || return 1
-    git rev-parse --abbrev-ref HEAD || return 1
+    cd "${1}" 2>/dev/null || { echo "null"; return; }
+    git rev-parse --abbrev-ref HEAD || echo "null"
 }
 
 function get_local_version() {
     # Return active version
-    cd "${1}" 2>/dev/null || return 1
-    git describe --tags --always 2>/dev/null || return 1
+    cd "${1}" 2>/dev/null || { echo "null"; return; }
+    git describe --tags --always 2>/dev/null || echo "null"
 }
 
 function get_local_hash() {
-    cd "${1}" 2>/dev/null || return 1
-    git rev-parse --short=8 HEAD || return 1
+    cd "${1}" 2>/dev/null || { echo "null"; return; }
+    git rev-parse --short=8 HEAD || echo "null"
 }
 
 function get_remote_version() {
     # if ${2} is = "master" we need to use the "latest" endpoint, otherwise, we simply return null
     if [[ "${2}" == "master" ]]; then
-        curl -s "https://api.github.com/repos/pi-hole/${1}/releases/latest" 2>/dev/null | jq --raw-output .tag_name || return 1
+        curl -s "https://api.github.com/repos/pi-hole/${1}/releases/latest" 2>/dev/null | jq --raw-output .tag_name || echo "null"
     else
         echo "null"
     fi
 }
 
 function get_remote_hash() {
-    git ls-remote "https://github.com/pi-hole/${1}" --tags "${2}" | awk '{print substr($0, 1,8);}' || return 1
+    git ls-remote "https://github.com/pi-hole/${1}" --tags "${2}" | awk '{print substr($0, 1,8);}' || echo "null"
 }
 
 # Source the utils file for addOrEditKeyValPair()
-# shellcheck disable=SC1091
+# shellcheck source="./advanced/Scripts/utils.sh"
 . /opt/pihole/utils.sh
 
+ADMIN_INTERFACE_DIR=$(getFTLConfigValue "webserver.paths.webroot")$(getFTLConfigValue "webserver.paths.webhome")
+readonly ADMIN_INTERFACE_DIR
+
 # Remove the below three legacy files if they exist
 rm -f "/etc/pihole/GitHubVersions"
 rm -f "/etc/pihole/localbranches"
 rm -f "/etc/pihole/localversions"
 
-# Create new versions file if it does not exist
 VERSION_FILE="/etc/pihole/versions"
-touch "${VERSION_FILE}"
+
+# Truncates the file to zero length if it exists to clear it up, otherwise creates an empty file.
+truncate -s 0 "${VERSION_FILE}"
 chmod 644 "${VERSION_FILE}"
 
 # if /pihole.docker.tag file exists, we will use it's value later in this script
@@ -85,13 +89,13 @@ addOrEditKeyValPair "${VERSION_FILE}" "GITHUB_CORE_HASH" "${GITHUB_CORE_HASH}"
 
 # get Web versions
 
-WEB_VERSION="$(get_local_version /var/www/html/admin)"
+WEB_VERSION="$(get_local_version "${ADMIN_INTERFACE_DIR}")"
 addOrEditKeyValPair "${VERSION_FILE}" "WEB_VERSION" "${WEB_VERSION}"
 
-WEB_BRANCH="$(get_local_branch /var/www/html/admin)"
+WEB_BRANCH="$(get_local_branch "${ADMIN_INTERFACE_DIR}")"
 addOrEditKeyValPair "${VERSION_FILE}" "WEB_BRANCH" "${WEB_BRANCH}"
 
-WEB_HASH="$(get_local_hash /var/www/html/admin)"
+WEB_HASH="$(get_local_hash "${ADMIN_INTERFACE_DIR}")"
 addOrEditKeyValPair "${VERSION_FILE}" "WEB_HASH" "${WEB_HASH}"
 
 GITHUB_WEB_VERSION="$(get_remote_version web "${WEB_BRANCH}")"

advanced/Scripts/utils.sh

@@ -1,5 +1,4 @@
 #!/usr/bin/env sh
-# shellcheck disable=SC3043 #https://github.com/koalaman/shellcheck/wiki/SC3043#exceptions
 
 # Pi-hole: A black hole for Internet advertisements
 # (c) 2017 Pi-hole, LLC (https://pi-hole.net)
@@ -31,9 +30,6 @@ addOrEditKeyValPair() {
   local key="${2}"
   local value="${3}"
 
-  # touch file to prevent grep error if file does not exist yet
-  touch "${file}"
-
   if grep -q "^${key}=" "${file}"; then
     # Key already exists in file, modify the value
     sed -i "/^${key}=/c\\${key}=${value}" "${file}"
@@ -74,7 +70,9 @@ getFTLPID() {
 # Example getFTLConfigValue dns.piholePTR
 #######################
 getFTLConfigValue(){
-  pihole-FTL --config -q "${1}"
+  # Pipe to cat to avoid pihole-FTL assuming this is an interactive command
+  # returning colored output.
+  pihole-FTL --config -q "${1}" | cat
 }
 
 #######################
@@ -87,9 +85,17 @@ getFTLConfigValue(){
 # setFTLConfigValue dns.upstreams '[ "8.8.8.8" , "8.8.4.4" ]'
 #######################
 setFTLConfigValue(){
-  pihole-FTL --config "${1}" "${2}" >/dev/null
-  if [[ $? -eq 5 ]]; then
-    echo -e "  ${CROSS} ${1} set by environment variable. Please unset it to use this function"
-    exit 5
-  fi
+    local err
+    { pihole-FTL --config "${1}" "${2}" >/dev/null; err="$?"; } || true
+
+    case $err in
+    0) ;;
+    5)
+        # FTL returns 5 if the value was set by an environment variable and is therefore read-only
+        printf "  %s %s set by environment variable. Please unset it to use this function\n" "${CROSS}" "${1}";
+        exit 5;;
+    *)
+        printf "  %s Failed to set %s. Try with sudo power\n" "${CROSS}" "${1}"
+        exit 1
+    esac
 }

advanced/Scripts/version.sh

@@ -8,23 +8,28 @@
 # This file is copyright under the latest version of the EUPL.
 # Please see LICENSE file for your rights under this license.
 
-# Ignore warning about `local` being undefinded in POSIX
-# shellcheck disable=SC3043
-# https://github.com/koalaman/shellcheck/wiki/SC3043#exceptions
-
-# Source the versions file poupulated by updatechecker.sh
+# Source the versions file populated by updatechecker.sh
 cachedVersions="/etc/pihole/versions"
 
 if [ -f ${cachedVersions} ]; then
-    # shellcheck disable=SC1090
-    . "$cachedVersions"
+    # shellcheck source=/dev/null
+    . "${cachedVersions}"
 else
     echo "Could not find /etc/pihole/versions. Running update now."
     pihole updatechecker
-    # shellcheck disable=SC1090
-    . "$cachedVersions"
+     # shellcheck source=/dev/null
+    . "${cachedVersions}"
 fi
 
+# Convert "null" or empty values to "N/A" for display
+normalize_version() {
+    if [ -z "${1}" ] || [ "${1}" = "null" ]; then
+        echo "N/A"
+    else
+        echo "${1}"
+    fi
+}
+
 main() {
     local details
     details=false
@@ -37,21 +42,21 @@ main() {
 
     if [ "${details}" = true ]; then
         echo "Core"
-        echo "    Version is ${CORE_VERSION:=N/A} (Latest: ${GITHUB_CORE_VERSION:=N/A})"
-        echo "    Branch is ${CORE_BRANCH:=N/A}"
-        echo "    Hash is ${CORE_HASH:=N/A} (Latest: ${GITHUB_CORE_HASH:=N/A})"
+        echo "    Version is $(normalize_version "${CORE_VERSION}") (Latest: $(normalize_version "${GITHUB_CORE_VERSION}"))"
+        echo "    Branch is $(normalize_version "${CORE_BRANCH}")"
+        echo "    Hash is $(normalize_version "${CORE_HASH}") (Latest: $(normalize_version "${GITHUB_CORE_HASH}"))"
         echo "Web"
-        echo "    Version is ${WEB_VERSION:=N/A} (Latest: ${GITHUB_WEB_VERSION:=N/A})"
-        echo "    Branch is ${WEB_BRANCH:=N/A}"
-        echo "    Hash is ${WEB_HASH:=N/A} (Latest: ${GITHUB_WEB_HASH:=N/A})"
+        echo "    Version is $(normalize_version "${WEB_VERSION}") (Latest: $(normalize_version "${GITHUB_WEB_VERSION}"))"
+        echo "    Branch is $(normalize_version "${WEB_BRANCH}")"
+        echo "    Hash is $(normalize_version "${WEB_HASH}") (Latest: $(normalize_version "${GITHUB_WEB_HASH}"))"
         echo "FTL"
-        echo "    Version is ${FTL_VERSION:=N/A} (Latest: ${GITHUB_FTL_VERSION:=N/A})"
-        echo "    Branch is ${FTL_BRANCH:=N/A}"
-        echo "    Hash is ${FTL_HASH:=N/A} (Latest: ${GITHUB_FTL_HASH:=N/A})"
+        echo "    Version is $(normalize_version "${FTL_VERSION}") (Latest: $(normalize_version "${GITHUB_FTL_VERSION}"))"
+        echo "    Branch is $(normalize_version "${FTL_BRANCH}")"
+        echo "    Hash is $(normalize_version "${FTL_HASH}") (Latest: $(normalize_version "${GITHUB_FTL_HASH}"))"
     else
-        echo "Core version is ${CORE_VERSION:=N/A} (Latest: ${GITHUB_CORE_VERSION:=N/A})"
-        echo "Web version is ${WEB_VERSION:=N/A} (Latest: ${GITHUB_WEB_VERSION:=N/A})"
-        echo "FTL version is ${FTL_VERSION:=N/A} (Latest: ${GITHUB_FTL_VERSION:=N/A})"
+        echo "Core version is $(normalize_version "${CORE_VERSION}") (Latest: $(normalize_version "${GITHUB_CORE_VERSION}"))"
+        echo "Web version is $(normalize_version "${WEB_VERSION}") (Latest: $(normalize_version "${GITHUB_WEB_VERSION}"))"
+        echo "FTL version is $(normalize_version "${FTL_VERSION}") (Latest: $(normalize_version "${GITHUB_FTL_VERSION}"))"
     fi
 }
 

advanced/Templates/gravity.db.sql

@@ -43,8 +43,8 @@ CREATE TABLE adlist
 
 CREATE TABLE adlist_by_group
 (
-    adlist_id INTEGER NOT NULL REFERENCES adlist (id),
-    group_id INTEGER NOT NULL REFERENCES "group" (id),
+    adlist_id INTEGER NOT NULL REFERENCES adlist (id) ON DELETE CASCADE,
+    group_id INTEGER NOT NULL REFERENCES "group" (id) ON DELETE CASCADE,
     PRIMARY KEY (adlist_id, group_id)
 );
 
@@ -66,7 +66,7 @@ CREATE TABLE info
     value TEXT NOT NULL
 );
 
-INSERT INTO "info" VALUES('version','19');
+INSERT INTO "info" VALUES('version','20');
 /* This is a flag to indicate if gravity was restored from a backup
     false = not restored,
     failed = restoration failed due to no backup
@@ -75,8 +75,8 @@ INSERT INTO "info" VALUES('gravity_restored','false');
 
 CREATE TABLE domainlist_by_group
 (
-    domainlist_id INTEGER NOT NULL REFERENCES domainlist (id),
-    group_id INTEGER NOT NULL REFERENCES "group" (id),
+    domainlist_id INTEGER NOT NULL REFERENCES domainlist (id) ON DELETE CASCADE,
+    group_id INTEGER NOT NULL REFERENCES "group" (id) ON DELETE CASCADE,
     PRIMARY KEY (domainlist_id, group_id)
 );
 
@@ -91,8 +91,8 @@ CREATE TABLE client
 
 CREATE TABLE client_by_group
 (
-    client_id INTEGER NOT NULL REFERENCES client (id),
-    group_id INTEGER NOT NULL REFERENCES "group" (id),
+    client_id INTEGER NOT NULL REFERENCES client (id) ON DELETE CASCADE,
+    group_id INTEGER NOT NULL REFERENCES "group" (id) ON DELETE CASCADE,
     PRIMARY KEY (client_id, group_id)
 );
 
@@ -111,7 +111,7 @@ CREATE TRIGGER tr_domainlist_update AFTER UPDATE ON domainlist
       UPDATE domainlist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain;
     END;
 
-CREATE VIEW vw_whitelist AS SELECT domain, domainlist.id AS id, domainlist_by_group.group_id AS group_id
+CREATE VIEW vw_allowlist AS SELECT domain, domainlist.id AS id, domainlist_by_group.group_id AS group_id
     FROM domainlist
     LEFT JOIN domainlist_by_group ON domainlist_by_group.domainlist_id = domainlist.id
     LEFT JOIN "group" ON "group".id = domainlist_by_group.group_id
@@ -119,7 +119,7 @@ CREATE VIEW vw_whitelist AS SELECT domain, domainlist.id AS id, domainlist_by_gr
     AND domainlist.type = 0
     ORDER BY domainlist.id;
 
-CREATE VIEW vw_blacklist AS SELECT domain, domainlist.id AS id, domainlist_by_group.group_id AS group_id
+CREATE VIEW vw_denylist AS SELECT domain, domainlist.id AS id, domainlist_by_group.group_id AS group_id
     FROM domainlist
     LEFT JOIN domainlist_by_group ON domainlist_by_group.domainlist_id = domainlist.id
     LEFT JOIN "group" ON "group".id = domainlist_by_group.group_id
@@ -127,7 +127,7 @@ CREATE VIEW vw_blacklist AS SELECT domain, domainlist.id AS id, domainlist_by_gr
     AND domainlist.type = 1
     ORDER BY domainlist.id;
 
-CREATE VIEW vw_regex_whitelist AS SELECT domain, domainlist.id AS id, domainlist_by_group.group_id AS group_id
+CREATE VIEW vw_regex_allowlist AS SELECT domain, domainlist.id AS id, domainlist_by_group.group_id AS group_id
     FROM domainlist
     LEFT JOIN domainlist_by_group ON domainlist_by_group.domainlist_id = domainlist.id
     LEFT JOIN "group" ON "group".id = domainlist_by_group.group_id
@@ -135,7 +135,7 @@ CREATE VIEW vw_regex_whitelist AS SELECT domain, domainlist.id AS id, domainlist
     AND domainlist.type = 2
     ORDER BY domainlist.id;
 
-CREATE VIEW vw_regex_blacklist AS SELECT domain, domainlist.id AS id, domainlist_by_group.group_id AS group_id
+CREATE VIEW vw_regex_denylist AS SELECT domain, domainlist.id AS id, domainlist_by_group.group_id AS group_id
     FROM domainlist
     LEFT JOIN domainlist_by_group ON domainlist_by_group.domainlist_id = domainlist.id
     LEFT JOIN "group" ON "group".id = domainlist_by_group.group_id

advanced/Templates/pihole-FTL-poststop.sh

@@ -3,7 +3,7 @@
 # Source utils.sh for getFTLConfigValue()
 PI_HOLE_SCRIPT_DIR='/opt/pihole'
 utilsfile="${PI_HOLE_SCRIPT_DIR}/utils.sh"
-# shellcheck disable=SC1090
+# shellcheck source="./advanced/Scripts/utils.sh"
 . "${utilsfile}"
 
 # Get file paths

advanced/Templates/pihole-FTL-prestart.sh

@@ -3,32 +3,40 @@
 # Source utils.sh for getFTLConfigValue()
 PI_HOLE_SCRIPT_DIR='/opt/pihole'
 utilsfile="${PI_HOLE_SCRIPT_DIR}/utils.sh"
-# shellcheck disable=SC1090
+# shellcheck source="./advanced/Scripts/utils.sh"
 . "${utilsfile}"
 
 # Get file paths
 FTL_PID_FILE="$(getFTLConfigValue files.pid)"
+FTL_LOG_FILE="$(getFTLConfigValue files.log.ftl)"
+PIHOLE_LOG_FILE="$(getFTLConfigValue files.log.dnsmasq)"
+WEBSERVER_LOG_FILE="$(getFTLConfigValue files.log.webserver)"
+FTL_PID_FILE="${FTL_PID_FILE:-/run/pihole-FTL.pid}"
+FTL_LOG_FILE="${FTL_LOG_FILE:-/var/log/pihole/FTL.log}"
+PIHOLE_LOG_FILE="${PIHOLE_LOG_FILE:-/var/log/pihole/pihole.log}"
+WEBSERVER_LOG_FILE="${WEBSERVER_LOG_FILE:-/var/log/pihole/webserver.log}"
 
 # Ensure that permissions are set so that pihole-FTL can edit all necessary files
-# shellcheck disable=SC2174
-mkdir -pm 0640 /var/log/pihole
-chown -R pihole:pihole /etc/pihole /var/log/pihole
-chmod -R 0640 /var/log/pihole
-chmod -R 0660 /etc/pihole
+mkdir -p /var/log/pihole
+chown -R pihole:pihole /etc/pihole/ /var/log/pihole/
 
-# Logrotate config file need to be owned by root and must not be writable by group and others
-chown root:root /etc/pihole/logrotate
-chmod 0644 /etc/pihole/logrotate
-
-# allow all users to enter the directories
-chmod 0755 /etc/pihole /var/log/pihole
+# allow all users read version file (and use pihole -v)
+touch /etc/pihole/versions
+chmod 0644 /etc/pihole/versions
 
 # allow pihole to access subdirs in /etc/pihole (sets execution bit on dirs)
-# credits https://stackoverflow.com/a/11512211
-find /etc/pihole -type d -exec chmod 0755 {} \;
+find /etc/pihole/ /var/log/pihole/ -type d -exec chmod 0755 {} +
+# Set all files (except TLS-related ones) to u+rw g+r
+find /etc/pihole/ /var/log/pihole/ -type f ! \( -name '*.pem' -o -name '*.crt' \) -exec chmod 0640 {} +
+# Set TLS-related files to a more restrictive u+rw *only* (they may contain private keys)
+find /etc/pihole/ -type f \( -name '*.pem' -o -name '*.crt' \) -exec chmod 0600 {} +
+
+# Logrotate config file need to be owned by root
+chown root:root /etc/pihole/logrotate
 
 # Touch files to ensure they exist (create if non-existing, preserve if existing)
 [ -f "${FTL_PID_FILE}" ] || install -D -m 644 -o pihole -g pihole /dev/null "${FTL_PID_FILE}"
-[ -f /var/log/pihole/FTL.log ] || install -m 640 -o pihole -g pihole /dev/null /var/log/pihole/FTL.log
-[ -f /var/log/pihole/pihole.log ] || install -m 640 -o pihole -g pihole /dev/null /var/log/pihole/pihole.log
+[ -f "${FTL_LOG_FILE}" ] || install -m 640 -o pihole -g pihole /dev/null "${FTL_LOG_FILE}"
+[ -f "${PIHOLE_LOG_FILE}" ] || install -m 640 -o pihole -g pihole /dev/null "${PIHOLE_LOG_FILE}"
+[ -f "${WEBSERVER_LOG_FILE}" ] || install -m 640 -o pihole -g pihole /dev/null "${WEBSERVER_LOG_FILE}"
 [ -f /etc/pihole/dhcp.leases ] || install -m 644 -o pihole -g pihole /dev/null /etc/pihole/dhcp.leases

advanced/Templates/pihole-FTL.openrc

@@ -0,0 +1,40 @@
+#!/sbin/openrc-run
+# shellcheck shell=sh disable=SC2034
+
+: "${PI_HOLE_SCRIPT_DIR:=/opt/pihole}"
+
+command="/usr/bin/pihole-FTL"
+command_user="pihole:pihole"
+supervisor=supervise-daemon
+command_args_foreground="-f"
+command_background=true
+pidfile="/run/${RC_SVCNAME}_openrc.pid"
+extra_started_commands="reload"
+
+respawn_max=5
+respawn_period=60
+capabilities="^CAP_NET_BIND_SERVICE,^CAP_NET_RAW,^CAP_NET_ADMIN,^CAP_SYS_NICE,^CAP_IPC_LOCK,^CAP_CHOWN,^CAP_SYS_TIME"
+
+depend() {
+    want net
+    provide dns
+}
+
+checkconfig() {
+    $command -f test
+}
+
+start_pre() {
+    sh "${PI_HOLE_SCRIPT_DIR}/pihole-FTL-prestart.sh"
+}
+
+stop_post() {
+    sh "${PI_HOLE_SCRIPT_DIR}/pihole-FTL-poststop.sh"
+}
+
+reload() {
+    checkconfig || return $?
+    ebegin "Reloading ${RC_SVCNAME}"
+    start-stop-daemon --signal HUP --pidfile "${pidfile}"
+    eend $?
+}

advanced/Templates/pihole-FTL.service

@@ -12,7 +12,7 @@
 # Source utils.sh for getFTLConfigValue(), getFTLPID()
 PI_HOLE_SCRIPT_DIR="/opt/pihole"
 utilsfile="${PI_HOLE_SCRIPT_DIR}/utils.sh"
-# shellcheck disable=SC1090
+# shellcheck source="./advanced/Scripts/utils.sh"
 . "${utilsfile}"
 
 
@@ -57,13 +57,16 @@ start() {
 stop() {
   if is_running; then
     kill "${FTL_PID}"
-    for i in 1 2 3 4 5; do
+    # Give FTL 120 seconds to gracefully stop
+    i=1
+    while [ "${i}" -le 120 ]; do
       if ! is_running; then
         break
       fi
 
       printf "."
       sleep 1
+      i=$((i + 1))
     done
     echo
 

advanced/Templates/pihole-FTL.systemd

@@ -17,18 +17,18 @@ StartLimitIntervalSec=60s
 
 [Service]
 User=pihole
-PermissionsStartOnly=true
 AmbientCapabilities=CAP_NET_BIND_SERVICE CAP_NET_RAW CAP_NET_ADMIN CAP_SYS_NICE CAP_IPC_LOCK CAP_CHOWN CAP_SYS_TIME
 
-ExecStartPre=/opt/pihole/pihole-FTL-prestart.sh
+# Run prestart with elevated permissions
+ExecStartPre=+/opt/pihole/pihole-FTL-prestart.sh
 ExecStart=/usr/bin/pihole-FTL -f
 Restart=on-failure
 RestartSec=5s
 ExecReload=/bin/kill -HUP $MAINPID
-ExecStopPost=/opt/pihole/pihole-FTL-poststop.sh
+ExecStopPost=+/opt/pihole/pihole-FTL-poststop.sh
 
 # Use graceful shutdown with a reasonable timeout
-TimeoutStopSec=10s
+TimeoutStopSec=120s
 
 # Make /usr, /boot, /etc and possibly some more folders read-only...
 ProtectSystem=full

advanced/Templates/pihole.cron

@@ -24,7 +24,7 @@
 #          The flush script will use logrotate if available
 #          parameter "once": logrotate only once (default is twice)
 #          parameter "quiet": don't print messages
-00 00   * * *   root    PATH="$PATH:/usr/sbin:/usr/local/bin/" pihole flush once quiet
+00 00   * * *   root    PATH="$PATH:/usr/sbin:/usr/local/bin/" pihole logrotate quiet
 
 @reboot root /usr/sbin/logrotate --state /var/lib/logrotate/pihole /etc/pihole/logrotate
 

advanced/bash-completion/pihole

@@ -1,51 +0,0 @@
-_pihole() {
-    local cur prev opts opts_checkout opts_debug opts_logging opts_query opts_update opts_version
-    COMPREPLY=()
-    cur="${COMP_WORDS[COMP_CWORD]}"
-    prev="${COMP_WORDS[COMP_CWORD-1]}"
-    prev2="${COMP_WORDS[COMP_CWORD-2]}"
-
-    case "${prev}" in
-        "pihole")
-            opts="allow allow-regex allow-wild deny checkout debug disable enable flush help logging query reconfigure regex reloaddns reloadlists status tail uninstall updateGravity updatePihole version wildcard arpflush api"
-            COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
-        ;;
-        "allow"|"deny"|"wildcard"|"regex"|"allow-regex"|"allow-wild")
-            opts_lists="\not \--delmode \--quiet \--list \--help"
-            COMPREPLY=( $(compgen -W "${opts_lists}" -- ${cur}) )
-        ;;
-        "checkout")
-            opts_checkout="core ftl web master dev"
-            COMPREPLY=( $(compgen -W "${opts_checkout}" -- ${cur}) )
-        ;;
-        "debug")
-            opts_debug="-a"
-            COMPREPLY=( $(compgen -W "${opts_debug}" -- ${cur}) )
-        ;;
-        "logging")
-            opts_logging="on off 'off noflush'"
-            COMPREPLY=( $(compgen -W "${opts_logging}" -- ${cur}) )
-        ;;
-        "query")
-            opts_query="--partial --all"
-            COMPREPLY=( $(compgen -W "${opts_query}" -- ${cur}) )
-        ;;
-        "updatePihole"|"-up")
-            opts_update="--check-only"
-            COMPREPLY=( $(compgen -W "${opts_update}" -- ${cur}) )
-        ;;
-        "core"|"admin"|"ftl")
-            if [[ "$prev2" == "checkout" ]]; then
-                opts_checkout="master dev"
-                COMPREPLY=( $(compgen -W "${opts_checkout}" -- ${cur}) )
-            else
-                return 1
-            fi
-        ;;
-        *)
-        return 1
-        ;;
-    esac
-    return 0
-}
-complete -F _pihole pihole

advanced/bash-completion/pihole-ftl.bash

@@ -0,0 +1,9 @@
+#!/bin/bash
+#
+# Bash completion script for pihole-FTL
+#
+# This completion script provides tab completion for pihole-FTL CLI flags and commands.
+# It uses the `pihole-FTL --complete` command to generate the completion options.
+_complete_FTL() { mapfile -t COMPREPLY < <(pihole-FTL --complete "${COMP_WORDS[@]}"); }
+
+complete -F _complete_FTL pihole-FTL

advanced/bash-completion/pihole.bash

@@ -0,0 +1,59 @@
+#!/bin/bash
+#
+# Bash completion script for pihole
+#
+_pihole() {
+    local cur prev prev2 opts opts_lists opts_checkout opts_debug opts_logging opts_query opts_update opts_networkflush
+    COMPREPLY=()
+    cur="${COMP_WORDS[COMP_CWORD]}"
+    prev="${COMP_WORDS[COMP_CWORD-1]}"
+    prev2="${COMP_WORDS[COMP_CWORD-2]}"
+
+    case "${prev}" in
+        "pihole")
+            opts="allow allow-regex allow-wild deny checkout debug disable enable flush help logging query repair regex reloaddns reloadlists setpassword status tail uninstall updateGravity updatePihole version wildcard networkflush api"
+            mapfile -t COMPREPLY < <(compgen -W "${opts}" -- "${cur}")
+        ;;
+        "allow"|"deny"|"wildcard"|"regex"|"allow-regex"|"allow-wild")
+            opts_lists="\not \--delmode \--quiet \--list \--help"
+            mapfile -t COMPREPLY < <(compgen -W "${opts_lists}" -- "${cur}")
+        ;;
+        "checkout")
+            opts_checkout="core ftl web master dev"
+            mapfile -t COMPREPLY < <(compgen -W "${opts_checkout}" -- "${cur}")
+        ;;
+        "debug")
+            opts_debug="-a"
+            mapfile -t COMPREPLY < <(compgen -W "${opts_debug}" -- "${cur}")
+        ;;
+        "logging")
+            opts_logging="on off 'off noflush'"
+            mapfile -t COMPREPLY < <(compgen -W "${opts_logging}" -- "${cur}")
+        ;;
+        "query")
+            opts_query="--partial --all"
+            mapfile -t COMPREPLY < <(compgen -W "${opts_query}" -- "${cur}")
+        ;;
+        "updatePihole"|"-up")
+            opts_update="--check-only"
+            mapfile -t COMPREPLY < <(compgen -W "${opts_update}" -- "${cur}")
+        ;;
+        "networkflush")
+            opts_networkflush="--arp"
+            mapfile -t COMPREPLY < <(compgen -W "${opts_networkflush}" -- "${cur}")
+        ;;
+        "core"|"web"|"ftl")
+            if [[ "$prev2" == "checkout" ]]; then
+                opts_checkout="master development"
+                mapfile -t COMPREPLY < <(compgen -W "${opts_checkout}" -- "${cur}")
+            else
+                return 1
+            fi
+        ;;
+        *)
+        return 1
+        ;;
+    esac
+    return 0
+}
+complete -F _pihole pihole

automated install/uninstall.sh

@@ -8,13 +8,17 @@
 # This file is copyright under the latest version of the EUPL.
 # Please see LICENSE file for your rights under this license.
 
+# shellcheck source="./advanced/Scripts/COL_TABLE"
 source "/opt/pihole/COL_TABLE"
+# shellcheck source="./advanced/Scripts/utils.sh"
+source "/opt/pihole/utils.sh"
+# getFTLConfigValue() from utils.sh
 
 while true; do
-    read -rp "  ${QST} Are you sure you would like to remove ${COL_WHITE}Pi-hole${COL_NC}? [y/N] " answer
+    read -rp "  ${QST} Are you sure you would like to remove ${COL_BOLD}Pi-hole${COL_NC}? [y/N] " answer
     case ${answer} in
         [Yy]* ) break;;
-        * ) echo -e "${OVER}  ${COL_LIGHT_GREEN}Uninstall has been canceled${COL_NC}"; exit 0;;
+        * ) echo -e "${OVER}  ${COL_GREEN}Uninstall has been canceled${COL_NC}"; exit 0;;
     esac
 done
 
@@ -23,141 +27,200 @@ str="Root user check"
 if [[ ${EUID} -eq 0 ]]; then
     echo -e "  ${TICK} ${str}"
 else
-    # Check if sudo is actually installed
-    # If it isn't, exit because the uninstall can not complete
-    if [ -x "$(command -v sudo)" ]; then
-        export SUDO="sudo"
-    else
-        echo -e "  ${CROSS} ${str}
-            Script called with non-root privileges
-            The Pi-hole requires elevated privileges to uninstall"
-        exit 1
-    fi
+    echo -e "  ${CROSS} ${str}
+        Script called with non-root privileges
+        The Pi-hole requires elevated privileges to uninstall"
+    exit 1
 fi
 
-readonly PI_HOLE_FILES_DIR="/etc/.pihole"
+# Get paths for admin interface, log files and database files,
+# to allow deletion where user has specified a non-default location
+ADMIN_INTERFACE_DIR=$(getFTLConfigValue "webserver.paths.webroot")$(getFTLConfigValue "webserver.paths.webhome")
+FTL_LOG=$(getFTLConfigValue "files.log.ftl")
+DNSMASQ_LOG=$(getFTLConfigValue "files.log.dnsmasq")
+WEBSERVER_LOG=$(getFTLConfigValue "files.log.webserver")
+PIHOLE_DB=$(getFTLConfigValue "files.database")
+GRAVITY_DB=$(getFTLConfigValue "files.gravity")
+MACVENDOR_DB=$(getFTLConfigValue "files.macvendor")
+
+PI_HOLE_LOCAL_REPO="/etc/.pihole"
+# Setting SKIP_INSTALL="true" to source the installer functions without running them
 SKIP_INSTALL="true"
-source "${PI_HOLE_FILES_DIR}/automated install/basic-install.sh"
-
-# package_manager_detect() sourced from basic-install.sh
-package_manager_detect
-
+# shellcheck source="./automated install/basic-install.sh"
+source "${PI_HOLE_LOCAL_REPO}/automated install/basic-install.sh"
+# Functions and Variables sources from basic-install:
+# package_manager_detect(), disable_service(), stop_service(),
+# restart service() and is_command()
+# PI_HOLE_CONFIG_DIR PI_HOLE_INSTALL_DIR PI_HOLE_LOCAL_REPO
 
 removeMetaPackage() {
     # Purge Pi-hole meta package
     echo ""
     echo -ne "  ${INFO} Removing Pi-hole meta package...";
-    eval "${SUDO}" "${PKG_REMOVE}" "pihole-meta" &> /dev/null;
+    eval "${PKG_REMOVE}" "pihole-meta" &> /dev/null;
     echo -e "${OVER}  ${INFO} Removed Pi-hole meta package";
-
 }
 
-removePiholeFiles() {
-    # Only web directories/files that are created by Pi-hole should be removed
+removeWebInterface() {
+    # Remove the web interface of Pi-hole
     echo -ne "  ${INFO} Removing Web Interface..."
-    ${SUDO} rm -rf /var/www/html/admin &> /dev/null
-
-
-    # If the web directory is empty after removing these files, then the parent html directory can be removed.
-    if [ -d "/var/www/html" ]; then
-        if [[ ! "$(ls -A /var/www/html)" ]]; then
-            ${SUDO} rm -rf /var/www/html &> /dev/null
-        fi
-    fi
+    rm -rf "${ADMIN_INTERFACE_DIR:-/var/www/html/admin/}" &> /dev/null
     echo -e "${OVER}  ${TICK} Removed Web Interface"
+}
 
-    # Attempt to preserve backwards compatibility with older versions
-    # to guarantee no additional changes were made to /etc/crontab after
-    # the installation of pihole, /etc/crontab.pihole should be permanently
-    # preserved.
-    if [[ -f /etc/crontab.orig ]]; then
-        ${SUDO} mv /etc/crontab /etc/crontab.pihole
-        ${SUDO} mv /etc/crontab.orig /etc/crontab
-        ${SUDO} service cron restart
-        echo -e "  ${TICK} Restored the default system cron"
-    fi
+removeFTL() {
+    # Remove FTL and stop any running FTL service
+    if is_command "pihole-FTL"; then
+        # service stop & disable from basic_install.sh
+        stop_service pihole-FTL
+        disable_service pihole-FTL
 
-    # Attempt to preserve backwards compatibility with older versions
-    if [[ -f /etc/cron.d/pihole ]];then
-        ${SUDO} rm -f /etc/cron.d/pihole &> /dev/null
-        echo -e "  ${TICK} Removed /etc/cron.d/pihole"
-    fi
-
-    ${SUDO} rm -rf /var/log/*pihole* &> /dev/null
-    ${SUDO} rm -rf /var/log/pihole/*pihole* &> /dev/null
-    ${SUDO} rm -rf /etc/pihole/ &> /dev/null
-    ${SUDO} rm -rf /etc/.pihole/ &> /dev/null
-    ${SUDO} rm -rf /opt/pihole/ &> /dev/null
-    ${SUDO} rm -f /usr/local/bin/pihole &> /dev/null
-    ${SUDO} rm -f /etc/bash_completion.d/pihole &> /dev/null
-    ${SUDO} rm -f /etc/sudoers.d/pihole &> /dev/null
-    echo -e "  ${TICK} Removed config files"
-
-    # Restore Resolved
-    if [[ -e /etc/systemd/resolved.conf.orig ]] || [[ -e /etc/systemd/resolved.conf.d/90-pi-hole-disable-stub-listener.conf ]]; then
-        ${SUDO} cp -p /etc/systemd/resolved.conf.orig /etc/systemd/resolved.conf &> /dev/null || true
-        ${SUDO} rm -f /etc/systemd/resolved.conf.d/90-pi-hole-disable-stub-listener.conf
-        systemctl reload-or-restart systemd-resolved
-    fi
-
-    # Remove FTL
-    if command -v pihole-FTL &> /dev/null; then
         echo -ne "  ${INFO} Removing pihole-FTL..."
-        if [[ -x "$(command -v systemctl)" ]]; then
-            systemctl stop pihole-FTL
-        else
-            service pihole-FTL stop
-        fi
-        ${SUDO} rm -f /etc/systemd/system/pihole-FTL.service
+        rm -f /etc/systemd/system/pihole-FTL.service &> /dev/null
         if [[ -d '/etc/systemd/system/pihole-FTL.service.d' ]]; then
             read -rp "  ${QST} FTL service override directory /etc/systemd/system/pihole-FTL.service.d detected. Do you wish to remove this from your system? [y/N] " answer
             case $answer in
                 [yY]*)
                     echo -ne "  ${INFO} Removing /etc/systemd/system/pihole-FTL.service.d..."
-                    ${SUDO} rm -R /etc/systemd/system/pihole-FTL.service.d
+                    rm -R /etc/systemd/system/pihole-FTL.service.d &> /dev/null
                     echo -e "${OVER}  ${INFO} Removed /etc/systemd/system/pihole-FTL.service.d"
                 ;;
                 *) echo -e "  ${INFO} Leaving /etc/systemd/system/pihole-FTL.service.d in place.";;
             esac
         fi
-        ${SUDO} rm -f /etc/init.d/pihole-FTL
-        ${SUDO} rm -f /usr/bin/pihole-FTL
+        rm -f /etc/init.d/pihole-FTL &> /dev/null
+        rm -f /usr/bin/pihole-FTL &> /dev/null
         echo -e "${OVER}  ${TICK} Removed pihole-FTL"
+
+        # Force systemd reload after service files are removed
+        if is_command "systemctl"; then
+            echo -ne "  ${INFO} Restarting systemd..."
+            systemctl daemon-reload
+            echo -e "${OVER}  ${TICK} Restarted systemd..."
+        fi
+    fi
+}
+
+removeCronFiles() {
+    # Attempt to preserve backwards compatibility with older versions
+    # to guarantee no additional changes were made to /etc/crontab after
+    # the installation of pihole, /etc/crontab.pihole should be permanently
+    # preserved.
+    if [[ -f /etc/crontab.orig ]]; then
+        mv /etc/crontab /etc/crontab.pihole
+        mv /etc/crontab.orig /etc/crontab
+        restart_service cron
+        echo -e "  ${TICK} Restored the default system cron"
+        echo -e "  ${INFO} A backup of the most recent crontab is saved at /etc/crontab.pihole"
     fi
 
-    # If the pihole manpage exists, then delete and rebuild man-db
+    # Attempt to preserve backwards compatibility with older versions
+    if [[ -f /etc/cron.d/pihole ]];then
+        rm -f /etc/cron.d/pihole &> /dev/null
+        echo -e "  ${TICK} Removed /etc/cron.d/pihole"
+    fi
+}
+
+removePiholeFiles() {
+    # Remove databases (including user specified non-default paths)
+    rm -f "${PIHOLE_DB:-/etc/pihole/pihole-FTL.db}" &> /dev/null
+    rm -f "${GRAVITY_DB:-/etc/pihole/gravity.db}" &> /dev/null
+    rm -f "${MACVENDOR_DB:-/etc/pihole/macvendor.db}" &> /dev/null
+
+    # Remove pihole config, repo and local files
+    rm -rf "${PI_HOLE_CONFIG_DIR:-/etc/pihole}" &> /dev/null
+    rm -rf "${PI_HOLE_LOCAL_REPO:-/etc/.pihole}" &> /dev/null
+    rm -rf "${PI_HOLE_INSTALL_DIR:-/opt/pihole}" &> /dev/null
+
+    # Remove log files (including user specified non-default paths)
+    # and rotated logs
+    # Explicitly escape spaces, in case of trailing space in path before wildcard
+    rm -f "$(printf '%q' "${FTL_LOG:-/var/log/pihole/FTL.log}")*" &> /dev/null
+    rm -f "$(printf '%q' "${DNSMASQ_LOG:-/var/log/pihole/pihole.log}")*" &> /dev/null
+    rm -f "$(printf '%q' "${WEBSERVER_LOG:-/var/log/pihole/webserver.log}")*" &> /dev/null
+
+    # remove any remnant log-files from old versions
+    rm -rf /var/log/*pihole* &> /dev/null
+
+    # remove log directory
+    rm -rf /var/log/pihole &> /dev/null
+
+    # remove the pihole command
+    rm -f /usr/local/bin/pihole &> /dev/null
+
+    # remove Pi-hole's bash completion
+    rm -f /etc/bash_completion.d/pihole &> /dev/null
+    rm -f /etc/bash_completion.d/pihole-FTL &> /dev/null
+
+    # Remove pihole from sudoers for compatibility with old versions
+    rm -f /etc/sudoers.d/pihole &> /dev/null
+
+    echo -e "  ${TICK} Removed config files"
+}
+
+removeManPage() {
+    # If the pihole manpage exists, then delete
     if [[ -f /usr/local/share/man/man8/pihole.8 ]]; then
-        ${SUDO} rm -f /usr/local/share/man/man8/pihole.8 /usr/local/share/man/man8/pihole-FTL.8 /usr/local/share/man/man5/pihole-FTL.conf.5
-        ${SUDO} mandb -q &>/dev/null
+        rm -f /usr/local/share/man/man8/pihole.8 /usr/local/share/man/man8/pihole-FTL.8 /usr/local/share/man/man5/pihole-FTL.conf.5
+        # Rebuild man-db if present
+        if is_command "mandb"; then
+            mandb -q &>/dev/null
+        fi
         echo -e "  ${TICK} Removed pihole man page"
     fi
+}
 
+removeUser() {
     # If the pihole user exists, then remove
     if id "pihole" &> /dev/null; then
-        if ${SUDO} userdel -r pihole 2> /dev/null; then
+        if userdel -r pihole 2> /dev/null; then
             echo -e "  ${TICK} Removed 'pihole' user"
         else
             echo -e "  ${CROSS} Unable to remove 'pihole' user"
         fi
     fi
+
     # If the pihole group exists, then remove
     if getent group "pihole" &> /dev/null; then
-        if ${SUDO} groupdel pihole 2> /dev/null; then
+        if groupdel pihole 2> /dev/null; then
             echo -e "  ${TICK} Removed 'pihole' group"
         else
             echo -e "  ${CROSS} Unable to remove 'pihole' group"
         fi
     fi
+}
 
+restoreResolved() {
+    # Restore Resolved from saved configuration, if present
+    if [[ -e /etc/systemd/resolved.conf.orig ]] || [[ -e /etc/systemd/resolved.conf.d/90-pi-hole-disable-stub-listener.conf ]]; then
+        cp -p /etc/systemd/resolved.conf.orig /etc/systemd/resolved.conf &> /dev/null || true
+        rm -f /etc/systemd/resolved.conf.d/90-pi-hole-disable-stub-listener.conf &> /dev/null
+        systemctl reload-or-restart systemd-resolved
+    fi
+}
+
+completionMessage() {
     echo -e "\\n   We're sorry to see you go, but thanks for checking out Pi-hole!
        If you need help, reach out to us on GitHub, Discourse, Reddit or Twitter
-       Reinstall at any time: ${COL_WHITE}curl -sSL https://install.pi-hole.net | bash${COL_NC}
+       Reinstall at any time: ${COL_BOLD}curl -sSL https://install.pi-hole.net | bash${COL_NC}
 
-      ${COL_LIGHT_RED}Please reset the DNS on your router/clients to restore internet connectivity${COL_NC}
+      ${COL_RED}Please reset the DNS on your router/clients to restore internet connectivity${COL_NC}
       ${INFO} Pi-hole's meta package has been removed, use the 'autoremove' function from your package manager to remove unused dependencies${COL_NC}
-      ${COL_LIGHT_GREEN}Uninstallation Complete! ${COL_NC}"
+      ${COL_GREEN}Uninstallation Complete! ${COL_NC}"
 }
 
 ######### SCRIPT ###########
+# The ordering here allows clean uninstallation with nothing
+# removed before anything that depends upon it.
+# eg removeFTL relies on scripts removed by removePiholeFiles
+# removeUser relies on commands removed by removeMetaPackage
+package_manager_detect
+removeWebInterface
+removeCronFiles
+restoreResolved
+removeManPage
+removeFTL
+removeUser
 removeMetaPackage
 removePiholeFiles
+completionMessage

gravity.sh

@@ -1,5 +1,4 @@
 #!/usr/bin/env bash
-# shellcheck disable=SC1090
 
 # Pi-hole: A black hole for Internet advertisements
 # (c) 2017 Pi-hole, LLC (https://pi-hole.net)
@@ -16,13 +15,13 @@ export LC_ALL=C
 PI_HOLE_SCRIPT_DIR="/opt/pihole"
 # Source utils.sh for GetFTLConfigValue
 utilsfile="${PI_HOLE_SCRIPT_DIR}/utils.sh"
-# shellcheck disable=SC1090
+# shellcheck source=./advanced/Scripts/utils.sh
 . "${utilsfile}"
 
 coltable="${PI_HOLE_SCRIPT_DIR}/COL_TABLE"
-# shellcheck disable=SC1090
+# shellcheck source=./advanced/Scripts/COL_TABLE
 . "${coltable}"
-# shellcheck disable=SC1091
+# shellcheck source=./advanced/Scripts/database_migration/gravity-db.sh
 . "/etc/.pihole/advanced/Scripts/database_migration/gravity-db.sh"
 
 basename="pihole"
@@ -51,14 +50,14 @@ etag_support=false
 
 # Check gravity temp directory
 if [ ! -d "${GRAVITY_TMPDIR}" ] || [ ! -w "${GRAVITY_TMPDIR}" ]; then
-  echo -e "  ${COL_LIGHT_RED}Gravity temporary directory does not exist or is not a writeable directory, falling back to /tmp. ${COL_NC}"
+  echo -e "  ${COL_RED}Gravity temporary directory does not exist or is not a writeable directory, falling back to /tmp. ${COL_NC}"
   GRAVITY_TMPDIR="/tmp"
 fi
 
 # Set this only after sourcing pihole-FTL.conf as the gravity database path may
 # have changed
 gravityDBfile="${GRAVITYDB}"
-gravityDBfile_default="/etc/pihole/gravity.db"
+gravityDBfile_default="${piholeDir}/gravity.db"
 gravityTEMPfile="${GRAVITYDB}_temp"
 gravityDIR="$(dirname -- "${gravityDBfile}")"
 gravityOLDfile="${gravityDIR}/gravity_old.db"
@@ -119,15 +118,18 @@ gravity_swap_databases() {
 
   # Swap databases and remove or conditionally rename old database
   # Number of available blocks on disk
-  availableBlocks=$(stat -f --format "%a" "${gravityDIR}")
+  # Busybox Compat: `stat` long flags unsupported
+  #   -f flag is short form of --file-system.
+  #   -c flag is short form of --format.
+  availableBlocks=$(stat -f -c "%a" "${gravityDIR}")
   # Number of blocks, used by gravity.db
-  gravityBlocks=$(stat --format "%b" "${gravityDBfile}")
+  gravityBlocks=$(stat -c "%b" "${gravityDBfile}")
   # Only keep the old database if available disk space is at least twice the size of the existing gravity.db.
   # Better be safe than sorry...
   oldAvail=false
   if [ "${availableBlocks}" -gt "$((gravityBlocks * 2))" ] && [ -f "${gravityDBfile}" ]; then
     oldAvail=true
-    cp "${gravityDBfile}" "${gravityOLDfile}"
+    cp -p "${gravityDBfile}" "${gravityOLDfile}"
   fi
 
   # Drop the gravity and antigravity tables + subsequent VACUUM the current
@@ -140,7 +142,7 @@ gravity_swap_databases() {
   else
     # Check if the backup directory exists
     if [ ! -d "${gravityBCKdir}" ]; then
-      mkdir -p "${gravityBCKdir}"
+      mkdir -p "${gravityBCKdir}" && chown pihole:pihole "${gravityBCKdir}"
     fi
 
     # If multiple gravityBCKfile's are present (appended with a number), rotate them
@@ -306,7 +308,7 @@ migrate_to_database() {
     fi
 
     # Check if gravity database needs to be updated
-    upgrade_gravityDB "${gravityDBfile}" "${piholeDir}"
+    upgrade_gravityDB "${gravityDBfile}"
 
     # Migrate list files to new database
     if [ -e "${adListFile}" ]; then
@@ -334,7 +336,7 @@ migrate_to_database() {
   fi
 
   # Check if gravity database needs to be updated
-  upgrade_gravityDB "${gravityDBfile}" "${piholeDir}"
+  upgrade_gravityDB "${gravityDBfile}"
 }
 
 # Determine if DNS resolution is available before proceeding
@@ -349,17 +351,24 @@ gravity_CheckDNSResolutionAvailable() {
     echo -e "  ${CROSS} DNS resolution is currently unavailable"
   fi
 
-  str="Waiting until DNS resolution is available..."
+  str="Waiting up to 120 seconds for DNS resolution..."
   echo -ne "  ${INFO} ${str}"
-  until getent hosts github.com &> /dev/null; do
-  # Append one dot for each second waiting
-    str="${str}."
-    echo -ne "  ${OVER}  ${INFO} ${str}"
-    sleep 1
+
+ # Default DNS timeout is two seconds, plus 1 second for each dot > 120 seconds
+  for ((i = 0; i < 40; i++)); do
+      if getent hosts github.com &> /dev/null; then
+        # If we reach this point, DNS resolution is available
+        echo -e "${OVER}  ${TICK} DNS resolution is available"
+        return 0
+      fi
+      # Append one dot for each second waiting
+      echo -ne "."
+      sleep 1
   done
 
-  # If we reach this point, DNS resolution is available
-  echo -e "${OVER}  ${TICK} DNS resolution is available"
+  # DNS resolution is still unavailable after 120 seconds
+  return 1
+
 }
 
 # Function: try_restore_backup
@@ -418,7 +427,7 @@ gravity_DownloadBlocklists() {
     echo -e "  ${INFO} Storing gravity database in ${COL_BOLD}${gravityDBfile}${COL_NC}"
   fi
 
-  local url domain str target compression adlist_type directory success
+  local url domain str compression adlist_type directory success
   echo ""
 
   # Prepare new gravity database
@@ -567,12 +576,12 @@ gravity_DownloadBlocklists() {
     if [[ "${check_url}" =~ ${regex} ]]; then
       echo -e "  ${CROSS} Invalid Target"
     else
-      timeit gravity_DownloadBlocklistFromUrl "${url}" "${sourceIDs[$i]}" "${saveLocation}" "${target}" "${compression}" "${adlist_type}" "${domain}"
+      timeit gravity_DownloadBlocklistFromUrl "${url}" "${sourceIDs[$i]}" "${saveLocation}" "${compression}" "${adlist_type}" "${domain}"
     fi
     echo ""
   done
 
-  gravity_Blackbody=true
+  DownloadBlocklists_done=true
 }
 
 compareLists() {
@@ -601,9 +610,11 @@ compareLists() {
 
 # Download specified URL and perform checks on HTTP status and file content
 gravity_DownloadBlocklistFromUrl() {
-  local url="${1}" adlistID="${2}" saveLocation="${3}" target="${4}" compression="${5}" gravity_type="${6}" domain="${7}"
-  local modifiedOptions="" listCurlBuffer str httpCode success="" ip cmd_ext
-  local file_path permissions ip_addr port blocked=false download=true
+  local url="${1}" adlistID="${2}" saveLocation="${3}" compression="${4}" gravity_type="${5}" domain="${6}"
+  local listCurlBuffer str httpCode success="" ip customUpstreamResolver=""
+  local file_path ip_addr port blocked=false download=true
+  # modifiedOptions is an array to store all the options used to check if the adlist has been changed upstream
+  local modifiedOptions=()
 
   # Create temp file to store content on disk instead of RAM
   # We don't use '--suffix' here because not all implementations of mktemp support it, e.g. on Alpine
@@ -620,14 +631,14 @@ gravity_DownloadBlocklistFromUrl() {
       # Save HTTP ETag to the specified file. An ETag is a caching related header,
       # usually returned in a response. If no ETag is sent by the server, an empty
       # file is created and can later be used consistently.
-      modifiedOptions="--etag-save ${saveLocation}.etag"
+      modifiedOptions=("${modifiedOptions[@]}" --etag-save "${saveLocation}".etag)
 
       if [[ -f "${saveLocation}.etag" ]]; then
         # This option makes a conditional HTTP request for the specific ETag read
         # from the given file by sending a custom If-None-Match header using the
         # stored ETag. This way, the server will only send the file if it has
         # changed since the last request.
-        modifiedOptions="${modifiedOptions} --etag-compare ${saveLocation}.etag"
+        modifiedOptions=("${modifiedOptions[@]}" --etag-compare "${saveLocation}".etag)
       fi
     fi
 
@@ -640,39 +651,13 @@ gravity_DownloadBlocklistFromUrl() {
       # Interstingly, this option is not supported by raw.githubusercontent.com
       # URLs, however, it is still supported by many older web servers which may
       # not support the HTTP ETag method so we keep it as a fallback.
-      modifiedOptions="${modifiedOptions} -z ${saveLocation}"
+      modifiedOptions=("${modifiedOptions[@]}" -z "${saveLocation}")
     fi
   fi
 
   str="Status:"
   echo -ne "  ${INFO} ${str} Pending..."
   blocked=false
-  case $(getFTLConfigValue dns.blocking.mode) in
-  "IP-NODATA-AAAA" | "IP")
-    # Get IP address of this domain
-    ip="$(dig "${domain}" +short)"
-    # Check if this IP matches any IP of the system
-    if [[ -n "${ip}" && $(grep -Ec "inet(|6) ${ip}" <<<"$(ip a)") -gt 0 ]]; then
-      blocked=true
-    fi
-    ;;
-  "NXDOMAIN")
-    if [[ $(dig "${domain}" | grep "NXDOMAIN" -c) -ge 1 ]]; then
-      blocked=true
-    fi
-    ;;
-  "NODATA")
-    if [[ $(dig "${domain}" | grep "NOERROR" -c) -ge 1 ]] && [[ -z $(dig +short "${domain}") ]]; then
-      blocked=true
-    fi
-    ;;
-  "NULL" | *)
-    if [[ $(dig "${domain}" +short | grep "0.0.0.0" -c) -ge 1 ]]; then
-      blocked=true
-    fi
-    ;;
-  esac
-
   # Check if this domain is blocked by Pi-hole but only if the domain is not a
   # local file or empty
   if [[ $url != "file"* ]] && [[ -n "${domain}" ]]; then
@@ -732,46 +717,56 @@ gravity_DownloadBlocklistFromUrl() {
       fi
       echo -e "${OVER}  ${CROSS} ${str} ${domain} is blocked by one of your lists. Using DNS server ${upstream} instead"
       echo -ne "  ${INFO} ${str} Pending..."
-      cmd_ext="--resolve $domain:$port:$ip"
+      customUpstreamResolver="--resolve $domain:$port:$ip"
     fi
   fi
 
-  # If we are going to "download" a local file, we first check if the target
-  # file has a+r permission. We explicitly check for all+read because we want
-  # to make sure that the file is readable by everyone and not just the user
-  # running the script.
-  if [[ $url == "file://"* ]]; then
+  # If we "download" a local file (file://), verify read access before using it.
+    # When running as root (e.g., via pihole -g), check that the 'pihole' user can read the file
+    # to match the effective runtime user of FTL; otherwise, check the current user's read access
+    # (e.g., in Docker or when invoked by a non-root user). The target must
+    # resolve to a regular file and be readable by the evaluated user.
+  if [[ "${url}" == "file:/"* ]]; then
     # Get the file path
-    file_path=$(echo "$url" | cut -d'/' -f3-)
+    file_path=$(echo "${url}" | cut -d'/' -f3-)
     # Check if the file exists and is a regular file (i.e. not a socket, fifo, tty, block). Might still be a symlink.
-    if [[ ! -f $file_path ]]; then
-      # Output that the file does not exist
-      echo -e "${OVER}  ${CROSS} ${file_path} does not exist"
-      download=false
-    else
-      # Check if the file or a file referenced by the symlink has a+r permissions
-      permissions=$(stat -L -c "%a" "$file_path")
-      if [[ $permissions == *4 || $permissions == *5 || $permissions == *6 || $permissions == *7 ]]; then
-        # Output that we are using the local file
-        echo -e "${OVER}  ${INFO} Using local file ${file_path}"
-      else
-        # Output that the file does not have the correct permissions
-        echo -e "${OVER}  ${CROSS} Cannot read file (file needs to have a+r permission)"
+    if [[ ! -f ${file_path} ]]; then
+        # Output that the file does not exist
+        echo -e "${OVER}  ${CROSS} ${file_path} does not exist"
         download=false
-      fi
+    else
+        if [ "$(id -un)" == "root" ]; then
+            # If we are root, we need to check if the pihole user has read permission
+            #  otherwise, we might read files that the pihole user should not be able to read
+            if sudo -u pihole test -r "${file_path}"; then
+                echo -e "${OVER}  ${INFO} Using local file ${file_path}"
+            else
+                echo -e "${OVER}  ${CROSS} Cannot read file (user 'pihole' lacks read permission)"
+                download=false
+            fi
+        else
+            # If we are not root, we just check if the current user has read permission
+            if [[ -r "${file_path}" ]]; then
+                # Output that we are using the local file
+                echo -e "${OVER}  ${INFO} Using local file ${file_path}"
+            else
+                # Output that the file is not readable by the current user
+                echo -e "${OVER}  ${CROSS} Cannot read file (current user '$(id -un)' lacks read permission)"
+                download=false
+            fi
+        fi
     fi
   fi
 
   # Check for allowed protocols
   if [[ $url != "http"* && $url != "https"* && $url != "file"* && $url != "ftp"* && $url != "ftps"* && $url != "sftp"* ]]; then
     echo -e "${OVER}  ${CROSS} ${str} Invalid protocol specified. Ignoring list."
-    echo -e "Ensure your URL starts with a valid protocol like http:// , https:// or file:// ."
+    echo -e "      Ensure your URL starts with a valid protocol like http:// , https:// or file:// ."
     download=false
   fi
 
   if [[ "${download}" == true ]]; then
-    # shellcheck disable=SC2086
-    httpCode=$(curl --connect-timeout ${curl_connect_timeout} -s -L ${compression} ${cmd_ext} ${modifiedOptions} -w "%{http_code}" "${url}" -o "${listCurlBuffer}" 2>/dev/null)
+    httpCode=$(curl --connect-timeout ${curl_connect_timeout} -s -L ${compression:+${compression}} ${customUpstreamResolver:+${customUpstreamResolver}} "${modifiedOptions[@]}" -w "%{http_code}" "${url}" -o "${listCurlBuffer}" 2>/dev/null)
   fi
 
   case $url in
@@ -821,12 +816,16 @@ gravity_DownloadBlocklistFromUrl() {
       done="true"
     # Check if $listCurlBuffer is a non-zero length file
     elif [[ -s "${listCurlBuffer}" ]]; then
-      # Determine if blocklist is non-standard and parse as appropriate
-      gravity_ParseFileIntoDomains "${listCurlBuffer}" "${saveLocation}"
-      # Remove curl buffer file after its use
-      rm "${listCurlBuffer}"
-      # Compare lists if are they identical
+      # Move the downloaded list to the final location
+      mv "${listCurlBuffer}" "${saveLocation}"
+      # Ensure the file has the correct permissions
+      fix_owner_permissions "${saveLocation}"
+      # Compare lists if they are identical
       compareLists "${adlistID}" "${saveLocation}"
+      # Set permissions for the *.etag file
+      if [[ -f "${saveLocation}.etag" ]]; then
+          fix_owner_permissions "${saveLocation}.etag"
+      fi
       # Add domains to database table file
       pihole-FTL "${gravity_type}" parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}"
       done="true"
@@ -840,13 +839,13 @@ gravity_DownloadBlocklistFromUrl() {
   if [[ "${done}" != "true" ]]; then
     # Determine if cached list has read permission
     if [[ -r "${saveLocation}" ]]; then
-      echo -e "  ${CROSS} List download failed: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}"
+      echo -e "  ${CROSS} List download failed: ${COL_GREEN}using previously cached list${COL_NC}"
       # Set list status to "download-failed/cached"
       database_adlist_status "${adlistID}" "3"
       # Add domains to database table file
       pihole-FTL "${gravity_type}" parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}"
     else
-      echo -e "  ${CROSS} List download failed: ${COL_LIGHT_RED}no cached list available${COL_NC}"
+      echo -e "  ${CROSS} List download failed: ${COL_RED}no cached list available${COL_NC}"
       # Manually reset these two numbers because we do not call parseList here
       database_adlist_number "${adlistID}" 0 0
       database_adlist_status "${adlistID}" "4"
@@ -854,37 +853,6 @@ gravity_DownloadBlocklistFromUrl() {
   fi
 }
 
-# Parse source files into domains format
-gravity_ParseFileIntoDomains() {
-  local src="${1}" destination="${2}"
-
-  # Remove comments and print only the domain name
-  # Most of the lists downloaded are already in hosts file format but the spacing/formatting is not contiguous
-  # This helps with that and makes it easier to read
-  # It also helps with debugging so each stage of the script can be researched more in depth
-  # 1) Convert all characters to lowercase
-  tr '[:upper:]' '[:lower:]' <"${src}" >"${destination}"
-
-  # 2) Remove carriage returns
-  # 3) Remove lines starting with ! (ABP Comments)
-  # 4) Remove lines starting with [ (ABP Header)
-  # 5) Remove lines containing ABP extended CSS selectors ("##", "#$#", "#@#", "#?#") and Adguard JavaScript (#%#) preceded by a letter
-  # 6) Remove comments (text starting with "#", include possible spaces before the hash sign)
-  # 7) Remove leading tabs, spaces, etc. (Also removes leading IP addresses)
-  # 8) Remove empty lines
-
-  sed -i -r \
-    -e 's/\r$//' \
-    -e 's/\s*!.*//g' \
-    -e 's/\s*\[.*//g' \
-    -e '/[a-z]\#[$?@%]{0,3}\#/d' \
-    -e 's/\s*#.*//g' \
-    -e 's/^.*\s+//g' \
-    -e '/^$/d' "${destination}"
-
-  fix_owner_permissions "${destination}"
-}
-
 # Report number of entries in a table
 gravity_Table_Count() {
   local table="${1}"
@@ -901,7 +869,7 @@ gravity_Table_Count() {
   fi
 }
 
-# Output count of blacklisted domains and regex filters
+# Output count of denied and allowed domains and regex filters
 gravity_ShowCount() {
   # Here we use the table "gravity" instead of the view "vw_gravity" for speed.
   # It's safe to replace it here, because right after a gravity run both will show the exactly same number of domains.
@@ -914,7 +882,7 @@ gravity_ShowCount() {
 
 # Trap Ctrl-C
 gravity_Trap() {
-  trap '{ echo -e "\\n\\n  ${INFO} ${COL_LIGHT_RED}User-abort detected${COL_NC}"; gravity_Cleanup "error"; }' INT
+  trap '{ echo -e "\\n\\n  ${INFO} ${COL_RED}User-abort detected${COL_NC}"; gravity_Cleanup "error"; }' INT
 }
 
 # Clean up after Gravity upon exit or cancellation
@@ -932,13 +900,13 @@ gravity_Cleanup() {
   # invalid_domains location
   rm "${GRAVITY_TMPDIR}"/*.ph-non-domains 2>/dev/null
 
-  # Ensure this function only runs when gravity_SetDownloadOptions() has completed
-  if [[ "${gravity_Blackbody:-}" == true ]]; then
-    # Remove any unused .domains files
-    for file in "${piholeDir}"/*."${domainsExtension}"; do
-      # If list is not in active array, then remove it
+  # Ensure this function only runs when gravity_DownloadBlocklists() has completed
+  if [[ "${DownloadBlocklists_done:-}" == true ]]; then
+    # Remove any unused .domains/.etag/.sha files
+    for file in "${listsCacheDir}"/*."${domainsExtension}"; do
+      # If list is not in active array, then remove it and all associated files
       if [[ ! "${activeDomains[*]}" == *"${file}"* ]]; then
-        rm -f "${file}" 2>/dev/null ||
+        rm -f "${file}"* 2>/dev/null ||
           echo -e "  ${CROSS} Failed to remove ${file##*/}"
       fi
     done
@@ -994,7 +962,7 @@ database_recovery() {
   else
     echo -e "${OVER}  ${CROSS} ${str} - the following errors happened:"
     while IFS= read -r line; do echo "  - $line"; done <<<"$result"
-    echo -e "  ${CROSS} Recovery failed. Try \"pihole -r recreate\" instead."
+    echo -e "  ${CROSS} Recovery failed. Try \"pihole -g -r recreate\" instead."
     exit 1
   fi
   echo ""
@@ -1072,7 +1040,7 @@ migrate_to_listsCache_dir() {
   # If not, we need to migrate the old files to the new directory
   local str="Migrating the list's cache directory to new location"
   echo -ne "  ${INFO} ${str}..."
-  mkdir -p "${listsCacheDir}"
+  mkdir -p "${listsCacheDir}" && chown pihole:pihole "${listsCacheDir}"
 
   # Move the old files to the new directory
   if mv "${piholeDir}"/list.* "${listsCacheDir}/" 2>/dev/null; then
@@ -1082,7 +1050,7 @@ migrate_to_listsCache_dir() {
   fi
 
   # Update the list's paths in the corresponding .sha1 files to the new location
-  sed -i "s|${piholeDir}/|${listsCacheDir}/|g" "${listsCacheDir}"/*.sha1
+  sed -i "s|${piholeDir}/|${listsCacheDir}/|g" "${listsCacheDir}"/*.sha1 2>/dev/null
 }
 
 helpFunc() {
@@ -1131,13 +1099,19 @@ for var in "$@"; do
   "-t" | "--timeit") timed=true ;;
   "-r" | "--repair") repairSelector "$3" ;;
   "-u" | "--upgrade")
-    upgrade_gravityDB "${gravityDBfile}" "${piholeDir}"
+    upgrade_gravityDB "${gravityDBfile}"
     exit 0
     ;;
   "-h" | "--help") helpFunc ;;
   esac
 done
 
+# Check if DNS is available, no need to do any database manipulation if we're not able to download adlists
+if ! timeit gravity_CheckDNSResolutionAvailable; then
+  echo -e "   ${CROSS} No DNS resolution available. Please contact support."
+  exit 1
+fi
+
 # Remove OLD (backup) gravity file, if it exists
 if [[ -f "${gravityOLDfile}" ]]; then
   rm "${gravityOLDfile}"
@@ -1171,18 +1145,13 @@ fi
 
 if [[ "${forceDelete:-}" == true ]]; then
   str="Deleting existing list cache"
-  echo -ne "${INFO} ${str}..."
+  echo -ne "  ${INFO} ${str}..."
 
   rm "${listsCacheDir}/list.*" 2>/dev/null || true
   echo -e "${OVER}  ${TICK} ${str}"
 fi
 
 # Gravity downloads blocklists next
-if ! timeit gravity_CheckDNSResolutionAvailable; then
-  echo -e "   ${CROSS} Can not complete gravity update, no DNS is available. Please contact support."
-  exit 1
-fi
-
 if ! gravity_DownloadBlocklists; then
   echo -e "   ${CROSS} Unable to create gravity database. Please try again later. If the problem persists, please contact support."
   exit 1

manpages/pihole.8

@@ -23,7 +23,7 @@ pihole -r
 .br
 \fBpihole -g\fR
 .br
-\fBpihole\fR -\fBq\fR [options]
+\fBpihole\fR \fB-q\fR [options]
 .br
 \fBpihole\fR \fB-l\fR (\fBon|off|off noflush\fR)
 .br
@@ -43,7 +43,7 @@ pihole -r
 .br
 \fBpihole\fR \fBcheckout\fR repo [branch]
 .br
-\fBpihole\fR \api\fR endpoint
+\fBpihole\fR \fBapi\fR [verbose] endpoint
 .br
 \fBpihole\fR \fBhelp\fR
 .br
@@ -100,14 +100,17 @@ Available commands and options:
       -c                Include a Pi-hole database integrity check
 .br
 
-\fB-f, flush\fR
+\fB-f, flush\fR [quite]
 .br
-    Flush the Pi-hole log
+    Flush the Pi-hole log and last 24h from the query database
 .br
 
-\fB-r, reconfigure\fR
+      quite           Suppress output
 .br
-    Reconfigure or Repair Pi-hole subsystems
+
+\fB-r, repair\fR
+.br
+    Repair Pi-hole subsystems
 .br
 
 \fB-t, tail\fR [arg]
@@ -234,10 +237,22 @@ Available commands and options:
       branchname        Update subsystems to the specified branchname
 .br
 
-\fBapi\fR endpoint
+\fBapi\fR [verbose] endpoint
 .br
     Query the Pi-hole API at <endpoint>
 .br
+
+      verbose           Show authentication and status messages
+.br
+
+\fBlogrotate\fR [quite]
+.br
+    Rotate Pi-hole's log files
+.br
+
+      quite           Suppress output
+.br
+
 .SH "EXAMPLE"
 
 Some usage examples
@@ -264,7 +279,7 @@ Allow-/denylist manipulation
 
 \fBpihole --regex "ad.*\\.example\\.com$"\fR
 .br
-    Adds "ad.*\\.example\\.com$" to the regex blacklist.
+    Adds "ad.*\\.example\\.com$" to the regex denylist.
     Would block all subdomains of example.com which start with "ad"
 .br
 
@@ -313,9 +328,10 @@ Switching Pi-hole subsystem branches
     Switch to core development branch
 .br
 
-\fBpihole arpflush\fR
+\fBpihole networkflush\fR
 .br
-    Flush information stored in Pi-hole's network tables
+    Flush information stored in Pi-hole's network table
+    Add '--arp' to additionally flush the ARP table
 .br
 
 \fBpihole api stats/summary\fR
@@ -323,6 +339,11 @@ Switching Pi-hole subsystem branches
     Queries FTL for the stats/summary endpoint
 .br
 
+\fBpihole api verbose stats/summary\fR
+.br
+    Same as above, but shows authentication and status messages
+.br
+
 .SH "COLOPHON"
 
 Get sucked into the latest news and community activity by entering Pi-hole's orbit. Information about Pi-hole, and the latest version of the software can be found at https://pi-hole.net.

pihole

@@ -9,7 +9,7 @@
 # This file is copyright under the latest version of the EUPL.
 # Please see LICENSE file for your rights under this license.
 
-readonly PI_HOLE_SCRIPT_DIR="/opt/pihole"
+PI_HOLE_SCRIPT_DIR="/opt/pihole"
 
 # PI_HOLE_BIN_DIR is not readonly here because in some functions (checkout),
 # they might get set again when the installer is sourced. This causes an
@@ -17,13 +17,16 @@ readonly PI_HOLE_SCRIPT_DIR="/opt/pihole"
 PI_HOLE_BIN_DIR="/usr/local/bin"
 
 readonly colfile="${PI_HOLE_SCRIPT_DIR}/COL_TABLE"
+# shellcheck source=./advanced/Scripts/COL_TABLE
 source "${colfile}"
 
-readonly utilsfile="${PI_HOLE_SCRIPT_DIR}/utils.sh"
+utilsfile="${PI_HOLE_SCRIPT_DIR}/utils.sh"
+# shellcheck source=./advanced/Scripts/utils.sh
 source "${utilsfile}"
 
 # Source api functions
 readonly apifile="${PI_HOLE_SCRIPT_DIR}/api.sh"
+# shellcheck source=./advanced/Scripts/api.sh
 source "${apifile}"
 
 versionsfile="/etc/pihole/versions"
@@ -31,6 +34,7 @@ if [ -f "${versionsfile}" ]; then
     # Only source versionsfile if the file exits
     # fixes a warning during installation where versionsfile does not exist yet
     # but gravity calls `pihole -status` and thereby sourcing the file
+    # shellcheck source=/dev/null
     source "${versionsfile}"
 fi
 
@@ -88,12 +92,32 @@ debugFunc() {
 }
 
 flushFunc() {
-  "${PI_HOLE_SCRIPT_DIR}"/piholeLogFlush.sh "$@"
+    # unsupported in docker because it requires restarting FTL
+    if [ -n "${DOCKER_VERSION}" ]; then
+        unsupportedFunc
+    else
+        "${PI_HOLE_SCRIPT_DIR}"/piholeLogFlush.sh "$@"
+        exit 0
+    fi
+}
+
+# Deprecated function, should be removed in the future
+# use networkFlush instead
+arpFunc() {
+  shift
+  echo -e "  ${INFO} The 'arpflush' command is deprecated, use 'networkflush' instead"
+  "${PI_HOLE_SCRIPT_DIR}"/piholeNetworkFlush.sh "$@"
   exit 0
 }
 
-arpFunc() {
-  "${PI_HOLE_SCRIPT_DIR}"/piholeARPTable.sh "$@"
+logrotateFunc() {
+   "${PI_HOLE_SCRIPT_DIR}"/piholeLogRotate.sh "$@"
+    exit 0
+}
+
+networkFlush() {
+  shift
+  "${PI_HOLE_SCRIPT_DIR}"/piholeNetworkFlush.sh "$@"
   exit 0
 }
 
@@ -107,11 +131,26 @@ updatePiholeFunc() {
   fi
 }
 
-reconfigurePiholeFunc() {
+repairPiholeFunc() {
   if [ -n "${DOCKER_VERSION}" ]; then
     unsupportedFunc
   else
-    /etc/.pihole/automated\ install/basic-install.sh --reconfigure
+    local skipFTL additionalFlag
+    skipFTL=false
+    # Check arguments
+    for var in "$@"; do
+        case "$var" in
+            "--skipFTL") skipFTL=true ;;
+        esac
+    done
+
+    if [ "${skipFTL}" == true ]; then
+        additionalFlag="--skipFTL"
+    else
+        additionalFlag=""
+    fi
+
+    /etc/.pihole/automated\ install/basic-install.sh --repair ${additionalFlag}
     exit 0;
   fi
 }
@@ -143,10 +182,11 @@ uninstallFunc() {
 
 versionFunc() {
   exec "${PI_HOLE_SCRIPT_DIR}"/version.sh
+  exit 0
 }
 
 reloadDNS() {
-  local svcOption svc str output status pid icon FTL_PID_FILE
+  local svcOption svc str output status pid icon FTL_PID_FILE sigrtmin
   svcOption="${1:-reload}"
 
   # get the current path to the pihole-FTL.pid
@@ -165,7 +205,10 @@ reloadDNS() {
       str="FTL is not running"
       icon="${INFO}"
     else
-      svc="kill -RTMIN ${pid}"
+      sigrtmin="$(pihole-FTL sigrtmin 2>/dev/null)"
+      # Make sure sigrtmin is a number, otherwise fallback to RTMIN
+      [[ "${sigrtmin}" =~ ^[0-9]+$ ]] || unset sigrtmin
+      svc="kill -${sigrtmin:-RTMIN} ${pid}"
       str="Reloading DNS lists"
       icon="${TICK}"
     fi
@@ -234,7 +277,7 @@ Time:
     fi
 
     if [[ ${error} == true ]];then
-      echo -e "  ${COL_LIGHT_RED}Unknown format for blocking timer!${COL_NC}"
+      echo -e "  ${COL_RED}Unknown format for blocking timer!${COL_NC}"
       echo -e "  Try 'pihole disable --help' for more information."
       exit 1
     fi
@@ -247,17 +290,20 @@ Time:
   data=$(PostFTLData "dns/blocking" "{ \"blocking\": ${1}, \"timer\": ${tt} }")
 
   # Check the response
-  local extra=" forever"
-  local timer="$(echo "${data}"| jq --raw-output '.timer' )"
+  local extra timer
+  extra=" forever"
+  timer="$(echo "${data}"| jq --raw-output '.timer' )"
   if [[ "${timer}" != "null" ]]; then
     extra=" for ${timer}s"
   fi
-  local str="Pi-hole $(echo "${data}" | jq --raw-output '.blocking')${extra}"
+  local str
+  str="Pi-hole $(echo "${data}" | jq --raw-output '.blocking')${extra}"
 
   # Logout from the API
   LogoutAPI
 
   echo -e "${OVER}  ${TICK} ${str}"
+  exit 0
 }
 
 piholeLogging() {
@@ -287,7 +333,7 @@ Options:
     echo -e "  ${INFO} Enabling logging..."
     local str="Logging has been enabled!"
   else
-    echo -e "  ${COL_LIGHT_RED}Invalid option${COL_NC}
+    echo -e "  ${COL_RED}Invalid option${COL_NC}
   Try 'pihole logging --help' for more information."
     exit 1
   fi
@@ -375,20 +421,22 @@ statusFunc() {
 
 tailFunc() {
   # Warn user if Pi-hole's logging is disabled
-  local logging_enabled=$(getFTLConfigValue dns.queryLogging)
+  local logging_enabled
+  logging_enabled=$(getFTLConfigValue dns.queryLogging)
   if [[ "${logging_enabled}" != "true" ]]; then
     echo "  ${CROSS} Warning: Query logging is disabled"
   fi
   echo -e "  ${INFO} Press Ctrl-C to exit"
 
   # Get logfile path
-  readonly LOGFILE=$(getFTLConfigValue files.log.dnsmasq)
+  LOGFILE=$(getFTLConfigValue files.log.dnsmasq)
+  readonly LOGFILE
 
   # Strip date from each line
   # Color blocklist/denylist/wildcard entries as red
   # Color A/AAAA/DHCP strings as white
   # Color everything else as gray
-  tail -f $LOGFILE | grep --line-buffered "${1}" | sed -E \
+  tail -f $LOGFILE | grep --line-buffered -- "${1}" | sed -E \
     -e "s,($(date +'%b %d ')| dnsmasq\[[0-9]*\]),,g" \
     -e "s,(.*(denied |gravity blocked ).*),${COL_RED}&${COL_NC}," \
     -e "s,.*(query\\[A|DHCP).*,${COL_NC}&${COL_NC}," \
@@ -398,7 +446,10 @@ tailFunc() {
 
 piholeCheckoutFunc() {
   if [ -n "${DOCKER_VERSION}" ]; then
-    unsupportedFunc
+    echo -e "${CROSS} Function not supported in Docker images"
+    echo "Please build a custom image following the steps at"
+    echo "https://github.com/pi-hole/docker-pi-hole?tab=readme-ov-file#building-the-image-locally"
+    exit 0
   else
     if [[ "$2" == "-h" ]] || [[ "$2" == "--help" ]]; then
       echo "Switch Pi-hole subsystems to a different GitHub branch
@@ -420,6 +471,7 @@ piholeCheckoutFunc() {
       exit 0
     fi
 
+    #shellcheck source=./advanced/Scripts/piholeCheckout.sh
     source "${PI_HOLE_SCRIPT_DIR}"/piholeCheckout.sh
     shift
     checkout "$@"
@@ -475,12 +527,14 @@ Debugging Options:
   -d, debug           Start a debugging session
                         Add '-c' or '--check-database' to include a Pi-hole database integrity check
                         Add '-a' to automatically upload the log to tricorder.pi-hole.net
-  -f, flush           Flush the Pi-hole log
-  -r, reconfigure     Reconfigure or Repair Pi-hole subsystems
+  -f, flush           Flush the Pi-hole logs and last 24h from the query database
+                        Add 'quiet' to suppress output messages
+  -r, repair          Repair Pi-hole subsystems
   -t, tail [arg]      View the live output of the Pi-hole log.
                       Add an optional argument to filter the log
                       (regular expressions are supported)
   api <endpoint>      Query the Pi-hole API at <endpoint>
+                        Precede <endpoint> with 'verbose' option to show authentication and status messages
 
 
 Options:
@@ -506,7 +560,10 @@ Options:
   reloadlists         Update the lists WITHOUT flushing the cache or restarting the DNS server
   checkout            Switch Pi-hole subsystems to a different GitHub branch
                         Add '-h' for more info on checkout usage
-  arpflush            Flush information stored in Pi-hole's network tables";
+  networkflush        Flush information stored in Pi-hole's network tables
+                        Add '--arp' to additionally flush the ARP table
+  logrotate          Rotate Pi-hole's log files
+                        Add 'quiet' to suppress output messages";
   exit 0
 }
 
@@ -515,7 +572,7 @@ if [[ $# = 0 ]]; then
 fi
 
 # functions that do not require sudo power
-need_root=1
+need_root=
 case "${1}" in
   "-h" | "help" | "--help"        ) helpFunc;;
   "-v" | "version"                ) versionFunc;;
@@ -523,31 +580,33 @@ case "${1}" in
   "-q" | "query"                  ) queryFunc "$@";;
   "status"                        ) statusFunc "$2";;
   "tricorder"                     ) tricorderFunc;;
+  "allow" | "allowlist"           ) listFunc "$@";;
+  "deny" | "denylist"             ) listFunc "$@";;
+  "--wild" | "wildcard"           ) listFunc "$@";;
+  "--regex" | "regex"             ) listFunc "$@";;
+  "--allow-regex" | "allow-regex" ) listFunc "$@";;
+  "--allow-wild" | "allow-wild"   ) listFunc "$@";;
+  "enable"                        ) piholeEnable true "$2";;
+  "disable"                       ) piholeEnable false "$2";;
+  "api"                           ) shift; apiFunc "$@"; exit 0;;
 
   # we need to add all arguments that require sudo power to not trigger the * argument
-  "allow" | "allowlist"           ) need_root=0;;
-  "deny" | "denylist"             ) need_root=0;;
-  "--wild" | "wildcard"           ) need_root=0;;
-  "--regex" | "regex"             ) need_root=0;;
-  "--allow-regex" | "allow-regex" ) need_root=0;;
-  "--allow-wild" | "allow-wild"   ) need_root=0;;
-  "-f" | "flush"                  ) ;;
-  "-up" | "updatePihole"          ) ;;
-  "-r"  | "reconfigure"           ) ;;
-  "-l" | "logging"                ) ;;
-  "uninstall"                     ) ;;
-  "enable"                        ) need_root=0;;
-  "disable"                       ) need_root=0;;
-  "-d" | "debug"                  ) ;;
-  "-g" | "updateGravity"          ) ;;
-  "reloaddns"                     ) ;;
-  "reloadlists"                   ) ;;
-  "setpassword"                   ) ;;
-  "checkout"                      ) ;;
-  "updatechecker"                 ) ;;
-  "arpflush"                      ) ;;
-  "-t" | "tail"                   ) ;;
-  "api"                           ) need_root=0;;
+  "-f" | "flush"                  ) need_root=true;;
+  "-up" | "updatePihole"          ) need_root=true;;
+  "-r"  | "repair"                ) need_root=true;;
+  "-l" | "logging"                ) need_root=true;;
+  "uninstall"                     ) need_root=true;;
+  "-d" | "debug"                  ) need_root=true;;
+  "-g" | "updateGravity"          ) need_root=true;;
+  "reloaddns"                     ) need_root=true;;
+  "reloadlists"                   ) need_root=true;;
+  "setpassword"                   ) need_root=true;;
+  "checkout"                      ) need_root=true;;
+  "updatechecker"                 ) need_root=true;;
+  "arpflush"                      ) need_root=true;; # Deprecated, use networkflush instead
+  "networkflush"                  ) need_root=true;;
+  "-t" | "tail"                   ) need_root=true;;
+  "logrotate"                    ) need_root=true;;
   *                               ) helpFunc;;
 esac
 
@@ -557,38 +616,32 @@ if [[ -z ${USER} ]]; then
   USER=$(whoami)
 fi
 
-# Check if the current user is neither root nor pihole and if the command
+# Check if the current user is not root and if the command
 # requires root. If so, exit with an error message.
-if [[ $EUID -ne 0 && ${USER} != "pihole" && need_root -eq 1 ]];then
-  echo -e "  ${CROSS} The Pi-hole command requires root privileges, try:"
+# Add an exception for the user "pihole" to allow the webserver running gravity
+if [[ ( $EUID -ne 0 && ${USER} != "pihole" ) && -n "${need_root}" ]]; then
+  echo -e "  ${CROSS} This Pi-hole command requires root privileges, try:"
   echo -e "      ${COL_GREEN}sudo pihole $*${COL_NC}"
   exit 1
 fi
 
 # Handle redirecting to specific functions based on arguments
 case "${1}" in
-  "allow" | "allowlist"           ) listFunc "$@";;
-  "deny" | "denylist"             ) listFunc "$@";;
-  "--wild" | "wildcard"           ) listFunc "$@";;
-  "--regex" | "regex"             ) listFunc "$@";;
-  "--allow-regex" | "allow-regex" ) listFunc "$@";;
-  "--allow-wild" | "allow-wild"   ) listFunc "$@";;
   "-d" | "debug"                  ) debugFunc "$@";;
   "-f" | "flush"                  ) flushFunc "$@";;
   "-up" | "updatePihole"          ) updatePiholeFunc "$@";;
-  "-r"  | "reconfigure"           ) reconfigurePiholeFunc;;
+  "-r"  | "repair"                ) repairPiholeFunc "$@";;
   "-g" | "updateGravity"          ) updateGravityFunc "$@";;
   "-l" | "logging"                ) piholeLogging "$@";;
   "uninstall"                     ) uninstallFunc;;
-  "enable"                        ) piholeEnable true "$2";;
-  "disable"                       ) piholeEnable false "$2";;
   "reloaddns"                     ) reloadDNS "reload";;
   "reloadlists"                   ) reloadDNS "reload-lists";;
   "setpassword"                   ) SetWebPassword "$@";;
   "checkout"                      ) piholeCheckoutFunc "$@";;
   "updatechecker"                 ) shift; updateCheckFunc "$@";;
-  "arpflush"                      ) arpFunc "$@";;
+  "arpflush"                      ) arpFunc "$@";; # Deprecated, use networkflush instead
+  "networkflush"                  ) networkFlush "$@";;
   "-t" | "tail"                   ) tailFunc "$2";;
-  "api"                           ) apiFunc "$2";;
+  "logrotate"                     ) logrotateFunc "$@";;
   *                               ) helpFunc;;
 esac

test/_alpine_3_21.Dockerfile

@@ -0,0 +1,18 @@
+FROM alpine:3.21
+
+ENV GITDIR=/etc/.pihole
+ENV SCRIPTDIR=/opt/pihole
+RUN sed -i 's/#\(.*\/community\)/\1/' /etc/apk/repositories
+RUN apk --no-cache add bash coreutils curl git jq openrc shadow
+
+RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole
+ADD . $GITDIR
+RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $GITDIR/advanced/Scripts/COL_TABLE $SCRIPTDIR/
+ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR
+
+RUN true && \
+    chmod +x $SCRIPTDIR/*
+
+ENV SKIP_INSTALL=true
+
+#sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \

test/_alpine_3_22.Dockerfile

@@ -0,0 +1,18 @@
+FROM alpine:3.22
+
+ENV GITDIR=/etc/.pihole
+ENV SCRIPTDIR=/opt/pihole
+RUN sed -i 's/#\(.*\/community\)/\1/' /etc/apk/repositories
+RUN apk --no-cache add bash coreutils curl git jq openrc shadow
+
+RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole
+ADD . $GITDIR
+RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $GITDIR/advanced/Scripts/COL_TABLE $SCRIPTDIR/
+ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR
+
+RUN true && \
+    chmod +x $SCRIPTDIR/*
+
+ENV SKIP_INSTALL=true
+
+#sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \

test/_alpine_3_23.Dockerfile

@@ -0,0 +1,18 @@
+FROM alpine:3.23
+
+ENV GITDIR=/etc/.pihole
+ENV SCRIPTDIR=/opt/pihole
+RUN sed -i 's/#\(.*\/community\)/\1/' /etc/apk/repositories
+RUN apk --no-cache add bash coreutils curl git jq openrc shadow
+
+RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole
+ADD . $GITDIR
+RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $GITDIR/advanced/Scripts/COL_TABLE $SCRIPTDIR/
+ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR
+
+RUN true && \
+    chmod +x $SCRIPTDIR/*
+
+ENV SKIP_INSTALL=true
+
+#sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \

test/_centos_10.Dockerfile

@@ -0,0 +1,19 @@
+FROM quay.io/centos/centos:stream10
+# Disable SELinux
+RUN echo "SELINUX=disabled" > /etc/selinux/config
+RUN yum install -y --allowerasing curl git initscripts
+
+ENV GITDIR=/etc/.pihole
+ENV SCRIPTDIR=/opt/pihole
+
+RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole
+ADD . $GITDIR
+RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $GITDIR/advanced/Scripts/COL_TABLE $SCRIPTDIR/
+ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR
+
+RUN true && \
+    chmod +x $SCRIPTDIR/*
+
+ENV SKIP_INSTALL=true
+
+#sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \

test/_centos_9.Dockerfile

@@ -15,6 +15,5 @@ RUN true && \
     chmod +x $SCRIPTDIR/*
 
 ENV SKIP_INSTALL=true
-ENV OS_CHECK_DOMAIN_NAME=dev-supportedos.pi-hole.net
 
 #sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \

test/_debian_11.Dockerfile

@@ -12,6 +12,5 @@ RUN true && \
     chmod +x $SCRIPTDIR/*
 
 ENV SKIP_INSTALL=true
-ENV OS_CHECK_DOMAIN_NAME=dev-supportedos.pi-hole.net
 
 #sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \

test/_debian_12.Dockerfile

@@ -12,6 +12,5 @@ RUN true && \
     chmod +x $SCRIPTDIR/*
 
 ENV SKIP_INSTALL=true
-ENV OS_CHECK_DOMAIN_NAME=dev-supportedos.pi-hole.net
 
 #sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \

test/_debian_13.Dockerfile

@@ -0,0 +1,16 @@
+FROM buildpack-deps:trixie-scm
+
+ENV GITDIR=/etc/.pihole
+ENV SCRIPTDIR=/opt/pihole
+
+RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole
+ADD . $GITDIR
+RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $GITDIR/advanced/Scripts/COL_TABLE $SCRIPTDIR/
+ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR
+
+RUN true && \
+    chmod +x $SCRIPTDIR/*
+
+ENV SKIP_INSTALL=true
+
+#sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \

test/_fedora_40.Dockerfile

@@ -13,6 +13,5 @@ RUN true && \
     chmod +x $SCRIPTDIR/*
 
 ENV SKIP_INSTALL=true
-ENV OS_CHECK_DOMAIN_NAME=dev-supportedos.pi-hole.net
 
 #sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \

test/_fedora_41.Dockerfile

@@ -13,6 +13,5 @@ RUN true && \
     chmod +x $SCRIPTDIR/*
 
 ENV SKIP_INSTALL=true
-ENV OS_CHECK_DOMAIN_NAME=dev-supportedos.pi-hole.net
 
 #sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \

test/_fedora_42.Dockerfile

@@ -0,0 +1,17 @@
+FROM fedora:42
+RUN dnf install -y git initscripts
+
+ENV GITDIR=/etc/.pihole
+ENV SCRIPTDIR=/opt/pihole
+
+RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole
+ADD . $GITDIR
+RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $GITDIR/advanced/Scripts/COL_TABLE $SCRIPTDIR/
+ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR
+
+RUN true && \
+    chmod +x $SCRIPTDIR/*
+
+ENV SKIP_INSTALL=true
+
+#sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \

test/_fedora_43.Dockerfile

@@ -0,0 +1,17 @@
+FROM fedora:43
+RUN dnf install -y git initscripts
+
+ENV GITDIR=/etc/.pihole
+ENV SCRIPTDIR=/opt/pihole
+
+RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole
+ADD . $GITDIR
+RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $GITDIR/advanced/Scripts/COL_TABLE $SCRIPTDIR/
+ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR
+
+RUN true && \
+    chmod +x $SCRIPTDIR/*
+
+ENV SKIP_INSTALL=true
+
+#sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \

test/_ubuntu_20.Dockerfile

@@ -12,6 +12,5 @@ RUN true && \
     chmod +x $SCRIPTDIR/*
 
 ENV SKIP_INSTALL=true
-ENV OS_CHECK_DOMAIN_NAME=dev-supportedos.pi-hole.net
 
 #sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \

test/_ubuntu_22.Dockerfile

@@ -13,6 +13,5 @@ RUN true && \
     chmod +x $SCRIPTDIR/*
 
 ENV SKIP_INSTALL=true
-ENV OS_CHECK_DOMAIN_NAME=dev-supportedos.pi-hole.net
 
 #sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \

test/_ubuntu_24.Dockerfile

@@ -13,6 +13,5 @@ RUN true && \
     chmod +x $SCRIPTDIR/*
 
 ENV SKIP_INSTALL=true
-ENV OS_CHECK_DOMAIN_NAME=dev-supportedos.pi-hole.net
 
 #sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \

test/conftest.py

@@ -51,29 +51,19 @@ def mock_command(script, args, container):
     in unit tests
     """
     full_script_path = "/usr/local/bin/{}".format(script)
-    mock_script = dedent(
-        r"""\
+    mock_script = dedent(r"""\
     #!/bin/bash -e
     echo "\$0 \$@" >> /var/log/{script}
-    case "\$1" in""".format(
-            script=script
-        )
-    )
+    case "\$1" in""".format(script=script))
     for k, v in args.items():
-        case = dedent(
-            """
+        case = dedent("""
         {arg})
         echo {res}
         exit {retcode}
-        ;;""".format(
-                arg=k, res=v[0], retcode=v[1]
-            )
-        )
+        ;;""".format(arg=k, res=v[0], retcode=v[1]))
         mock_script += case
-    mock_script += dedent(
-        """
-    esac"""
-    )
+    mock_script += dedent("""
+    esac""")
     container.run(
         """
     cat <<EOF> {script}\n{content}\nEOF
@@ -94,37 +84,23 @@ def mock_command_passthrough(script, args, container):
     """
     orig_script_path = container.check_output("command -v {}".format(script))
     full_script_path = "/usr/local/bin/{}".format(script)
-    mock_script = dedent(
-        r"""\
+    mock_script = dedent(r"""\
     #!/bin/bash -e
     echo "\$0 \$@" >> /var/log/{script}
-    case "\$1" in""".format(
-            script=script
-        )
-    )
+    case "\$1" in""".format(script=script))
     for k, v in args.items():
-        case = dedent(
-            """
+        case = dedent("""
         {arg})
         echo {res}
         exit {retcode}
-        ;;""".format(
-                arg=k, res=v[0], retcode=v[1]
-            )
-        )
+        ;;""".format(arg=k, res=v[0], retcode=v[1]))
         mock_script += case
-    mock_script += dedent(
-        r"""
+    mock_script += dedent(r"""
     *)
     {orig_script_path} "\$@"
-    ;;""".format(
-            orig_script_path=orig_script_path
-        )
-    )
-    mock_script += dedent(
-        """
-    esac"""
-    )
+    ;;""".format(orig_script_path=orig_script_path))
+    mock_script += dedent("""
+    esac""")
     container.run(
         """
     cat <<EOF> {script}\n{content}\nEOF
@@ -141,29 +117,19 @@ def mock_command_run(script, args, container):
     in unit tests
     """
     full_script_path = "/usr/local/bin/{}".format(script)
-    mock_script = dedent(
-        r"""\
+    mock_script = dedent(r"""\
     #!/bin/bash -e
     echo "\$0 \$@" >> /var/log/{script}
-    case "\$1 \$2" in""".format(
-            script=script
-        )
-    )
+    case "\$1 \$2" in""".format(script=script))
     for k, v in args.items():
-        case = dedent(
-            """
+        case = dedent("""
         \"{arg}\")
         echo {res}
         exit {retcode}
-        ;;""".format(
-                arg=k, res=v[0], retcode=v[1]
-            )
-        )
+        ;;""".format(arg=k, res=v[0], retcode=v[1]))
         mock_script += case
-    mock_script += dedent(
-        """
-    esac"""
-    )
+    mock_script += dedent(r"""
+    esac""")
     container.run(
         """
     cat <<EOF> {script}\n{content}\nEOF
@@ -180,29 +146,19 @@ def mock_command_2(script, args, container):
     in unit tests
     """
     full_script_path = "/usr/local/bin/{}".format(script)
-    mock_script = dedent(
-        r"""\
+    mock_script = dedent(r"""\
     #!/bin/bash -e
     echo "\$0 \$@" >> /var/log/{script}
-    case "\$1 \$2" in""".format(
-            script=script
-        )
-    )
+    case "\$1 \$2" in""".format(script=script))
     for k, v in args.items():
-        case = dedent(
-            """
+        case = dedent("""
         \"{arg}\")
         echo \"{res}\"
         exit {retcode}
-        ;;""".format(
-                arg=k, res=v[0], retcode=v[1]
-            )
-        )
+        ;;""".format(arg=k, res=v[0], retcode=v[1]))
         mock_script += case
-    mock_script += dedent(
-        """
-    esac"""
-    )
+    mock_script += dedent(r"""
+    esac""")
     container.run(
         """
     cat <<EOF> {script}\n{content}\nEOF

test/requirements.txt

@@ -1,6 +1,6 @@
-pyyaml == 6.0.2
-pytest == 8.3.4
-pytest-xdist == 3.6.1
-pytest-testinfra == 10.1.1
-tox == 4.24.1
+pyyaml == 6.0.3
+pytest == 9.0.2
+pytest-xdist == 3.8.0
+pytest-testinfra == 10.2.2
+tox == 4.49.1
 pytest-clarity == 1.0.1

test/test_any_automated_install.py

@@ -6,10 +6,8 @@ from .conftest import (
     info_box,
     cross_box,
     mock_command,
-    mock_command_run,
     mock_command_2,
     mock_command_passthrough,
-    run_script,
 )
 
 FTL_BRANCH = "development"
@@ -22,12 +20,11 @@ def test_supported_package_manager(host):
     # break supported package managers
     host.run("rm -rf /usr/bin/apt-get")
     host.run("rm -rf /usr/bin/rpm")
-    package_manager_detect = host.run(
-        """
+    host.run("rm -rf /sbin/apk")
+    package_manager_detect = host.run("""
     source /opt/pihole/basic-install.sh
     package_manager_detect
-    """
-    )
+    """)
     expected_stdout = cross_box + " No supported package manager found"
     assert expected_stdout in package_manager_detect.stdout
     # assert package_manager_detect.rc == 1
@@ -37,13 +34,11 @@ def test_selinux_not_detected(host):
     """
     confirms installer continues when SELinux configuration file does not exist
     """
-    check_selinux = host.run(
-        """
+    check_selinux = host.run("""
     rm -f /etc/selinux/config
     source /opt/pihole/basic-install.sh
     checkSelinux
-    """
-    )
+    """)
     expected_stdout = info_box + " SELinux not detected"
     assert expected_stdout in check_selinux.stdout
     assert check_selinux.rc == 0
@@ -77,26 +72,33 @@ def test_installPihole_fresh_install_readableFiles(host):
         },
         host,
     )
+    mock_command_2(
+        "rc-service",
+        {
+            "rc-service pihole-FTL enable": ("", "0"),
+            "rc-service pihole-FTL restart": ("", "0"),
+            "rc-service pihole-FTL start": ("", "0"),
+            "*": ('echo "rc-service call with $@"', "0"),
+        },
+        host,
+    )
     # try to install man
     host.run("command -v apt-get > /dev/null && apt-get install -qq man")
     host.run("command -v dnf > /dev/null && dnf install -y man")
     host.run("command -v yum > /dev/null && yum install -y man")
+    host.run("command -v apk > /dev/null && apk add mandoc man-pages")
     # Workaround to get FTLv6 installed until it reaches master branch
     host.run('echo "' + FTL_BRANCH + '" > /etc/pihole/ftlbranch')
-    install = host.run(
-        """
+    install = host.run("""
     export TERM=xterm
     export DEBIAN_FRONTEND=noninteractive
     umask 0027
     runUnattended=true
-    useUpdateVars=true
     source /opt/pihole/basic-install.sh > /dev/null
     runUnattended=true
-    useUpdateVars=true
     main
     /opt/pihole/pihole-FTL-prestart.sh
-    """
-    )
+    """)
     assert 0 == install.rc
     maninstalled = True
     if (info_box + " man not installed") in install.stdout:
@@ -105,7 +107,7 @@ def test_installPihole_fresh_install_readableFiles(host):
         maninstalled = False
     piholeuser = "pihole"
     exit_status_success = 0
-    test_cmd = 'su --shell /bin/bash --command "test -{0} {1}" -p {2}'
+    test_cmd = 'su -s /bin/bash -c "test -{0} {1}" -p {2}'
     # check files in /etc/pihole for read, write and execute permission
     check_etc = test_cmd.format("r", "/etc/pihole", piholeuser)
     actual_rc = host.run(check_etc).rc
@@ -127,10 +129,6 @@ def test_installPihole_fresh_install_readableFiles(host):
     check_localversion = test_cmd.format("r", "/etc/pihole/versions", piholeuser)
     actual_rc = host.run(check_localversion).rc
     assert exit_status_success == actual_rc
-    # readable logrotate
-    check_logrotate = test_cmd.format("r", "/etc/pihole/logrotate", piholeuser)
-    actual_rc = host.run(check_logrotate).rc
-    assert exit_status_success == actual_rc
     # readable macvendor.db
     check_macvendor = test_cmd.format("r", "/etc/pihole/macvendor.db", piholeuser)
     actual_rc = host.run(check_macvendor).rc
@@ -156,12 +154,6 @@ def test_installPihole_fresh_install_readableFiles(host):
         check_man = test_cmd.format("r", "/usr/local/share/man/man8", piholeuser)
         actual_rc = host.run(check_man).rc
         assert exit_status_success == actual_rc
-        check_man = test_cmd.format("x", "/usr/local/share/man/man5", piholeuser)
-        actual_rc = host.run(check_man).rc
-        assert exit_status_success == actual_rc
-        check_man = test_cmd.format("r", "/usr/local/share/man/man5", piholeuser)
-        actual_rc = host.run(check_man).rc
-        assert exit_status_success == actual_rc
         check_man = test_cmd.format(
             "r", "/usr/local/share/man/man8/pihole.8", piholeuser
         )
@@ -195,13 +187,11 @@ def test_update_package_cache_success_no_errors(host):
     """
     confirms package cache was updated without any errors
     """
-    updateCache = host.run(
-        """
+    updateCache = host.run("""
     source /opt/pihole/basic-install.sh
     package_manager_detect
     update_package_cache
-    """
-    )
+    """)
     expected_stdout = tick_box + " Update local cache of available packages"
     assert expected_stdout in updateCache.stdout
     assert "error" not in updateCache.stdout.lower()
@@ -212,13 +202,11 @@ def test_update_package_cache_failure_no_errors(host):
     confirms package cache was not updated
     """
     mock_command("apt-get", {"update": ("", "1")}, host)
-    updateCache = host.run(
-        """
+    updateCache = host.run("""
     source /opt/pihole/basic-install.sh
     package_manager_detect
     update_package_cache
-    """
-    )
+    """)
     expected_stdout = cross_box + " Update local cache of available packages"
     assert expected_stdout in updateCache.stdout
     assert "Error: Unable to update package cache." in updateCache.stdout
@@ -249,20 +237,19 @@ def test_FTL_detect_no_errors(host, arch, detected_string, supported):
         {
             "-A /bin/sh": ("Tag_CPU_arch: " + arch, "0"),
             "-A /usr/bin/sh": ("Tag_CPU_arch: " + arch, "0"),
+            "-A /usr/sbin/sh": ("Tag_CPU_arch: " + arch, "0"),
         },
         host,
     )
     host.run('echo "' + FTL_BRANCH + '" > /etc/pihole/ftlbranch')
-    detectPlatform = host.run(
-        """
+    detectPlatform = host.run("""
     source /opt/pihole/basic-install.sh
     create_pihole_user
     funcOutput=$(get_binary_name)
     binary="pihole-FTL${funcOutput##*pihole-FTL}"
     theRest="${funcOutput%pihole-FTL*}"
     FTLdetect "${binary}" "${theRest}"
-    """
-    )
+    """)
     if supported:
         expected_stdout = info_box + " FTL Checks..."
         assert expected_stdout in detectPlatform.stdout
@@ -282,22 +269,18 @@ def test_FTL_development_binary_installed_and_responsive_no_errors(host):
     confirms FTL development binary is copied and functional in installed location
     """
     host.run('echo "' + FTL_BRANCH + '" > /etc/pihole/ftlbranch')
-    host.run(
-        """
+    host.run("""
     source /opt/pihole/basic-install.sh
     create_pihole_user
     funcOutput=$(get_binary_name)
     binary="pihole-FTL${funcOutput##*pihole-FTL}"
     theRest="${funcOutput%pihole-FTL*}"
     FTLdetect "${binary}" "${theRest}"
-    """
-    )
-    version_check = host.run(
-        """
+    """)
+    version_check = host.run("""
     VERSION=$(pihole-FTL version)
     echo ${VERSION:0:1}
-    """
-    )
+    """)
     expected_stdout = "v"
     assert expected_stdout in version_check.stdout
 
@@ -312,12 +295,10 @@ def test_IPv6_only_link_local(host):
         {"-6 address": ("inet6 fe80::d210:52fa:fe00:7ad7/64 scope link", "0")},
         host,
     )
-    detectPlatform = host.run(
-        """
+    detectPlatform = host.run("""
     source /opt/pihole/basic-install.sh
     find_IPv6_information
-    """
-    )
+    """)
     expected_stdout = "Unable to find IPv6 ULA/GUA address"
     assert expected_stdout in detectPlatform.stdout
 
@@ -337,12 +318,10 @@ def test_IPv6_only_ULA(host):
         },
         host,
     )
-    detectPlatform = host.run(
-        """
+    detectPlatform = host.run("""
     source /opt/pihole/basic-install.sh
     find_IPv6_information
-    """
-    )
+    """)
     expected_stdout = "Found IPv6 ULA address"
     assert expected_stdout in detectPlatform.stdout
 
@@ -362,12 +341,10 @@ def test_IPv6_only_GUA(host):
         },
         host,
     )
-    detectPlatform = host.run(
-        """
+    detectPlatform = host.run("""
     source /opt/pihole/basic-install.sh
     find_IPv6_information
-    """
-    )
+    """)
     expected_stdout = "Found IPv6 GUA address"
     assert expected_stdout in detectPlatform.stdout
 
@@ -388,12 +365,10 @@ def test_IPv6_GUA_ULA_test(host):
         },
         host,
     )
-    detectPlatform = host.run(
-        """
+    detectPlatform = host.run("""
     source /opt/pihole/basic-install.sh
     find_IPv6_information
-    """
-    )
+    """)
     expected_stdout = "Found IPv6 ULA address"
     assert expected_stdout in detectPlatform.stdout
 
@@ -414,12 +389,10 @@ def test_IPv6_ULA_GUA_test(host):
         },
         host,
     )
-    detectPlatform = host.run(
-        """
+    detectPlatform = host.run("""
     source /opt/pihole/basic-install.sh
     find_IPv6_information
-    """
-    )
+    """)
     expected_stdout = "Found IPv6 ULA address"
     assert expected_stdout in detectPlatform.stdout
 
@@ -430,14 +403,10 @@ def test_validate_ip(host):
     """
 
     def test_address(addr, success=True):
-        output = host.run(
-            """
+        output = host.run("""
         source /opt/pihole/basic-install.sh
         valid_ip "{addr}"
-        """.format(
-                addr=addr
-            )
-        )
+        """.format(addr=addr))
 
         assert output.rc == 0 if success else 1
 
@@ -469,61 +438,16 @@ def test_validate_ip(host):
     test_address("0.0.0.0#00001", False)
 
 
-def test_os_check_fails(host):
-    """Confirms install fails on unsupported OS"""
-    host.run(
-        """
-    source /opt/pihole/basic-install.sh
-    package_manager_detect
-    build_dependency_package
-    install_dependent_packages
-    cat <<EOT > /etc/os-release
-ID=UnsupportedOS
-VERSION_ID="2"
-EOT
-    """
-    )
-    detectOS = host.run(
-        """t
-    source /opt/pihole/basic-install.sh
-    os_check
-    """
-    )
-    expected_stdout = "Unsupported OS detected: UnsupportedOS"
-    assert expected_stdout in detectOS.stdout
-
-
-def test_os_check_passes(host):
-    """Confirms OS meets the requirements"""
-    host.run(
-        """
-    source /opt/pihole/basic-install.sh
-    package_manager_detect
-    build_dependency_package
-    install_dependent_packages
-    """
-    )
-    detectOS = host.run(
-        """
-    source /opt/pihole/basic-install.sh
-    os_check
-    """
-    )
-    expected_stdout = "Supported OS detected"
-    assert expected_stdout in detectOS.stdout
-
-
 def test_package_manager_has_pihole_deps(host):
     """Confirms OS is able to install the required packages for Pi-hole"""
     mock_command("dialog", {"*": ("", "0")}, host)
-    output = host.run(
-        """
+    output = host.run("""
     source /opt/pihole/basic-install.sh
     package_manager_detect
+    update_package_cache
     build_dependency_package
     install_dependent_packages
-    """
-    )
+    """)
 
     assert "No package" not in output.stdout
     assert output.rc == 0
@@ -532,20 +456,17 @@ def test_package_manager_has_pihole_deps(host):
 def test_meta_package_uninstall(host):
     """Confirms OS is able to install and uninstall the Pi-hole meta package"""
     mock_command("dialog", {"*": ("", "0")}, host)
-    install = host.run(
-        """
+    install = host.run("""
     source /opt/pihole/basic-install.sh
     package_manager_detect
+    update_package_cache
     build_dependency_package
     install_dependent_packages
-    """
-    )
+    """)
     assert install.rc == 0
 
-    uninstall = host.run(
-        """
+    uninstall = host.run("""
     source /opt/pihole/uninstall.sh
     removeMetaPackage
-    """
-    )
+    """)
     assert uninstall.rc == 0

test/test_any_utils.py

@@ -1,31 +1,26 @@
 def test_key_val_replacement_works(host):
     """Confirms addOrEditKeyValPair either adds or replaces a key value pair in a given file"""
-    host.run(
-        """
+    host.run("""
     source /opt/pihole/utils.sh
+    touch ./testoutput
     addOrEditKeyValPair "./testoutput" "KEY_ONE" "value1"
     addOrEditKeyValPair "./testoutput" "KEY_TWO" "value2"
     addOrEditKeyValPair "./testoutput" "KEY_ONE" "value3"
     addOrEditKeyValPair "./testoutput" "KEY_FOUR" "value4"
-    """
-    )
-    output = host.run(
-        """
+    """)
+    output = host.run("""
     cat ./testoutput
-    """
-    )
+    """)
     expected_stdout = "KEY_ONE=value3\nKEY_TWO=value2\nKEY_FOUR=value4\n"
     assert expected_stdout == output.stdout
 
 
 def test_getFTLPID_default(host):
     """Confirms getFTLPID returns the default value if FTL is not running"""
-    output = host.run(
-        """
+    output = host.run("""
     source /opt/pihole/utils.sh
     getFTLPID
-    """
-    )
+    """)
     expected_stdout = "-1\n"
     assert expected_stdout == output.stdout
 
@@ -36,8 +31,7 @@ def test_setFTLConfigValue_getFTLConfigValue(host):
     Requires FTL to be installed, so we do that first
     (taken from test_FTL_development_binary_installed_and_responsive_no_errors)
     """
-    host.run(
-        """
+    host.run("""
     source /opt/pihole/basic-install.sh
     create_pihole_user
     funcOutput=$(get_binary_name)
@@ -45,15 +39,12 @@ def test_setFTLConfigValue_getFTLConfigValue(host):
     binary="pihole-FTL${funcOutput##*pihole-FTL}"
     theRest="${funcOutput%pihole-FTL*}"
     FTLdetect "${binary}" "${theRest}"
-    """
-    )
+    """)
 
-    output = host.run(
-        """
+    output = host.run("""
     source /opt/pihole/utils.sh
     setFTLConfigValue "dns.upstreams" '["9.9.9.9"]' > /dev/null
     getFTLConfigValue "dns.upstreams"
-    """
-    )
+    """)
 
     assert "[ 9.9.9.9 ]" in output.stdout

test/test_centos_fedora_common_support.py

@@ -15,14 +15,10 @@ def mock_selinux_config(state, host):
     # getenforce returns the running state of SELinux
     mock_command("getenforce", {"*": (state.capitalize(), "0")}, host)
     # create mock configuration with desired content
-    host.run(
-        """
+    host.run("""
     mkdir /etc/selinux
     echo "SELINUX={state}" > /etc/selinux/config
-    """.format(
-            state=state.lower()
-        )
-    )
+    """.format(state=state.lower()))
 
 
 def test_selinux_enforcing_exit(host):
@@ -30,12 +26,10 @@ def test_selinux_enforcing_exit(host):
     confirms installer prompts to exit when SELinux is Enforcing by default
     """
     mock_selinux_config("enforcing", host)
-    check_selinux = host.run(
-        """
+    check_selinux = host.run("""
     source /opt/pihole/basic-install.sh
     checkSelinux
-    """
-    )
+    """)
     expected_stdout = cross_box + " Current SELinux: enforcing"
     assert expected_stdout in check_selinux.stdout
     expected_stdout = "SELinux Enforcing detected, exiting installer"
@@ -48,12 +42,10 @@ def test_selinux_permissive(host):
     confirms installer continues when SELinux is Permissive
     """
     mock_selinux_config("permissive", host)
-    check_selinux = host.run(
-        """
+    check_selinux = host.run("""
     source /opt/pihole/basic-install.sh
     checkSelinux
-    """
-    )
+    """)
     expected_stdout = tick_box + " Current SELinux: permissive"
     assert expected_stdout in check_selinux.stdout
     assert check_selinux.rc == 0
@@ -64,12 +56,10 @@ def test_selinux_disabled(host):
     confirms installer continues when SELinux is Disabled
     """
     mock_selinux_config("disabled", host)
-    check_selinux = host.run(
-        """
+    check_selinux = host.run("""
     source /opt/pihole/basic-install.sh
     checkSelinux
-    """
-    )
+    """)
     expected_stdout = tick_box + " Current SELinux: disabled"
     assert expected_stdout in check_selinux.stdout
     assert check_selinux.rc == 0

test/tox.alpine_3_21.ini

@@ -0,0 +1,10 @@
+[tox]
+envlist = py3
+
+[testenv:py3]
+allowlist_externals = docker
+deps = -rrequirements.txt
+setenv =
+    COLUMNS=120
+commands = docker buildx build --load --progress plain -f _alpine_3_21.Dockerfile -t pytest_pihole:test_container ../
+           pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py

test/tox.alpine_3_22.ini

@@ -0,0 +1,10 @@
+[tox]
+envlist = py3
+
+[testenv:py3]
+allowlist_externals = docker
+deps = -rrequirements.txt
+setenv =
+    COLUMNS=120
+commands = docker buildx build --load --progress plain -f _alpine_3_22.Dockerfile -t pytest_pihole:test_container ../
+           pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py

test/tox.alpine_3_23.ini

@@ -0,0 +1,10 @@
+[tox]
+envlist = py3
+
+[testenv:py3]
+allowlist_externals = docker
+deps = -rrequirements.txt
+setenv =
+    COLUMNS=120
+commands = docker buildx build --load --progress plain -f _alpine_3_23.Dockerfile -t pytest_pihole:test_container ../
+           pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py

test/tox.centos_10.ini

@@ -0,0 +1,10 @@
+[tox]
+envlist = py3
+
+[testenv:py3]
+allowlist_externals = docker
+deps = -rrequirements.txt
+setenv =
+    COLUMNS=120
+commands = docker buildx build --load --progress plain -f _centos_10.Dockerfile -t pytest_pihole:test_container ../
+           pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py

test/tox.debian_13.ini

@@ -0,0 +1,10 @@
+[tox]
+envlist = py3
+
+[testenv:py3]
+allowlist_externals = docker
+deps = -rrequirements.txt
+setenv =
+    COLUMNS=120
+commands = docker buildx build --load --progress plain -f _debian_13.Dockerfile -t pytest_pihole:test_container ../
+           pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py

test/tox.fedora_42.ini

@@ -0,0 +1,10 @@
+[tox]
+envlist = py3
+
+[testenv]
+allowlist_externals = docker
+deps = -rrequirements.txt
+setenv =
+    COLUMNS=120
+commands = docker buildx build --load --progress plain -f _fedora_42.Dockerfile -t pytest_pihole:test_container ../
+           pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py

test/tox.fedora_43.ini

@@ -0,0 +1,10 @@
+[tox]
+envlist = py3
+
+[testenv]
+allowlist_externals = docker
+deps = -rrequirements.txt
+setenv =
+    COLUMNS=120
+commands = docker buildx build --load --progress plain -f _fedora_43.Dockerfile -t pytest_pihole:test_container ../
+           pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py